{ "seed": 20261014, "sources": [ { "task": "turkish_mmlu", "repo": "AYueksel/TurkishMMLU", "revision": "fcbe5bb19af2714186fdd5752671e0748efa0024", "config": "All", "split": "test", "available": 900, "selected": 900, "skipped": {} }, { "task": "belebele_tr", "repo": "facebook/belebele", "revision": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2", "config": "tur_Latn", "split": "test", "available": 900, "selected": 900, "skipped": {} }, { "task": "gsm8k_tr", "repo": "malhajar/gsm8k-tr", "revision": "1046702fc5d8e14330ae90fdfd98735194ffa4ce", "config": "main", "split": "test", "available": 1318, "selected": 200, "skipped": {} }, { "task": "ifeval_tr_subset", "repo": "bezir/IFEval_tr", "revision": "07e8c2cf4ae66a86fba57d389a73cbc10e865d5d", "config": null, "split": "train", "available": 350, "selected": 120, "skipped": { "Unsupported Turkish rule: combination:repeat_prompt": 15, "Unsupported Turkish rule: change_case:capital_word_frequency": 10, "Unsupported Turkish rule: keywords:letter_frequency": 19, "Unsupported Turkish rule: detectable_content:postscript": 18, "Unsupported Turkish rule: length_constraints:number_sentences": 29, "Unsupported Turkish rule: length_constraints:nth_paragraph_first_word": 9, "Unsupported Turkish rule: detectable_format:constrained_response": 12 } } ], "models": [ { "id": "jamba_sft", "repo": "serda-dev/Jamba2-3B-Turkish-SFT-20261013", "revision": "0b22f86c21610f620826067f573aa5ef2821e9cf", "subfolder": "checkpoints/final" }, { "id": "qwen25_3b", "repo": "Qwen/Qwen2.5-3B-Instruct", "revision": "aa8e72537993ba99e69dfaafa59ed015b17504d1" }, { "id": "qwen3_4b_2507", "repo": "Qwen/Qwen3-4B-Instruct-2507", "revision": "cdbee75f17c01a7cc42f958dc650907174af0554" }, { "id": "jamba_cpt", "repo": "serda-dev/Jamba2-3B-Turkish-CPT-1B-20261007", "revision": "8b1a9c812791fa7930e64b9444b41519c023269e", "subfolder": "checkpoints/final" }, { "id": "jamba_original", "repo": "ai21labs/AI21-Jamba2-3B", "revision": "525c6c8e1d9f5bddedfbdc1dbb0ade2df84230c9" } ], "case_sha256": "d68619d919432e6139baa3749558a8705f2bcc381ed1d136e9f87f8584ba6b8e", "scorer_sha256": { "instruction_following_eval/instructions.py": "60e086f5342a03ce8e18b64bbcccf86308f523c08aa826707a562150a52f3edf", "instruction_following_eval/instructions_util.py": "a73797261eee5bf447e279d82a2b700b1bdd3cb1193412dbab1270a85832bc6b", "instruction_following_eval/instructions_registry.py": "ec92d72c264f6d906978613085db262356174300370a3fffe6fefd5969ce9cfc", "instruction_following_eval/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, "protocol": "0-shot choice-letter likelihood; greedy generation; BF16 Transformers SDPA; common fixtures", "gsm8k_new_tokens": 512, "ifeval_new_tokens": 2048, "generation_batch": 16, "ifeval_notice": "Supported deterministic Turkish-safe subset, not the full published leaderboard metric.", "mmlu_notice": "Public AYueksel subset; not the complete 10K paper dataset or a reproduced 5-shot leaderboard.", "speed_notice": "Native BF16 Transformers on one GPU; excludes download/compile warmup; tokens and characters per second reported.", "generation_batch_schedule": "Initial samples batch 4; remaining samples batch 16. Same weights, BF16, greedy decoding and token budgets.", "gsm8k_cot8_new_tokens": 1024, "gsm8k_cot8_shots": 8, "gsm8k_cot8_source": "https://github.com/EleutherAI/lm-evaluation-harness/blob/main/lm_eval/tasks/gsm8k/gsm8k-cot.yaml", "jamba_baseline_quality_batch": 32, "jamba_baseline_batch_notice": "CPT and original quality runs use batch 32; fixed speed protocol remains batch 1 and 4.", "eos_fix": { "affected_models": [ "jamba_cpt", "jamba_original" ], "upstream_generation_eos": 2, "model_and_tokenizer_eos": 519, "correction": "Union of tokenizer, model config and generation config terminal IDs", "scope": "Rerun 1040 generative evaluations; preserve valid MCQ results and common original GPU speed cohort" }, "speed_instance_id": 55841484, "corrected_generation_cohort": { "instance_id": 55857180, "gpu": "RTX 4090", "machine_id": 47283, "public_ipaddr": "95.3.33.46", "scope": "Only original/CPT generation EOS correction; same physical GPU host as common speed cohort", "numeric_notice": "Same BF16 Transformers backend and hardware; batching may cause small numerical differences" } }