Update root model to step 84000

config.json

@@ -5,7 +5,7 @@
   "attention_bias": false,
   "attention_dropout": 0.0,
   "bos_token_id": 1,
-  "dtype": "float32",
+  "dtype": "bfloat16",
   "eos_token_id": 2,
   "head_dim": 64,
   "hidden_act": "silu",
@@ -18,12 +18,13 @@
   "num_attention_heads": 6,
   "num_hidden_layers": 15,
   "num_key_value_heads": 1,
+  "pad_token_id": 2,
   "pretraining_tp": 1,
   "rms_norm_eps": 1e-05,
   "rope_scaling": null,
   "rope_theta": 10000.0,
   "tie_word_embeddings": true,
   "transformers_version": "4.57.6",
-  "use_cache": true,
+  "use_cache": false,
   "vocab_size": 32000
 }

continual_state.json

@@ -0,0 +1,71 @@
+{
+  "global_step": 84000,
+  "sequence_tokens_emitted": 14909685760,
+  "mix": [
+    0.8,
+    0.09,
+    0.05,
+    0.05,
+    0.01
+  ],
+  "fineweb": "HuggingFaceFW/fineweb-edu:default",
+  "ultra_multi": "openbmb/Ultra-FineWeb-L3:Ultra-FineWeb-L3-en-Multi-Style-Synthetic",
+  "ultra_qa": "openbmb/Ultra-FineWeb-L3:Ultra-FineWeb-L3-en-QA-Synthetic",
+  "rewrite": "appvoid/rewrite6",
+  "noprompt": "appvoid/no-prompt-75k",
+  "early_mix": [
+    0.9,
+    0.05,
+    0.03,
+    0.0175,
+    0.0025
+  ],
+  "late_mix": [
+    0.8,
+    0.09,
+    0.05,
+    0.05,
+    0.01
+  ],
+  "formatter_version": 9,
+  "rewrite6_format": "join non-empty instruction/text/output; empty components valid",
+  "noprompt_format": "stored text field exactly; empty rows skipped",
+  "curated_sampling": "shuffled exhaustive cycles without replacement",
+  "curated_progress": {
+    "rewrite_epoch": 0,
+    "rewrite_cursor": 24333,
+    "rewrite_total": 205404,
+    "rewrite_fraction": 0.11846410001752644,
+    "noprompt_epoch": 0,
+    "noprompt_cursor": 4852,
+    "noprompt_total": 75000,
+    "noprompt_fraction": 0.06469333333333334
+  },
+  "batch_history": [
+    {
+      "start_step": 0,
+      "start_tokens": 0,
+      "batch_size": 128,
+      "gradient_accumulation": 1,
+      "tokens_per_update": 131072
+    },
+    {
+      "start_step": 49000,
+      "start_tokens": 6422528000,
+      "batch_size": 128,
+      "gradient_accumulation": 1,
+      "tokens_per_update": 131072
+    },
+    {
+      "start_step": 50000,
+      "start_tokens": 6553600000,
+      "batch_size": 240,
+      "gradient_accumulation": 1,
+      "tokens_per_update": 245760
+    }
+  ],
+  "current_batch_size": 240,
+  "current_gradient_accumulation": 1,
+  "current_tokens_per_update": 245760,
+  "cumulative_tokens_at_save": 14909440000
+}

generation_config.json

@@ -4,5 +4,6 @@
   "eos_token_id": [
     2
   ],
+  "pad_token_id": 2,
   "transformers_version": "4.57.6"
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:04b88033e1475ded1b606254082140ba945ee741465a4048f79714c67a15c125
-size 140637408
+oid sha256:dba05b9fa1a922b062f440b1993e179436c7bf633fdcf78d45a22892684b66c0
+size 70326256

special_tokens_map.json

@@ -13,6 +13,7 @@
     "rstrip": false,
     "single_word": false
   },
+  "pad_token": "</s>",
   "unk_token": {
     "content": "<unk>",
     "lstrip": false,

tokenizer_config.json

@@ -33,8 +33,8 @@
   "eos_token": "</s>",
   "extra_special_tokens": {},
   "legacy": false,
-  "model_max_length": 1000000000000000019884624838656,
-  "pad_token": null,
+  "model_max_length": 1000000000000000000000000000000,
+  "pad_token": "</s>",
   "padding_side": "right",
   "sp_model_kwargs": {},
   "tokenizer_class": "LlamaTokenizer",