picur-tokenizer a2x: 49,152-piece Hungarian Unigram vocabulary

.gitattributes

@@ -1,35 +1,3 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
 *.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
 *.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2026 Lajos Bencz
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

NOTICE

@@ -0,0 +1,48 @@
+picur-tokenizer
+Copyright 2026 Lajos Bencz
+Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+This tokenizer is a derivative work. Its seed lexicon and its morpheme
+segmentation were derived from the two resources below, whose terms require
+attribution and are reproduced here.
+
+--------------------------------------------------------------------------
+morphdb.hu -- Creative Commons Attribution 2.5
+Copyright (c) Nemeth Laszlo, Godo Ferenc
+https://github.com/morphdb/morphdb.hu
+http://creativecommons.org/licenses/by/2.5/
+
+Supplied the declared lexicon: 351,917 headwords, 37,042 compound seams,
+derivational splits, alternation classes and vowel harmony per entry. These
+decided which pieces the fit protects and which it evicts.
+
+--------------------------------------------------------------------------
+hunmorph-foma -- BSD 2-Clause
+Copyright (c) 2016 r0ller. All rights reserved.
+https://github.com/r0ller/hunmorph-foma
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice,
+   this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+The finite-state transducers analysed corpus words into lemma plus inflection
+tags. Those analyses bootstrapped the morpheme segmentation the fit is scored
+against.
+--------------------------------------------------------------------------

README.md

@@ -0,0 +1,155 @@
+---
+language:
+- hu
+license: apache-2.0
+library_name: tokenizers
+tags:
+- picur
+- tokenizer
+- hungarian
+- magyar
+- morphology
+- sentencepiece
+- unigram
+---
+
+# picur-tokenizer
+
+A 49,152-piece SentencePiece Unigram tokenizer for Hungarian,
+fitted so that it segments on real morpheme boundaries,
+rather than on whatever the compression objective prefers.
+
+```python
+from transformers import AutoTokenizer
+
+tok = AutoTokenizer.from_pretrained("picur/picur-tokenizer")
+tok.tokenize("a házakban lakó emberekkel")
+# ['▁a', '▁ház', 'ak', 'ban', '▁lakó', '▁ember', 'ek', 'kel']
+```
+
+## Why another tokenizer
+
+Hungarian is agglutinative: one stem carries a long tail of suffixes, so `ház` (house) surfaces as `házban`, `házakban`, `házaimban`.
+A tokenizer fitted on compression alone shatters that paradigm across unrelated rows,
+and the model then has to learn `ház` several times over - once per spelling of the same word.
+Training effort that should have gone into knowledge goes into re-learning morphology instead.
+
+This vocabulary is fitted to keep a stem on one row and let the suffixes be their own pieces,
+so a paradigm shares its knowledge instead of spreading it.
+
+## How it segments
+
+| word | pieces | |
+|---|---|---|
+| `házakban` | `▁ház` `ak` `ban` | stem + plural + inessive |
+| `könyvekkel` | `▁könyv` `ek` `kel` | stem + plural + instrumental |
+| `emberekkel` | `▁ember` `ek` `kel` | the same two suffixes, the same rows |
+| `magyarokat` | `▁magyar` `ok` `at` | back-vowel harmony of the pair above |
+| `csapadékvíz` | `▁csapadék` `víz` | compound, cut at the declared seam |
+| `művek` | `▁műv` `ek` | v-stem alternation (`mű` -> `műv`) |
+
+Vowel harmony is why `ban`/`ben`, `kel`/`kal` and `hoz`/`hez`/`höz` all exist as separate rows:
+they are the same grammatical function wearing different vowels.
+See [Tied rows](#tied-rows) for what is shipped to help a model learn them as one.
+
+## Measured
+
+Scored on 20,000 held-out documents (1.45M weighted pairs),
+against morpheme segmentations derived from finite-state analyses of the same corpus:
+
+| metric | value | what it means |
+|---|---|---|
+| **morph-f1** | **75.2%** | precision 89.4% / recall 64.9% against verified morpheme boundaries |
+| **stem seam** | **71.7%** | inflected words cut at the stem boundary |
+| **one-piece function** | **69.0%** | grammatical functions that are a single row (99 functions) |
+| **single-token lemma** | **96.7%** | dictionary headwords that survive as one piece |
+| **paradigm spread** | **1.30 rows** | distinct stem rows a paradigm spends; 1.0 is ideal |
+| chars/token | 3.748 | fertility, on Hungarian prose |
+| round-trip failures | 0 | on the full evaluation sample |
+| byte fallback | 0.070% | |
+
+Fertility is reported for reference only.
+It was **not** the objective, and this vocabulary deliberately trades a little of it for grammar:
+the fit that maximised chars/token scored 53.2% morph-f1, versus 75.2% here.
+
+### Special tokens
+
+| id | token | |
+|---|---|---|
+| 0 | `<\|unk\|>` | |
+| 1 | `<\|bos\|>` | one per document |
+| 2 | `<\|eos\|>` | |
+| 3 | `<\|pad\|>` | |
+| 4 | `<\|im_start\|>` | ChatML turn open |
+| 5 | `<\|im_end\|>` | ChatML turn close; the stop token for chat |
+| 6-21 | `<\|reserved_0..15\|>` | unused, for later extension |
+
+Tool-call markers (`<tools>`, `<tool_call>`, `<tool_response>` and their closers) are present as ordinary pieces, following the Hermes convention.
+
+Note the ChatML split: `<|im_end|>` terminates a *turn*, `<|eos|>` terminates a *sequence*. A chat model should stop on `<|im_end|>`.
+
+### Tied rows
+
+`ties.json` maps 929 allomorph rows onto the one row a model should learn first: `ban` -> `ben`, `höz` -> `hoz`, across 101 suffix families.
+Vowel harmony makes these variants of one morpheme,
+so initialising their embeddings from a shared base gives the model the paradigm for free rather than making it discover the relationship from data.
+
+This is an initialisation aid, not part of tokenization, nothing reads it automatically.
+
+## How it was fitted
+
+The vocabulary is overfitted well past its target size and then **selected down**,
+with a declared Hungarian lexicon deciding what survives:
+
+1. **Seed** - 22,361 pieces required up front: 12,000 frequent stems, 4,000 named entities,
+   4,000 capitalised forms, 445 suffixes, 2,011 alternating stems and 448 programming keywords.
+2. **Overfit** - a large Unigram vocabulary is trained on the corpus.
+3. **Evict** - any piece that is a regular inflection of a declared headword is removed. `okat` is not a row; `ok` and `at` are.
+4. **Spare** - an eviction is reversed if the word would then segment worse than its morphology says it should.
+5. **Prune** - rows the corpus never reaches are dropped.
+
+Corpus: 605 Mchar drawn across 21 Hungarian sources:
+encyclopedic, news, legal, parliamentary, academic, literary, subtitles, web and forum.
+Plus small code and mathematics slices, so that technical text does not fall back to bytes.
+
+## Limitations
+
+- **Hungarian-first.** English and code are handled but not optimised; other languages fall back to bytes.
+- **Coverage, not rules, sets the floor.** A stem outside the 12,000-stem seed can still be mis-split:
+  `gyeplőhöz` cuts the `-höz` correctly but splits the stem into `gyep` + `lő`.
+- **Compound seams come from a declared lexicon**, so a compound absent from it gets no guaranteed seam.
+- **Cased entity handling is partial.** 4,000 capitalised forms are seeded; beyond those, capitalisation costs a row.
+- **Two losses are declared, not accidental.** Metaspace writes the word mark as
+  U+2581, so a literal U+2581 in the input is indistinguishable from a space. The
+  tokenizer declares a `Replace(U+2581, " ")` normalizer ahead of Metaspace, which
+  makes the collision part of the contract instead of a surprise - text carrying
+  block-drawing characters (a sparkline, say) round-trips with that one character
+  turned into a space. Leading whitespace on an input is dropped for the same
+  reason: Metaspace prepends a mark unconditionally. Everything else round-trips
+  byte-exactly, including every script outside Latin, via byte fallback.
+
+## Attribution
+
+This tokenizer is a derivative work of two resources whose licences require attribution.
+See [NOTICE](NOTICE).
+
+- **[morphdb.hu](https://github.com/morphdb/morphdb.hu)** - CC-BY 2.5, (c) Nemeth
+  Laszlo and Godo Ferenc. The declared lexicon: 351,917 headwords, 37,042 compound
+  seams, alternation classes and vowel harmony. This decided what the fit protects
+  and what it evicts.
+- **[hunmorph-foma](https://github.com/r0ller/hunmorph-foma)** - BSD-2-Clause,
+  (c) 2016 r0ller. Finite-state transducers that analysed corpus words into lemma
+  plus tags, bootstrapping the segmentation this vocabulary is scored against.
+
+The tokenizer itself is Apache-2.0.
+
+## Citation
+
+```bibtex
+@misc{picur-tokenizer,
+  title  = {picur-tokenizer: a morphology-aware Hungarian tokenizer},
+  author = {Bencz, Lajos},
+  year   = {2026},
+  url    = {https://huggingface.co/picur/picur-tokenizer}
+}
+```

chat_template.jinja

@@ -0,0 +1,42 @@
+{{- bos_token -}}
+{%- if messages[0].role == 'system' -%}
+    {%- set system_message = messages[0].content -%}
+    {%- set messages = messages[1:] -%}
+{%- else -%}
+    {%- set system_message = '' -%}
+{%- endif -%}
+{{- '<|im_start|>system\n' + system_message -}}
+{%- if tools -%}
+    {{- '\n\n# Eszközök\n\nA felhasználói kérés megválaszolásához függvényeket hívhatsz. Az elérhető függvények szignatúrái <tools></tools> XML címkék között:\n<tools>' -}}
+    {%- for tool in tools -%}
+        {%- set fn = tool.function if tool.function is defined else tool -%}
+        {{- '\n' + (fn | tojson(ensure_ascii=False)) -}}
+    {%- endfor -%}
+    {{- '\n</tools>\n\nFüggvényhíváskor adj vissza egy json objektumot a függvény nevével és argumentumaival <tool_call></tool_call> XML címkék között:\n<tool_call>\n{"name": <függvény-név>, "arguments": <argumentum-json>}\n</tool_call>' -}}
+{%- endif -%}
+{{- '<|im_end|>\n' -}}
+{%- for message in messages -%}
+    {%- if message.role == 'user' -%}
+        {{- '<|im_start|>user\n' + message.content + '<|im_end|>\n' -}}
+    {%- elif message.role == 'assistant' -%}
+        {{- '<|im_start|>assistant\n' -}}
+        {%- generation -%}
+        {{- message.content or '' -}}
+        {%- if message.tool_calls -%}
+            {%- for tc in message.tool_calls -%}
+                {%- set fn = tc.function -%}
+                {{- '\n<tool_call>\n{"name": "' + fn.name + '", "arguments": ' -}}
+                {%- if fn.arguments is string -%}{{- fn.arguments -}}{%- else -%}{{- fn.arguments | tojson(ensure_ascii=False) -}}{%- endif -%}
+                {{- '}\n</tool_call>' -}}
+            {%- endfor -%}
+        {%- endif -%}
+        {{- '<|im_end|>' -}}
+        {%- endgeneration -%}
+        {{- '\n' -}}
+    {%- elif message.role == 'tool' -%}
+        {{- '<|im_start|>user\n<tool_response>\n' + message.content + '\n</tool_response><|im_end|>\n' -}}
+    {%- endif -%}
+{%- endfor -%}
+{%- if add_generation_prompt -%}
+    {{- '<|im_start|>assistant\n' -}}
+{%- endif -%}

ties.json

@@ -0,0 +1 @@
+{"430": 402, "835": 290, "927": 1581, "1004": 362, "1425": 1279, "1503": 44722, "1974": 1342, "1996": 1342, "2008": 1545, "2077": 535, "2280": 2267, "2551": 1279, "3225": 844, "3400": 2527, "3655": 393, "3829": 535, "3945": 535, "4159": 1712, "4895": 1015, "4927": 2729, "4947": 1279, "5038": 5003, "5202": 2361, "5223": 8103, "5335": 47222, "5367": 3924, "5490": 4552, "5863": 343, "5958": 1522, "6012": 606, "6185": 4701, "6817": 6144, "6937": 2152, "7312": 6564, "7406": 2729, "7804": 6250, "7837": 1419, "8052": 2928, "8133": 6564, "8245": 3312, "8486": 1279, "8580": 2729, "8820": 8264, "8821": 4571, "9364": 585, "9378": 4166, "9412": 7719, "9490": 6966, "9496": 7846, "9534": 4403, "9622": 17271, "9730": 4445, "9821": 2346, "10343": 2299, "10543": 10734, "10579": 6567, "10646": 12129, "10709": 1678, "10775": 7846, "10801": 5194, "10843": 29767, "10911": 3405, "11056": 11872, "11175": 4535, "11344": 10730, "11476": 18829, "11708": 7846, "11855": 8788, "11879": 1543, "12033": 4992, "12090": 1574, "12146": 6103, "12188": 658, "12218": 5960, "12265": 5852, "12425": 6381, "12511": 4147, "12643": 9530, "12876": 46849, "13170": 17307, "13178": 8019, "13198": 2074, "13499": 7873, "13516": 6525, "13553": 3017, "13620": 10800, "13636": 1166, "13820": 6564, "13941": 1659, "13944": 6824, "14179": 2267, "14335": 10725, "14431": 2611, "14450": 15239, "14468": 11109, "14478": 47133, "14589": 4101, "14652": 10789, "14735": 26904, "14884": 10189, "14890": 13038, "14917": 5347, "14996": 808, "15044": 10057, "15063": 28998, "15093": 11450, "15166": 9459, "15215": 4527, "15351": 12413, "15419": 4044, "15443": 41316, "15677": 8475, "15744": 15231, "16095": 6920, "16100": 18545, "16170": 14172, "16253": 8381, "16349": 1136, "16605": 9530, "16634": 8565, "16924": 4997, "17074": 14473, "17327": 11258, "17398": 47323, "17449": 8610, "17605": 4552, "17685": 11840, "17745": 12820, "17817": 15204, "17831": 6000, "18032": 45720, "18229": 28962, "18354": 2527, "18547": 4445, "18724": 3113, "18747": 25721, "18768": 2486, "18785": 9581, "18851": 17077, "18900": 12820, "19018": 3373, "19046": 1279, "19181": 41899, "19197": 11906, "19230": 24813, "19405": 45400, "19462": 9459, "19491": 13168, "19563": 5467, "19585": 7891, "19692": 8180, "19694": 2718, "19701": 38439, "19753": 23597, "19861": 1185, "19928": 8750, "19988": 42523, "19993": 22005, "20085": 38833, "20201": 2499, "20265": 38628, "20529": 10571, "20565": 8966, "20573": 28759, "20789": 22520, "21101": 9530, "21346": 19410, "21441": 42635, "21538": 1252, "21577": 6712, "21643": 36718, "21913": 31359, "21929": 1545, "22274": 6314, "22424": 39531, "22435": 13861, "22445": 4445, "22473": 9207, "22558": 16083, "22870": 41067, "22915": 5159, "23102": 4717, "23856": 10535, "24000": 47515, "24083": 17409, "24293": 5224, "24478": 4106, "24628": 1522, "24654": 5685, "24678": 31387, "24717": 3924, "24775": 21173, "24800": 42886, "24904": 39411, "24956": 42504, "25053": 36956, "25080": 3924, "25139": 23395, "25205": 15955, "25261": 16858, "25278": 36401, "25305": 43253, "25352": 7752, "25438": 5960, "25472": 17287, "25525": 11408, "25660": 18300, "25690": 38099, "26177": 7773, "26245": 6050, "27000": 42247, "27058": 16139, "27177": 40143, "27192": 30614, "27483": 4106, "27494": 30598, "27544": 5409, "27701": 14555, "27779": 3737, "28098": 16131, "28105": 6805, "28297": 23990, "28334": 26733, "28404": 11880, "28847": 32117, "29077": 25372, "29099": 46385, "29587": 5413, "29736": 31148, "29920": 39326, "30045": 39863, "30059": 39579, "30064": 47321, "30340": 932, "30403": 28383, "30615": 39847, "30648": 27727, "30656": 46200, "30679": 12820, "30725": 40180, "30940": 26350, "31085": 38862, "31141": 6966, "31172": 42275, "31334": 1767, "31620": 48855, "32080": 10461, "32246": 32663, "32294": 4195, "32918": 43409, "33084": 31675, "33432": 3926, "33557": 48370, "33614": 44437, "33831": 29617, "33890": 12201, "33926": 2729, "33938": 41917, "34159": 9906, "34363": 38868, "34678": 33694, "34810": 13858, "34907": 36874, "35873": 24448, "35919": 45870, "35990": 20252, "36319": 46865, "36345": 46367, "36356": 40085, "36389": 48244, "36527": 45185, "36621": 22534, "36865": 38413, "37048": 39669, "37111": 44715, "37191": 4106, "37217": 12532, "37649": 46197, "37688": 36671, "37802": 39395, "37916": 48199, "38080": 47122, "38086": 29984, "38093": 48167, "38105": 46323, "38154": 22002, "38160": 38307, "38164": 32719, "38185": 6811, "38212": 39677, "38222": 1521, "38229": 27012, "38237": 23005, "38306": 13781, "38464": 20285, "38476": 3651, "38524": 38987, "38548": 46377, "38552": 42470, "38573": 43925, "38588": 46837, "38612": 47618, "38680": 10145, "38683": 43894, "38719": 40925, "38724": 37525, "38729": 39776, "38805": 46285, "38820": 34323, "38823": 4246, "38854": 1965, "38864": 38645, "38896": 49062, "38937": 16390, "38943": 46927, "38954": 3583, "38989": 39484, "38990": 20205, "39034": 28157, "39036": 22898, "39042": 3412, "39117": 14934, "39131": 47688, "39206": 1522, "39221": 39156, "39224": 46762, "39246": 32842, "39249": 5395, "39251": 40184, "39262": 39667, "39263": 48161, "39273": 12191, "39289": 18864, "39297": 33634, "39317": 46840, "39319": 40122, "39374": 24552, "39396": 44748, "39404": 16168, "39441": 26021, "39469": 22766, "39470": 11159, "39517": 12463, "39523": 6737, "39542": 13822, "39553": 48365, "39563": 3736, "39567": 22284, "39592": 17898, "39621": 12820, "39634": 20852, "39657": 39513, "39706": 20673, "39709": 28022, "39757": 18880, "39802": 38659, "39810": 777, "39841": 42209, "39844": 8777, "39872": 25924, "39915": 46607, "39946": 2984, "39952": 39164, "39961": 47047, "39970": 39261, "39971": 32808, "39985": 7977, "40014": 39195, "40032": 39687, "40033": 29367, "40046": 39641, "40048": 2453, "40050": 30861, "40076": 37866, "40099": 36227, "40101": 46981, "40104": 31860, "40118": 33695, "40121": 2593, "40174": 30063, "40189": 2460, "40219": 2496, "40229": 33959, "40253": 1536, "40334": 4030, "40342": 5741, "40422": 2547, "40425": 1808, "40466": 18276, "40475": 2949, "40502": 1314, "40512": 704, "40527": 8951, "40539": 5716, "40560": 4106, "40574": 10408, "40602": 1185, "40612": 4571, "40657": 4778, "40661": 7399, "40662": 2984, "40747": 16254, "40770": 8543, "40787": 1406, "40795": 9315, "40802": 4702, "40825": 9932, "40870": 10134, "40881": 1181, "40950": 20620, "40966": 1149, "40980": 6801, "40989": 5476, "41032": 44997, "41040": 12621, "41047": 8289, "41059": 6878, "41090": 5718, "41115": 9738, "41153": 27162, "41169": 11156, "41196": 3759, "41200": 16855, "41220": 47828, "41310": 3019, "41327": 16195, "41330": 16539, "41342": 40330, "41360": 3264, "41396": 18348, "41402": 23093, "41411": 4492, "41424": 3036, "41431": 5013, "41448": 15096, "41488": 22741, "41499": 3756, "41548": 24823, "41610": 42770, "41628": 16826, "41631": 14516, "41664": 2118, "41729": 42100, "41730": 1836, "41735": 4899, "41751": 30352, "41754": 3671, "41761": 13160, "41795": 40634, "41801": 21466, "41878": 5135, "41895": 30829, "41925": 16241, "41968": 38415, "41979": 38354, "41985": 42778, "41986": 41822, "41990": 43366, "41994": 2267, "42028": 26478, "42035": 47359, "42042": 10235, "42043": 1661, "42061": 39736, "42114": 44092, "42129": 17877, "42141": 43507, "42146": 42122, "42156": 38327, "42164": 4552, "42201": 43150, "42213": 5403, "42214": 6417, "42228": 23649, "42233": 465, "42240": 43724, "42260": 41629, "42280": 13617, "42296": 42090, "42334": 43427, "42361": 14124, "42382": 27677, "42415": 42210, "42420": 42235, "42445": 14722, "42454": 47788, "42482": 42087, "42492": 17483, "42500": 13624, "42507": 48126, "42537": 20223, "42544": 47851, "42552": 26184, "42557": 19043, "42563": 33575, "42569": 6907, "42629": 13289, "42678": 31091, "42707": 15866, "42735": 29586, "42751": 32172, "42756": 41844, "42759": 48130, "42793": 18861, "42808": 40228, "42841": 22676, "42858": 47749, "42859": 7514, "42862": 13304, "42867": 16658, "42875": 4805, "42894": 32584, "42907": 38779, "42919": 34095, "42965": 10862, "43020": 17077, "43039": 15220, "43046": 42704, "43083": 23657, "43117": 20602, "43256": 577, "43257": 25951, "43286": 21521, "43287": 848, "43303": 667, "43341": 13868, "43351": 591, "43445": 44436, "43451": 21170, "43469": 47497, "43477": 43721, "43502": 12544, "43513": 38563, "43552": 42685, "43557": 10464, "43580": 41806, "43603": 41857, "43631": 34331, "43640": 5852, "43642": 41479, "43657": 6144, "43717": 40149, "43759": 43196, "43762": 25647, "43776": 41576, "43787": 41126, "43807": 8501, "43808": 9530, "43828": 2129, "43873": 3765, "43877": 19412, "43911": 23124, "43923": 48398, "43945": 21410, "43957": 15374, "43968": 49042, "43984": 4431, "43987": 18536, "43988": 42952, "43991": 39323, "44015": 30003, "44038": 14320, "44061": 9840, "44064": 40130, "44071": 6966, "44073": 37963, "44146": 44351, "44163": 42180, "44167": 34014, "44179": 48262, "44233": 934, "44235": 26722, "44254": 24192, "44318": 8312, "44324": 20590, "44353": 39335, "44408": 3564, "44415": 6966, "44432": 30167, "44482": 18536, "44505": 41184, "44507": 32754, "44530": 473, "44534": 5852, "44558": 44729, "44560": 13137, "44571": 44744, "44626": 44724, "44628": 28520, "44673": 2527, "44678": 41970, "44680": 4201, "44691": 8279, "44703": 1085, "44707": 35703, "44708": 2256, "44710": 1844, "44714": 7157, "44725": 1545, "44755": 3540, "44757": 11078, "44759": 3872, "44780": 1417, "44808": 1659, "44825": 44930, "44830": 7597, "44835": 2815, "44842": 26973, "44851": 46117, "44865": 16718, "44866": 33071, "44875": 3396, "44895": 45864, "44900": 7612, "44926": 41312, "44941": 3736, "44945": 12504, "44954": 1407, "44955": 11372, "44961": 20630, "45023": 32023, "45107": 1870, "45109": 17992, "45115": 42337, "45132": 13118, "45134": 13788, "45145": 1723, "45160": 21622, "45163": 2591, "45164": 17316, "45171": 1674, "45181": 40384, "45189": 22153, "45192": 31240, "45210": 45547, "45216": 7534, "45222": 9780, "45231": 40655, "45232": 20118, "45241": 43056, "45253": 34756, "45256": 20898, "45273": 531, "45274": 19808, "45280": 2971, "45303": 13310, "45311": 7933, "45329": 45528, "45335": 5179, "45338": 35460, "45343": 6718, "45358": 23990, "45359": 26722, "45361": 22096, "45442": 1500, "45463": 45258, "45469": 13451, "45472": 12210, "45483": 33134, "45490": 39817, "45498": 4244, "45514": 23857, "45515": 4721, "45529": 33443, "45534": 5479, "45555": 23990, "45563": 44489, "45668": 26171, "45671": 3717, "45672": 20256, "45702": 1718, "45706": 45495, "45717": 41948, "45733": 15492, "45752": 39826, "45768": 18822, "45792": 42788, "45793": 15931, "45813": 26557, "45818": 33543, "45822": 45518, "45836": 11627, "45840": 39032, "45848": 39074, "45874": 41753, "45897": 1201, "45904": 7693, "45967": 45726, "45968": 8282, "45974": 42744, "45975": 1068, "45985": 4753, "46000": 46369, "46009": 3477, "46015": 3422, "46016": 45461, "46049": 20135, "46069": 4599, "46071": 13420, "46120": 39557, "46147": 44166, "46170": 46081, "46183": 17057, "46210": 14143, "46226": 45519, "46284": 42823, "46293": 39649, "46302": 40039, "46305": 46018, "46309": 38273, "46328": 38835, "46336": 1268, "46340": 41765, "46352": 40626, "46363": 46086, "46372": 29699, "46389": 39804, "46401": 23954, "46419": 45875, "46454": 39278, "46458": 26389, "46460": 11732, "46464": 39610, "46476": 7306, "46490": 45973, "46511": 8894, "46524": 2910, "46540": 41355, "46562": 21708, "46563": 45769, "46565": 39318, "46587": 47696, "46591": 7240, "46602": 21226, "46647": 28269, "46668": 18976, "46696": 45920, "46708": 33943, "46717": 13730, "46728": 46265, "46766": 47198, "46772": 13237, "46779": 5168, "46782": 44660, "46786": 15910, "46803": 28111, "46811": 26002, "46816": 12346, "46833": 16540, "46852": 44554, "46864": 25567, "46870": 17183, "46873": 44728, "46885": 9861, "46908": 46455, "46923": 46942, "46952": 15815, "46977": 8128, "46984": 47456, "46990": 38678, "47010": 43719, "47011": 47112, "47018": 47346, "47032": 3883, "47062": 11095, "47064": 39070, "47068": 11347, "47075": 19544, "47081": 40465, "47088": 38511, "47097": 42788, "47130": 10622, "47165": 16943, "47171": 14856, "47172": 42029, "47193": 5187, "47205": 6502, "47225": 22585, "47228": 45996, "47235": 11246, "47265": 1435, "47340": 39643, "47354": 2126, "47408": 38486, "47416": 30367, "47466": 38551, "47470": 6856, "47478": 5953, "47488": 38194, "47495": 13407, "47496": 9230, "47510": 47102, "47520": 16336, "47524": 12633, "47552": 12519, "47572": 47154, "47597": 46060, "47619": 2622, "47637": 44304, "47645": 25081, "47662": 2793, "47668": 2182, "47678": 41295, "47709": 6292, "47732": 31699, "47780": 21986, "47785": 46048, "47806": 39381, "47858": 46543, "47895": 10396, "47896": 47781, "47942": 39373, "47957": 46233, "47960": 19675, "47961": 7104, "47964": 40956, "47965": 3820, "47985": 46730, "47988": 18260, "47992": 42895, "47994": 46842, "47995": 30687, "48000": 27543, "48002": 39244, "48003": 18625, "48006": 46689, "48012": 44144, "48013": 16116, "48021": 37071, "48034": 20964, "48059": 46742, "48060": 9358, "48079": 14302, "48159": 34419, "48194": 42831, "48195": 47607, "48196": 7021, "48212": 25028, "48220": 5375, "48232": 40906, "48250": 16271, "48265": 26804, "48281": 16043, "48284": 7704, "48298": 30002, "48304": 43335, "48309": 6889, "48319": 5506, "48320": 38752, "48331": 23052, "48332": 47291, "48349": 18860, "48355": 13843, "48375": 14585, "48382": 47224, "48408": 27464, "48413": 47657, "48427": 6198, "48434": 47556, "48438": 22993, "48457": 9731, "48464": 15548, "48475": 46376, "48482": 48494, "48510": 40549, "48524": 42016, "48525": 35430, "48534": 26240, "48574": 15580, "48576": 47870, "48580": 39850, "48582": 30047, "48585": 46625, "48589": 6486, "48590": 28496, "48592": 47832, "48603": 45554, "48611": 37892, "48617": 34829, "48618": 43793, "48632": 37886, "48651": 38906, "48658": 48411, "48682": 37383, "48684": 10032, "48685": 39957, "48691": 7949, "48710": 46042, "48717": 47210, "48725": 6735, "48734": 3079, "48760": 47948, "48782": 38685, "48784": 10876, "48785": 36752, "48787": 31755, "48826": 41570, "48831": 26431, "48834": 16911, "48835": 46822, "48842": 41849, "48848": 48663, "48851": 18941, "48860": 47846, "48866": 13343, "48902": 15983, "48915": 47899, "48946": 39490, "48954": 38651, "48972": 39165, "48973": 48674, "48977": 32587, "48991": 39616, "48997": 36138, "49041": 29236, "49045": 36767, "49083": 34811, "49089": 15962, "49091": 43707, "49092": 43252, "49102": 35423, "49108": 49043, "49110": 38994, "49112": 37881, "49119": 48924, "49124": 48985, "49128": 24255, "49142": 38499, "49145": 46010}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f3860bc6db18cd0c59539b006748bc010b221cec939293e1a7c07942451e40fd
+size 994909

tokenizer_config.json

@@ -0,0 +1,35 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|bos|>",
+  "eos_token": "<|eos|>",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<tools>",
+    "</tools>",
+    "<tool_call>",
+    "</tool_call>",
+    "<tool_response>",
+    "</tool_response>",
+    "<|reserved_0|>",
+    "<|reserved_1|>",
+    "<|reserved_2|>",
+    "<|reserved_3|>",
+    "<|reserved_4|>",
+    "<|reserved_5|>",
+    "<|reserved_6|>",
+    "<|reserved_7|>",
+    "<|reserved_8|>",
+    "<|reserved_9|>",
+    "<|reserved_10|>",
+    "<|reserved_11|>",
+    "<|reserved_12|>",
+    "<|reserved_13|>",
+    "<|reserved_14|>",
+    "<|reserved_15|>"
+  ],
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|pad|>",
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "<|unk|>"
+}