Pre release update.

README.md

@@ -17,26 +17,63 @@ Granite Docling 258M builds upon the IDEFICS3 architecture, but introduces two k
 Granite-docling-258M is fully integrated into the Docling pipelines, carrying over existing [features](https://huggingface.co/ds4sd/SmolDocling-256M-preview) while introducing a number of powerful new features, including:
 
 - 🔢 Enhanced Equation Recognition: More accurate detection and formatting of mathematical formulas
-- 🧮 Enhanceed Inline Equations: Better inline math recognition
-- 📸 Robust OCR for Documents in the wild: Accurately extracts text from handheld scans, photos, and low-quality images
-- 🗝️ Key-Value Pair Extraction: Identifies structured key-value relationships (e.g., forms, receipts)
+- 🧩 Flexible Inference Modes: Choose between full-page inference, bbox-guided region inference
 - 🧘 Improved Stability: Tends to avoid infinite loops more effectively
+- 🧮 Enhanceed Inline Equations: Better inline math recognition
+- 🧾 Document Element QA: Answer questions about a document’s structure such as the presence and order of document elements
 - 🌍 Japanese, Arabic and Chinese support (_experimental_)
 
 
 ## Evaluations
 
-|     |  smoldocling-256m-preview   |   granite-docling-258m   |  
-|------|------|-------|
-| | OCR |      |      |      
-| | Layout   |      |      |
-| | Code |  |  |
-| | Equation |  |  |
-| | Table |  |  |
-| | Loop |  |  |
-| | MMStar |  |  |
-| | OCRBench |  |  |
+<table>
+  <thead>
+    <tr>
+      <th></th>
+      <th><b>smoldocling-256m-preview</b></th>
+      <th><b>granite-docling-258m</b></th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr><td colspan="3"><b>Layout</b></td></tr>
+    <tr><td>MAP ↑</td><td>0.21</td><td><b>0.28</b></td></tr>
+    <tr><td>F1 ↑</td><td>0.79</td><td><b>0.85</b></td></tr>
+    <tr><td>Precision ↑</td><td>0.86</td><td><b>0.87</b></td></tr>
+    <tr><td>Recall ↑</td><td>0.82</td><td><b>0.89</b></td></tr>
+
+    <tr><td colspan="3"><b>Full Page OCR</b></td></tr>
+    <tr><td>Edit-distance ↓</td><td>0.48 (0.46)</td><td><b>0.46</b> (<b>0.44</b>)</td></tr>
+    <tr><td>F1 ↑</td><td><b>0.80</b> (0.76)</td><td>0.75 (<b>0.78</b>)</td></tr>
+    <tr><td>Precision ↑</td><td><b>0.89</b> (0.85)</td><td>0.81 (0.85)</td></tr>
+    <tr><td>Recall ↑</td><td><b>0.79</b> (0.74)</td><td>0.73 (<b>0.77</b>)</td></tr>
+    <tr><td>BLEU ↑</td><td><b>0.58</b> (0.54)</td><td>0.56 (<b>0.59</b>)</td></tr>
+    <tr><td>Meteor ↑</td><td>0.67 (0.67)</td><td>0.67 (<b>0.70</b>)</td></tr>
+
+    <tr><td colspan="3"><b>Code Recognition</b></td></tr>
+    <tr><td>Edit-distance ↓</td><td>0.114</td><td><b>0.013</b></td></tr>
+    <tr><td>F1 ↑</td><td>0.915</td><td><b>0.988</b></td></tr>
+    <tr><td>Precision ↑</td><td>0.94</td><td><b>0.99</b></td></tr>
+    <tr><td>Recall ↑</td><td>0.909</td><td><b>0.988</b></td></tr>
+    <tr><td>BLEU ↑</td><td>0.875</td><td><b>0.983</b></td></tr>
+    <tr><td>Meteor ↑</td><td>0.889</td><td><b>0.986</b></td></tr>
+
+    <tr><td colspan="3"><b>Equation Recognition</b></td></tr>
+    <tr><td>Edit-distance ↓</td><td>0.119</td><td><b>0.073</b></td></tr>
+    <tr><td>F1 ↑</td><td>0.947</td><td><b>0.968</b></td></tr>
+    <tr><td>Precision ↑</td><td>0.959</td><td><b>0.968</b></td></tr>
+    <tr><td>Recall ↑</td><td>0.941</td><td><b>0.969</b></td></tr>
+    <tr><td>BLEU ↑</td><td>0.824</td><td><b>0.893</b></td></tr>
+    <tr><td>Meteor ↑</td><td>0.878</td><td><b>0.927</b></td></tr>
+
+    <tr><td colspan="3"><b>Table Recognition (FinTabNet 150dpi)</b></td></tr>
+    <tr><td>TEDS (structure) ↑</td><td>0.82</td><td><b>0.97</b></td></tr>
+    <tr><td>TEDS (w/content) ↑</td><td>0.76</td><td><b>0.96</b></td></tr>
+    <tr><td colspan="3"><b>Other Benchmarks</b></td></tr>
+    <tr><td>MMStar ↑</td><td>0.17</td><td><b>0.3</b></td></tr>
+    <tr><td>OCRBench ↑</td><td>338</td><td><b>500</b></td></tr>
+
 
+</table>
 
 ## Getting started
 
@@ -134,7 +171,7 @@ from pathlib import Path
 MODEL_PATH = "ibm-granite/granite-docling-258M"
 IMAGE_DIR = "img/"  # Place your page images here
 OUTPUT_DIR = "out/"
-PROMPT_TEXT = "Convert page to Docling."
+PROMPT_TEXT = "Convert page to docling."
 
 # Ensure output directory exists
 os.makedirs(OUTPUT_DIR, exist_ok=True)
@@ -154,9 +191,13 @@ image_names = []
 for img_file in sorted(os.listdir(IMAGE_DIR)):
     if img_file.lower().endswith((".png", ".jpg", ".jpeg")):
         img_path = os.path.join(IMAGE_DIR, img_file)
-        image = Image.open(img_path).convert("RGB")
+        with Image.open(img_path) as im:
+            image = im.convert("RGB")
 
-        prompt = f"<|start_of_role|>user:<image>{PROMPT_TEXT}<end_of_utterance>\nassistant:"
+        prompt = (
+            f"<|start_of_role|>user<|end_of_role|><image>{PROMPT_TEXT}<|end_of_text|>\n"
+            f"<|start_of_role|>assistant<|end_of_role|>"
+        )
         batched_inputs.append({"prompt": prompt, "multi_modal_data": {"image": image}})
         image_names.append(os.path.splitext(img_file)[0])
 
@@ -242,14 +283,14 @@ print(f"Total time: {time.time() - start_time:.2f} sec")
 
 The architecture of granite-docling-258m consists of the following components:
 
-(1) Vision encoder: siglip2-base-patch16-512 (https://huggingface.co/google/siglip2-base-patch16-512).
+(1) Vision encoder: [siglip2-base-patch16-512](https://huggingface.co/google/siglip2-base-patch16-512).
 
 (2) Vision-language connector: pixel shuffle projector (as in idefics3) 
 
 (3) Large language model: Granite 165M.
 
-We built upon Idefics3 (https://huggingface.co/docs/transformers/en/model_doc/idefics3) to train our model. We incorporated DocTags into our LLM’s supervised fine-tuning (SFT) data to help the model become familiar with the format, enabling faster convergence and mitigating issues previously observed with SmolDocling.
-
+We built upon [Idefics3](https://huggingface.co/docs/transformers/en/model_doc/idefics3) to train our model. We incorporated DocTags into our LLM’s supervised fine-tuning (SFT) data to help the model become familiar with the format, enabling faster convergence and mitigating issues previously observed with SmolDocling.
+The model was trained using the [nanoVLM](https://github.com/huggingface/nanoVLM) framework, which provides a lightweight and efficient training setup for vision-language models
 # Training Data:
 
 Overall, our training data is largely comprised of two key sources: (1) publicly available datasets (2) internally created synthetic data targeting specific capabilities.

added_tokens.json

@@ -1,38 +1,3 @@
 {
-  "<row_1_col_1>": 100352,
-  "<row_1_col_2>": 100353,
-  "<row_1_col_3>": 100354,
-  "<row_1_col_4>": 100355,
-  "<row_1_col_5>": 100356,
-  "<row_1_col_6>": 100357,
-  "<row_2_col_1>": 100358,
-  "<row_2_col_2>": 100359,
-  "<row_2_col_3>": 100360,
-  "<row_2_col_4>": 100361,
-  "<row_2_col_5>": 100362,
-  "<row_2_col_6>": 100363,
-  "<row_3_col_1>": 100364,
-  "<row_3_col_2>": 100365,
-  "<row_3_col_3>": 100366,
-  "<row_3_col_4>": 100367,
-  "<row_3_col_5>": 100368,
-  "<row_3_col_6>": 100369,
-  "<row_4_col_1>": 100370,
-  "<row_4_col_2>": 100371,
-  "<row_4_col_3>": 100372,
-  "<row_4_col_4>": 100373,
-  "<row_4_col_5>": 100374,
-  "<row_4_col_6>": 100375,
-  "<row_5_col_1>": 100376,
-  "<row_5_col_2>": 100377,
-  "<row_5_col_3>": 100378,
-  "<row_5_col_4>": 100379,
-  "<row_5_col_5>": 100380,
-  "<row_5_col_6>": 100381,
-  "<row_6_col_1>": 100382,
-  "<row_6_col_2>": 100383,
-  "<row_6_col_3>": 100384,
-  "<row_6_col_4>": 100385,
-  "<row_6_col_5>": 100386,
-  "<row_6_col_6>": 100387
+  "<end_of_utterance>": 100352
 }

chat_template.jinja

@@ -1,2 +1,21 @@
-<|start_of_role|>{% for message in messages %}{{message['role']}}{% if message['content'][0]['type'] == 'image' %}{{':'}}{% else %}{{': '}}{% endif %}{% for line in message['content'] %}{% if line['type'] == 'text' %}{{line['text']}}{% elif line['type'] == 'image' %}{{ '<image>' }}{% endif %}{% endfor %}<end_of_utterance>
-{% endfor %}{% if add_generation_prompt %}{{ 'assistant:' }}{% endif %}
+{%- for message in messages -%}
+{{- '<|start_of_role|>' + message['role'] + '<|end_of_role|>' -}}
+{%- if message['content'] is string -%}
+{{- message['content'] -}}
+{%- else -%}
+{%- for part in message['content'] -%}
+{%- if part['type'] == 'text' -%}
+{{- part['text'] -}}
+{%- elif part['type'] == 'image' -%}
+{{- '<image>' -}}
+{%- endif -%}
+{%- endfor -%}
+{%- endif -%}
+{{- '<|end_of_text|>
+' -}}
+{%- endfor -%}
+{%- if add_generation_prompt -%}
+{{- '<|start_of_role|>assistant' -}}
+{%- if controls -%}{{- ' ' + controls | tojson() -}}{%- endif -%}
+{{- '<|end_of_role|>' -}}
+{%- endif -%}

config.json

@@ -1,52 +1,24 @@
 {
-  "_flash_attn_2_enabled": true,
   "architectures": [
     "Idefics3ForConditionalGeneration"
   ],
-  "attention_bias": false,
-  "attention_dropout": 0.0,
   "bos_token_id": 100264,
-  "eos_token_id": 100338,
-  "head_dim": 64,
-  "hidden_act": "silu",
-  "hidden_size": 576,
+  "dtype": "bfloat16",
+  "eos_token_id": 100257,
   "image_token_id": 100270,
-  "initializer_range": 0.02,
-  "intermediate_size": 1536,
-  "max_position_embeddings": 8192,
-  "mlp_bias": false,
   "model_type": "idefics3",
-  "neftune_noise_alpha": 0.0,
-  "num_attention_heads": 9,
-  "num_hidden_layers": 30,
-  "num_key_value_heads": 3,
-  "pad_token_id": 128002,
-  "perceiver_config": {
-    "attention_dropout": 0.0,
-    "hidden_act": "silu",
-    "model_type": "vllama3",
-    "num_key_value_heads": 1,
-    "qk_layer_norms_perceiver": false,
-    "resampler_depth": 6,
-    "resampler_head_dim": 96,
-    "resampler_n_heads": 16,
-    "resampler_n_latents": 64
-  },
-  "pixel_shuffle_factor": 4,
-  "pretraining_tp": 1,
-  "qk_layer_norms": false,
-  "rms_norm_eps": 1e-05,
-  "rope_scaling": null,
-  "rope_theta": 100000.0,
+  "pad_token_id": 100257,
   "scale_factor": 4,
   "text_config": {
+    "_name_or_path": "models/granitev06_hf_ai4k_sft_data_v4",
     "architectures": [
-      "VLlama3ForCausalLM"
+      "LlamaForCausalLM"
     ],
     "attention_bias": false,
     "attention_dropout": 0.0,
     "bos_token_id": 100264,
-    "eos_token_id": 100338,
+    "dtype": "bfloat16",
+    "eos_token_id": 100257,
     "head_dim": 64,
     "hidden_act": "silu",
     "hidden_size": 576,
@@ -58,20 +30,18 @@
     "num_attention_heads": 9,
     "num_hidden_layers": 30,
     "num_key_value_heads": 3,
+    "pad_token_id": 100257,
     "pretraining_tp": 1,
     "rms_norm_eps": 1e-05,
     "rope_scaling": null,
-    "rope_theta": 10000.0,
+    "rope_theta": 100000.0,
     "tie_word_embeddings": true,
-    "torch_dtype": "bfloat16",
-    "use_cache": true,
-    "vocab_size": 100480
+    "use_cache": false,
+    "vocab_size": 100352
   },
   "tie_word_embeddings": true,
-  "torch_dtype": "bfloat16",
-  "transformers_version": "4.53.0.dev0",
+  "transformers_version": "4.56.1",
   "use_cache": true,
-  "use_resampler": false,
   "vision_config": {
     "attention_dropout": 0.0,
     "hidden_act": "gelu_pytorch_tanh",
@@ -89,11 +59,8 @@
     "num_hidden_layers": 12,
     "patch_size": 16,
     "size": {
-      "longest_edge": 2048
-    },
-    "tie_word_embeddings": false,
-    "torch_dtype": "bfloat16",
-    "use_base_siglip": true
+      "longest_edge": 512
+    }
   },
-  "vocab_size": 100480
+  "vocab_size": 100352
 }

generation_config.json

@@ -1,7 +1,8 @@
 {
   "_from_model_config": true,
   "bos_token_id": 100264,
-  "eos_token_id": 100338,
-  "pad_token_id": 128002,
-  "transformers_version": "4.53.0.dev0"
+  "eos_token_id": 100257,
+  "pad_token_id": 100257,
+  "transformers_version": "4.56.1",
+  "use_cache": false
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:da69b13b7e145c8ece853ab911e3009dd5f4348388aa52ce9da9103d023fd78c
-size 515240560
+oid sha256:1cdad234deb1cde18ee6a586f849057f19851daf1fedce2e40aff791dbe46f61
+size 515093104

special_tokens_map.json

@@ -23,7 +23,7 @@
     }
   ],
   "bos_token": {
-    "content": "<|end_of_text|>",
+    "content": "<|start_of_role|>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,
@@ -36,13 +36,7 @@
     "rstrip": false,
     "single_word": false
   },
-  "pad_token": {
-    "content": "<|pad|>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
+  "pad_token": "<|end_of_text|>",
   "unk_token": {
     "content": "<|unk|>",
     "lstrip": false,

tokenizer.json

@@ -23,7 +23,7 @@
     },
     {
       "id": 100258,
-      "content": "<|fim_prefix|>",
+      "content": "<row_1_col_1>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -32,7 +32,7 @@
     },
     {
       "id": 100259,
-      "content": "<|fim_middle|>",
+      "content": "<row_1_col_2>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -41,7 +41,7 @@
     },
     {
       "id": 100260,
-      "content": "<|fim_suffix|>",
+      "content": "<text>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -50,7 +50,7 @@
     },
     {
       "id": 100261,
-      "content": "<|fim_pad|>",
+      "content": "<row_1_col_3>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -59,7 +59,7 @@
     },
     {
       "id": 100262,
-      "content": "<|filename|>",
+      "content": "<row_1_col_4>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -68,7 +68,7 @@
     },
     {
       "id": 100263,
-      "content": "<|reponame|>",
+      "content": "<row_2_col_1>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -95,7 +95,7 @@
     },
     {
       "id": 100266,
-      "content": "<|tool_call|>",
+      "content": "</title>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -104,7 +104,7 @@
     },
     {
       "id": 100267,
-      "content": "<|start_of_plugin|>",
+      "content": "<row_2_col_2>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -113,7 +113,7 @@
     },
     {
       "id": 100268,
-      "content": "<|end_of_plugin|>",
+      "content": "<row_2_col_3>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -122,7 +122,7 @@
     },
     {
       "id": 100269,
-      "content": "<|unk|>",
+      "content": "<title>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -554,7 +554,7 @@
     },
     {
       "id": 100317,
-      "content": "<loc_",
+      "content": "</text>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -563,7 +563,7 @@
     },
     {
       "id": 100318,
-      "content": "<paragraph",
+      "content": "<paragraph>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -662,7 +662,7 @@
     },
     {
       "id": 100329,
-      "content": "<text_break>",
+      "content": "<rec_",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -743,7 +743,7 @@
     },
     {
       "id": 100338,
-      "content": "<end_of_utterance>",
+      "content": "<|unk|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -770,7 +770,7 @@
     },
     {
       "id": 100341,
-      "content": "<|unused_72|>",
+      "content": "<row_2_col_4>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -779,7 +779,7 @@
     },
     {
       "id": 100342,
-      "content": "<|unused_73|>",
+      "content": "<row_3_col_1>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -788,7 +788,7 @@
     },
     {
       "id": 100343,
-      "content": "<|unused_74|>",
+      "content": "<row_3_col_2>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -797,7 +797,7 @@
     },
     {
       "id": 100344,
-      "content": "<|unused_75|>",
+      "content": "<row_3_col_3>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -806,7 +806,7 @@
     },
     {
       "id": 100345,
-      "content": "<|unused_76|>",
+      "content": "<row_3_col_4>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -815,7 +815,7 @@
     },
     {
       "id": 100346,
-      "content": "<|unused_77|>",
+      "content": "<row_4_col_1>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -824,7 +824,7 @@
     },
     {
       "id": 100347,
-      "content": "<|unused_78|>",
+      "content": "<row_4_col_2>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -833,7 +833,7 @@
     },
     {
       "id": 100348,
-      "content": "<|unused_79|>",
+      "content": "<row_4_col_3>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -842,7 +842,7 @@
     },
     {
       "id": 100349,
-      "content": "<|unused_80|>",
+      "content": "<row_4_col_4>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -851,7 +851,7 @@
     },
     {
       "id": 100350,
-      "content": "<|unused_81|>",
+      "content": "<code>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -860,7 +860,7 @@
     },
     {
       "id": 100351,
-      "content": "<|unused_82|>",
+      "content": "</code>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -869,322 +869,7 @@
     },
     {
       "id": 100352,
-      "content": "<row_1_col_1>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 100353,
-      "content": "<row_1_col_2>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 100354,
-      "content": "<row_1_col_3>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 100355,
-      "content": "<row_1_col_4>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 100356,
-      "content": "<row_1_col_5>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 100357,
-      "content": "<row_1_col_6>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 100358,
-      "content": "<row_2_col_1>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 100359,
-      "content": "<row_2_col_2>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 100360,
-      "content": "<row_2_col_3>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 100361,
-      "content": "<row_2_col_4>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 100362,
-      "content": "<row_2_col_5>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 100363,
-      "content": "<row_2_col_6>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 100364,
-      "content": "<row_3_col_1>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 100365,
-      "content": "<row_3_col_2>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 100366,
-      "content": "<row_3_col_3>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 100367,
-      "content": "<row_3_col_4>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 100368,
-      "content": "<row_3_col_5>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 100369,
-      "content": "<row_3_col_6>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 100370,
-      "content": "<row_4_col_1>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 100371,
-      "content": "<row_4_col_2>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 100372,
-      "content": "<row_4_col_3>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 100373,
-      "content": "<row_4_col_4>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 100374,
-      "content": "<row_4_col_5>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 100375,
-      "content": "<row_4_col_6>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 100376,
-      "content": "<row_5_col_1>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 100377,
-      "content": "<row_5_col_2>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 100378,
-      "content": "<row_5_col_3>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 100379,
-      "content": "<row_5_col_4>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 100380,
-      "content": "<row_5_col_5>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 100381,
-      "content": "<row_5_col_6>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 100382,
-      "content": "<row_6_col_1>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 100383,
-      "content": "<row_6_col_2>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 100384,
-      "content": "<row_6_col_3>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 100385,
-      "content": "<row_6_col_4>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 100386,
-      "content": "<row_6_col_5>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 100387,
-      "content": "<row_6_col_6>",
+      "content": "<end_of_utterance>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -101479,18 +101164,18 @@
       "ĠConveyor": 100255,
       "<|pad|>": 100256,
       "<|end_of_text|>": 100257,
-      "<|fim_prefix|>": 100258,
-      "<|fim_middle|>": 100259,
-      "<|fim_suffix|>": 100260,
-      "<|fim_pad|>": 100261,
-      "<|filename|>": 100262,
-      "<|reponame|>": 100263,
+      "<row_1_col_1>": 100258,
+      "<row_1_col_2>": 100259,
+      "<text>": 100260,
+      "<row_1_col_3>": 100261,
+      "<row_1_col_4>": 100262,
+      "<row_2_col_1>": 100263,
       "<|start_of_role|>": 100264,
       "<|end_of_role|>": 100265,
-      "<|tool_call|>": 100266,
-      "<|start_of_plugin|>": 100267,
-      "<|end_of_plugin|>": 100268,
-      "<|unk|>": 100269,
+      "</title>": 100266,
+      "<row_2_col_2>": 100267,
+      "<row_2_col_3>": 100268,
+      "<title>": 100269,
       "<image>": 100270,
       "<caption>": 100271,
       "</caption>": 100272,
@@ -101538,8 +101223,8 @@
       "<page_break>": 100314,
       "<smiles>": 100315,
       "</smiles>": 100316,
-      "<loc_": 100317,
-      "<paragraph": 100318,
+      "</text>": 100317,
+      "<paragraph>": 100318,
       "</paragraph>": 100319,
       "<references>": 100320,
       "</references>": 100321,
@@ -101550,7 +101235,7 @@
       "<group>": 100326,
       "<doctag>": 100327,
       "</doctag>": 100328,
-      "<text_break>": 100329,
+      "<rec_": 100329,
       "<fcel>": 100330,
       "<ecel>": 100331,
       "<lcel>": 100332,
@@ -101559,20 +101244,20 @@
       "<nl>": 100335,
       "<ched>": 100336,
       "<rhed>": 100337,
-      "<end_of_utterance>": 100338,
+      "<|unk|>": 100338,
       "<fake_token_around_image>": 100339,
       "<global-img>": 100340,
-      "<|unused_72|>": 100341,
-      "<|unused_73|>": 100342,
-      "<|unused_74|>": 100343,
-      "<|unused_75|>": 100344,
-      "<|unused_76|>": 100345,
-      "<|unused_77|>": 100346,
-      "<|unused_78|>": 100347,
-      "<|unused_79|>": 100348,
-      "<|unused_80|>": 100349,
-      "<|unused_81|>": 100350,
-      "<|unused_82|>": 100351
+      "<row_2_col_4>": 100341,
+      "<row_3_col_1>": 100342,
+      "<row_3_col_2>": 100343,
+      "<row_3_col_3>": 100344,
+      "<row_3_col_4>": 100345,
+      "<row_4_col_1>": 100346,
+      "<row_4_col_2>": 100347,
+      "<row_4_col_3>": 100348,
+      "<row_4_col_4>": 100349,
+      "<code>": 100350,
+      "</code>": 100351
     },
     "merges": [
       [

tokenizer_config.json

@@ -19,7 +19,7 @@
       "special": true
     },
     "100258": {
-      "content": "<|fim_prefix|>",
+      "content": "<row_1_col_1>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -27,7 +27,7 @@
       "special": true
     },
     "100259": {
-      "content": "<|fim_middle|>",
+      "content": "<row_1_col_2>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -35,7 +35,7 @@
       "special": true
     },
     "100260": {
-      "content": "<|fim_suffix|>",
+      "content": "<text>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -43,7 +43,7 @@
       "special": true
     },
     "100261": {
-      "content": "<|fim_pad|>",
+      "content": "<row_1_col_3>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -51,7 +51,7 @@
       "special": true
     },
     "100262": {
-      "content": "<|filename|>",
+      "content": "<row_1_col_4>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -59,7 +59,7 @@
       "special": true
     },
     "100263": {
-      "content": "<|reponame|>",
+      "content": "<row_2_col_1>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -83,7 +83,7 @@
       "special": true
     },
     "100266": {
-      "content": "<|tool_call|>",
+      "content": "</title>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -91,7 +91,7 @@
       "special": true
     },
     "100267": {
-      "content": "<|start_of_plugin|>",
+      "content": "<row_2_col_2>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -99,7 +99,7 @@
       "special": true
     },
     "100268": {
-      "content": "<|end_of_plugin|>",
+      "content": "<row_2_col_3>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -107,7 +107,7 @@
       "special": true
     },
     "100269": {
-      "content": "<|unk|>",
+      "content": "<title>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -491,7 +491,7 @@
       "special": true
     },
     "100317": {
-      "content": "<loc_",
+      "content": "</text>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -499,7 +499,7 @@
       "special": true
     },
     "100318": {
-      "content": "<paragraph",
+      "content": "<paragraph>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -587,7 +587,7 @@
       "special": true
     },
     "100329": {
-      "content": "<text_break>",
+      "content": "<rec_",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -659,7 +659,7 @@
       "special": true
     },
     "100338": {
-      "content": "<end_of_utterance>",
+      "content": "<|unk|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -683,7 +683,7 @@
       "special": true
     },
     "100341": {
-      "content": "<|unused_72|>",
+      "content": "<row_2_col_4>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -691,7 +691,7 @@
       "special": true
     },
     "100342": {
-      "content": "<|unused_73|>",
+      "content": "<row_3_col_1>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -699,7 +699,7 @@
       "special": true
     },
     "100343": {
-      "content": "<|unused_74|>",
+      "content": "<row_3_col_2>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -707,7 +707,7 @@
       "special": true
     },
     "100344": {
-      "content": "<|unused_75|>",
+      "content": "<row_3_col_3>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -715,7 +715,7 @@
       "special": true
     },
     "100345": {
-      "content": "<|unused_76|>",
+      "content": "<row_3_col_4>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -723,7 +723,7 @@
       "special": true
     },
     "100346": {
-      "content": "<|unused_77|>",
+      "content": "<row_4_col_1>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -731,7 +731,7 @@
       "special": true
     },
     "100347": {
-      "content": "<|unused_78|>",
+      "content": "<row_4_col_2>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -739,7 +739,7 @@
       "special": true
     },
     "100348": {
-      "content": "<|unused_79|>",
+      "content": "<row_4_col_3>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -747,7 +747,7 @@
       "special": true
     },
     "100349": {
-      "content": "<|unused_80|>",
+      "content": "<row_4_col_4>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -755,7 +755,7 @@
       "special": true
     },
     "100350": {
-      "content": "<|unused_81|>",
+      "content": "<code>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -763,7 +763,7 @@
       "special": true
     },
     "100351": {
-      "content": "<|unused_82|>",
+      "content": "</code>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -771,287 +771,7 @@
       "special": true
     },
     "100352": {
-      "content": "<row_1_col_1>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "100353": {
-      "content": "<row_1_col_2>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "100354": {
-      "content": "<row_1_col_3>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "100355": {
-      "content": "<row_1_col_4>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "100356": {
-      "content": "<row_1_col_5>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "100357": {
-      "content": "<row_1_col_6>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "100358": {
-      "content": "<row_2_col_1>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "100359": {
-      "content": "<row_2_col_2>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "100360": {
-      "content": "<row_2_col_3>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "100361": {
-      "content": "<row_2_col_4>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "100362": {
-      "content": "<row_2_col_5>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "100363": {
-      "content": "<row_2_col_6>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "100364": {
-      "content": "<row_3_col_1>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "100365": {
-      "content": "<row_3_col_2>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "100366": {
-      "content": "<row_3_col_3>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "100367": {
-      "content": "<row_3_col_4>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "100368": {
-      "content": "<row_3_col_5>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "100369": {
-      "content": "<row_3_col_6>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "100370": {
-      "content": "<row_4_col_1>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "100371": {
-      "content": "<row_4_col_2>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "100372": {
-      "content": "<row_4_col_3>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "100373": {
-      "content": "<row_4_col_4>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "100374": {
-      "content": "<row_4_col_5>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "100375": {
-      "content": "<row_4_col_6>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "100376": {
-      "content": "<row_5_col_1>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "100377": {
-      "content": "<row_5_col_2>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "100378": {
-      "content": "<row_5_col_3>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "100379": {
-      "content": "<row_5_col_4>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "100380": {
-      "content": "<row_5_col_5>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "100381": {
-      "content": "<row_5_col_6>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "100382": {
-      "content": "<row_6_col_1>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "100383": {
-      "content": "<row_6_col_2>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "100384": {
-      "content": "<row_6_col_3>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "100385": {
-      "content": "<row_6_col_4>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "100386": {
-      "content": "<row_6_col_5>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "100387": {
-      "content": "<row_6_col_6>",
+      "content": "<end_of_utterance>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -1066,11 +786,11 @@
   ],
   "bos_token": "<|start_of_role|>",
   "clean_up_tokenization_spaces": false,
-  "eos_token": "<|end_of_utterance|>",
+  "eos_token": "<|end_of_text|>",
   "errors": "replace",
   "extra_special_tokens": {},
   "model_max_length": 8192,
-  "pad_token": "<|pad|>",
+  "pad_token": "<|end_of_text|>",
   "padding_side": "left",
   "processor_class": "Idefics3Processor",
   "tokenizer_class": "GPT2Tokenizer",