Add Sentence Transformers MultiVectorEncoder integration

1_Dense/config.json

@@ -0,0 +1,8 @@
+{
+  "in_features": 4096,
+  "out_features": 4096,
+  "bias": true,
+  "activation_function": "torch.nn.modules.linear.Identity",
+  "module_input_name": "token_embeddings",
+  "module_output_name": "token_embeddings"
+}

1_Dense/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b0dc39705e9cfd9610dd3cdeaa0c6ac0540c5780e5f08d860e259f4854c887f
+size 67125416

2_Normalize/config.json

@@ -0,0 +1,4 @@
+{
+  "module_input_name": "token_embeddings",
+  "module_output_name": "token_embeddings"
+}

3_MultiVectorMask/config.json

@@ -0,0 +1,3 @@
+{
+  "skiplist_words": []
+}

README.md

@@ -13,6 +13,7 @@ tags:
 - vidore
 - document-retrieval
 - multimodal
+- sentence-transformers
 datasets:
 - vidore/vidore_benchmark
 - vidore/vidore_benchmark_v2
@@ -165,7 +166,9 @@ Protocol `paired-all-pages-dedup+process_queries+ndcg2r-20260827` ($\text{MVT} =
 
 ## ⚡ Quick Start
 
-### Installation
+### ColPali Engine
+
+#### Installation
 
 ```bash
 git clone https://github.com/Tencent/EVIE.git
@@ -174,7 +177,7 @@ pip install -r requirements.txt
 export PYTHONPATH="$(pwd)/colpali${PYTHONPATH:+:$PYTHONPATH}"
 ```
 
-### Python Inference
+#### Python Inference
 
 ```python
 import torch
@@ -210,6 +213,52 @@ scores = processor.score(query_embeddings, image_embeddings)
 print("Late-interaction MaxSim Relevance Score:", scores)
 ```
 
+### Sentence Transformers
+
+Load EVIE with [Sentence Transformers](https://www.sbert.net/) to encode queries and document images and compute MaxSim scores. Bidirectional attention is configured automatically.
+
+```bash
+pip install -U "sentence-transformers[image]>=6.0.0"
+```
+
+```python
+from sentence_transformers import MultiVectorEncoder
+
+model = MultiVectorEncoder("tencent/EVIE-8B")
+
+queries = [
+    "What is the variable represented on the y-axis of the graph?",
+    "Total outlay is maximum in which year?",
+]
+documents = [
+    "https://huggingface.co/datasets/sentence-transformers/example-documents/resolve/main/doc1.jpg",
+    "https://huggingface.co/datasets/sentence-transformers/example-documents/resolve/main/doc2.jpg",
+    "https://huggingface.co/datasets/sentence-transformers/example-documents/resolve/main/doc3.jpg",
+    "https://huggingface.co/datasets/sentence-transformers/example-documents/resolve/main/doc4.jpg",
+]
+
+query_embeddings = model.encode_query(queries)
+document_embeddings = model.encode_document(documents, batch_size=1)
+print(query_embeddings[0].shape, document_embeddings[0].shape)
+# torch.Size([23, 4096]) torch.Size([3161, 4096])
+
+scores = model.similarity(query_embeddings, document_embeddings)
+print(scores)
+# tensor([[14.3594,  6.1758,  4.5137,  3.2979],
+#         [ 1.8079, 10.9180,  1.9668,  1.9541]])
+```
+
+Documents can be URLs, local image paths, or `PIL.Image` objects. The output contains 4096-dimensional token embeddings. Scores can vary slightly with dtype and attention backend.
+
+The default page budget allows up to 16,384 visual tokens. Encoding images one at a time reduces peak memory use. To use the 1,024-token budget from the evaluation protocol above, set the image processor's pixel budget:
+
+```python
+model = MultiVectorEncoder(
+    "tencent/EVIE-8B",
+    processor_kwargs={"size": {"longest_edge": 1024 * 32 * 32, "shortest_edge": 65536}},
+)
+```
+
 ---
 
 ## 📂 Repository Layout
@@ -218,6 +267,8 @@ print("Late-interaction MaxSim Relevance Score:", scores)
 Evie-8B/
 ├── model.safetensors         # EVIE-8B weights (safetensors)
 ├── config.json               # Model configuration (4096D)
+├── 1_Dense/                  # Sentence Transformers 4096D projection
+├── modules.json              # MultiVectorEncoder module graph
 ├── infer.py                  # Standalone inference & scoring CLI
 ├── colpali/                  # ColQwen3.5 inference engine
 ├── code/shared/              # 138-task eval harness

additional_chat_templates/sentence_transformers.jinja

@@ -0,0 +1,19 @@
+{%- for message in messages -%}
+{%- set ns = namespace(has_image=false, text='') -%}
+{%- if message['content'] is string -%}
+{%- set ns.text = message['content'] -%}
+{%- else -%}
+{%- for item in message['content'] -%}
+{%- if 'image' in item or 'image_url' in item or item.type == 'image' -%}
+{%- set ns.has_image = true -%}
+{%- elif 'text' in item -%}
+{%- set ns.text = ns.text + item.text -%}
+{%- endif -%}
+{%- endfor -%}
+{%- endif -%}
+{%- if ns.has_image -%}
+{{- '<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>Describe the image.<|im_end|><|endoftext|>' -}}
+{%- else -%}
+{{- ns.text + '<|endoftext|>' * 10 -}}
+{%- endif -%}
+{%- endfor -%}

config_sentence_transformers.json

@@ -0,0 +1,9 @@
+{
+  "__version__": {
+    "sentence_transformers": "6.0.0"
+  },
+  "model_type": "MultiVectorEncoder",
+  "similarity_fn_name": "maxsim",
+  "prompts": {},
+  "default_prompt_name": null
+}

modules.json

@@ -0,0 +1,26 @@
+[
+  {
+    "idx": 0,
+    "name": "0",
+    "path": "",
+    "type": "sentence_transformers.base.modules.transformer.Transformer"
+  },
+  {
+    "idx": 1,
+    "name": "1",
+    "path": "1_Dense",
+    "type": "sentence_transformers.base.modules.dense.Dense"
+  },
+  {
+    "idx": 2,
+    "name": "2",
+    "path": "2_Normalize",
+    "type": "sentence_transformers.sentence_transformer.modules.normalize.Normalize"
+  },
+  {
+    "idx": 3,
+    "name": "3",
+    "path": "3_MultiVectorMask",
+    "type": "sentence_transformers.multi_vector_encoder.modules.multi_vector_mask.MultiVectorMask"
+  }
+]

processor_config.json

@@ -25,7 +25,7 @@
     },
     "temporal_patch_size": 2
   },
-  "processor_class": "ColQwen3_5Processor",
+  "processor_class": "Qwen3VLProcessor",
   "video_processor": {
     "do_convert_rgb": true,
     "do_normalize": true,

sentence_bert_config.json

@@ -0,0 +1,30 @@
+{
+  "transformer_task": "feature-extraction",
+  "modality_config": {
+    "text": {
+      "method": "forward",
+      "method_output_name": "last_hidden_state"
+    },
+    "image": {
+      "method": "forward",
+      "method_output_name": "last_hidden_state"
+    },
+    "message": {
+      "method": "forward",
+      "method_output_name": "last_hidden_state",
+      "format": "structured"
+    }
+  },
+  "module_output_name": "token_embeddings",
+  "unpad_inputs": false,
+  "config_kwargs": {
+    "text_config": {
+      "is_causal": false
+    }
+  },
+  "processing_kwargs": {
+    "chat_template": {
+      "chat_template": "sentence_transformers"
+    }
+  }
+}

tokenizer_config.json

@@ -21,7 +21,8 @@
   },
   "pad_token": "<|endoftext|>",
   "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
-  "processor_class": "ColQwen3_5Processor",
+  "processor_class": "Qwen3VLProcessor",
+  "padding_side": "left",
   "split_special_tokens": false,
   "tokenizer_class": "Qwen2Tokenizer",
   "unk_token": null,