feat(embedding): optionally truncate oversized embeddings to configured dims

Matryoshka (MRL) embedding models served behind OpenAI-compatible endpoints
(llama.cpp, some vLLM builds) ignore the `dimensions=` request parameter and
always return their native dimension. Honcho then hard-fails on the dimension
mismatch, making these models unusable even though truncating an MRL vector to
a smaller size is valid.

Add an opt-in `EMBEDDING_TRUNCATE_TO_DIMENSIONS` flag. When enabled and the
provider returns a vector larger than `EMBEDDING_VECTOR_DIMENSIONS`, keep the
leading components and L2-renormalize so cosine/inner-product similarity stays
well-defined. Off by default, so a genuinely misconfigured model still fails
loudly.

Also keeps embeddings at or below pgvector's 2000-dimension HNSW index cap when
the native model dimension exceeds it.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Author: JustinBacher

.env.template

@@ -19,6 +19,12 @@ PERFORMANCE_LOG_FORMAT=compact # compact|rich
 # EMBEDDING_VECTOR_DIMENSIONS=1536
 # EMBEDDING_MAX_INPUT_TOKENS=8192
 # EMBEDDING_MAX_TOKENS_PER_REQUEST=300000
+# Truncate + L2-renormalize oversized embeddings down to EMBEDDING_VECTOR_DIMENSIONS
+# instead of erroring on a dimension mismatch. Use with Matryoshka (MRL) models
+# served behind OpenAI-compatible endpoints (llama.cpp, some vLLM builds) that
+# ignore the `dimensions=` request parameter and always emit their native size.
+# Off by default so a genuinely misconfigured model still fails loudly.
+# EMBEDDING_TRUNCATE_TO_DIMENSIONS=false
 # EMBEDDING_MODEL_CONFIG__TRANSPORT=openai
 # EMBEDDING_MODEL_CONFIG__MODEL=text-embedding-3-small
 # EMBEDDING_MODEL_CONFIG__OVERRIDES__BASE_URL=

src/config.py

@@ -758,6 +758,14 @@ def _MODEL_CONFIG_DEFAULT() -> ConfiguredEmbeddingModelSettings:
     # saturated, message creation skips the fast path entirely and the
     # reconciler embeds on its next cycle. 0 disables the fast path.
     MAX_PENDING_EMBED_TASKS: Annotated[int, Field(default=50, ge=0)] = 50
+    # When the embedding provider returns a vector larger than
+    # VECTOR_DIMENSIONS, truncate it to VECTOR_DIMENSIONS and L2-renormalize
+    # instead of raising a dimension-mismatch error. Intended for MRL-trained
+    # models (e.g. Qwen3-Embedding) served behind OpenAI-compatible endpoints
+    # (llama.cpp, some vLLM builds) that ignore the `dimensions=` request
+    # parameter and always emit their native dimension. Off by default so a
+    # genuinely misconfigured model still fails loudly.
+    TRUNCATE_TO_DIMENSIONS: bool = False
 
     @model_validator(mode="before")
     @classmethod

src/embedding_client.py

@@ -1,5 +1,6 @@
 import asyncio
 import logging
+import math
 import threading
 import time
 from collections import defaultdict
@@ -173,11 +174,13 @@ def __init__(
         max_input_tokens: int,
         max_tokens_per_request: int,
         send_dimensions: bool,
+        truncate_to_dimensions: bool = False,
     ):
         self.transport: str = config.transport
         self.model: str = config.model
         self.vector_dimensions: int = vector_dimensions
         self.send_dimensions: bool = send_dimensions
+        self.truncate_to_dimensions: bool = truncate_to_dimensions
 
         if self.transport == "gemini":
             if not config.api_key:
@@ -223,6 +226,20 @@ def _validate_embedding_dimensions(self, embedding: list[float]) -> list[float]:
             )
         return embedding
 
+    def _fit_embedding_dimensions(self, embedding: list[float]) -> list[float]:
+        """Coerce a provider embedding to ``vector_dimensions``, then validate.
+
+        When ``truncate_to_dimensions`` is enabled and the provider returned a
+        larger vector than configured (common with MRL-trained models served
+        behind OpenAI-compatible endpoints that ignore ``dimensions=``), keep
+        the leading ``vector_dimensions`` components and L2-renormalize so
+        cosine similarity stays well-defined. Otherwise, fall through to the
+        strict validator, which raises on any mismatch.
+        """
+        if self.truncate_to_dimensions and len(embedding) > self.vector_dimensions:
+            embedding = _l2_truncate(embedding, self.vector_dimensions)
+        return self._validate_embedding_dimensions(embedding)
+
     async def embed(self, query: str) -> list[float]:
         token_count = len(self.encoding.encode(query))
 
@@ -264,7 +281,7 @@ async def _call_openai() -> list[float]:
             if self.send_dimensions:
                 openai_kwargs["dimensions"] = self.vector_dimensions
             response = await openai_client.embeddings.create(**openai_kwargs)
-            return self._validate_embedding_dimensions(response.data[0].embedding)
+            return self._fit_embedding_dimensions(response.data[0].embedding)
 
         return await _emit_embedding_call(
             provider=self.transport,
@@ -469,7 +486,7 @@ async def _call_provider() -> dict[str, dict[int, list[float]]]:
                 response = await self.client.embeddings.create(**openai_kwargs)
                 for item, embedding_data in zip(batch, response.data, strict=True):
                     result[item.text_id][item.chunk_index] = (
-                        self._validate_embedding_dimensions(embedding_data.embedding)
+                        self._fit_embedding_dimensions(embedding_data.embedding)
                     )
             return result
 
@@ -531,6 +548,21 @@ def _accumulate_embeddings(
         }
 
 
+def _l2_truncate(embedding: list[float], target_dimensions: int) -> list[float]:
+    """Keep the leading ``target_dimensions`` components and L2-renormalize.
+
+    Valid for Matryoshka (MRL) embeddings, where the leading subvector is a
+    trained lower-dimensional representation. Renormalizing restores unit norm
+    so cosine/inner-product similarity is preserved. A zero-norm slice is
+    returned unchanged (degenerate input; nothing to normalize).
+    """
+    sliced = embedding[:target_dimensions]
+    norm = math.sqrt(sum(value * value for value in sliced))
+    if norm == 0.0:
+        return sliced
+    return [value / norm for value in sliced]
+
+
 def _chunk_text_with_tokens(
     text: str,
     encoded_tokens: list[int],
@@ -603,6 +635,7 @@ def _get_client(self) -> _EmbeddingClient:
                         max_input_tokens=settings.EMBEDDING.MAX_INPUT_TOKENS,
                         max_tokens_per_request=settings.EMBEDDING.MAX_TOKENS_PER_REQUEST,
                         send_dimensions=settings.EMBEDDING.resolve_send_dimensions(),
+                        truncate_to_dimensions=settings.EMBEDDING.TRUNCATE_TO_DIMENSIONS,
                     )
                     self._instance_signature = signature
                     logger.debug(
@@ -627,6 +660,7 @@ def _get_settings_signature(self) -> tuple[object, ...]:
             settings.EMBEDDING.MAX_INPUT_TOKENS,
             settings.EMBEDDING.MAX_TOKENS_PER_REQUEST,
             settings.EMBEDDING.resolve_send_dimensions(),
+            settings.EMBEDDING.TRUNCATE_TO_DIMENSIONS,
         )
 
     async def embed(self, query: str) -> list[float]:

tests/llm/test_embedding_client.py

@@ -1,10 +1,14 @@
+import math
 from types import SimpleNamespace
 from typing import Any
 
 import pytest
 
 from src.config import EmbeddingModelConfig
-from src.embedding_client import _EmbeddingClient  # pyright: ignore[reportPrivateUsage]
+from src.embedding_client import (  # pyright: ignore[reportPrivateUsage]
+    _EmbeddingClient,
+    _l2_truncate,
+)
 
 
 class FakeOpenAIEmbeddingsAPI:
@@ -92,6 +96,79 @@ def __init__(self, *, api_key: str | None, base_url: str | None) -> None:
         await client.embed("hello world")
 
 
+def test_l2_truncate_slices_and_renormalizes() -> None:
+    truncated = _l2_truncate([0.1] * 16, 8)
+
+    assert len(truncated) == 8
+    assert math.isclose(math.sqrt(sum(v * v for v in truncated)), 1.0, abs_tol=1e-9)
+
+
+def test_l2_truncate_leaves_zero_norm_slice_untouched() -> None:
+    assert _l2_truncate([0.0] * 4, 2) == [0.0, 0.0]
+
+
+@pytest.mark.asyncio
+async def test_openai_embedding_client_truncates_oversized_when_enabled(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    # Provider returns a native 16-dim vector (e.g. an MRL model behind a server
+    # that ignores `dimensions=`); we configured 8 and enabled truncation.
+    fake_embeddings = FakeOpenAIEmbeddingsAPI([0.1] * 16)
+
+    class FakeOpenAIClient:
+        def __init__(self, *, api_key: str | None, base_url: str | None) -> None:
+            self.embeddings: FakeOpenAIEmbeddingsAPI = fake_embeddings
+
+    monkeypatch.setattr("src.embedding_client.AsyncOpenAI", FakeOpenAIClient)
+
+    client = _EmbeddingClient(
+        EmbeddingModelConfig(
+            transport="openai",
+            model="qwen3-embedding",
+            api_key="test-key",
+        ),
+        vector_dimensions=8,
+        max_input_tokens=8192,
+        max_tokens_per_request=300_000,
+        send_dimensions=True,
+        truncate_to_dimensions=True,
+    )
+
+    embedding = await client.embed("hello world")
+
+    assert len(embedding) == 8
+    assert math.isclose(math.sqrt(sum(v * v for v in embedding)), 1.0, abs_tol=1e-9)
+
+
+@pytest.mark.asyncio
+async def test_openai_embedding_client_rejects_oversized_without_truncation(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    fake_embeddings = FakeOpenAIEmbeddingsAPI([0.1] * 16)
+
+    class FakeOpenAIClient:
+        def __init__(self, *, api_key: str | None, base_url: str | None) -> None:
+            self.embeddings: FakeOpenAIEmbeddingsAPI = fake_embeddings
+
+    monkeypatch.setattr("src.embedding_client.AsyncOpenAI", FakeOpenAIClient)
+
+    client = _EmbeddingClient(
+        EmbeddingModelConfig(
+            transport="openai",
+            model="qwen3-embedding",
+            api_key="test-key",
+        ),
+        vector_dimensions=8,
+        max_input_tokens=8192,
+        max_tokens_per_request=300_000,
+        send_dimensions=True,
+        truncate_to_dimensions=False,
+    )
+
+    with pytest.raises(ValueError, match="Embedding dimension mismatch"):
+        await client.embed("hello world")
+
+
 @pytest.mark.asyncio
 async def test_gemini_embedding_client_uses_output_dimensionality(
     monkeypatch: pytest.MonkeyPatch,