Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions src/deriver/prompts.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,10 @@ def minimal_deriver_prompt(
- EXPLICIT: "I took my dog for a walk in NYC" → "alice has a dog", "alice lives in NYC"
- EXPLICIT: "alice attended college" + general knowledge → "alice completed high school or equivalent"

Respond with a json object in the following format:
{{"explicit": [{{"content": "fact 1"}}, {{"content": "fact 2"}}]}}
Each item in "explicit" must be an object with a "content" field, not a bare string.

{custom_instructions_section}

Target peer:
Expand Down
7 changes: 6 additions & 1 deletion src/utils/representation.py
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,12 @@ class PromptRepresentation(BaseModel):
"""

explicit: list[ExplicitObservationBase] = Field(
description="Facts LITERALLY stated by the user - direct quotes or clear paraphrases only, no interpretation or inference. Example: ['The user is 25 years old', 'The user has a dog named Rover']",
description=(
"Facts LITERALLY stated by the user - direct quotes or clear "
"paraphrases only, no interpretation or inference. "
'Example: [{"content": "The user is 25 years old"}, '
'{"content": "The user has a dog named Rover"}]'
),
default_factory=list,
)

Expand Down
39 changes: 39 additions & 0 deletions tests/deriver/test_prompts.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,27 @@
from collections.abc import AsyncGenerator, Generator
from unittest.mock import patch

import pytest
from pydantic import ValidationError

from src.deriver.prompts import (
estimate_deriver_prompt_tokens,
estimate_minimal_deriver_prompt_tokens,
minimal_deriver_prompt,
)
from src.utils.representation import PromptRepresentation


@pytest.fixture(autouse=True)
async def clean_queue_tables() -> AsyncGenerator[None, None]:
"""Prompt-only tests do not need queue table cleanup."""
yield


@pytest.fixture(autouse=True)
def mock_tracked_db() -> Generator[None, None, None]:
"""Prompt-only tests do not need tracked_db patching."""
yield


def test_minimal_deriver_prompt_includes_custom_instructions_when_present() -> None:
Expand All @@ -30,6 +45,30 @@ def test_minimal_deriver_prompt_omits_custom_instructions_when_absent() -> None:
assert "CUSTOM INSTRUCTIONS:" not in prompt


def test_minimal_deriver_prompt_describes_explicit_json_object_shape() -> None:
prompt = minimal_deriver_prompt(
peer_id="alice",
messages="alice: I just had my 25th birthday",
custom_instructions=None,
)

assert '"explicit": [{"content": "fact 1"}' in prompt
assert 'Each item in "explicit" must be an object with a "content" field' in prompt
assert "not a bare string" in prompt
with pytest.raises(ValidationError):
PromptRepresentation.model_validate_json(
'{"explicit": ["alice is 25 years old"]}'
)
assert PromptRepresentation.model_validate_json(
'{"explicit": [{"content": "alice is 25 years old"}]}'
).explicit[0].content == "alice is 25 years old"
explicit_schema_description = PromptRepresentation.model_json_schema()["properties"][
"explicit"
]["description"]
assert '{"content": "The user is 25 years old"}' in explicit_schema_description
assert "['The user is 25 years old'" not in explicit_schema_description


def test_estimate_deriver_prompt_tokens_increases_with_custom_instructions() -> None:
base_tokens = estimate_minimal_deriver_prompt_tokens()
custom_tokens = estimate_deriver_prompt_tokens(
Expand Down
32 changes: 32 additions & 0 deletions tests/llm/test_backends/test_openai.py
Original file line number Diff line number Diff line change
Expand Up @@ -813,6 +813,38 @@ async def test_structured_output_json_object_mode_request_shape() -> None:
assert result.content.answer == "ok"


@pytest.mark.asyncio
async def test_structured_output_json_object_mode_prompt_representation_schema_shape() -> None:
"""PromptRepresentation schema hints explicit observations as content objects."""
client = Mock()
client.chat.completions.parse = AsyncMock()
client.chat.completions.create = AsyncMock(
return_value=_structured_create_return(
'{"explicit": [{"content": "alice likes coffee"}]}'
)
)

backend = OpenAIBackend(client)
result = await backend.complete(
model="glm-4.6",
messages=[{"role": "user", "content": "Hello"}],
max_tokens=100,
response_format=PromptRepresentation,
extra_params={"structured_output_mode": "json_object"},
)

call = _await_kwargs(client.chat.completions.create)
system_messages = [m for m in call["messages"] if m["role"] == "system"]
assert system_messages, "expected a system message carrying the schema"
system_content = system_messages[0]["content"]
schema = json.loads(system_content.split("JSON schema:\n", 1)[1])
explicit_description = schema["properties"]["explicit"]["description"]
assert '{"content": "The user is 25 years old"}' in explicit_description
assert "['The user is 25 years old'" not in explicit_description
assert isinstance(result.content, PromptRepresentation)
assert result.content.explicit[0].content == "alice likes coffee"


@pytest.mark.asyncio
async def test_structured_output_json_object_mode_repairs_markdown() -> None:
"""A provider that ignores json_object and returns prose must not crash —
Expand Down