feat(rag): lite-model document summaries — non-markdown docs summarized at import, summary chunk retrieves and resolves to the full source doc
This commit is contained in:
@@ -47,18 +47,23 @@ def _doc(title: str, content: str) -> Document:
|
||||
|
||||
|
||||
def _chunk(
|
||||
doc: Document, score: float, cosine: float | None = None, fts_hit: bool = False
|
||||
doc: Document,
|
||||
score: float,
|
||||
cosine: float | None = None,
|
||||
fts_hit: bool = False,
|
||||
is_summary: bool = False,
|
||||
) -> RetrievedChunk:
|
||||
"""Fake candidate: *score* is the fused rank score; *cosine* (defaults to
|
||||
*score*) is the vector-similarity gate input."""
|
||||
return RetrievedChunk(
|
||||
chunk_id=uuid.uuid4(),
|
||||
position=0,
|
||||
position=-1 if is_summary else 0,
|
||||
content=doc.content[:32],
|
||||
score=score,
|
||||
document=doc,
|
||||
cosine=score if cosine is None else cosine,
|
||||
fts_hit=fts_hit,
|
||||
is_summary=is_summary,
|
||||
)
|
||||
|
||||
|
||||
@@ -167,6 +172,70 @@ def test_gate_zero_chunks_deflects_with_fallback_chips() -> None:
|
||||
assert 2 <= len(plan.suggestions) <= MAX_SUGGESTIONS
|
||||
|
||||
|
||||
# ---------- summary hits (phase 30: summary → full source document) ----------
|
||||
|
||||
|
||||
def test_summary_hit_on_selected_top_doc_counts() -> None:
|
||||
"""HIGH branch: the top document was hit via its summary chunk ⇒ 1.
|
||||
|
||||
Context assembly is unchanged (A7 revised): the *source* document's
|
||||
full content lands in the prompt, not the summary text alone.
|
||||
"""
|
||||
a = _doc("Alpha", "ALPHA_FULL_SOURCE_CONTENT")
|
||||
b = _doc("Beta", "BETA_FULL_SOURCE_CONTENT")
|
||||
chunks = [
|
||||
_chunk(a, 0.90, is_summary=True), # top doc reached through its summary
|
||||
_chunk(b, 0.50),
|
||||
]
|
||||
plan = chat_api.plan_turn(chunks, _settings(threshold=0.30))
|
||||
assert plan.deflected is False
|
||||
assert plan.summary_hits == 1
|
||||
# The full source document is what the LLM sees (phase 24 contract).
|
||||
assert "ALPHA_FULL_SOURCE_CONTENT" in plan.system_prompt
|
||||
|
||||
|
||||
def test_summary_hit_outside_top_n_selection_not_counted() -> None:
|
||||
"""A summary chunk on a document outside the top-N (default 2) selection
|
||||
does not count — only hits that landed in the selected context do."""
|
||||
a = _doc("Alpha", "ALPHA_CONTENT")
|
||||
b = _doc("Beta", "BETA_CONTENT")
|
||||
c = _doc("Gamma", "GAMMA_CONTENT")
|
||||
chunks = [
|
||||
_chunk(a, 0.90),
|
||||
_chunk(b, 0.80),
|
||||
_chunk(c, 0.70, is_summary=True), # 3rd-ranked doc — not selected
|
||||
]
|
||||
plan = chat_api.plan_turn(chunks, _settings(threshold=0.30))
|
||||
assert plan.deflected is False
|
||||
assert [d.title for d in plan.docs] == ["Alpha", "Beta"]
|
||||
assert plan.summary_hits == 0
|
||||
|
||||
|
||||
def test_low_branch_counts_summary_hit_on_selected_doc() -> None:
|
||||
"""LOW (deflected) branch records ``summary_hits`` too: the weak hit's
|
||||
parent is still the selected (weak-hit) document."""
|
||||
a = _doc("Gamma", "GAMMA_DOC_CONTENT")
|
||||
b = _doc("Delta", "DELTA_DOC_CONTENT")
|
||||
chunks = [
|
||||
_chunk(a, 0.05, cosine=0.05, is_summary=True), # weak cosine, no FTS
|
||||
_chunk(b, 0.03, cosine=0.03),
|
||||
]
|
||||
plan = chat_api.plan_turn(chunks, _settings(threshold=0.30))
|
||||
assert plan.deflected is True
|
||||
assert plan.summary_hits == 1
|
||||
|
||||
|
||||
def test_no_summary_chunks_yields_zero_summary_hits() -> None:
|
||||
"""Legacy chunks (``is_summary=false``) keep ``summary_hits == 0``."""
|
||||
a = _doc("Alpha", "ALPHA_CONTENT")
|
||||
b = _doc("Beta", "BETA_CONTENT")
|
||||
plan = chat_api.plan_turn([_chunk(a, 0.90), _chunk(b, 0.40)], _settings(threshold=0.30))
|
||||
assert plan.summary_hits == 0
|
||||
plan_low = chat_api.plan_turn([_chunk(a, 0.05, cosine=0.05)], _settings(threshold=0.30))
|
||||
assert plan_low.deflected is True
|
||||
assert plan_low.summary_hits == 0
|
||||
|
||||
|
||||
# ---------- prompt content (LOW vs HIGH) ----------
|
||||
|
||||
|
||||
|
||||
@@ -25,6 +25,9 @@ def test_defaults_match_locked_decisions(monkeypatch: pytest.MonkeyPatch) -> Non
|
||||
s = _settings()
|
||||
assert s.llm_chat_model == "turbo"
|
||||
assert s.llm_embed_model == "embed"
|
||||
# A5 extended (phase 30): one-shot completions default to the ``lite``
|
||||
# model on the same endpoint.
|
||||
assert s.llm_summary_model == "lite"
|
||||
assert s.embedding_dim == 768
|
||||
assert s.llm_base_url.endswith("/v1")
|
||||
# A8 (revised): the honesty gate input is the best cosine, default 0.62.
|
||||
@@ -53,6 +56,22 @@ def test_env_override(monkeypatch) -> None:
|
||||
assert s.llm_chat_model == "juggernaut"
|
||||
|
||||
|
||||
def test_llm_summary_model_env_override(monkeypatch) -> None:
|
||||
"""Phase 30: ``BOR_LLM_SUMMARY_MODEL`` overrides the ``lite`` default"""
|
||||
monkeypatch.setenv("BOR_LLM_SUMMARY_MODEL", "mini")
|
||||
s = _settings()
|
||||
assert s.llm_summary_model == "mini"
|
||||
|
||||
|
||||
def test_summary_max_chars_default_and_env_override(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Phase 30: document content sent to the ``lite`` model is capped at
|
||||
``BOR_SUMMARY_MAX_CHARS`` (default 12 000 chars per call)."""
|
||||
monkeypatch.delenv("BOR_SUMMARY_MAX_CHARS", raising=False)
|
||||
assert _settings().summary_max_chars == 12_000
|
||||
monkeypatch.setenv("BOR_SUMMARY_MAX_CHARS", "5000")
|
||||
assert _settings().summary_max_chars == 5000
|
||||
|
||||
|
||||
def test_max_output_tokens_env_override(monkeypatch) -> None:
|
||||
monkeypatch.setenv("BOR_MAX_OUTPUT_TOKENS", "1234")
|
||||
s = _settings()
|
||||
|
||||
+211
-4
@@ -1,12 +1,18 @@
|
||||
"""Unit tests: importer directory walk + sha256 delta logic.
|
||||
"""Unit tests: importer directory walk + sha256 delta logic + summaries.
|
||||
|
||||
The walk tests are pure filesystem (``tmp_path``); the delta tests run
|
||||
against the local compose Postgres (preferred — a real vector table),
|
||||
skipping with clear instructions when the stack is not up.
|
||||
The walk tests are pure filesystem (``tmp_path``); the delta and summary
|
||||
tests run against the local compose Postgres (preferred — a real vector
|
||||
table), skipping with clear instructions when the stack is not up.
|
||||
|
||||
Summaries (phase 30): non-markdown files get a ``lite``-model summary via
|
||||
the fake's deterministic ``chat`` (``"Summary of <first token>"``); the
|
||||
sentinel word ``SUMMARY-BLOWUP`` makes ``chat`` raise :class:`LLMError`
|
||||
for the fail-soft path.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
@@ -15,6 +21,8 @@ from sqlalchemy import func, select
|
||||
from app.models import Chunk, Document
|
||||
from app.rag.importer import (
|
||||
EXCLUDED_DIRS,
|
||||
ImportSummary,
|
||||
_store_summary,
|
||||
import_sources,
|
||||
iter_importable_files,
|
||||
)
|
||||
@@ -350,6 +358,205 @@ def test_multi_format_import_counts_per_format_and_titles_stem(db, tmp_path: Pat
|
||||
_cleanup_source(db, root.name)
|
||||
|
||||
|
||||
# ---------- phase 30: lite-model summaries for non-markdown files ----------
|
||||
|
||||
|
||||
def test_non_markdown_file_gets_stored_and_indexed_summary(db, tmp_path: Path) -> None:
|
||||
"""A ``.yaml`` file is summarized: ``documents.summary`` is set and one
|
||||
``is_summary`` chunk (position −1, embedded) is indexed alongside the
|
||||
content chunks."""
|
||||
root = tmp_path / "sumsrc"
|
||||
root.mkdir()
|
||||
(root / "svc.yaml").write_text("alpha services:\n gitlab:\n port: 8929\n")
|
||||
llm = FakeEmbedder()
|
||||
try:
|
||||
summary = asyncio.run(import_sources([root], llm, session=db))
|
||||
assert summary.added == 1
|
||||
assert summary.summaries == 1
|
||||
assert summary.summary_errors == 0
|
||||
doc = db.scalar(
|
||||
select(Document).where(Document.source == root.name, Document.path == "svc.yaml")
|
||||
)
|
||||
assert doc is not None
|
||||
assert doc.summary is not None
|
||||
# Deterministic fake reply + the code-appended pointer line.
|
||||
assert doc.summary.startswith("Summary of alpha")
|
||||
assert doc.summary.endswith(f"Source: {root.name}/svc.yaml")
|
||||
schunks = [c for c in doc.chunks if c.is_summary]
|
||||
assert len(schunks) == 1
|
||||
assert schunks[0].position == -1
|
||||
assert schunks[0].content == doc.summary
|
||||
assert schunks[0].embedding is not None and len(schunks[0].embedding) == 768
|
||||
# Content chunks stay 0-based and are never flagged as summaries.
|
||||
content = [c for c in doc.chunks if not c.is_summary]
|
||||
assert sorted(c.position for c in content) == list(range(len(content)))
|
||||
finally:
|
||||
_cleanup_source(db, root.name)
|
||||
|
||||
|
||||
def test_markdown_file_never_gets_summary(db, tmp_path: Path) -> None:
|
||||
"""Markdown is already natural language: no summary, no ``is_summary``
|
||||
chunk, and the ``lite`` model is never called."""
|
||||
root = tmp_path / "mdsrc"
|
||||
root.mkdir()
|
||||
(root / "note.md").write_text("# Note\n\nmarkdown body\n")
|
||||
llm = FakeEmbedder()
|
||||
try:
|
||||
summary = asyncio.run(import_sources([root], llm, session=db))
|
||||
assert summary.added == 1
|
||||
assert summary.summaries == 0 and summary.summary_errors == 0
|
||||
assert llm.chat_calls == [] # the model was never asked
|
||||
doc = db.scalar(
|
||||
select(Document).where(Document.source == root.name, Document.path == "note.md")
|
||||
)
|
||||
assert doc is not None
|
||||
assert doc.summary is None
|
||||
assert doc.chunks and all(not c.is_summary for c in doc.chunks)
|
||||
finally:
|
||||
_cleanup_source(db, root.name)
|
||||
|
||||
|
||||
def test_summary_failure_is_fail_soft(db, tmp_path: Path) -> None:
|
||||
"""A ``lite``-model failure must never lose the document: the file is
|
||||
fully indexed (content chunks + embeddings), ``documents.summary`` stays
|
||||
NULL, and the failure is counted in ``summary_errors``."""
|
||||
root = tmp_path / "blowup"
|
||||
root.mkdir()
|
||||
(root / "bad.txt").write_text("SUMMARY-BLOWUP the lite model chokes on this\n")
|
||||
llm = FakeEmbedder()
|
||||
try:
|
||||
summary = asyncio.run(import_sources([root], llm, session=db))
|
||||
assert summary.errors == 0 # the document itself imported fine
|
||||
assert summary.added == 1
|
||||
assert summary.summaries == 0
|
||||
assert summary.summary_errors == 1
|
||||
doc = db.scalar(
|
||||
select(Document).where(Document.source == root.name, Document.path == "bad.txt")
|
||||
)
|
||||
assert doc is not None
|
||||
assert doc.summary is None
|
||||
assert len(doc.chunks) == 1
|
||||
assert doc.chunks[0].embedding is not None # content chunk embedded
|
||||
assert all(not c.is_summary for c in doc.chunks)
|
||||
finally:
|
||||
_cleanup_source(db, root.name)
|
||||
|
||||
|
||||
def test_summary_chunk_is_replaced_on_reimport(db, tmp_path: Path) -> None:
|
||||
"""Re-importing a changed non-markdown file keeps exactly one
|
||||
``is_summary`` chunk — the old one is gone, the new summary is stored
|
||||
and embedded, and the content chunks stay 0-based."""
|
||||
root = tmp_path / "repl"
|
||||
root.mkdir()
|
||||
path = root / "cfg.yaml"
|
||||
path.write_text("alpha settings:\n host: one\n")
|
||||
llm = FakeEmbedder()
|
||||
try:
|
||||
asyncio.run(import_sources([root], llm, session=db))
|
||||
path.write_text("bravo settings:\n host: two\n")
|
||||
summary = asyncio.run(import_sources([root], llm, session=db))
|
||||
assert summary.updated == 1
|
||||
assert summary.summaries == 1 and summary.summary_errors == 0
|
||||
doc = db.scalar(
|
||||
select(Document).where(Document.source == root.name, Document.path == "cfg.yaml")
|
||||
)
|
||||
assert doc is not None
|
||||
assert doc.summary is not None and doc.summary.startswith("Summary of bravo")
|
||||
schunks = [c for c in doc.chunks if c.is_summary]
|
||||
assert len(schunks) == 1 # the old one was deleted
|
||||
assert schunks[0].position == -1
|
||||
assert schunks[0].content == doc.summary
|
||||
assert schunks[0].embedding is not None
|
||||
assert "alpha" not in schunks[0].content # no stale summary text
|
||||
assert sorted(c.position for c in doc.chunks if not c.is_summary) == [0]
|
||||
finally:
|
||||
_cleanup_source(db, root.name)
|
||||
|
||||
|
||||
def test_store_summary_replaces_an_existing_summary_chunk(db, tmp_path: Path) -> None:
|
||||
"""Replacement unit, driven directly: with a pre-existing
|
||||
``is_summary`` chunk in place, ``_store_summary`` deletes the old one
|
||||
and leaves exactly one (new) summary chunk + updated
|
||||
``documents.summary`` — the at-most-one-summary invariant."""
|
||||
root = tmp_path / "direct"
|
||||
root.mkdir()
|
||||
(root / "a.yaml").write_text("alpha x\n")
|
||||
llm = FakeEmbedder()
|
||||
try:
|
||||
asyncio.run(import_sources([root], llm, session=db))
|
||||
doc = db.scalar(
|
||||
select(Document).where(Document.source == root.name, Document.path == "a.yaml")
|
||||
)
|
||||
assert doc is not None and doc.summary is not None
|
||||
assert any(c.is_summary for c in doc.chunks) # the first import's summary
|
||||
counters = ImportSummary()
|
||||
asyncio.run(
|
||||
_store_summary(
|
||||
session=db, doc=doc, source=root.name, rel="a.yaml",
|
||||
content=doc.content, llm=llm, summary=counters,
|
||||
)
|
||||
)
|
||||
assert counters.summaries == 1 and counters.summary_errors == 0
|
||||
schunks = [c for c in doc.chunks if c.is_summary]
|
||||
assert len(schunks) == 1 # the old one was deleted
|
||||
assert schunks[0].position == -1
|
||||
assert schunks[0].content == doc.summary
|
||||
assert schunks[0].embedding is not None
|
||||
finally:
|
||||
_cleanup_source(db, root.name)
|
||||
|
||||
|
||||
def test_store_summary_fail_soft_leaves_document_untouched(db, tmp_path: Path) -> None:
|
||||
"""A ``lite`` failure inside ``_store_summary`` rolls back only the
|
||||
summary rows: the previous summary (if any) and the document survive,
|
||||
and the failure is counted."""
|
||||
root = tmp_path / "directfail"
|
||||
root.mkdir()
|
||||
(root / "a.yaml").write_text("alpha x\n")
|
||||
llm = FakeEmbedder()
|
||||
try:
|
||||
asyncio.run(import_sources([root], llm, session=db))
|
||||
doc = db.scalar(
|
||||
select(Document).where(Document.source == root.name, Document.path == "a.yaml")
|
||||
)
|
||||
assert doc is not None and doc.summary is not None
|
||||
previous_summary = doc.summary
|
||||
(root / "a.yaml").write_text("SUMMARY-BLOWUP now the lite model fails\n")
|
||||
counters = ImportSummary()
|
||||
asyncio.run(
|
||||
_store_summary(
|
||||
session=db, doc=doc, source=root.name, rel="a.yaml",
|
||||
content="SUMMARY-BLOWUP now the lite model fails\n",
|
||||
llm=llm, summary=counters,
|
||||
)
|
||||
)
|
||||
assert counters.summaries == 0 and counters.summary_errors == 1
|
||||
assert doc.summary == previous_summary # rolled back, not nulled
|
||||
schunks = [c for c in doc.chunks if c.is_summary]
|
||||
assert len(schunks) == 1 # the old one survived the rollback
|
||||
assert schunks[0].content == previous_summary
|
||||
finally:
|
||||
_cleanup_source(db, root.name)
|
||||
|
||||
|
||||
def test_import_summary_log_line_includes_summary_counters(
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
"""PLAN §9 summary line: the phase-30 counters sit between
|
||||
``embed_batches`` and ``formats``."""
|
||||
s = ImportSummary()
|
||||
s.files, s.added, s.chunks, s.embed_batches = 3, 3, 5, 4
|
||||
s.summaries, s.summary_errors = 2, 1
|
||||
s.formats = {"md": 1, "yaml": 2}
|
||||
with caplog.at_level(logging.INFO, logger="app.importer"):
|
||||
s.log()
|
||||
line = caplog.records[-1].getMessage()
|
||||
assert line == (
|
||||
"import: summary files=3 added=3 updated=0 unchanged=0 pruned=0 errors=0 "
|
||||
"chunks=5 embed_batches=4 summaries=2 summary_errors=1 formats=yaml:2,md:1"
|
||||
)
|
||||
|
||||
|
||||
def test_prune_removes_files_now_excluded_by_format_filter(db, tmp_path: Path) -> None:
|
||||
"""Previously-imported junk leaves the index: a file that no longer
|
||||
matches the A9 extension filter is pruned on the next ``prune=True`` run.
|
||||
|
||||
@@ -275,17 +275,42 @@ class _FakeChatStream:
|
||||
return chunk
|
||||
|
||||
|
||||
class _FakeCompletion:
|
||||
"""One fake non-streaming ChatCompletion (``choices[].message`` shape).
|
||||
|
||||
``content=None`` mirrors the real wire where the field can be absent or
|
||||
empty (reasoning-only replies, provider quirks).
|
||||
"""
|
||||
|
||||
def __init__(self, content: str | None, empty_choices: bool = False) -> None:
|
||||
if empty_choices:
|
||||
self.choices = []
|
||||
else:
|
||||
self.choices = [SimpleNamespace(message=SimpleNamespace(content=content))]
|
||||
|
||||
|
||||
class _FakeCompletions:
|
||||
def __init__(self, chunks: list | None = None, fail: Exception | None = None) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
chunks: list | None = None,
|
||||
fail: Exception | None = None,
|
||||
completion: _FakeCompletion | None = None,
|
||||
) -> None:
|
||||
self.chunks = chunks or []
|
||||
self.fail = fail
|
||||
self.completion = completion
|
||||
self.kwargs: dict | None = None
|
||||
self.chat_kwargs: dict | None = None
|
||||
|
||||
async def create(self, **kwargs) -> _FakeChatStream:
|
||||
async def create(self, **kwargs) -> _FakeChatStream | _FakeCompletion:
|
||||
self.kwargs = kwargs
|
||||
if self.fail is not None:
|
||||
raise self.fail
|
||||
return _FakeChatStream(self.chunks)
|
||||
if kwargs.get("stream"):
|
||||
return _FakeChatStream(self.chunks)
|
||||
self.chat_kwargs = kwargs
|
||||
assert self.completion is not None
|
||||
return self.completion
|
||||
|
||||
|
||||
def _make_stream_client(
|
||||
@@ -428,3 +453,85 @@ def test_chat_stream_llm_error_passes_through_unwrapped() -> None:
|
||||
llm, _ = _make_stream_client(fail=LLMError("already wrapped"))
|
||||
with pytest.raises(LLMError, match="already wrapped"):
|
||||
asyncio.run(_collect(llm, [{"role": "user", "content": "q"}]))
|
||||
|
||||
|
||||
# ---------- one-shot chat: LLMClient.chat (phase 30, task 01) ----------
|
||||
|
||||
|
||||
def _make_chat_client(
|
||||
completion: _FakeCompletion | None = None,
|
||||
fail: Exception | None = None,
|
||||
**settings_kwargs: Any,
|
||||
) -> tuple[LLMClient, _FakeCompletions]:
|
||||
completions = _FakeCompletions(fail=fail, completion=completion)
|
||||
fake_openai = SimpleNamespace(chat=SimpleNamespace(completions=completions))
|
||||
llm = LLMClient(_settings(**settings_kwargs))
|
||||
llm._client = fake_openai # pyright: ignore[reportAttributeAccessIssue]
|
||||
return llm, completions
|
||||
|
||||
|
||||
def test_chat_returns_trimmed_content_with_locked_params() -> None:
|
||||
"""Default model is ``lite`` (BOR_LLM_SUMMARY_MODEL), non-streaming,
|
||||
low temperature, fixed 2048-token budget — summaries are short."""
|
||||
llm, completions = _make_chat_client(_FakeCompletion(" Summary text.\n"))
|
||||
messages = [{"role": "system", "content": "s"}, {"role": "user", "content": "u"}]
|
||||
out = asyncio.run(llm.chat(messages))
|
||||
assert out == "Summary text."
|
||||
assert completions.chat_kwargs is not None
|
||||
assert completions.chat_kwargs["model"] == "lite"
|
||||
assert completions.chat_kwargs["stream"] is False
|
||||
assert completions.chat_kwargs["temperature"] == 0.2
|
||||
assert completions.chat_kwargs["max_tokens"] == 2048
|
||||
assert completions.chat_kwargs["messages"] == messages
|
||||
|
||||
|
||||
def test_chat_default_model_comes_from_llm_summary_model_setting() -> None:
|
||||
llm, completions = _make_chat_client(
|
||||
_FakeCompletion("x"), llm_summary_model="tiny"
|
||||
)
|
||||
asyncio.run(llm.chat([{"role": "user", "content": "q"}]))
|
||||
assert completions.chat_kwargs is not None
|
||||
assert completions.chat_kwargs["model"] == "tiny"
|
||||
|
||||
|
||||
def test_chat_explicit_model_overrides_the_default() -> None:
|
||||
llm, completions = _make_chat_client(
|
||||
_FakeCompletion("x"), llm_summary_model="tiny"
|
||||
)
|
||||
asyncio.run(llm.chat([{"role": "user", "content": "q"}], model="special"))
|
||||
assert completions.chat_kwargs is not None
|
||||
assert completions.chat_kwargs["model"] == "special"
|
||||
|
||||
|
||||
def test_chat_transport_failure_wrapped_as_llm_error_with_base_url() -> None:
|
||||
"""HTTP/transport failures (incl. >=400 surfaced by the SDK) are wrapped
|
||||
with the base URL in the message — same style as chat_stream."""
|
||||
llm, _ = _make_chat_client(fail=RuntimeError("HTTP 502 Bad Gateway"))
|
||||
with pytest.raises(LLMError, match="HTTP 502") as exc:
|
||||
asyncio.run(llm.chat([{"role": "user", "content": "q"}]))
|
||||
assert "aipi.reeseapps.com" in str(exc.value)
|
||||
|
||||
|
||||
def test_chat_llm_error_passes_through_unwrapped() -> None:
|
||||
llm, _ = _make_chat_client(fail=LLMError("already wrapped"))
|
||||
with pytest.raises(LLMError, match="already wrapped"):
|
||||
asyncio.run(llm.chat([{"role": "user", "content": "q"}]))
|
||||
|
||||
|
||||
def test_chat_empty_choices_raises_llm_error() -> None:
|
||||
llm, _ = _make_chat_client(_FakeCompletion(None, empty_choices=True))
|
||||
with pytest.raises(LLMError, match="no choices"):
|
||||
asyncio.run(llm.chat([{"role": "user", "content": "q"}]))
|
||||
|
||||
|
||||
def test_chat_missing_content_raises_llm_error() -> None:
|
||||
"""A silent empty summary must never be stored — None content fails."""
|
||||
llm, _ = _make_chat_client(_FakeCompletion(None))
|
||||
with pytest.raises(LLMError, match="empty content"):
|
||||
asyncio.run(llm.chat([{"role": "user", "content": "q"}]))
|
||||
|
||||
|
||||
def test_chat_whitespace_only_content_raises_llm_error() -> None:
|
||||
llm, _ = _make_chat_client(_FakeCompletion(" \n\t "))
|
||||
with pytest.raises(LLMError, match="empty content"):
|
||||
asyncio.run(llm.chat([{"role": "user", "content": "q"}]))
|
||||
|
||||
@@ -7,6 +7,7 @@ chat integration tests against real Postgres; the pure mapping logic in
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -97,7 +98,8 @@ from app.rag.retriever import fuse, lexical_tsquery # noqa: E402
|
||||
|
||||
|
||||
def _rc(
|
||||
doc_path: str, cosine: float = 0.0, fts_hit: bool = False, position: int = 0
|
||||
doc_path: str, cosine: float = 0.0, fts_hit: bool = False, position: int = 0,
|
||||
is_summary: bool = False,
|
||||
) -> RetrievedChunk:
|
||||
return RetrievedChunk(
|
||||
chunk_id=uuid.uuid4(),
|
||||
@@ -107,6 +109,7 @@ def _rc(
|
||||
document=_doc(doc_path, "x" * 20),
|
||||
cosine=cosine,
|
||||
fts_hit=fts_hit,
|
||||
is_summary=is_summary,
|
||||
)
|
||||
|
||||
|
||||
@@ -186,3 +189,137 @@ def test_fuse_rejects_nonpositive_k() -> None:
|
||||
|
||||
def test_fuse_empty_lists() -> None:
|
||||
assert fuse([], [], k=60) == []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 30: is_summary survives both candidate lists and the fusion
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
from app.models import Chunk # noqa: E402
|
||||
from app.rag.retriever import _lexical_candidates, _vector_candidates # noqa: E402
|
||||
|
||||
|
||||
class _FakeResult:
|
||||
"""Stands in for SQLAlchemy's RowMapping result (``.all()`` only)."""
|
||||
|
||||
def __init__(self, rows: list) -> None:
|
||||
self._rows = rows
|
||||
|
||||
def all(self) -> list:
|
||||
return self._rows
|
||||
|
||||
|
||||
class _FakeSession:
|
||||
"""Returns canned rows from ``execute`` without touching Postgres."""
|
||||
|
||||
def __init__(self, rows: list) -> None:
|
||||
self._rows = rows
|
||||
self.statements: list = []
|
||||
|
||||
def execute(self, stmt, params: dict | None = None) -> _FakeResult:
|
||||
self.statements.append((stmt, params))
|
||||
return _FakeResult(self._rows)
|
||||
|
||||
|
||||
def _chunk_row(is_summary: bool) -> Chunk:
|
||||
doc = _doc("summary-src.yaml", "RAW_YAML_CONTENT")
|
||||
return Chunk(
|
||||
id=uuid.uuid4(),
|
||||
document_id=doc.id,
|
||||
position=-1, # the summary chunk's position (phase 30)
|
||||
content="Summary text",
|
||||
is_summary=is_summary,
|
||||
)
|
||||
|
||||
|
||||
def test_vector_candidates_carry_is_summary_flag() -> None:
|
||||
"""The vector list copies ``Chunk.is_summary`` onto each candidate."""
|
||||
doc = _doc("summary-src.yaml", "RAW_YAML_CONTENT")
|
||||
summary = _chunk_row(is_summary=True)
|
||||
ordinary = _chunk_row(is_summary=False)
|
||||
ordinary.position = 0
|
||||
ordinary.content = "ordinary content"
|
||||
rows = [
|
||||
(summary, 0.123456, doc),
|
||||
(ordinary, 0.2, doc),
|
||||
]
|
||||
out = _vector_candidates(_FakeSession(rows), [0.0] * 768, limit=5) # pyright: ignore[reportArgumentType]
|
||||
assert len(out) == 2
|
||||
by_pos = {rc.position: rc for rc in out}
|
||||
assert by_pos[-1].is_summary is True # the summary chunk (position −1)
|
||||
assert by_pos[0].is_summary is False # ordinary content chunk
|
||||
assert by_pos[-1].cosine == pytest.approx(0.876544) # 1 − distance, still rounded
|
||||
|
||||
|
||||
def test_vector_candidates_default_is_summary_false_for_legacy_chunks() -> None:
|
||||
"""Pre-phase-30 rows have ``is_summary=false`` — candidates stay False."""
|
||||
doc = _doc("legacy.md", "LEGACY")
|
||||
legacy = Chunk(
|
||||
id=uuid.uuid4(),
|
||||
document_id=doc.id,
|
||||
position=0,
|
||||
content="legacy content",
|
||||
is_summary=False,
|
||||
)
|
||||
out = _vector_candidates(_FakeSession([(legacy, 0.5, doc)]), [0.0] * 768, limit=5) # pyright: ignore[reportArgumentType]
|
||||
assert out[0].is_summary is False
|
||||
|
||||
|
||||
def _lexical_row(is_summary: bool, doc_path: str) -> object:
|
||||
"""One row of ``_LEXICAL_SQL`` (attribute access, as SQLAlchemy returns)."""
|
||||
doc = _doc(doc_path, "DOC_BODY")
|
||||
return SimpleNamespace(
|
||||
chunk_id=uuid.uuid4(),
|
||||
position=-1 if is_summary else 0,
|
||||
content="summary chunk text" if is_summary else "content chunk text",
|
||||
doc_id=doc.id,
|
||||
source=doc.source,
|
||||
path=doc.path,
|
||||
full_path=doc.full_path,
|
||||
title=doc.title,
|
||||
doc_content=doc.content,
|
||||
content_hash=doc.content_hash,
|
||||
indexed_at=None,
|
||||
is_summary=is_summary,
|
||||
rank=0.33,
|
||||
)
|
||||
|
||||
|
||||
def test_lexical_candidates_carry_is_summary_flag() -> None:
|
||||
"""The lexical list reads ``c.is_summary`` from the raw row."""
|
||||
rows = [_lexical_row(True, "summary-src.yaml"), _lexical_row(False, "other.md")]
|
||||
out = _lexical_candidates(_FakeSession(rows), "how do i configure the thing", limit=10) # pyright: ignore[reportArgumentType]
|
||||
assert len(out) == 2
|
||||
by_path = {rc.document.path: rc for rc in out}
|
||||
assert by_path["summary-src.yaml"].is_summary is True
|
||||
assert by_path["summary-src.yaml"].position == -1
|
||||
assert by_path["other.md"].is_summary is False
|
||||
assert all(rc.fts_hit is True for rc in out)
|
||||
|
||||
|
||||
def test_fuse_keeps_is_summary_on_double_hit() -> None:
|
||||
"""A summary chunk in both lists keeps the flag after fusion."""
|
||||
v1 = _rc("s.yaml", cosine=0.9, is_summary=True)
|
||||
l1 = _rc("s.yaml", cosine=0.9, is_summary=True) # lexical copy of the same chunk
|
||||
l1.chunk_id = v1.chunk_id
|
||||
out = fuse([v1], [l1], k=60)
|
||||
assert len(out) == 1
|
||||
assert out[0].is_summary is True
|
||||
assert out[0].fts_hit is True
|
||||
assert out[0].score == pytest.approx(2 / 61)
|
||||
|
||||
|
||||
def test_fuse_keeps_is_summary_on_lexical_only_hit() -> None:
|
||||
"""A summary-only lexical hit (no vector rank) keeps the flag."""
|
||||
out = fuse([], [_rc("s.yaml", is_summary=True)], k=60)
|
||||
assert len(out) == 1
|
||||
assert out[0].is_summary is True
|
||||
assert out[0].fts_hit is True
|
||||
assert out[0].cosine == 0.0
|
||||
|
||||
|
||||
def test_fuse_default_is_summary_stays_false_for_legacy_chunks() -> None:
|
||||
"""Neither list flagged ⇒ fusion never invents a summary flag."""
|
||||
out = fuse([_rc("a.md", cosine=0.8)], [_rc("b.md")], k=60)
|
||||
assert len(out) == 2
|
||||
assert all(rc.is_summary is False for rc in out)
|
||||
|
||||
@@ -0,0 +1,173 @@
|
||||
"""Unit: document summarizer (phase 30, task 03).
|
||||
|
||||
Covers the ``SUMMARY_MODE`` prompt (marker + instruction, capped user
|
||||
content), the code-deterministic ``Source: <source>/<path>`` pointer,
|
||||
and the rejection of empty/whitespace model output.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
from app.config import Settings, get_settings
|
||||
from app.rag.llm import LLMError
|
||||
from app.rag.retriever import TRUNCATION_MARKER
|
||||
from app.rag.summarizer import (
|
||||
SUMMARY_INSTRUCTION,
|
||||
SUMMARY_MODE,
|
||||
SYSTEM_PROMPT,
|
||||
build_summary_prompt,
|
||||
generate_summary,
|
||||
)
|
||||
|
||||
|
||||
class _FakeLLM:
|
||||
"""Duck-typed stand-in for ``LLMClient`` (``chat`` + ``settings``).
|
||||
|
||||
Records the messages and the ``model`` kwarg it was called with; can
|
||||
return a canned reply or raise (e.g. :class:`LLMError`).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
reply: str | None = "Backups run nightly at 02:00 via the borg schedule.",
|
||||
fail: Exception | None = None,
|
||||
) -> None:
|
||||
self._reply = reply
|
||||
self._fail = fail
|
||||
self.settings = Settings(_env_file=None) # pyright: ignore[reportCallIssue]
|
||||
self.messages: list[dict[str, str]] = []
|
||||
self.model: str | None = None
|
||||
|
||||
async def chat(
|
||||
self, messages: list[dict[str, str]], model: str | None = None
|
||||
) -> str:
|
||||
self.messages = list(messages)
|
||||
self.model = model
|
||||
if self._fail is not None:
|
||||
raise self._fail
|
||||
assert self._reply is not None
|
||||
return self._reply
|
||||
|
||||
|
||||
# ---------- build_summary_prompt: system ----------
|
||||
|
||||
|
||||
def test_system_prompt_has_marker_and_locked_instruction() -> None:
|
||||
assert SYSTEM_PROMPT.startswith(SUMMARY_MODE)
|
||||
assert SUMMARY_INSTRUCTION in SYSTEM_PROMPT
|
||||
for fragment in (
|
||||
"plain-text summary of this document in natural",
|
||||
"what it configures/defines",
|
||||
"Do not use markdown",
|
||||
"Do not invent anything that is not in the document",
|
||||
):
|
||||
assert fragment in SYSTEM_PROMPT
|
||||
system, _ = build_summary_prompt("Homelab", "a.yaml", "content")
|
||||
assert system == SYSTEM_PROMPT
|
||||
assert SUMMARY_MODE in system # the marker the E2E mock keys on
|
||||
|
||||
|
||||
# ---------- build_summary_prompt: user (capped content) ----------
|
||||
|
||||
|
||||
def test_user_prompt_is_full_content_when_under_cap() -> None:
|
||||
content = "services:\n borg:\n port: 9999"
|
||||
_, user = build_summary_prompt("Homelab", "a.yaml", content, max_chars=12_000)
|
||||
assert user == content
|
||||
assert TRUNCATION_MARKER not in user
|
||||
|
||||
|
||||
def test_user_prompt_at_exact_cap_is_not_truncated() -> None:
|
||||
content = "z" * 64
|
||||
_, user = build_summary_prompt("Homelab", "a.yaml", content, max_chars=64)
|
||||
assert user == content
|
||||
assert TRUNCATION_MARKER not in user
|
||||
|
||||
|
||||
def test_user_prompt_truncated_with_marker_when_over_custom_cap() -> None:
|
||||
content = "x" * 100 + "TAIL"
|
||||
_, user = build_summary_prompt("Homelab", "a.yaml", content, max_chars=100)
|
||||
assert user == "x" * 100 + "\n" + TRUNCATION_MARKER
|
||||
assert "TAIL" not in user # overflow is gone, not squeezed in
|
||||
assert user.endswith(TRUNCATION_MARKER)
|
||||
|
||||
|
||||
def test_user_prompt_truncated_at_default_cap() -> None:
|
||||
"""No explicit cap → ``BOR_SUMMARY_MAX_CHARS`` (read from the live
|
||||
settings, so the test holds for any configured value)."""
|
||||
cap = get_settings().summary_max_chars
|
||||
content = "y" * (cap + 50)
|
||||
_, user = build_summary_prompt("Homelab", "a.yaml", content)
|
||||
assert user == "y" * cap + "\n" + TRUNCATION_MARKER
|
||||
|
||||
|
||||
# ---------- generate_summary: pointer + validation ----------
|
||||
|
||||
|
||||
def test_generate_summary_returns_model_text_plus_deterministic_pointer() -> None:
|
||||
llm = _FakeLLM(reply="Backups run nightly at 02:00 via the borg schedule.")
|
||||
out = asyncio.run(
|
||||
generate_summary(llm, source="Homelab", path="backups/borg.yaml", content="c")
|
||||
)
|
||||
expected = (
|
||||
"Backups run nightly at 02:00 via the borg schedule.\n"
|
||||
"Source: Homelab/backups/borg.yaml"
|
||||
)
|
||||
assert out == expected
|
||||
assert out.splitlines()[-1] == "Source: Homelab/backups/borg.yaml"
|
||||
|
||||
|
||||
def test_generate_summary_calls_the_configured_summary_model() -> None:
|
||||
llm = _FakeLLM(reply="s")
|
||||
asyncio.run(generate_summary(llm, source="Homelab", path="a.yaml", content="c"))
|
||||
assert llm.model == llm.settings.llm_summary_model # the ``lite`` default
|
||||
assert llm.model == "lite"
|
||||
assert [m["role"] for m in llm.messages] == ["system", "user"]
|
||||
assert SUMMARY_MODE in llm.messages[0]["content"]
|
||||
assert llm.messages[1] == {"role": "user", "content": "c"}
|
||||
|
||||
|
||||
def test_generate_summary_strips_model_text_before_appending_pointer() -> None:
|
||||
llm = _FakeLLM(reply=" padded summary. \n")
|
||||
out = asyncio.run(generate_summary(llm, source="Deployments", path="f.txt", content="c"))
|
||||
assert out == "padded summary.\nSource: Deployments/f.txt"
|
||||
|
||||
|
||||
def test_pointer_is_code_deterministic_even_if_model_writes_its_own() -> None:
|
||||
"""The pointer must never be model-generated: even a model reply that
|
||||
contains a bogus 'Source:' line ends with the code-appended one."""
|
||||
llm = _FakeLLM(reply="The document itself says Source: fake/other.yaml inside.")
|
||||
out = asyncio.run(generate_summary(llm, source="Homelab", path="real.yaml", content="c"))
|
||||
assert out.splitlines()[-1] == "Source: Homelab/real.yaml"
|
||||
|
||||
|
||||
def test_generate_summary_sends_capped_content_to_the_model() -> None:
|
||||
"""The cap applies to what the model actually receives (overflow cut
|
||||
at the cap + marker) — read from the live settings for any value."""
|
||||
llm = _FakeLLM(reply="s")
|
||||
content = "w" * (get_settings().summary_max_chars + 50)
|
||||
asyncio.run(generate_summary(llm, source="Homelab", path="a.yaml", content=content))
|
||||
cap = get_settings().summary_max_chars
|
||||
assert llm.messages[1]["content"] == "w" * cap + "\n" + TRUNCATION_MARKER
|
||||
|
||||
|
||||
def test_generate_summary_rejects_whitespace_only_reply() -> None:
|
||||
llm = _FakeLLM(reply=" \n\t ")
|
||||
with pytest.raises(LLMError, match="empty content"):
|
||||
asyncio.run(generate_summary(llm, source="Homelab", path="a.yaml", content="c"))
|
||||
|
||||
|
||||
def test_generate_summary_rejects_empty_reply() -> None:
|
||||
llm = _FakeLLM(reply="")
|
||||
with pytest.raises(LLMError, match="empty content"):
|
||||
asyncio.run(generate_summary(llm, source="Homelab", path="a.yaml", content="c"))
|
||||
|
||||
|
||||
def test_generate_summary_propagates_llm_error_from_client() -> None:
|
||||
llm = _FakeLLM(
|
||||
fail=LLMError("chat completion from https://aipi.reeseapps.com/v1 failed: boom")
|
||||
)
|
||||
with pytest.raises(LLMError, match="boom"):
|
||||
asyncio.run(generate_summary(llm, source="Homelab", path="a.yaml", content="c"))
|
||||
Reference in New Issue
Block a user