From 572a4190a62f7fbad32d6fdd3d7e5f68b2b4a775 Mon Sep 17 00:00:00 2001 From: ducoterra Date: Tue, 25 Aug 2026 17:48:37 -0400 Subject: [PATCH] =?UTF-8?q?feat(rag):=20lite-model=20document=20summaries?= =?UTF-8?q?=20=E2=80=94=20non-markdown=20docs=20summarized=20at=20import,?= =?UTF-8?q?=20summary=20chunk=20retrieves=20and=20resolves=20to=20the=20fu?= =?UTF-8?q?ll=20source=20doc?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../01_lite_model_client.md | 0 .../02_migration_summary_columns.md | 0 .../03_summarizer_module.md | 0 .../04_importer_summary_integration.md | 0 .../05_pipeline_summary_resolution.md | 0 .agent/user_stories/document-summaries.md | 70 +++++ .env.example | 2 + README.md | 28 ++ alembic/versions/0004_summary_columns.py | 45 +++ app/api/chat.py | 28 +- app/config.py | 10 + app/models.py | 10 + app/rag/importer.py | 83 +++++- app/rag/llm.py | 54 +++- app/rag/retriever.py | 13 +- app/rag/summarizer.py | 123 +++++++++ scripts/import_docs.py | 3 +- tests/e2e/mock_llm.py | 16 ++ tests/e2e/test_document_summaries.py | 257 ++++++++++++++++++ tests/e2e/test_import_documents.py | 6 +- tests/fakes.py | 18 +- .../summary_kb/notes/qwen-llamacpp-notes.md | 16 ++ .../summary_kb/quadlet/qwen-llamacpp.yaml | 143 ++++++++++ tests/integration/test_chat_api.py | 9 + tests/integration/test_importer_e2e.py | 22 +- tests/integration/test_migration_0004.py | 144 ++++++++++ tests/unit/test_chat_gate.py | 73 ++++- tests/unit/test_config.py | 19 ++ tests/unit/test_importer.py | 215 ++++++++++++++- tests/unit/test_llm_client.py | 113 +++++++- tests/unit/test_retriever.py | 139 +++++++++- tests/unit/test_summarizer.py | 173 ++++++++++++ 32 files changed, 1806 insertions(+), 26 deletions(-) rename .agent/phases/{todo => complete}/30_document_summaries/01_lite_model_client.md (100%) rename .agent/phases/{todo => complete}/30_document_summaries/02_migration_summary_columns.md (100%) rename .agent/phases/{todo => complete}/30_document_summaries/03_summarizer_module.md (100%) rename .agent/phases/{todo => complete}/30_document_summaries/04_importer_summary_integration.md (100%) rename .agent/phases/{todo => complete}/30_document_summaries/05_pipeline_summary_resolution.md (100%) create mode 100644 .agent/user_stories/document-summaries.md create mode 100644 alembic/versions/0004_summary_columns.py create mode 100644 app/rag/summarizer.py create mode 100644 tests/e2e/test_document_summaries.py create mode 100644 tests/fixtures/summary_kb/notes/qwen-llamacpp-notes.md create mode 100644 tests/fixtures/summary_kb/quadlet/qwen-llamacpp.yaml create mode 100644 tests/integration/test_migration_0004.py create mode 100644 tests/unit/test_summarizer.py diff --git a/.agent/phases/todo/30_document_summaries/01_lite_model_client.md b/.agent/phases/complete/30_document_summaries/01_lite_model_client.md similarity index 100% rename from .agent/phases/todo/30_document_summaries/01_lite_model_client.md rename to .agent/phases/complete/30_document_summaries/01_lite_model_client.md diff --git a/.agent/phases/todo/30_document_summaries/02_migration_summary_columns.md b/.agent/phases/complete/30_document_summaries/02_migration_summary_columns.md similarity index 100% rename from .agent/phases/todo/30_document_summaries/02_migration_summary_columns.md rename to .agent/phases/complete/30_document_summaries/02_migration_summary_columns.md diff --git a/.agent/phases/todo/30_document_summaries/03_summarizer_module.md b/.agent/phases/complete/30_document_summaries/03_summarizer_module.md similarity index 100% rename from .agent/phases/todo/30_document_summaries/03_summarizer_module.md rename to .agent/phases/complete/30_document_summaries/03_summarizer_module.md diff --git a/.agent/phases/todo/30_document_summaries/04_importer_summary_integration.md b/.agent/phases/complete/30_document_summaries/04_importer_summary_integration.md similarity index 100% rename from .agent/phases/todo/30_document_summaries/04_importer_summary_integration.md rename to .agent/phases/complete/30_document_summaries/04_importer_summary_integration.md diff --git a/.agent/phases/todo/30_document_summaries/05_pipeline_summary_resolution.md b/.agent/phases/complete/30_document_summaries/05_pipeline_summary_resolution.md similarity index 100% rename from .agent/phases/todo/30_document_summaries/05_pipeline_summary_resolution.md rename to .agent/phases/complete/30_document_summaries/05_pipeline_summary_resolution.md diff --git a/.agent/user_stories/document-summaries.md b/.agent/user_stories/document-summaries.md new file mode 100644 index 0000000..f0ed0c9 --- /dev/null +++ b/.agent/user_stories/document-summaries.md @@ -0,0 +1,70 @@ +# Story: Document Summaries (lite-model summaries for non-markdown docs) + +**Phase:** `30_document_summaries.md` · **E2E:** `tests/e2e/test_document_summaries.py` + +## Narrative + +As **any user of Reese**, I ask questions about notes that live in +machine-formatted files (yaml, json, py, txt). Raw flags and keys embed +badly, so retrieval misses those documents. I want a small model to +**analyze every non-markdown document at import time** and store a +natural-language summary with a pointer back to the source. When +retrieval hits a summary, Reese follows the pointer and answers from the +**full source document** — the summary is a retrieval target, never the +answer. + +- **Given** a knowledge base containing non-markdown A9 documents (yaml, + yml, json, py, txt) +- **When** I run `import_docs` and then ask a question those documents + answer +- **Then** each non-markdown document carries a `lite`-model summary + (`documents.summary` plus one embedded `is_summary` chunk at position + −1), markdown documents carry neither, and a question whose best match + is a summary chunk gets an answer grounded in the **full source + document**, with the hit counted in the per-turn log line + (`summary_hits=N`). + +## Acceptance criteria +1. `BOR_LLM_SUMMARY_MODEL` (default `lite`) + non-streaming + `LLMClient.chat()` — A5 extended (same OpenAI-compatible endpoint), + empty/missing content raises `LLMError` (no silent summaries). +2. Migration 0004: `documents.summary TEXT NULL` + + `chunks.is_summary BOOLEAN NOT NULL DEFAULT FALSE` (reversible, + integration-tested up/down). +3. `app/rag/summarizer.py`: the `SUMMARY_MODE` prompt (document content + capped at `BOR_SUMMARY_MAX_CHARS`, overflow marked with the shared + truncation marker), the lite call, output validation, and the + **code-deterministic** `Source: /` pointer line (never + model-generated). +4. Importer: every non-markdown file gets its summary stored + indexed + (best-effort — a lite failure logs, counts in `summary_errors`, and + still leaves the document fully indexed); markdown files get neither; + re-import replaces the old summary chunk (exactly one at a time). +5. `is_summary` flows through the retriever's vector and lexical + candidate lists and `fuse`; `TurnPlan.summary_hits` counts hit chunks + with `is_summary` whose parent document made the selected top-N + context; the per-turn log line records `summary_hits=N` after + `fts_hits=N` (PLAN §9 extension). +6. A summary hit resolves to its parent through the unchanged + chunk→document mapping — the LLM receives the full source document + (A7 revised: never truncated). +7. `.env.example` + README document `BOR_LLM_SUMMARY_MODEL` / + `BOR_SUMMARY_MAX_CHARS` and the fail-soft behavior; the deterministic + mock LLM answers `SUMMARY_MODE` with a byte-stable 24-token digest. +8. Unit + integration green, `app/` coverage >90%, story E2E green in + isolation, `ruff` + `pyright` clean, one `--no-gpg-sign` commit. + +## Playwright Mapping Rule +`tests/e2e/test_document_summaries.py` — one story, one file, run in +isolation. It imports the story-dedicated fixture KB +(`tests/fixtures/summary_kb/`: one yaml document with a tail sentinel + +one markdown control document) through the real importer against the +deterministic mock LLM, asserts the yaml document's import state (one +embedded `is_summary` chunk at position −1 whose text is the byte-stable +mock digest + the deterministic `Source:` pointer), asserts the summary +chunk is the yaml document's best fused chunk, then asks the question +whose best yaml match is that summary chunk and asserts the rendered +answer quotes the document's tail sentinel `RESE-SUMMARY-SENTINEL-7f3a` +(only possible if the full source document — not the summary digest — +reached the LLM), the source chip cites the yaml path, the turn is not +deflected, and the markdown control document carries no summary chunk. diff --git a/.env.example b/.env.example index 8517575..ee6239b 100644 --- a/.env.example +++ b/.env.example @@ -15,6 +15,7 @@ BOR_LLM_BASE_URL=https://aipi.reeseapps.com/v1 BOR_LLM_API_KEY= # falls back to $AIPI_KEY, then "not-needed" BOR_LLM_CHAT_MODEL=turbo BOR_LLM_EMBED_MODEL=embed +BOR_LLM_SUMMARY_MODEL=lite # one-shot completions: document summaries (phase 30), KB overview (phase 31) BOR_EMBEDDING_DIM=768 # verified 2026-08 via scripts/llm_probe.py BOR_STREAM_THINKING=1 # stream the model's thinking as `thinking` SSE events (0 to suppress) @@ -23,6 +24,7 @@ BOR_TOP_N_DOCS=2 BOR_RELEVANCE_THRESHOLD=0.62 # answer when best cosine >= this OR an FTS hit; else honest deflection BOR_MAX_OUTPUT_TOKENS=32768 # max answer length in tokens (answers must not be cut off) BOR_STEERING_MAX_CHARS=8000 # char budget for the (steering notes) prompt section +BOR_SUMMARY_MAX_CHARS=12000 # cap on document content sent to the lite summary model (phase 30) BOR_CHUNK_TARGET_CHARS=2000 BOR_CHUNK_OVERLAP_CHARS=200 BOR_EMBED_BATCH_SIZE=16 diff --git a/README.md b/README.md index 093a2fa..8d8fc92 100644 --- a/README.md +++ b/README.md @@ -316,6 +316,32 @@ absent. The top `BOR_TOP_N_DOCS` full documents are still what the LLM sees. `chunk_hits`, `deflected`, `sources`, `latency_ms`) — the raw material for tuning: `psql … -c 'SELECT question, top_score, fts_hits, deflected FROM query_log ORDER BY created_at DESC LIMIT 20'`. +## Document summaries (non-markdown) + +Raw yaml/json/py/txt embeds badly — flags and keys are not language, so +retrieval can miss exactly the documents that are all configuration. At +import time, every **non-markdown** A9 document is summarized by the aipi +`lite` model (one-shot `LLMClient.chat`, `BOR_LLM_SUMMARY_MODEL`, default +`lite`): + +* the summary is stored on `documents.summary` **and** indexed as one + extra embedded chunk (`chunks.is_summary`, position −1), so hybrid + search has a natural-language target to hit instead of the raw text; +* the last line is a **code-deterministic** pointer — `Source: + /` — appended by the app, never model-generated; +* the model only sees the first `BOR_SUMMARY_MAX_CHARS` (default 12000) + characters of the document; overflow is cut and marked with the shared + `[…truncated…]` marker. + +A summary hit resolves through the normal chunk→document mapping: the +LLM receives the **full source document** (never the summary alone, +never truncated). Markdown files are natural language already and get no +summary. Summary generation is best-effort: if `lite` fails for a file, +the document is still indexed (without a summary), the failure is logged +and counted in the import summary line (`summaries=N summary_errors=N`). +On a chat turn, the per-turn log line records `summary_hits=N` — how many +summary chunks of the selected context the question landed on. + ## Debugging `debugpy` is **off by default** and *never imported* unless you opt in — @@ -416,6 +442,7 @@ served locally (no CDN), `BOR_ENVIRONMENT=production`. | `BOR_LLM_API_KEY` | — (falls back to `$AIPI_KEY`) | aipi API key | | `BOR_LLM_CHAT_MODEL` | `turbo` | chat model | | `BOR_LLM_EMBED_MODEL` | `embed` | embedding model | +| `BOR_LLM_SUMMARY_MODEL` | `lite` | one-shot (non-streaming) completions: document summaries at import (phase 30) and the KB overview (phase 31) | | `BOR_EMBEDDING_DIM` | `768` | vector dimension (fixed at table creation) | | `BOR_TOP_N_DOCS` | `2` | full documents fed to the LLM | | `BOR_RELEVANCE_THRESHOLD` | `0.62` | answer when best cosine ≥ this **or** an FTS hit; below + no FTS ⇒ honest deflection | @@ -426,6 +453,7 @@ served locally (no CDN), `BOR_ENVIRONMENT=production`. | `BOR_GIT_SOURCES` | — (empty) | csv of git repo URLs; `import_docs` clones/pulls them into `BOR_SOURCES_DIR` and indexes the checkouts (see *Git-based sources*) | | `BOR_SOURCES_DIR` | `~/bor-sources` | where the `BOR_GIT_SOURCES` repos are cloned/pulled (one subdirectory per repo) | | `BOR_STEERING_MAX_CHARS` | `8000` | char budget for the `` (steering notes) prompt section | +| `BOR_SUMMARY_MAX_CHARS` | `12000` | cap on document content sent to the `lite` summary model at import (see *Document summaries*) | | `BOR_SUGGESTIONS` | built-in list | JSON list of onboarding chips | | `BOR_ADMIN_PASSWORD` | *(required)* | the single admin's password (plaintext, `.env`); app refuses to start when empty | | `BOR_SESSION_SECRET` | *(required)* | signing key for the `bor_session` cookie; `python -c 'import secrets;print(secrets.token_hex(32))'` | diff --git a/alembic/versions/0004_summary_columns.py b/alembic/versions/0004_summary_columns.py new file mode 100644 index 0000000..2679f68 --- /dev/null +++ b/alembic/versions/0004_summary_columns.py @@ -0,0 +1,45 @@ +"""document summaries: documents.summary + chunks.is_summary + +Revision ID: 0004 +Revises: 0003 +Create Date: 2026-08-25 + +Phase 30 (document-summaries story, A13 — two additive, reversible +columns, no table rework): + +* ``documents.summary`` — TEXT, nullable. Natural-language summary of the + document produced at import time by the aipi ``lite`` model (non-markdown + A9 documents only). NULL for markdown docs and for documents imported + before summaries existed (or whose summary generation failed — the + fail-soft path still indexes the document without a summary). +* ``chunks.is_summary`` — BOOLEAN NOT NULL DEFAULT false. Marks the single + extra summary chunk (position -1) that mirrors ``documents.summary`` into + the embedding space, so hybrid search has a well-embedding + natural-language target for badly-formatted raw text. The default keeps + every pre-0004 row valid; the summary chunk flows through the unchanged + A7 retrieval path (cosine ∪ FTS ∪ RRF) and the existing chunk→document + mapping resolves a summary hit to its full source document. +""" +from __future__ import annotations + +import sqlalchemy as sa + +from alembic import op + +revision = "0004" +down_revision = "0003" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column("documents", sa.Column("summary", sa.Text(), nullable=True)) + op.add_column( + "chunks", + sa.Column("is_summary", sa.Boolean(), nullable=False, server_default=sa.text("false")), + ) + + +def downgrade() -> None: + op.drop_column("chunks", "is_summary") + op.drop_column("documents", "summary") diff --git a/app/api/chat.py b/app/api/chat.py index 361a2ac..42eeb4e 100644 --- a/app/api/chat.py +++ b/app/api/chat.py @@ -28,6 +28,14 @@ Steering (phase 15): the owner's stored tuning notes are loaded per turn (oldest first) and injected into the system prompt as a ```` section — both the HIGH and the LOW prompt carry it. The per-turn log line records ``tuning=N`` (the number of injected notes). + +Summaries (phase 30): a lite-model summary chunk's parent *is* the +source document, so a summary hit resolves to the full source document +through the unchanged chunk→document mapping (A7 revised) — context +assembly is untouched. ``TurnPlan.summary_hits`` counts the hit chunks +with ``is_summary`` whose parent document landed in the selected +top-N context, and the per-turn log line records ``summary_hits=N`` +after ``fts_hits`` (PLAN §9 line extension). """ from __future__ import annotations @@ -96,6 +104,9 @@ class TurnPlan: docs: list[Document] # cited sources (weak hits when deflected) suggestions: list[str] # "Maybe try" chips (deflected turns only) tuning_count: int = 0 # steering notes injected into the system prompt + #: Hit chunks with ``is_summary`` whose parent document made it into + #: *docs* (phase 30; per-turn log line ``summary_hits=N``). + summary_hits: int = 0 def plan_turn( @@ -121,12 +132,18 @@ def plan_turn( *notes* are the owner's steering notes (phase 15, oldest first): when non-empty, both the HIGH and the LOW prompt carry the ```` section; with no notes the prompts are unchanged. + + ``summary_hits`` (phase 30) counts the hit chunks with + ``is_summary`` whose parent document is among the selected + top-N documents — both the HIGH and the LOW branch record it. """ steering = list(notes or []) best_cosine = max((c.cosine for c in chunks), default=0.0) fts_hits = sum(1 for c in chunks if c.fts_hit) + docs = select_documents(chunks, n=settings.top_n_docs) + selected_ids = {d.id for d in docs} + summary_hits = sum(1 for c in chunks if c.is_summary and c.document.id in selected_ids) if best_cosine >= settings.relevance_threshold or fts_hits > 0: - docs = select_documents(chunks, n=settings.top_n_docs) return TurnPlan( best_cosine, fts_hits, @@ -135,6 +152,7 @@ def plan_turn( docs, [], len(steering), + summary_hits, ) titles = weak_hit_titles(chunks) return TurnPlan( @@ -142,9 +160,10 @@ def plan_turn( fts_hits, True, build_deflect_prompt(titles, notes=steering), - select_documents(chunks, n=settings.top_n_docs), + docs, derive_suggestions(titles, settings.suggestions), len(steering), + summary_hits, ) @@ -264,12 +283,13 @@ async def chat( logger.exception("chat: failed to write query_log question=%r", request.message) logger.info( - "question=%r embed_ms=%d top_score=%.3f fts_hits=%d tuning=%d threshold=%.2f " - "deflected=%s sources=%r thinking_chars=%d total_ms=%d", + "question=%r embed_ms=%d top_score=%.3f fts_hits=%d summary_hits=%d tuning=%d " + "threshold=%.2f deflected=%s sources=%r thinking_chars=%d total_ms=%d", request.message, embed_ms, plan.top_score, plan.fts_hits, + plan.summary_hits, plan.tuning_count, settings.relevance_threshold, plan.deflected, diff --git a/app/config.py b/app/config.py index bff3a00..e16c62b 100644 --- a/app/config.py +++ b/app/config.py @@ -41,6 +41,11 @@ class Settings(BaseSettings): llm_api_key: str = "" llm_chat_model: str = "turbo" llm_embed_model: str = "embed" + #: One-shot (non-streaming) completion model (A5 extended, phase 30): + #: document summaries at import time and the KB overview (phase 31). + #: Served by the same OpenAI-compatible endpoint — no new model + #: management. Called via ``LLMClient.chat()``. + llm_summary_model: str = "lite" #: Operator kill-switch for the ``thinking`` SSE events (phase 17, #: ``BOR_STREAM_THINKING``; ``0``/``false`` → off). When off, thinking #: pieces are still counted for the per-turn log line but never @@ -66,6 +71,11 @@ class Settings(BaseSettings): #: (phase 15, steering notes). The newest-fitting notes are kept and the #: overflow is replaced by the ``[…truncated…]`` marker. steering_max_chars: int = 8_000 + #: Cap on the document content sent to the ``lite`` summary model in one + #: call (phase 30, ``BOR_SUMMARY_MAX_CHARS``). Overflow is cut at the cap + #: and the shared ``[…truncated…]`` marker is appended (see + #: ``app.rag.summarizer``). + summary_max_chars: int = 12_000 # --- Hybrid retrieval (A7, revised 2026-08-21) --- # cosine top-N ∪ Postgres FTS top-N, fused with Reciprocal Rank Fusion diff --git a/app/models.py b/app/models.py index 736a087..7b14601 100644 --- a/app/models.py +++ b/app/models.py @@ -50,6 +50,12 @@ class Document(Base): content: Mapped[str] = mapped_column(Text) # full markdown — the RAG context content_hash: Mapped[str] = mapped_column(String(64), index=True) # sha256 for change detection indexed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) + #: Lite-model summary, phase 30. Natural-language summary of the + #: document (non-markdown A9 docs only, generated at import time by the + #: aipi ``lite`` model). NULL for markdown docs, pre-phase-30 rows, and + #: the fail-soft path where summary generation failed but the document + #: was still indexed. + summary: Mapped[str | None] = mapped_column(Text, default=None) chunks: Mapped[list[Chunk]] = relationship( back_populates="document", cascade="all, delete-orphan" @@ -66,6 +72,10 @@ class Chunk(Base): position: Mapped[int] = mapped_column(Integer) content: Mapped[str] = mapped_column(Text) embedding: Mapped[list[float] | None] = mapped_column(Vector(EMBEDDING_DIM)) + #: Summary chunk, position −1, phase 30. Marks the single extra embedded + #: chunk mirroring ``Document.summary``; default False keeps every + #: pre-phase-30 row (and ordinary content chunks) valid. + is_summary: Mapped[bool] = mapped_column(Boolean, default=False) document: Mapped[Document] = relationship(back_populates="chunks") diff --git a/app/rag/importer.py b/app/rag/importer.py index 1baeb82..aa1cdfb 100644 --- a/app/rag/importer.py +++ b/app/rag/importer.py @@ -9,6 +9,12 @@ the two-phase upsert: 2. embed the new chunks in batches and attach the vectors 3. commit — one transaction per file, so a failed embedding leaves the database untouched and the file is simply retried on the next run +4. non-markdown files only (phase 30): generate a ``lite``-model summary + and, best-effort, store it on ``documents.summary`` plus one extra + embedded chunk (``is_summary``, position −1). The document row and its + content chunks are already committed at this point, so a summary + failure only means the file is indexed without a summary (logged and + counted in ``summary_errors``) — it is never lost. Scope (A9, revised 2026-08-21): any path containing a dot-prefixed component (hidden dirs — vendored caches like ``.esphome/.espressif/**`` — @@ -36,7 +42,8 @@ from app.config import Settings from app.db import SessionLocal from app.models import Chunk, Document from app.rag.chunker import chunk_document, extract_title -from app.rag.llm import EmbeddingError +from app.rag.llm import EmbeddingError, LLMError +from app.rag.summarizer import generate_summary logger = logging.getLogger("app.importer") @@ -54,6 +61,10 @@ class Embedder(Protocol): async def embed(self, texts: list[str]) -> list[list[float]]: ... + async def chat(self, messages: list[dict[str, str]], model: str | None = None) -> str: ... + # ^ the one-shot completion the summarizer uses for the ``lite`` model + # (phase 30, task 01); :class:`app.rag.llm.LLMClient` satisfies it. + @dataclass class ImportSummary: @@ -67,6 +78,12 @@ class ImportSummary: errors: int = 0 chunks: int = 0 embed_batches: int = 0 + #: Non-markdown files whose lite summary was generated + indexed + #: (phase 30). One ``is_summary`` chunk per success. + summaries: int = 0 + #: Non-markdown files whose summary generation failed (best-effort — + #: the document is still indexed, without a summary). + summary_errors: int = 0 #: Files walked, keyed by lowercased extension (``md``, ``yaml``, …). formats: dict[str, int] = field(default_factory=dict) @@ -80,7 +97,8 @@ class ImportSummary: def log(self) -> None: logger.info( "import: summary files=%d added=%d updated=%d unchanged=%d pruned=%d " - "errors=%d chunks=%d embed_batches=%d formats=%s", + "errors=%d chunks=%d embed_batches=%d summaries=%d summary_errors=%d " + "formats=%s", self.files, self.added, self.updated, @@ -89,6 +107,8 @@ class ImportSummary: self.errors, self.chunks, self.embed_batches, + self.summaries, + self.summary_errors, self.format_counts(), ) @@ -268,6 +288,65 @@ async def _index_file( summary.chunks += len(chunks_text) logger.info("import: %s source=%s path=%s chunks=%d", verb, source, rel, len(chunks_text)) + # Phase 30: markdown is already natural language, so only the other A9 + # formats (txt, yaml, yml, json, py) get a ``lite``-model summary. + if full_path.suffix.lower() in (".md", ".markdown"): + return + await _store_summary( + session, doc=doc, source=source, rel=rel, content=content, llm=llm, summary=summary + ) + + +async def _store_summary( + session: Session, + *, + doc: Document, + source: str, + rel: str, + content: str, + llm: Embedder, + summary: ImportSummary, +) -> None: + """Best-effort ``lite`` summary for one already-committed document. + + Generates the summary (task 03), stores it on ``documents.summary`` + and indexes it as one extra embedded chunk (``is_summary``, + position −1) that hybrid search can hit instead of badly-formatted + raw text. Replacement is guaranteed: any pre-existing ``is_summary`` + chunk of this document is deleted first, so at most one summary chunk + exists per document at a time. + + Best-effort by contract: the doc row + content chunks are committed + by the caller before this runs, so an :class:`LLMError` / + :class:`EmbeddingError` only rolls back the summary rows — the file + stays indexed, without a summary, and the failure is counted in + ``summary_errors`` (PLAN phase 30). + """ + try: + text = await generate_summary(llm, source=source, path=rel, content=content) + # Replacement: at most one summary chunk per document at a time. + # Removing from the collection is what the ``delete-orphan`` + # cascade turns into a row delete on flush — and it keeps the + # in-memory collection consistent (this session runs with + # ``expire_on_commit=False``). + for old in [c for c in doc.chunks if c.is_summary]: + doc.chunks.remove(old) + chunk = Chunk(document_id=doc.id, position=-1, content=text, is_summary=True) + vector = (await llm.embed([text]))[0] + chunk.embedding = vector + doc.summary = text + # Append through the relationship (the ``all`` cascade persists the + # row) so the collection — live in this session because of + # ``expire_on_commit=False`` — reflects the committed state. + doc.chunks.append(chunk) + session.commit() + summary.summaries += 1 + logger.info("import: summary source=%s path=%s chars=%d", source, rel, len(text)) + except (LLMError, EmbeddingError) as e: + session.rollback() + summary.summary_errors += 1 + logger.error("import: summary failed source=%s path=%s — %s", source, rel, e) + def _prune(session: Session, source_names: set[str], seen: set[tuple[str, str]]) -> int: """Delete documents of *source_names* whose file is no longer in *seen*.""" diff --git a/app/rag/llm.py b/app/rag/llm.py index 03e7841..4fcb016 100644 --- a/app/rag/llm.py +++ b/app/rag/llm.py @@ -1,10 +1,12 @@ """Async OpenAI-compatible client for the self-hosted aipi endpoint (PLAN A5). -Provides the embeddings surface (importer, retrieval) and chat streaming -(PLAN A15) for the RAG pipeline. Chat streaming yields typed -:class:`StreamPiece` values (phase 17): aipi's ``turbo`` model streams -its reasoning as ``delta.reasoning_content`` chunks (deepseek/litellm -wire convention, verified live 2026-08-23) **before** the answer's +Provides the embeddings surface (importer, retrieval), one-shot chat +completions (phase 30: the ``lite`` model summarizes non-markdown +documents at import time), and chat streaming (PLAN A15) for the RAG +pipeline. Chat streaming yields typed :class:`StreamPiece` values +(phase 17): aipi's ``turbo`` model streams its reasoning as +``delta.reasoning_content`` chunks (deepseek/litellm wire convention, +verified live 2026-08-23) **before** the answer's ``delta.content`` chunks, and reasoning counts against ``max_tokens`` (an answer can in principle be empty). @@ -187,6 +189,48 @@ class LLMClient: (vec,) = await self.embed([text]) return vec + async def chat( + self, messages: list[dict[str, str]], model: str | None = None + ) -> str: + """One-shot (non-streaming) completion (A5 extended, phase 30). + + Short, low-temperature request (``temperature=0.2``, 2048-token + cap — summaries and outlines are small, so a fixed budget is + enough) against ``BOR_LLM_SUMMARY_MODEL`` (default ``lite``) + unless *model* names another. Used by the document summarizer + (phase 30) and the KB overview generator (phase 31). + + Any transport/HTTP/malformed failure, a choiceless reply, or an + empty/missing ``content`` field raises :class:`LLMError` — a + silent empty summary must never be stored. + """ + try: + resp = await self._client.chat.completions.create( + model=model or self.settings.llm_summary_model, + messages=cast("list[ChatCompletionMessageParam]", messages), + temperature=0.2, + max_tokens=2048, + stream=False, + ) + except LLMError: + raise + except Exception as e: # noqa: BLE001 — wrap transport-level failures + raise LLMError( + f"chat completion from {self.settings.llm_base_url} failed: {e}" + ) from e + if not resp.choices: + raise LLMError( + f"chat completion from {self.settings.llm_base_url} " + "returned no choices" + ) + content = resp.choices[0].message.content + if content is None or not content.strip(): + raise LLMError( + f"chat completion from {self.settings.llm_base_url} returned " + "empty content — refusing to store a silent summary" + ) + return content.strip() + async def chat_stream( self, messages: list[dict[str, str]] ) -> AsyncIterator[StreamPiece]: diff --git a/app/rag/retriever.py b/app/rag/retriever.py index 63e7e37..56c25fc 100644 --- a/app/rag/retriever.py +++ b/app/rag/retriever.py @@ -56,6 +56,7 @@ _LEXICAL_SQL = text( d.content AS doc_content, d.content_hash AS content_hash, d.indexed_at AS indexed_at, + c.is_summary AS is_summary, ts_rank(c.tsv, to_tsquery('english', :tsquery)) AS rank FROM chunks c JOIN documents d ON d.id = c.document_id @@ -75,6 +76,11 @@ class RetrievedChunk: * ``cosine`` — vector similarity ``1 − distance`` (the honesty-gate input; ``0.0`` for lexical-only hits that have no vector rank). * ``fts_hit`` — the chunk matched the question's OR-tsquery. + * ``is_summary`` — True for the lite-model summary chunk (phase 30, + position −1): its parent *is* the source document, so a summary hit + resolves to the full source document through the unchanged + chunk→document mapping (A7 revised). Default ``False`` keeps every + ordinary content chunk valid. """ chunk_id: uuid.UUID @@ -84,6 +90,7 @@ class RetrievedChunk: document: Document cosine: float = 0.0 fts_hit: bool = False + is_summary: bool = False def lexical_tsquery(question: str) -> str | None: @@ -148,7 +155,9 @@ def _vector_candidates( """Top-*limit* chunks by pgvector cosine distance (``<=>``). ``cosine = 1 − distance``. Chunks whose embedding is still NULL - (two-phase import in progress) are skipped. + (two-phase import in progress) are skipped. Each candidate carries + its ``Chunk.is_summary`` flag (phase 30) so a summary hit stays + identifiable after fusion. """ distance = Chunk.embedding.cosine_distance(question_embedding) rows = db.execute( @@ -166,6 +175,7 @@ def _vector_candidates( score=0.0, # fused score is filled in by :func:`fuse` document=doc, cosine=round(1.0 - float(dist), 6), + is_summary=chunk.is_summary, ) for chunk, dist, doc in rows ] @@ -205,6 +215,7 @@ def _lexical_candidates(db: Session, question: str, limit: int) -> list[Retrieve document=doc, cosine=0.0, # no vector rank — lexical-only hit fts_hit=True, + is_summary=row.is_summary, ) ) return out diff --git a/app/rag/summarizer.py b/app/rag/summarizer.py new file mode 100644 index 0000000..6a2604e --- /dev/null +++ b/app/rag/summarizer.py @@ -0,0 +1,123 @@ +"""Document summarizer (phase 30, task 03). + +Builds the ``SUMMARY_MODE`` prompt for one document, calls the aipi +``lite`` model through the one-shot ``LLMClient.chat`` (phase 30, +task 01), and returns the validated summary text with a +**code-deterministic** pointer line back to the source:: + + Source: / + +The pointer is appended by this module, never model-generated — the +model is told what to summarize, not to cite. + +Quality contracts enforced here: + +* **Capped input** — the document content is cut at + ``BOR_SUMMARY_MAX_CHARS`` (default 12 000) before the single model + call; overflow is cut exactly at the cap and the shared + ``TRUNCATION_MARKER`` (``[…truncated…]``) is appended, so the model + never sees more than the cap and the cut is visible. +* **No silent summaries** — a reply that is empty after trimming raises + :class:`LLMError` (the client already rejects empty content; the + summarizer re-asserts defensively and never hands the importer a + pointer-only row). + +The ``SUMMARY_MODE`` marker follows the ``DEFLECT_MODE`` convention: +the deterministic E2E mock LLM keys on it in the system prompt +(``tests/e2e/mock_llm.py`` — wired in task 06). +""" +from __future__ import annotations + +from typing import Protocol + +from app.config import Settings, get_settings +from app.rag.llm import LLMError +from app.rag.retriever import TRUNCATION_MARKER + +#: System-prompt marker for summary generation — the E2E mock LLM keys on +#: it (same convention as ``DEFLECT_MODE``, PLAN §6). +SUMMARY_MODE = "SUMMARY_MODE" + +#: Locked instruction for the ``lite`` model (phase 30): the summary is a +#: natural-language retrieval target, so it must be plain, concrete, and +#: strictly grounded in the document. +SUMMARY_INSTRUCTION = ( + "Write a 3-6 sentence plain-text summary of this document in natural " + "language. Cover what it configures/defines and its most important " + "values. Do not use markdown. Do not invent anything that is not in " + "the document." +) + +#: Full system prompt: marker first (the mock's key), then the instruction. +SYSTEM_PROMPT = f"{SUMMARY_MODE}: {SUMMARY_INSTRUCTION}" + + +class SummaryLLM(Protocol): + """The one-shot chat surface the summarizer needs. + + :class:`app.rag.llm.LLMClient` satisfies it; unit tests pass a + duck-typed fake (``chat`` + ``settings``) instead — same pattern as + the importer's ``Embedder`` protocol. + """ + + settings: Settings + + async def chat( + self, messages: list[dict[str, str]], model: str | None = None + ) -> str: ... + + +def _capped_content(content: str, max_chars: int | None) -> str: + """Document content for the user message, capped at *max_chars*. + + The default cap is ``BOR_SUMMARY_MAX_CHARS``. Overflow is cut exactly + at the cap and the shared ``TRUNCATION_MARKER`` is appended on its + own line; content that fits (length ≤ cap) passes through unchanged. + """ + limit = max_chars if max_chars is not None else get_settings().summary_max_chars + if len(content) <= limit: + return content + return content[:limit] + "\n" + TRUNCATION_MARKER + + +def build_summary_prompt( + source: str, path: str, content: str, max_chars: int | None = None +) -> tuple[str, str]: + """The ``(system, user)`` message pair for one summary call. + + * ``system`` — :data:`SYSTEM_PROMPT`: the ``SUMMARY_MODE`` marker + + the locked instruction. + * ``user`` — the document content, capped (see :func:`_capped_content`). + + *source* and *path* are part of the signature so the call site reads + like the document it summarizes (and for :func:`generate_summary`'s + pointer) — the pointer is built in code and deliberately **not** part + of the prompt, so the model cannot echo or mangle it. + """ + return SYSTEM_PROMPT, _capped_content(content, max_chars) + + +async def generate_summary( + llm: SummaryLLM, *, source: str, path: str, content: str +) -> str: + """One-shot ``lite`` summary of *content*, ending in the pointer line. + + Returns the model's text (trimmed) plus the deterministic + ``Source: /`` line — the pointer is appended by code, + never model-generated. Raises :class:`LLMError` when the model + returns nothing usable after trimming, and propagates any + :class:`LLMError` the client raises (the importer's fail-soft path + turns that into a logged, counted ``summary_errors`` entry). + """ + system, user = build_summary_prompt(source, path, content) + raw = await llm.chat( + [{"role": "system", "content": system}, {"role": "user", "content": user}], + model=llm.settings.llm_summary_model, + ) + summary = raw.strip() + if not summary: + raise LLMError( + f"summary model returned empty content for {source}/{path} — " + "refusing to store a silent summary" + ) + return f"{summary}\nSource: {source}/{path}" diff --git a/scripts/import_docs.py b/scripts/import_docs.py index 66e4e2a..ad6302f 100644 --- a/scripts/import_docs.py +++ b/scripts/import_docs.py @@ -149,7 +149,8 @@ def main(argv: list[str] | None = None) -> int: f"import_docs: files={summary.files} added={summary.added} " f"updated={summary.updated} unchanged={summary.unchanged} " f"pruned={summary.pruned} errors={summary.errors} chunks={summary.chunks} " - f"embed_batches={summary.embed_batches} formats={summary.format_counts()}" + f"embed_batches={summary.embed_batches} summaries={summary.summaries} " + f"summary_errors={summary.summary_errors} formats={summary.format_counts()}" ) # Non-zero if any file failed, so cron/CI notice — the rest of the KB # was imported and the failed files are retried on the next run. diff --git a/tests/e2e/mock_llm.py b/tests/e2e/mock_llm.py index e0ea614..3dddb5f 100644 --- a/tests/e2e/mock_llm.py +++ b/tests/e2e/mock_llm.py @@ -11,6 +11,10 @@ Implements just enough of the aipi surface: off markers in the system prompt: - user message containing ``write a long answer`` -> a ~900-word deterministic numbered answer (long-answers story, phase 11) + - ``SUMMARY_MODE`` -> the deterministic summary digest: the first 24 + tokens of the user message (the summarizer puts the capped document + content there) — byte-stable for a given fixture (document summaries, + phase 30) - ``DEFLECT_MODE`` -> honest "I haven't done anything like that" answer - otherwise -> upbeat answer quoting the provided document context - user message containing ``pretend to think slowly`` -> 3s warm-up delay @@ -156,6 +160,18 @@ def compose_answer(body: dict[str, Any]) -> str: user = _user(body) if LONG_ANSWER_TRIGGER in user.lower(): answer = long_answer() + elif "SUMMARY_MODE" in system: + # Document summaries (phase 30): the ``lite`` stand-in returns a + # deterministic digest — the first 24 tokens of the user message + # (the summarizer puts the capped document content there). Byte- + # stable for a given fixture, so the summary chunk's retrieval + # rank is a pure function of the fixture text. Checked BEFORE the + # DEFLECT_MODE branch (task 06) so a deflection prompt that ever + # carries the marker cannot shadow the summary call. + answer = ( + f"This document covers " + f"{' '.join(TOKEN_RE.findall(user.lower())[:24])}." + ) elif "DEFLECT_MODE" in system: answer = ( "Ah — I haven't done anything like that, so I don't want to make stuff up! " diff --git a/tests/e2e/test_document_summaries.py b/tests/e2e/test_document_summaries.py new file mode 100644 index 0000000..2c6b654 --- /dev/null +++ b/tests/e2e/test_document_summaries.py @@ -0,0 +1,257 @@ +"""Phase 30 E2E (Playwright): a summary hit delivers the full source doc. + +Story: ``.agent/user_stories/document-summaries.md`` +Run in isolation (DB must be up: ``podman compose up -d db``): + + uv run pytest tests/e2e/test_document_summaries.py -v --no-cov + +The fixture KB is a story-dedicated directory +(``tests/fixtures/summary_kb/`` — the shared ``tests/fixtures/docs/`` +stays at its 8 pinned files) with two documents: + +* ``quadlet/qwen-llamacpp.yaml`` — a non-markdown A9 doc. At import the + mock ``lite`` model (``SUMMARY_MODE`` marker, ``tests/e2e/mock_llm.py``) + reduces it to a deterministic 24-token digest, stored on + ``documents.summary`` and indexed as one ``is_summary`` chunk. The raw + yaml body is deliberately token-diluted, so the document's best fused + chunk is its summary chunk. The sentinel ``RESE-SUMMARY-SENTINEL-7f3a`` + sits on the document's LAST line — outside the 24-token digest, + unreachable from the summary. +* ``notes/qwen-llamacpp-notes.md`` — a markdown control doc (never + summarized) that ranks first, which puts the yaml document LAST inside + ````. + +The mock LLM's tail-echo trigger (``END_OF_NOTES_TRIGGER``) makes the +answer quote the last 160 chars of the document context — the tail of +the LAST selected document. The sentinel therefore appears in the +rendered answer **iff the entire yaml source document (not the summary +digest) reached the LLM prompt** — the summary→parent-document resolution +through the unchanged chunk→document mapping (A7 revised: never +truncated), which is what this story is about. +""" +from __future__ import annotations + +import asyncio +from collections.abc import Callable, Sequence +from pathlib import Path +from threading import Thread +from typing import Any + +from playwright.sync_api import Page, expect +from sqlalchemy import select, text +from sqlalchemy.orm import Session + +from app.config import Settings +from app.db import SessionLocal +from app.models import Document, QueryLog +from app.rag.chunker import chunk_document +from app.rag.importer import ImportSummary, import_sources +from app.rag.llm import LLMClient +from app.rag.retriever import RetrievedChunk, retrieve +from tests.e2e.mock_llm import TOKEN_RE, embed_text + +REPO = Path(__file__).resolve().parents[2] +FIXTURES = REPO / "tests" / "fixtures" / "summary_kb" + +QUESTION = ( + "What are the optimal parameters for qwen 3.8 on llama.cpp? " + "show the end of your notes" +) +SENTINEL = "RESE-SUMMARY-SENTINEL-7f3a" +SOURCE = "summary_kb" +YAML_PATH = "quadlet/qwen-llamacpp.yaml" +MD_PATH = "notes/qwen-llamacpp-notes.md" +MOCK_ANSWER_MARKER = "Deterministic mock answer for E2E" + +#: The importer's chunk policy (Settings defaults; the mock never trips +#: the endpoint token-cap retry, so the target is never halved). +CHUNK_TARGET = 2_000 +CHUNK_OVERLAP = 200 + + +# --- Importer + thread helpers (test_whole_document_context.py pattern) --- + + +async def _import_fixtures(mock_port: int) -> ImportSummary: + kwargs: dict[str, Any] = { + "_env_file": None, + "llm_base_url": f"http://127.0.0.1:{mock_port}/v1", + } + settings = Settings(**kwargs) # pyright: ignore[reportCallIssue] + return await import_sources([FIXTURES], LLMClient(settings)) + + +def _run_in_thread(coro: Any) -> Any: + """Run a coroutine on a worker thread. + + Playwright's sync API keeps an asyncio loop running on the test thread, + so ``asyncio.run`` cannot be called directly from a test body. + """ + box: dict[str, Any] = {} + + def runner() -> None: + try: + box["value"] = asyncio.run(coro) + except BaseException as e: # noqa: BLE001 — re-raised on the test thread + box["error"] = e + + t = Thread(target=runner) + t.start() + t.join() + if "error" in box: + raise box["error"] + return box["value"] + + +def _reset_db(seed: Callable[[Session], None] | None = None) -> None: + """Truncate the KB (and query log + steering), then optionally seed.""" + with SessionLocal() as db: + db.execute(text("TRUNCATE chunks, documents, query_log, steering_notes")) + db.commit() + if seed is not None: + seed(db) + db.commit() + + +def _ask(page: Page, app_url: str, question: str) -> Any: + """Submit *question* and wait for the streamed brain bubble.""" + page.set_default_timeout(30_000) + page.goto(app_url) + page.fill("#message-input", question) + page.click("#send-btn") + bubble = page.locator(".msg.brain .bubble") + bubble.first.wait_for(state="visible", timeout=30_000) + return bubble.first + + +def _last_query_log() -> QueryLog: + with SessionLocal() as db: + rows = db.scalars(select(QueryLog)).all() + assert len(rows) == 1, f"expected exactly one query_log row, got {len(rows)}" + return rows[0] + + +def _doc(db: Session, path: str) -> Document: + doc = db.scalar(select(Document).where(Document.path == path)) + assert doc is not None, f"fixture doc {path!r} was not imported" + return doc + + +def _expected_summary(content: str, source: str, path: str) -> str: + """The mock lite model's byte-stable digest + the code pointer line. + + Mirrors ``mock_llm.compose_answer``'s ``SUMMARY_MODE`` branch (first + 24 tokens of the document content) plus the summarizer's deterministic + ``Source:`` line — no model output is ever trusted. + """ + digest = " ".join(TOKEN_RE.findall(content.lower())[:24]) + return f"This document covers {digest}.\nSource: {source}/{path}" + + +def _chunks_by_path(chunks: Sequence[RetrievedChunk], path: str) -> list[RetrievedChunk]: + return [c for c in chunks if c.document.path == path] + + +# --- Story tests ------------------------------------------------------------- + + +def test_summary_hit_retrieves_full_source_document( + page: Page, app_url: str, mock_llm: int, db_ready: None +) -> None: + """A question whose best yaml match is the summary chunk yields an + answer grounded in the FULL yaml source document: its tail sentinel — + which the summary digest cannot contain — is echoed back, and the + source chip cites the yaml path (deflected: false).""" + _reset_db() + summary = _run_in_thread(_import_fixtures(mock_llm)) + assert summary.added == 2 # yaml + md control + assert summary.summaries == 1 and summary.summary_errors == 0 + assert summary.errors == 0 + + # Import state: exactly one embedded ``is_summary`` chunk (position + # −1) whose text is the byte-stable mock digest + the deterministic + # pointer line. + yaml_content = (FIXTURES / YAML_PATH).read_text(encoding="utf-8") + assert SENTINEL in yaml_content.splitlines()[-1] # last line, by design + with SessionLocal() as db: + yaml_doc = _doc(db, YAML_PATH) + schunks = [c for c in yaml_doc.chunks if c.is_summary] + assert len(schunks) == 1 + assert schunks[0].position == -1 + assert schunks[0].embedding is not None + assert yaml_doc.summary == _expected_summary(yaml_content, SOURCE, YAML_PATH) + + # Retrieval state: the summary chunk is the yaml document's best fused + # chunk — the document enters the context through its summary, not + # through the diluted raw yaml chunks. + with SessionLocal() as db: + chunks = retrieve(db, QUESTION, embed_text(QUESTION)) + yaml_chunks = _chunks_by_path(chunks, YAML_PATH) + best_yaml = max(yaml_chunks, key=lambda c: c.score) + assert best_yaml.is_summary + assert len(yaml_chunks) >= 2 # summary + at least one raw candidate + + bubble = _ask(page, app_url, QUESTION) + + # The tail sentinel exists only on the document's last line and + # cannot be in the summary digest — its presence proves the entire + # source document was in the LLM prompt (summary→parent resolution). + expect(bubble).to_contain_text(SENTINEL, timeout=30_000) + expect(bubble).to_contain_text(MOCK_ANSWER_MARKER) + + # Grounded: the yaml source chip renders (the md doc ranks first, so + # both fixtures are cited). + chip = page.locator(".msg.brain .source-chip", has_text=YAML_PATH) + expect(chip).to_have_count(1) + expect(chip.first).to_contain_text(f"{SOURCE}/{YAML_PATH}") + expect(page.locator(".msg.brain .source-chip", has_text=MD_PATH)).to_have_count(1) + + # Button recovers (never stale) and the turn was grounded, not + # deflected. + expect(page.locator("#send-btn")).to_be_enabled() + expect(page.locator("#send-label")).to_have_text("Send") + row = _last_query_log() + assert row.question == QUESTION + assert row.deflected is False + assert f"{SOURCE}/{YAML_PATH}" in row.sources + assert f"{SOURCE}/{MD_PATH}" in row.sources + + +def test_markdown_control_doc_gets_no_summary_chunk( + mock_llm: int, db_ready: None +) -> None: + """Control: in the same KB the markdown doc gets no summary at all — + its chunk count is exactly the raw chunks; the yaml doc has exactly + one ``is_summary`` row and its raw chunk count is untouched by the + summary.""" + _reset_db() + summary = _run_in_thread(_import_fixtures(mock_llm)) + assert summary.added == 2 + + md_content = (FIXTURES / MD_PATH).read_text(encoding="utf-8") + yaml_content = (FIXTURES / YAML_PATH).read_text(encoding="utf-8") + with SessionLocal() as db: + md_doc = _doc(db, MD_PATH) + yaml_doc = _doc(db, YAML_PATH) + md_chunks = [c for c in md_doc.chunks if not c.is_summary] + yaml_raw = [c for c in yaml_doc.chunks if not c.is_summary] + yaml_summary = [c for c in yaml_doc.chunks if c.is_summary] + + # Markdown: never summarized (phase 30 scope — A9 non-markdown only). + assert md_doc.summary is None + assert len(md_chunks) == len( + chunk_document(md_content, MD_PATH, CHUNK_TARGET, CHUNK_OVERLAP) + ) + + # YAML: raw chunks exactly as chunked by the importer policy, plus + # exactly one summary chunk (position −1, embedded, on the doc row). + assert len(yaml_raw) == len( + chunk_document(yaml_content, YAML_PATH, CHUNK_TARGET, CHUNK_OVERLAP) + ) + assert sorted(c.position for c in yaml_raw) == list(range(len(yaml_raw))) + assert len(yaml_summary) == 1 + assert yaml_summary[0].position == -1 + assert yaml_summary[0].embedding is not None + assert yaml_doc.summary is not None + assert yaml_doc.summary == yaml_summary[0].content + assert yaml_doc.summary.endswith(f"\nSource: {SOURCE}/{YAML_PATH}") diff --git a/tests/e2e/test_import_documents.py b/tests/e2e/test_import_documents.py index 7475246..dbe48ca 100644 --- a/tests/e2e/test_import_documents.py +++ b/tests/e2e/test_import_documents.py @@ -92,7 +92,11 @@ def test_sources_page_lists_indexed_docs( login(page, app_url) # phase 16: the catalog is admin-only expect(page.locator("#stat-docs")).to_have_text("8") - expect(page.locator("#stat-chunks")).to_have_text(str(summary.chunks)) + # Phase 30: non-markdown fixtures each gained one ``is_summary`` chunk, + # so the Sources total is content chunks + summary chunks. + expect(page.locator("#stat-chunks")).to_have_text( + str(summary.chunks + summary.summaries) + ) expect(page.locator("#stat-last")).not_to_have_text("–") expect(page.locator("#sources-empty")).to_be_hidden() diff --git a/tests/fakes.py b/tests/fakes.py index 4b41697..3acefb5 100644 --- a/tests/fakes.py +++ b/tests/fakes.py @@ -2,6 +2,7 @@ from __future__ import annotations from app.config import Settings +from app.rag.llm import LLMError class FakeEmbedder: @@ -9,7 +10,11 @@ class FakeEmbedder: ``Embedder`` protocol in :mod:`app.rag.importer`). Returns deterministic vectors of *dim* dimensions; records every call - so tests can assert batching behaviour. + so tests can assert batching behaviour. ``chat`` is the deterministic + ``lite``-model stand-in (phase 30): it returns + ``"Summary of "`` and raises + :class:`LLMError` when the content contains the sentinel word + ``SUMMARY-BLOWUP`` (drives the importer's fail-soft summary path). """ def __init__(self, dim: int = 768) -> None: @@ -17,8 +22,19 @@ class FakeEmbedder: self.settings = Settings(_env_file=None) # pyright: ignore[reportCallIssue] self.embed_batches = 0 self.calls: list[list[str]] = [] + self.chat_calls: list[list[dict[str, str]]] = [] async def embed(self, texts: list[str]) -> list[list[float]]: self.calls.append(list(texts)) self.embed_batches += 1 return [[0.01 * (i % 97) for i in range(self.dim)] for _ in texts] + + async def chat( + self, messages: list[dict[str, str]], model: str | None = None + ) -> str: + self.chat_calls.append(list(messages)) + user = next((m["content"] for m in messages if m.get("role") == "user"), "") + if "SUMMARY-BLOWUP" in user: + raise LLMError("simulated lite-model failure (SUMMARY-BLOWUP sentinel)") + first = user.split() + return "Summary of " + (first[0] if first else "") diff --git a/tests/fixtures/summary_kb/notes/qwen-llamacpp-notes.md b/tests/fixtures/summary_kb/notes/qwen-llamacpp-notes.md new file mode 100644 index 0000000..c75c595 --- /dev/null +++ b/tests/fixtures/summary_kb/notes/qwen-llamacpp-notes.md @@ -0,0 +1,16 @@ +# Qwen 3.8 on llama.cpp — deployment notes + +## Optimal parameters + +The optimal parameters for qwen 3.8 on llama.cpp came out of a week of +benchmarks on the homelab GPU. Context length, flash attention and the +batch size matter more than the mmap knob. The full flag set lives in +`quadlet/qwen-llamacpp.yaml` — the yaml is the source of truth, these +notes are the reasoning behind each picked value. + +## What changed since last month + +Switched the server image to the 0.1.43 release and moved the model +files to the NVMe cache drive. Pinned the quant to q4_k_m; the repeat +penalty is the knob that kept the rambles honest. The webui now fronts +the raw server so chat sessions survive a container restart. diff --git a/tests/fixtures/summary_kb/quadlet/qwen-llamacpp.yaml b/tests/fixtures/summary_kb/quadlet/qwen-llamacpp.yaml new file mode 100644 index 0000000..d01e45a --- /dev/null +++ b/tests/fixtures/summary_kb/quadlet/qwen-llamacpp.yaml @@ -0,0 +1,143 @@ +# qwen 3.8 llama.cpp optimal parameters deployment notes +services: + llamacpp-server: + image: reg.local/ai/llamacpp-server:0.1.43 + restart: unless-stopped + network_mode: host + devices: + - /dev/dri:/dev/dri + volumes: + - /srv/models:/models:ro + - /srv/llamacpp/cache:/cache + environment: + - HOST=0.0.0.0 + - PORT=8081 + - MODEL=/models/qwen3-8b-instruct-q4_k_m.gguf + - N_CTX=32768 + - N_BATCH=512 + - N_THREADS=12 + - FLASH_ATTN=1 + - MAIN_GPU=1 + - REPEAT_PENALTY=1.1 + - TEMPERATURE=0.7 + - TOP_K=40 + - TOP_P=0.9 + - MIN_P=0.05 + - SEED=42 + deploy: + resources: + reservations: + devices: + - driver: gpu + count: 1 + capabilities: + - gpu + healthcheck: + test: ["CMD", "curl", "-fsS", "http://127.0.0.1:8081/health"] + interval: 30s + start_period: 90s + retries: 5 + webui: + image: reg.local/ai/open-webui:0.6.12 + restart: unless-stopped + network_mode: host + volumes: + - /srv/webui/data:/app/data + environment: + - PORT=3000 + - UPSTREAM=http://127.0.0.1:8081 + - AUTO_UPGRADE=0 + +models: + qwen3-8b-instruct-q4_k_m.gguf: + sha256: "9f2c1e07b5a4d6c8f1a3e9b7d2c4a6f0e8b1d3c5a7f9e2b4c6d8f0a1b3c5d7e9" + size_gb: 5.2 + quant: q4_k_m + context: 32768 + mistral-7b-instruct-v0-q5_k_s.gguf: + sha256: "b4d8f2c6a0e4f8b2d6c0a4e8f2b6d0c4a8e2f6b0d4c8a2e6f0b4d8c2a6e0f4b8" + size_gb: 4.9 + quant: q5_k_s + context: 16384 + gemma-2-2b-it-q4_k_m.gguf: + sha256: "c7a2e4f8b0d6c1a3e5f7b9d2c4a6e8f0b2d4c6a8e0f2b4d6c8a0e2f4b6d8c0a2" + size_gb: 1.6 + quant: q4_k_m + context: 8192 + phi-2-2.7b-q5_k_s.gguf: + sha256: "e1f3a5c7b9d1e3f5a7c9b1d3e5f7a9c1b3d5e7f9a1c3b5d7e9f1a3c5b7d9e1f3" + size_gb: 2.1 + quant: q5_k_s + context: 4096 + +benchmarks: + gtx-1080-ti: + q4_k_m: + tok_per_s: 21.4 + p50_ms: 148 + p95_ms: 412 + q5_k_s: + tok_per_s: 18.7 + p50_ms: 171 + p95_ms: 466 + rtx-4090: + q4_k_m: + tok_per_s: 88.2 + p50_ms: 34 + p95_ms: 96 + q5_k_s: + tok_per_s: 74.6 + p50_ms: 41 + p95_ms: 118 + rtx-4060-ti: + q4_k_m: + tok_per_s: 46.9 + p50_ms: 62 + p95_ms: 178 + +tuning: + repeat_penalty: + tried: [1.0, 1.05, 1.1, 1.2] + picked: 1.1 + why: 1.2 clipped mid-sentence twice + temperature: + tried: [0.5, 0.7, 0.9] + picked: 0.7 + top_p: + tried: [0.8, 0.9, 0.95] + picked: 0.9 + min_p: + tried: [0.0, 0.05, 0.1] + picked: 0.05 + context: + tried: [16384, 32768] + picked: 32768 + why: 16384 evicted early turns in long chats + +registry: + mirror: reg.local/ai + pull_policy: pinned + scan_interval: 24h + garbage_collect: weekly + access: local-network-only + +alerts: + gpu_memory_high: + threshold_pct: 92 + channel: ntfy + repeat_after: 6h + server_down: + channel: ntfy + repeat_after: 30m + model_stale_days: + value: 180 + channel: ntfy + +maintenance: + backup_cron: "0 4 * * *" + log_rotate_days: 14 + update_policy: manual + image_retention: 2 + model_download_mirror: reg.local/ai/models + rollback: keep previous tags pinned in registry +# RESE-SUMMARY-SENTINEL-7f3a diff --git a/tests/integration/test_chat_api.py b/tests/integration/test_chat_api.py index 407a576..e5909d2 100644 --- a/tests/integration/test_chat_api.py +++ b/tests/integration/test_chat_api.py @@ -72,6 +72,15 @@ class FakeRagLLM: self.embed_batches += 1 return [_token_vec(t) for t in texts] + async def chat( + self, messages: list[dict[str, str]], model: str | None = None + ) -> str: + """Deterministic ``lite`` stand-in for the import-time summaries + (phase 30) — same convention as ``tests.fakes.FakeEmbedder.chat``.""" + user = next((m["content"] for m in messages if m.get("role") == "user"), "") + first = user.split() + return "Summary of " + (first[0] if first else "") + async def embed_one(self, text: str) -> list[float]: if self.embed_error is not None: raise self.embed_error diff --git a/tests/integration/test_importer_e2e.py b/tests/integration/test_importer_e2e.py index 0dbdd0e..5dabf7a 100644 --- a/tests/integration/test_importer_e2e.py +++ b/tests/integration/test_importer_e2e.py @@ -61,11 +61,31 @@ def test_import_fixtures_end_to_end(admin_client, db) -> None: k8s = next(d for d in docs if d.path == "homelab/kubernetes.md") assert "Talos Linux" in k8s.content and k8s.content_hash + # Phase 30: the four non-markdown fixtures each gained one embedded + # ``is_summary`` chunk, so the DB holds content + summary chunks. n_chunks = db.scalar(select(func.count()).select_from(Chunk)) - assert n_chunks == summary.chunks + assert n_chunks == summary.chunks + summary.summaries for c in db.scalars(select(Chunk)).all(): assert c.embedding is not None and len(c.embedding) == 768 + assert summary.summary_errors == 0 + for d in docs: + non_md = Path(d.path).suffix.lower() not in (".md", ".markdown") + schunks = [c for c in d.chunks if c.is_summary] + if non_md: + # Lite summary stored + exactly one embedded summary chunk (−1). + assert d.summary is not None, f"{d.path} should have a summary" + assert len(schunks) == 1 + assert schunks[0].position == -1 + assert schunks[0].content == d.summary + assert schunks[0].embedding is not None + else: + # Markdown docs never get a summary (phase 30 scope). + assert d.summary is None and not schunks + assert summary.summaries == sum( + 1 for d in docs if Path(d.path).suffix.lower() not in (".md", ".markdown") + ) + # The Sources page consumes exactly this shape. r = admin_client.get("/api/docs") # phase 16: the catalog is admin-only assert r.status_code == 200 diff --git a/tests/integration/test_migration_0004.py b/tests/integration/test_migration_0004.py new file mode 100644 index 0000000..3a7b470 --- /dev/null +++ b/tests/integration/test_migration_0004.py @@ -0,0 +1,144 @@ +"""Integration: migration 0004 (document summaries) schema contract. + +Drives the **real Alembic engine** against the live dev database +(``podman compose up -d db``), mirroring the style of +``test_migration_0002.py`` (information_schema assertions on the state the +migration must leave): + +* upgrade to head → ``documents.summary`` (TEXT, nullable) and + ``chunks.is_summary`` (BOOLEAN NOT NULL, default false) both exist, and a + chunk inserted without the column gets ``is_summary = false`` (pre-0004 + insert paths stay valid); +* downgrade to 0003 → both columns are gone; +* upgrade to head again → both are back (round-trip). + +The ``alembic`` fixture guarantees the DB ends at head even if a test +fails or the process is interrupted. +""" +from __future__ import annotations + +from collections.abc import Iterator +from typing import Any + +import pytest +from alembic.config import Config +from sqlalchemy import text +from sqlalchemy.orm import Session + +from alembic import command +from app.db import db_available + + +@pytest.fixture() +def alembic(db: Session) -> Iterator[Config]: + """Real Alembic config bound to the dev DB (URL from app settings). + + Starts at head (repairs an interrupted earlier run); teardown upgrades + to head no matter what happened, so the dev DB is never left below + head. + """ + if not db_available(): + pytest.skip("Postgres not reachable — run `podman compose up -d db` first") + cfg = Config() # no alembic.ini file — env.py gets the URL from app config + cfg.set_main_option("script_location", "alembic") + command.upgrade(cfg, "head") + try: + yield cfg + finally: + command.upgrade(cfg, "head") + + +def _column(db: Session, table: str, column: str) -> tuple[Any, ...] | None: + """(data_type, is_nullable, column_default) for one column, or None.""" + row = db.execute( + text( + "SELECT data_type, is_nullable, column_default" + " FROM information_schema.columns" + " WHERE table_name = :t AND column_name = :c" + ), + {"t": table, "c": column}, + ).fetchone() + return tuple(row) if row is not None else None + + +def _version(db: Session) -> str | None: + return db.execute(text("SELECT version_num FROM alembic_version")).scalar() + + +def test_upgrade_to_head_adds_summary_columns(db: Session, alembic: Config) -> None: + """Upgrade to head: both columns exist with the locked types/defaults.""" + command.downgrade(alembic, "0003") # start from the pre-0004 state + assert _version(db) == "0003" + + command.upgrade(alembic, "head") + assert _version(db) == "0004", "alembic_version must be at 0004 (head)" + + summary = _column(db, "documents", "summary") + assert summary is not None, "documents.summary is missing" + assert summary[0] == "text", "documents.summary must be TEXT" + assert summary[1] == "YES", "documents.summary must be NULLABLE" + + is_summary = _column(db, "chunks", "is_summary") + assert is_summary is not None, "chunks.is_summary is missing" + assert is_summary[0] == "boolean", "chunks.is_summary must be BOOLEAN" + assert is_summary[1] == "NO", "chunks.is_summary must be NOT NULL" + assert is_summary[2] is not None and "false" in is_summary[2], ( + "chunks.is_summary must have server default false" + ) + + +def test_is_summary_defaults_false_for_new_chunks(db: Session, alembic: Config) -> None: + """The default keeps old rows/insert paths valid: a chunk inserted + without the column (the pre-0004 insert shape) lands as ``false``.""" + command.upgrade(alembic, "head") + doc_id = db.execute(text("SELECT gen_random_uuid()")).scalar() + try: + db.execute( + text( + "INSERT INTO documents (id, source, path, full_path, title, content," + " content_hash, indexed_at) VALUES" + " (:id, 'mig_test', 't.md', '/t.md', 'T', 'content here'," + " repeat('0', 64), now())" + ), + {"id": doc_id}, + ) + db.execute( + text( + "INSERT INTO chunks (id, document_id, position, content)" + " VALUES (gen_random_uuid(), :id, 0, 'content here')" + ), + {"id": doc_id}, + ) + db.commit() + flag = db.execute( + text("SELECT is_summary FROM chunks WHERE document_id = :id"), {"id": doc_id} + ).scalar() + assert flag is False, "chunks.is_summary must default to false" + finally: + db.execute(text("DELETE FROM chunks WHERE document_id = :id"), {"id": doc_id}) + db.execute(text("DELETE FROM documents WHERE id = :id"), {"id": doc_id}) + db.commit() + + +def test_downgrade_to_0003_removes_columns(db: Session, alembic: Config) -> None: + """Downgrade to 0003: both columns are dropped (A13 — reversible).""" + command.downgrade(alembic, "0003") + assert _version(db) == "0003" + + assert _column(db, "documents", "summary") is None, "documents.summary must be dropped" + assert _column(db, "chunks", "is_summary") is None, "chunks.is_summary must be dropped" + + +def test_upgrade_round_trip_restores_columns(db: Session, alembic: Config) -> None: + """Upgrade back to head after the downgrade: both columns are back.""" + command.upgrade(alembic, "head") + assert _version(db) == "0004", "round-trip upgrade must land at 0004 (head)" + + summary = _column(db, "documents", "summary") + assert summary is not None and summary[1] == "YES", "documents.summary must be back" + + is_summary = _column(db, "chunks", "is_summary") + assert is_summary is not None and is_summary[1] == "NO", "chunks.is_summary must be back" + assert is_summary[2] is not None and "false" in is_summary[2], ( + "chunks.is_summary must keep its server default false after the round-trip" + ) diff --git a/tests/unit/test_chat_gate.py b/tests/unit/test_chat_gate.py index 3972441..22f183d 100644 --- a/tests/unit/test_chat_gate.py +++ b/tests/unit/test_chat_gate.py @@ -47,18 +47,23 @@ def _doc(title: str, content: str) -> Document: def _chunk( - doc: Document, score: float, cosine: float | None = None, fts_hit: bool = False + doc: Document, + score: float, + cosine: float | None = None, + fts_hit: bool = False, + is_summary: bool = False, ) -> RetrievedChunk: """Fake candidate: *score* is the fused rank score; *cosine* (defaults to *score*) is the vector-similarity gate input.""" return RetrievedChunk( chunk_id=uuid.uuid4(), - position=0, + position=-1 if is_summary else 0, content=doc.content[:32], score=score, document=doc, cosine=score if cosine is None else cosine, fts_hit=fts_hit, + is_summary=is_summary, ) @@ -167,6 +172,70 @@ def test_gate_zero_chunks_deflects_with_fallback_chips() -> None: assert 2 <= len(plan.suggestions) <= MAX_SUGGESTIONS +# ---------- summary hits (phase 30: summary → full source document) ---------- + + +def test_summary_hit_on_selected_top_doc_counts() -> None: + """HIGH branch: the top document was hit via its summary chunk ⇒ 1. + + Context assembly is unchanged (A7 revised): the *source* document's + full content lands in the prompt, not the summary text alone. + """ + a = _doc("Alpha", "ALPHA_FULL_SOURCE_CONTENT") + b = _doc("Beta", "BETA_FULL_SOURCE_CONTENT") + chunks = [ + _chunk(a, 0.90, is_summary=True), # top doc reached through its summary + _chunk(b, 0.50), + ] + plan = chat_api.plan_turn(chunks, _settings(threshold=0.30)) + assert plan.deflected is False + assert plan.summary_hits == 1 + # The full source document is what the LLM sees (phase 24 contract). + assert "ALPHA_FULL_SOURCE_CONTENT" in plan.system_prompt + + +def test_summary_hit_outside_top_n_selection_not_counted() -> None: + """A summary chunk on a document outside the top-N (default 2) selection + does not count — only hits that landed in the selected context do.""" + a = _doc("Alpha", "ALPHA_CONTENT") + b = _doc("Beta", "BETA_CONTENT") + c = _doc("Gamma", "GAMMA_CONTENT") + chunks = [ + _chunk(a, 0.90), + _chunk(b, 0.80), + _chunk(c, 0.70, is_summary=True), # 3rd-ranked doc — not selected + ] + plan = chat_api.plan_turn(chunks, _settings(threshold=0.30)) + assert plan.deflected is False + assert [d.title for d in plan.docs] == ["Alpha", "Beta"] + assert plan.summary_hits == 0 + + +def test_low_branch_counts_summary_hit_on_selected_doc() -> None: + """LOW (deflected) branch records ``summary_hits`` too: the weak hit's + parent is still the selected (weak-hit) document.""" + a = _doc("Gamma", "GAMMA_DOC_CONTENT") + b = _doc("Delta", "DELTA_DOC_CONTENT") + chunks = [ + _chunk(a, 0.05, cosine=0.05, is_summary=True), # weak cosine, no FTS + _chunk(b, 0.03, cosine=0.03), + ] + plan = chat_api.plan_turn(chunks, _settings(threshold=0.30)) + assert plan.deflected is True + assert plan.summary_hits == 1 + + +def test_no_summary_chunks_yields_zero_summary_hits() -> None: + """Legacy chunks (``is_summary=false``) keep ``summary_hits == 0``.""" + a = _doc("Alpha", "ALPHA_CONTENT") + b = _doc("Beta", "BETA_CONTENT") + plan = chat_api.plan_turn([_chunk(a, 0.90), _chunk(b, 0.40)], _settings(threshold=0.30)) + assert plan.summary_hits == 0 + plan_low = chat_api.plan_turn([_chunk(a, 0.05, cosine=0.05)], _settings(threshold=0.30)) + assert plan_low.deflected is True + assert plan_low.summary_hits == 0 + + # ---------- prompt content (LOW vs HIGH) ---------- diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 5fae9c6..dd8fe32 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -25,6 +25,9 @@ def test_defaults_match_locked_decisions(monkeypatch: pytest.MonkeyPatch) -> Non s = _settings() assert s.llm_chat_model == "turbo" assert s.llm_embed_model == "embed" + # A5 extended (phase 30): one-shot completions default to the ``lite`` + # model on the same endpoint. + assert s.llm_summary_model == "lite" assert s.embedding_dim == 768 assert s.llm_base_url.endswith("/v1") # A8 (revised): the honesty gate input is the best cosine, default 0.62. @@ -53,6 +56,22 @@ def test_env_override(monkeypatch) -> None: assert s.llm_chat_model == "juggernaut" +def test_llm_summary_model_env_override(monkeypatch) -> None: + """Phase 30: ``BOR_LLM_SUMMARY_MODEL`` overrides the ``lite`` default""" + monkeypatch.setenv("BOR_LLM_SUMMARY_MODEL", "mini") + s = _settings() + assert s.llm_summary_model == "mini" + + +def test_summary_max_chars_default_and_env_override(monkeypatch: pytest.MonkeyPatch) -> None: + """Phase 30: document content sent to the ``lite`` model is capped at + ``BOR_SUMMARY_MAX_CHARS`` (default 12 000 chars per call).""" + monkeypatch.delenv("BOR_SUMMARY_MAX_CHARS", raising=False) + assert _settings().summary_max_chars == 12_000 + monkeypatch.setenv("BOR_SUMMARY_MAX_CHARS", "5000") + assert _settings().summary_max_chars == 5000 + + def test_max_output_tokens_env_override(monkeypatch) -> None: monkeypatch.setenv("BOR_MAX_OUTPUT_TOKENS", "1234") s = _settings() diff --git a/tests/unit/test_importer.py b/tests/unit/test_importer.py index 4a23cf4..e1d1302 100644 --- a/tests/unit/test_importer.py +++ b/tests/unit/test_importer.py @@ -1,12 +1,18 @@ -"""Unit tests: importer directory walk + sha256 delta logic. +"""Unit tests: importer directory walk + sha256 delta logic + summaries. -The walk tests are pure filesystem (``tmp_path``); the delta tests run -against the local compose Postgres (preferred — a real vector table), -skipping with clear instructions when the stack is not up. +The walk tests are pure filesystem (``tmp_path``); the delta and summary +tests run against the local compose Postgres (preferred — a real vector +table), skipping with clear instructions when the stack is not up. + +Summaries (phase 30): non-markdown files get a ``lite``-model summary via +the fake's deterministic ``chat`` (``"Summary of "``); the +sentinel word ``SUMMARY-BLOWUP`` makes ``chat`` raise :class:`LLMError` +for the fail-soft path. """ from __future__ import annotations import asyncio +import logging from pathlib import Path import pytest @@ -15,6 +21,8 @@ from sqlalchemy import func, select from app.models import Chunk, Document from app.rag.importer import ( EXCLUDED_DIRS, + ImportSummary, + _store_summary, import_sources, iter_importable_files, ) @@ -350,6 +358,205 @@ def test_multi_format_import_counts_per_format_and_titles_stem(db, tmp_path: Pat _cleanup_source(db, root.name) +# ---------- phase 30: lite-model summaries for non-markdown files ---------- + + +def test_non_markdown_file_gets_stored_and_indexed_summary(db, tmp_path: Path) -> None: + """A ``.yaml`` file is summarized: ``documents.summary`` is set and one + ``is_summary`` chunk (position −1, embedded) is indexed alongside the + content chunks.""" + root = tmp_path / "sumsrc" + root.mkdir() + (root / "svc.yaml").write_text("alpha services:\n gitlab:\n port: 8929\n") + llm = FakeEmbedder() + try: + summary = asyncio.run(import_sources([root], llm, session=db)) + assert summary.added == 1 + assert summary.summaries == 1 + assert summary.summary_errors == 0 + doc = db.scalar( + select(Document).where(Document.source == root.name, Document.path == "svc.yaml") + ) + assert doc is not None + assert doc.summary is not None + # Deterministic fake reply + the code-appended pointer line. + assert doc.summary.startswith("Summary of alpha") + assert doc.summary.endswith(f"Source: {root.name}/svc.yaml") + schunks = [c for c in doc.chunks if c.is_summary] + assert len(schunks) == 1 + assert schunks[0].position == -1 + assert schunks[0].content == doc.summary + assert schunks[0].embedding is not None and len(schunks[0].embedding) == 768 + # Content chunks stay 0-based and are never flagged as summaries. + content = [c for c in doc.chunks if not c.is_summary] + assert sorted(c.position for c in content) == list(range(len(content))) + finally: + _cleanup_source(db, root.name) + + +def test_markdown_file_never_gets_summary(db, tmp_path: Path) -> None: + """Markdown is already natural language: no summary, no ``is_summary`` + chunk, and the ``lite`` model is never called.""" + root = tmp_path / "mdsrc" + root.mkdir() + (root / "note.md").write_text("# Note\n\nmarkdown body\n") + llm = FakeEmbedder() + try: + summary = asyncio.run(import_sources([root], llm, session=db)) + assert summary.added == 1 + assert summary.summaries == 0 and summary.summary_errors == 0 + assert llm.chat_calls == [] # the model was never asked + doc = db.scalar( + select(Document).where(Document.source == root.name, Document.path == "note.md") + ) + assert doc is not None + assert doc.summary is None + assert doc.chunks and all(not c.is_summary for c in doc.chunks) + finally: + _cleanup_source(db, root.name) + + +def test_summary_failure_is_fail_soft(db, tmp_path: Path) -> None: + """A ``lite``-model failure must never lose the document: the file is + fully indexed (content chunks + embeddings), ``documents.summary`` stays + NULL, and the failure is counted in ``summary_errors``.""" + root = tmp_path / "blowup" + root.mkdir() + (root / "bad.txt").write_text("SUMMARY-BLOWUP the lite model chokes on this\n") + llm = FakeEmbedder() + try: + summary = asyncio.run(import_sources([root], llm, session=db)) + assert summary.errors == 0 # the document itself imported fine + assert summary.added == 1 + assert summary.summaries == 0 + assert summary.summary_errors == 1 + doc = db.scalar( + select(Document).where(Document.source == root.name, Document.path == "bad.txt") + ) + assert doc is not None + assert doc.summary is None + assert len(doc.chunks) == 1 + assert doc.chunks[0].embedding is not None # content chunk embedded + assert all(not c.is_summary for c in doc.chunks) + finally: + _cleanup_source(db, root.name) + + +def test_summary_chunk_is_replaced_on_reimport(db, tmp_path: Path) -> None: + """Re-importing a changed non-markdown file keeps exactly one + ``is_summary`` chunk — the old one is gone, the new summary is stored + and embedded, and the content chunks stay 0-based.""" + root = tmp_path / "repl" + root.mkdir() + path = root / "cfg.yaml" + path.write_text("alpha settings:\n host: one\n") + llm = FakeEmbedder() + try: + asyncio.run(import_sources([root], llm, session=db)) + path.write_text("bravo settings:\n host: two\n") + summary = asyncio.run(import_sources([root], llm, session=db)) + assert summary.updated == 1 + assert summary.summaries == 1 and summary.summary_errors == 0 + doc = db.scalar( + select(Document).where(Document.source == root.name, Document.path == "cfg.yaml") + ) + assert doc is not None + assert doc.summary is not None and doc.summary.startswith("Summary of bravo") + schunks = [c for c in doc.chunks if c.is_summary] + assert len(schunks) == 1 # the old one was deleted + assert schunks[0].position == -1 + assert schunks[0].content == doc.summary + assert schunks[0].embedding is not None + assert "alpha" not in schunks[0].content # no stale summary text + assert sorted(c.position for c in doc.chunks if not c.is_summary) == [0] + finally: + _cleanup_source(db, root.name) + + +def test_store_summary_replaces_an_existing_summary_chunk(db, tmp_path: Path) -> None: + """Replacement unit, driven directly: with a pre-existing + ``is_summary`` chunk in place, ``_store_summary`` deletes the old one + and leaves exactly one (new) summary chunk + updated + ``documents.summary`` — the at-most-one-summary invariant.""" + root = tmp_path / "direct" + root.mkdir() + (root / "a.yaml").write_text("alpha x\n") + llm = FakeEmbedder() + try: + asyncio.run(import_sources([root], llm, session=db)) + doc = db.scalar( + select(Document).where(Document.source == root.name, Document.path == "a.yaml") + ) + assert doc is not None and doc.summary is not None + assert any(c.is_summary for c in doc.chunks) # the first import's summary + counters = ImportSummary() + asyncio.run( + _store_summary( + session=db, doc=doc, source=root.name, rel="a.yaml", + content=doc.content, llm=llm, summary=counters, + ) + ) + assert counters.summaries == 1 and counters.summary_errors == 0 + schunks = [c for c in doc.chunks if c.is_summary] + assert len(schunks) == 1 # the old one was deleted + assert schunks[0].position == -1 + assert schunks[0].content == doc.summary + assert schunks[0].embedding is not None + finally: + _cleanup_source(db, root.name) + + +def test_store_summary_fail_soft_leaves_document_untouched(db, tmp_path: Path) -> None: + """A ``lite`` failure inside ``_store_summary`` rolls back only the + summary rows: the previous summary (if any) and the document survive, + and the failure is counted.""" + root = tmp_path / "directfail" + root.mkdir() + (root / "a.yaml").write_text("alpha x\n") + llm = FakeEmbedder() + try: + asyncio.run(import_sources([root], llm, session=db)) + doc = db.scalar( + select(Document).where(Document.source == root.name, Document.path == "a.yaml") + ) + assert doc is not None and doc.summary is not None + previous_summary = doc.summary + (root / "a.yaml").write_text("SUMMARY-BLOWUP now the lite model fails\n") + counters = ImportSummary() + asyncio.run( + _store_summary( + session=db, doc=doc, source=root.name, rel="a.yaml", + content="SUMMARY-BLOWUP now the lite model fails\n", + llm=llm, summary=counters, + ) + ) + assert counters.summaries == 0 and counters.summary_errors == 1 + assert doc.summary == previous_summary # rolled back, not nulled + schunks = [c for c in doc.chunks if c.is_summary] + assert len(schunks) == 1 # the old one survived the rollback + assert schunks[0].content == previous_summary + finally: + _cleanup_source(db, root.name) + + +def test_import_summary_log_line_includes_summary_counters( + caplog: pytest.LogCaptureFixture, +) -> None: + """PLAN §9 summary line: the phase-30 counters sit between + ``embed_batches`` and ``formats``.""" + s = ImportSummary() + s.files, s.added, s.chunks, s.embed_batches = 3, 3, 5, 4 + s.summaries, s.summary_errors = 2, 1 + s.formats = {"md": 1, "yaml": 2} + with caplog.at_level(logging.INFO, logger="app.importer"): + s.log() + line = caplog.records[-1].getMessage() + assert line == ( + "import: summary files=3 added=3 updated=0 unchanged=0 pruned=0 errors=0 " + "chunks=5 embed_batches=4 summaries=2 summary_errors=1 formats=yaml:2,md:1" + ) + + def test_prune_removes_files_now_excluded_by_format_filter(db, tmp_path: Path) -> None: """Previously-imported junk leaves the index: a file that no longer matches the A9 extension filter is pruned on the next ``prune=True`` run. diff --git a/tests/unit/test_llm_client.py b/tests/unit/test_llm_client.py index 329665a..67a81ce 100644 --- a/tests/unit/test_llm_client.py +++ b/tests/unit/test_llm_client.py @@ -275,17 +275,42 @@ class _FakeChatStream: return chunk +class _FakeCompletion: + """One fake non-streaming ChatCompletion (``choices[].message`` shape). + + ``content=None`` mirrors the real wire where the field can be absent or + empty (reasoning-only replies, provider quirks). + """ + + def __init__(self, content: str | None, empty_choices: bool = False) -> None: + if empty_choices: + self.choices = [] + else: + self.choices = [SimpleNamespace(message=SimpleNamespace(content=content))] + + class _FakeCompletions: - def __init__(self, chunks: list | None = None, fail: Exception | None = None) -> None: + def __init__( + self, + chunks: list | None = None, + fail: Exception | None = None, + completion: _FakeCompletion | None = None, + ) -> None: self.chunks = chunks or [] self.fail = fail + self.completion = completion self.kwargs: dict | None = None + self.chat_kwargs: dict | None = None - async def create(self, **kwargs) -> _FakeChatStream: + async def create(self, **kwargs) -> _FakeChatStream | _FakeCompletion: self.kwargs = kwargs if self.fail is not None: raise self.fail - return _FakeChatStream(self.chunks) + if kwargs.get("stream"): + return _FakeChatStream(self.chunks) + self.chat_kwargs = kwargs + assert self.completion is not None + return self.completion def _make_stream_client( @@ -428,3 +453,85 @@ def test_chat_stream_llm_error_passes_through_unwrapped() -> None: llm, _ = _make_stream_client(fail=LLMError("already wrapped")) with pytest.raises(LLMError, match="already wrapped"): asyncio.run(_collect(llm, [{"role": "user", "content": "q"}])) + + +# ---------- one-shot chat: LLMClient.chat (phase 30, task 01) ---------- + + +def _make_chat_client( + completion: _FakeCompletion | None = None, + fail: Exception | None = None, + **settings_kwargs: Any, +) -> tuple[LLMClient, _FakeCompletions]: + completions = _FakeCompletions(fail=fail, completion=completion) + fake_openai = SimpleNamespace(chat=SimpleNamespace(completions=completions)) + llm = LLMClient(_settings(**settings_kwargs)) + llm._client = fake_openai # pyright: ignore[reportAttributeAccessIssue] + return llm, completions + + +def test_chat_returns_trimmed_content_with_locked_params() -> None: + """Default model is ``lite`` (BOR_LLM_SUMMARY_MODEL), non-streaming, + low temperature, fixed 2048-token budget — summaries are short.""" + llm, completions = _make_chat_client(_FakeCompletion(" Summary text.\n")) + messages = [{"role": "system", "content": "s"}, {"role": "user", "content": "u"}] + out = asyncio.run(llm.chat(messages)) + assert out == "Summary text." + assert completions.chat_kwargs is not None + assert completions.chat_kwargs["model"] == "lite" + assert completions.chat_kwargs["stream"] is False + assert completions.chat_kwargs["temperature"] == 0.2 + assert completions.chat_kwargs["max_tokens"] == 2048 + assert completions.chat_kwargs["messages"] == messages + + +def test_chat_default_model_comes_from_llm_summary_model_setting() -> None: + llm, completions = _make_chat_client( + _FakeCompletion("x"), llm_summary_model="tiny" + ) + asyncio.run(llm.chat([{"role": "user", "content": "q"}])) + assert completions.chat_kwargs is not None + assert completions.chat_kwargs["model"] == "tiny" + + +def test_chat_explicit_model_overrides_the_default() -> None: + llm, completions = _make_chat_client( + _FakeCompletion("x"), llm_summary_model="tiny" + ) + asyncio.run(llm.chat([{"role": "user", "content": "q"}], model="special")) + assert completions.chat_kwargs is not None + assert completions.chat_kwargs["model"] == "special" + + +def test_chat_transport_failure_wrapped_as_llm_error_with_base_url() -> None: + """HTTP/transport failures (incl. >=400 surfaced by the SDK) are wrapped + with the base URL in the message — same style as chat_stream.""" + llm, _ = _make_chat_client(fail=RuntimeError("HTTP 502 Bad Gateway")) + with pytest.raises(LLMError, match="HTTP 502") as exc: + asyncio.run(llm.chat([{"role": "user", "content": "q"}])) + assert "aipi.reeseapps.com" in str(exc.value) + + +def test_chat_llm_error_passes_through_unwrapped() -> None: + llm, _ = _make_chat_client(fail=LLMError("already wrapped")) + with pytest.raises(LLMError, match="already wrapped"): + asyncio.run(llm.chat([{"role": "user", "content": "q"}])) + + +def test_chat_empty_choices_raises_llm_error() -> None: + llm, _ = _make_chat_client(_FakeCompletion(None, empty_choices=True)) + with pytest.raises(LLMError, match="no choices"): + asyncio.run(llm.chat([{"role": "user", "content": "q"}])) + + +def test_chat_missing_content_raises_llm_error() -> None: + """A silent empty summary must never be stored — None content fails.""" + llm, _ = _make_chat_client(_FakeCompletion(None)) + with pytest.raises(LLMError, match="empty content"): + asyncio.run(llm.chat([{"role": "user", "content": "q"}])) + + +def test_chat_whitespace_only_content_raises_llm_error() -> None: + llm, _ = _make_chat_client(_FakeCompletion(" \n\t ")) + with pytest.raises(LLMError, match="empty content"): + asyncio.run(llm.chat([{"role": "user", "content": "q"}])) diff --git a/tests/unit/test_retriever.py b/tests/unit/test_retriever.py index 10eea23..a673bac 100644 --- a/tests/unit/test_retriever.py +++ b/tests/unit/test_retriever.py @@ -7,6 +7,7 @@ chat integration tests against real Postgres; the pure mapping logic in from __future__ import annotations import uuid +from types import SimpleNamespace import pytest @@ -97,7 +98,8 @@ from app.rag.retriever import fuse, lexical_tsquery # noqa: E402 def _rc( - doc_path: str, cosine: float = 0.0, fts_hit: bool = False, position: int = 0 + doc_path: str, cosine: float = 0.0, fts_hit: bool = False, position: int = 0, + is_summary: bool = False, ) -> RetrievedChunk: return RetrievedChunk( chunk_id=uuid.uuid4(), @@ -107,6 +109,7 @@ def _rc( document=_doc(doc_path, "x" * 20), cosine=cosine, fts_hit=fts_hit, + is_summary=is_summary, ) @@ -186,3 +189,137 @@ def test_fuse_rejects_nonpositive_k() -> None: def test_fuse_empty_lists() -> None: assert fuse([], [], k=60) == [] + + +# --------------------------------------------------------------------------- +# Phase 30: is_summary survives both candidate lists and the fusion +# --------------------------------------------------------------------------- + +from app.models import Chunk # noqa: E402 +from app.rag.retriever import _lexical_candidates, _vector_candidates # noqa: E402 + + +class _FakeResult: + """Stands in for SQLAlchemy's RowMapping result (``.all()`` only).""" + + def __init__(self, rows: list) -> None: + self._rows = rows + + def all(self) -> list: + return self._rows + + +class _FakeSession: + """Returns canned rows from ``execute`` without touching Postgres.""" + + def __init__(self, rows: list) -> None: + self._rows = rows + self.statements: list = [] + + def execute(self, stmt, params: dict | None = None) -> _FakeResult: + self.statements.append((stmt, params)) + return _FakeResult(self._rows) + + +def _chunk_row(is_summary: bool) -> Chunk: + doc = _doc("summary-src.yaml", "RAW_YAML_CONTENT") + return Chunk( + id=uuid.uuid4(), + document_id=doc.id, + position=-1, # the summary chunk's position (phase 30) + content="Summary text", + is_summary=is_summary, + ) + + +def test_vector_candidates_carry_is_summary_flag() -> None: + """The vector list copies ``Chunk.is_summary`` onto each candidate.""" + doc = _doc("summary-src.yaml", "RAW_YAML_CONTENT") + summary = _chunk_row(is_summary=True) + ordinary = _chunk_row(is_summary=False) + ordinary.position = 0 + ordinary.content = "ordinary content" + rows = [ + (summary, 0.123456, doc), + (ordinary, 0.2, doc), + ] + out = _vector_candidates(_FakeSession(rows), [0.0] * 768, limit=5) # pyright: ignore[reportArgumentType] + assert len(out) == 2 + by_pos = {rc.position: rc for rc in out} + assert by_pos[-1].is_summary is True # the summary chunk (position −1) + assert by_pos[0].is_summary is False # ordinary content chunk + assert by_pos[-1].cosine == pytest.approx(0.876544) # 1 − distance, still rounded + + +def test_vector_candidates_default_is_summary_false_for_legacy_chunks() -> None: + """Pre-phase-30 rows have ``is_summary=false`` — candidates stay False.""" + doc = _doc("legacy.md", "LEGACY") + legacy = Chunk( + id=uuid.uuid4(), + document_id=doc.id, + position=0, + content="legacy content", + is_summary=False, + ) + out = _vector_candidates(_FakeSession([(legacy, 0.5, doc)]), [0.0] * 768, limit=5) # pyright: ignore[reportArgumentType] + assert out[0].is_summary is False + + +def _lexical_row(is_summary: bool, doc_path: str) -> object: + """One row of ``_LEXICAL_SQL`` (attribute access, as SQLAlchemy returns).""" + doc = _doc(doc_path, "DOC_BODY") + return SimpleNamespace( + chunk_id=uuid.uuid4(), + position=-1 if is_summary else 0, + content="summary chunk text" if is_summary else "content chunk text", + doc_id=doc.id, + source=doc.source, + path=doc.path, + full_path=doc.full_path, + title=doc.title, + doc_content=doc.content, + content_hash=doc.content_hash, + indexed_at=None, + is_summary=is_summary, + rank=0.33, + ) + + +def test_lexical_candidates_carry_is_summary_flag() -> None: + """The lexical list reads ``c.is_summary`` from the raw row.""" + rows = [_lexical_row(True, "summary-src.yaml"), _lexical_row(False, "other.md")] + out = _lexical_candidates(_FakeSession(rows), "how do i configure the thing", limit=10) # pyright: ignore[reportArgumentType] + assert len(out) == 2 + by_path = {rc.document.path: rc for rc in out} + assert by_path["summary-src.yaml"].is_summary is True + assert by_path["summary-src.yaml"].position == -1 + assert by_path["other.md"].is_summary is False + assert all(rc.fts_hit is True for rc in out) + + +def test_fuse_keeps_is_summary_on_double_hit() -> None: + """A summary chunk in both lists keeps the flag after fusion.""" + v1 = _rc("s.yaml", cosine=0.9, is_summary=True) + l1 = _rc("s.yaml", cosine=0.9, is_summary=True) # lexical copy of the same chunk + l1.chunk_id = v1.chunk_id + out = fuse([v1], [l1], k=60) + assert len(out) == 1 + assert out[0].is_summary is True + assert out[0].fts_hit is True + assert out[0].score == pytest.approx(2 / 61) + + +def test_fuse_keeps_is_summary_on_lexical_only_hit() -> None: + """A summary-only lexical hit (no vector rank) keeps the flag.""" + out = fuse([], [_rc("s.yaml", is_summary=True)], k=60) + assert len(out) == 1 + assert out[0].is_summary is True + assert out[0].fts_hit is True + assert out[0].cosine == 0.0 + + +def test_fuse_default_is_summary_stays_false_for_legacy_chunks() -> None: + """Neither list flagged ⇒ fusion never invents a summary flag.""" + out = fuse([_rc("a.md", cosine=0.8)], [_rc("b.md")], k=60) + assert len(out) == 2 + assert all(rc.is_summary is False for rc in out) diff --git a/tests/unit/test_summarizer.py b/tests/unit/test_summarizer.py new file mode 100644 index 0000000..0f7ebdd --- /dev/null +++ b/tests/unit/test_summarizer.py @@ -0,0 +1,173 @@ +"""Unit: document summarizer (phase 30, task 03). + +Covers the ``SUMMARY_MODE`` prompt (marker + instruction, capped user +content), the code-deterministic ``Source: /`` pointer, +and the rejection of empty/whitespace model output. +""" +from __future__ import annotations + +import asyncio + +import pytest + +from app.config import Settings, get_settings +from app.rag.llm import LLMError +from app.rag.retriever import TRUNCATION_MARKER +from app.rag.summarizer import ( + SUMMARY_INSTRUCTION, + SUMMARY_MODE, + SYSTEM_PROMPT, + build_summary_prompt, + generate_summary, +) + + +class _FakeLLM: + """Duck-typed stand-in for ``LLMClient`` (``chat`` + ``settings``). + + Records the messages and the ``model`` kwarg it was called with; can + return a canned reply or raise (e.g. :class:`LLMError`). + """ + + def __init__( + self, + reply: str | None = "Backups run nightly at 02:00 via the borg schedule.", + fail: Exception | None = None, + ) -> None: + self._reply = reply + self._fail = fail + self.settings = Settings(_env_file=None) # pyright: ignore[reportCallIssue] + self.messages: list[dict[str, str]] = [] + self.model: str | None = None + + async def chat( + self, messages: list[dict[str, str]], model: str | None = None + ) -> str: + self.messages = list(messages) + self.model = model + if self._fail is not None: + raise self._fail + assert self._reply is not None + return self._reply + + +# ---------- build_summary_prompt: system ---------- + + +def test_system_prompt_has_marker_and_locked_instruction() -> None: + assert SYSTEM_PROMPT.startswith(SUMMARY_MODE) + assert SUMMARY_INSTRUCTION in SYSTEM_PROMPT + for fragment in ( + "plain-text summary of this document in natural", + "what it configures/defines", + "Do not use markdown", + "Do not invent anything that is not in the document", + ): + assert fragment in SYSTEM_PROMPT + system, _ = build_summary_prompt("Homelab", "a.yaml", "content") + assert system == SYSTEM_PROMPT + assert SUMMARY_MODE in system # the marker the E2E mock keys on + + +# ---------- build_summary_prompt: user (capped content) ---------- + + +def test_user_prompt_is_full_content_when_under_cap() -> None: + content = "services:\n borg:\n port: 9999" + _, user = build_summary_prompt("Homelab", "a.yaml", content, max_chars=12_000) + assert user == content + assert TRUNCATION_MARKER not in user + + +def test_user_prompt_at_exact_cap_is_not_truncated() -> None: + content = "z" * 64 + _, user = build_summary_prompt("Homelab", "a.yaml", content, max_chars=64) + assert user == content + assert TRUNCATION_MARKER not in user + + +def test_user_prompt_truncated_with_marker_when_over_custom_cap() -> None: + content = "x" * 100 + "TAIL" + _, user = build_summary_prompt("Homelab", "a.yaml", content, max_chars=100) + assert user == "x" * 100 + "\n" + TRUNCATION_MARKER + assert "TAIL" not in user # overflow is gone, not squeezed in + assert user.endswith(TRUNCATION_MARKER) + + +def test_user_prompt_truncated_at_default_cap() -> None: + """No explicit cap → ``BOR_SUMMARY_MAX_CHARS`` (read from the live + settings, so the test holds for any configured value).""" + cap = get_settings().summary_max_chars + content = "y" * (cap + 50) + _, user = build_summary_prompt("Homelab", "a.yaml", content) + assert user == "y" * cap + "\n" + TRUNCATION_MARKER + + +# ---------- generate_summary: pointer + validation ---------- + + +def test_generate_summary_returns_model_text_plus_deterministic_pointer() -> None: + llm = _FakeLLM(reply="Backups run nightly at 02:00 via the borg schedule.") + out = asyncio.run( + generate_summary(llm, source="Homelab", path="backups/borg.yaml", content="c") + ) + expected = ( + "Backups run nightly at 02:00 via the borg schedule.\n" + "Source: Homelab/backups/borg.yaml" + ) + assert out == expected + assert out.splitlines()[-1] == "Source: Homelab/backups/borg.yaml" + + +def test_generate_summary_calls_the_configured_summary_model() -> None: + llm = _FakeLLM(reply="s") + asyncio.run(generate_summary(llm, source="Homelab", path="a.yaml", content="c")) + assert llm.model == llm.settings.llm_summary_model # the ``lite`` default + assert llm.model == "lite" + assert [m["role"] for m in llm.messages] == ["system", "user"] + assert SUMMARY_MODE in llm.messages[0]["content"] + assert llm.messages[1] == {"role": "user", "content": "c"} + + +def test_generate_summary_strips_model_text_before_appending_pointer() -> None: + llm = _FakeLLM(reply=" padded summary. \n") + out = asyncio.run(generate_summary(llm, source="Deployments", path="f.txt", content="c")) + assert out == "padded summary.\nSource: Deployments/f.txt" + + +def test_pointer_is_code_deterministic_even_if_model_writes_its_own() -> None: + """The pointer must never be model-generated: even a model reply that + contains a bogus 'Source:' line ends with the code-appended one.""" + llm = _FakeLLM(reply="The document itself says Source: fake/other.yaml inside.") + out = asyncio.run(generate_summary(llm, source="Homelab", path="real.yaml", content="c")) + assert out.splitlines()[-1] == "Source: Homelab/real.yaml" + + +def test_generate_summary_sends_capped_content_to_the_model() -> None: + """The cap applies to what the model actually receives (overflow cut + at the cap + marker) — read from the live settings for any value.""" + llm = _FakeLLM(reply="s") + content = "w" * (get_settings().summary_max_chars + 50) + asyncio.run(generate_summary(llm, source="Homelab", path="a.yaml", content=content)) + cap = get_settings().summary_max_chars + assert llm.messages[1]["content"] == "w" * cap + "\n" + TRUNCATION_MARKER + + +def test_generate_summary_rejects_whitespace_only_reply() -> None: + llm = _FakeLLM(reply=" \n\t ") + with pytest.raises(LLMError, match="empty content"): + asyncio.run(generate_summary(llm, source="Homelab", path="a.yaml", content="c")) + + +def test_generate_summary_rejects_empty_reply() -> None: + llm = _FakeLLM(reply="") + with pytest.raises(LLMError, match="empty content"): + asyncio.run(generate_summary(llm, source="Homelab", path="a.yaml", content="c")) + + +def test_generate_summary_propagates_llm_error_from_client() -> None: + llm = _FakeLLM( + fail=LLMError("chat completion from https://aipi.reeseapps.com/v1 failed: boom") + ) + with pytest.raises(LLMError, match="boom"): + asyncio.run(generate_summary(llm, source="Homelab", path="a.yaml", content="c"))