feat(rag): lite-model document summaries — non-markdown docs summarized at import, summary chunk retrieves and resolves to the full source doc

app/rag/summarizer.py

@@ -0,0 +1,123 @@
+"""Document summarizer (phase 30, task 03).
+
+Builds the ``SUMMARY_MODE`` prompt for one document, calls the aipi
+``lite`` model through the one-shot ``LLMClient.chat`` (phase 30,
+task 01), and returns the validated summary text with a
+**code-deterministic** pointer line back to the source::
+
+    Source: <source>/<path>
+
+The pointer is appended by this module, never model-generated — the
+model is told what to summarize, not to cite.
+
+Quality contracts enforced here:
+
+* **Capped input** — the document content is cut at
+  ``BOR_SUMMARY_MAX_CHARS`` (default 12 000) before the single model
+  call; overflow is cut exactly at the cap and the shared
+  ``TRUNCATION_MARKER`` (``[…truncated…]``) is appended, so the model
+  never sees more than the cap and the cut is visible.
+* **No silent summaries** — a reply that is empty after trimming raises
+  :class:`LLMError` (the client already rejects empty content; the
+  summarizer re-asserts defensively and never hands the importer a
+  pointer-only row).
+
+The ``SUMMARY_MODE`` marker follows the ``DEFLECT_MODE`` convention:
+the deterministic E2E mock LLM keys on it in the system prompt
+(``tests/e2e/mock_llm.py`` — wired in task 06).
+"""
+from __future__ import annotations
+
+from typing import Protocol
+
+from app.config import Settings, get_settings
+from app.rag.llm import LLMError
+from app.rag.retriever import TRUNCATION_MARKER
+
+#: System-prompt marker for summary generation — the E2E mock LLM keys on
+#: it (same convention as ``DEFLECT_MODE``, PLAN §6).
+SUMMARY_MODE = "SUMMARY_MODE"
+
+#: Locked instruction for the ``lite`` model (phase 30): the summary is a
+#: natural-language retrieval target, so it must be plain, concrete, and
+#: strictly grounded in the document.
+SUMMARY_INSTRUCTION = (
+    "Write a 3-6 sentence plain-text summary of this document in natural "
+    "language. Cover what it configures/defines and its most important "
+    "values. Do not use markdown. Do not invent anything that is not in "
+    "the document."
+)
+
+#: Full system prompt: marker first (the mock's key), then the instruction.
+SYSTEM_PROMPT = f"{SUMMARY_MODE}: {SUMMARY_INSTRUCTION}"
+
+
+class SummaryLLM(Protocol):
+    """The one-shot chat surface the summarizer needs.
+
+    :class:`app.rag.llm.LLMClient` satisfies it; unit tests pass a
+    duck-typed fake (``chat`` + ``settings``) instead — same pattern as
+    the importer's ``Embedder`` protocol.
+    """
+
+    settings: Settings
+
+    async def chat(
+        self, messages: list[dict[str, str]], model: str | None = None
+    ) -> str: ...
+
+
+def _capped_content(content: str, max_chars: int | None) -> str:
+    """Document content for the user message, capped at *max_chars*.
+
+    The default cap is ``BOR_SUMMARY_MAX_CHARS``. Overflow is cut exactly
+    at the cap and the shared ``TRUNCATION_MARKER`` is appended on its
+    own line; content that fits (length ≤ cap) passes through unchanged.
+    """
+    limit = max_chars if max_chars is not None else get_settings().summary_max_chars
+    if len(content) <= limit:
+        return content
+    return content[:limit] + "\n" + TRUNCATION_MARKER
+
+
+def build_summary_prompt(
+    source: str, path: str, content: str, max_chars: int | None = None
+) -> tuple[str, str]:
+    """The ``(system, user)`` message pair for one summary call.
+
+    * ``system`` — :data:`SYSTEM_PROMPT`: the ``SUMMARY_MODE`` marker +
+      the locked instruction.
+    * ``user`` — the document content, capped (see :func:`_capped_content`).
+
+    *source* and *path* are part of the signature so the call site reads
+    like the document it summarizes (and for :func:`generate_summary`'s
+    pointer) — the pointer is built in code and deliberately **not** part
+    of the prompt, so the model cannot echo or mangle it.
+    """
+    return SYSTEM_PROMPT, _capped_content(content, max_chars)
+
+
+async def generate_summary(
+    llm: SummaryLLM, *, source: str, path: str, content: str
+) -> str:
+    """One-shot ``lite`` summary of *content*, ending in the pointer line.
+
+    Returns the model's text (trimmed) plus the deterministic
+    ``Source: <source>/<path>`` line — the pointer is appended by code,
+    never model-generated. Raises :class:`LLMError` when the model
+    returns nothing usable after trimming, and propagates any
+    :class:`LLMError` the client raises (the importer's fail-soft path
+    turns that into a logged, counted ``summary_errors`` entry).
+    """
+    system, user = build_summary_prompt(source, path, content)
+    raw = await llm.chat(
+        [{"role": "system", "content": system}, {"role": "user", "content": user}],
+        model=llm.settings.llm_summary_model,
+    )
+    summary = raw.strip()
+    if not summary:
+        raise LLMError(
+            f"summary model returned empty content for {source}/{path} — "
+            "refusing to store a silent summary"
+        )
+    return f"{summary}\nSource: {source}/{path}"