feat(rag): lite-model document summaries — non-markdown docs summarized at import, summary chunk retrieves and resolves to the full source doc

app/rag/importer.py

@@ -9,6 +9,12 @@ the two-phase upsert:
 2. embed the new chunks in batches and attach the vectors
 3. commit — one transaction per file, so a failed embedding leaves the
    database untouched and the file is simply retried on the next run
+4. non-markdown files only (phase 30): generate a ``lite``-model summary
+   and, best-effort, store it on ``documents.summary`` plus one extra
+   embedded chunk (``is_summary``, position −1). The document row and its
+   content chunks are already committed at this point, so a summary
+   failure only means the file is indexed without a summary (logged and
+   counted in ``summary_errors``) — it is never lost.
 
 Scope (A9, revised 2026-08-21): any path containing a dot-prefixed
 component (hidden dirs — vendored caches like ``.esphome/.espressif/**`` —
@@ -36,7 +42,8 @@ from app.config import Settings
 from app.db import SessionLocal
 from app.models import Chunk, Document
 from app.rag.chunker import chunk_document, extract_title
-from app.rag.llm import EmbeddingError
+from app.rag.llm import EmbeddingError, LLMError
+from app.rag.summarizer import generate_summary
 
 logger = logging.getLogger("app.importer")
 
@@ -54,6 +61,10 @@ class Embedder(Protocol):
 
     async def embed(self, texts: list[str]) -> list[list[float]]: ...
 
+    async def chat(self, messages: list[dict[str, str]], model: str | None = None) -> str: ...
+    # ^ the one-shot completion the summarizer uses for the ``lite`` model
+    #   (phase 30, task 01); :class:`app.rag.llm.LLMClient` satisfies it.
+
 
 @dataclass
 class ImportSummary:
@@ -67,6 +78,12 @@ class ImportSummary:
     errors: int = 0
     chunks: int = 0
     embed_batches: int = 0
+    #: Non-markdown files whose lite summary was generated + indexed
+    #: (phase 30). One ``is_summary`` chunk per success.
+    summaries: int = 0
+    #: Non-markdown files whose summary generation failed (best-effort —
+    #: the document is still indexed, without a summary).
+    summary_errors: int = 0
     #: Files walked, keyed by lowercased extension (``md``, ``yaml``, …).
     formats: dict[str, int] = field(default_factory=dict)
 
@@ -80,7 +97,8 @@ class ImportSummary:
     def log(self) -> None:
         logger.info(
             "import: summary files=%d added=%d updated=%d unchanged=%d pruned=%d "
-            "errors=%d chunks=%d embed_batches=%d formats=%s",
+            "errors=%d chunks=%d embed_batches=%d summaries=%d summary_errors=%d "
+            "formats=%s",
             self.files,
             self.added,
             self.updated,
@@ -89,6 +107,8 @@ class ImportSummary:
             self.errors,
             self.chunks,
             self.embed_batches,
+            self.summaries,
+            self.summary_errors,
             self.format_counts(),
         )
 
@@ -268,6 +288,65 @@ async def _index_file(
     summary.chunks += len(chunks_text)
     logger.info("import: %s source=%s path=%s chunks=%d", verb, source, rel, len(chunks_text))
 
+    # Phase 30: markdown is already natural language, so only the other A9
+    # formats (txt, yaml, yml, json, py) get a ``lite``-model summary.
+    if full_path.suffix.lower() in (".md", ".markdown"):
+        return
+    await _store_summary(
+        session, doc=doc, source=source, rel=rel, content=content, llm=llm, summary=summary
+    )
+
+
+async def _store_summary(
+    session: Session,
+    *,
+    doc: Document,
+    source: str,
+    rel: str,
+    content: str,
+    llm: Embedder,
+    summary: ImportSummary,
+) -> None:
+    """Best-effort ``lite`` summary for one already-committed document.
+
+    Generates the summary (task 03), stores it on ``documents.summary``
+    and indexes it as one extra embedded chunk (``is_summary``,
+    position −1) that hybrid search can hit instead of badly-formatted
+    raw text. Replacement is guaranteed: any pre-existing ``is_summary``
+    chunk of this document is deleted first, so at most one summary chunk
+    exists per document at a time.
+
+    Best-effort by contract: the doc row + content chunks are committed
+    by the caller before this runs, so an :class:`LLMError` /
+    :class:`EmbeddingError` only rolls back the summary rows — the file
+    stays indexed, without a summary, and the failure is counted in
+    ``summary_errors`` (PLAN phase 30).
+    """
+    try:
+        text = await generate_summary(llm, source=source, path=rel, content=content)
+        # Replacement: at most one summary chunk per document at a time.
+        # Removing from the collection is what the ``delete-orphan``
+        # cascade turns into a row delete on flush — and it keeps the
+        # in-memory collection consistent (this session runs with
+        # ``expire_on_commit=False``).
+        for old in [c for c in doc.chunks if c.is_summary]:
+            doc.chunks.remove(old)
+        chunk = Chunk(document_id=doc.id, position=-1, content=text, is_summary=True)
+        vector = (await llm.embed([text]))[0]
+        chunk.embedding = vector
+        doc.summary = text
+        # Append through the relationship (the ``all`` cascade persists the
+        # row) so the collection — live in this session because of
+        # ``expire_on_commit=False`` — reflects the committed state.
+        doc.chunks.append(chunk)
+        session.commit()
+        summary.summaries += 1
+        logger.info("import: summary source=%s path=%s chars=%d", source, rel, len(text))
+    except (LLMError, EmbeddingError) as e:
+        session.rollback()
+        summary.summary_errors += 1
+        logger.error("import: summary failed source=%s path=%s — %s", source, rel, e)
+
 
 def _prune(session: Session, source_names: set[str], seen: set[tuple[str, str]]) -> int:
     """Delete documents of *source_names* whose file is no longer in *seen*."""