feat(rag): lite-generated KB overview in the system prompt — stored single row, regenerated on import, <knowledge_base> section in HIGH+LOW prompts

app/api/chat.py

@@ -36,6 +36,16 @@ assembly is untouched. ``TurnPlan.summary_hits`` counts the hit chunks
 with ``is_summary`` whose parent document landed in the selected
 top-N context, and the per-turn log line records ``summary_hits=N``
 after ``fts_hits`` (PLAN §9 line extension).
+
+KB overview (phase 31): the lite-generated outline of the knowledge
+base (single ``kb_overview`` row) is read per turn (one indexed PK
+lookup — no LLM call) and injected into **both** prompts as the
+``<knowledge_base>`` section, ordered ``<relevance>`` →
+``<knowledge_base>`` → ``<tuning>`` → mode body. With an empty row the
+prompts stay byte-identical to the pre-phase text (phase 15
+convention); ``TurnPlan.kb_chars`` records the length of the stored
+outline (0 when absent) and the per-turn log line records
+``kb_chars=N`` after ``tuning=N`` (PLAN §9 line extension).
 """
 from __future__ import annotations
 
@@ -60,6 +70,7 @@ from app.rag.llm import (
     LLMError,
     StreamPiece,  # noqa: F401  (phase 17 typing: chat_stream yields StreamPiece)
 )
+from app.rag.overview import load_kb_overview
 from app.rag.prompts import build_deflect_prompt, build_high_prompt
 from app.rag.retriever import RetrievedChunk, retrieve, select_documents, weak_hit_titles
 from app.rag.suggestions import derive_suggestions
@@ -107,12 +118,17 @@ class TurnPlan:
     #: Hit chunks with ``is_summary`` whose parent document made it into
     #: *docs* (phase 30; per-turn log line ``summary_hits=N``).
     summary_hits: int = 0
+    #: Length of the stored KB overview injected as the
+    #: ``<knowledge_base>`` section (phase 31; per-turn log line
+    #: ``kb_chars=N``). 0 when no non-empty row exists.
+    kb_chars: int = 0
 
 
 def plan_turn(
     chunks: Sequence[RetrievedChunk],
     settings: Settings,
     notes: Sequence[str] | None = None,
+    kb_overview: str | None = None,
 ) -> TurnPlan:
     """Apply the honesty gate (A8, revised) and assemble prompt + context.
 
@@ -133,11 +149,20 @@ def plan_turn(
     when non-empty, both the HIGH and the LOW prompt carry the
     ``<tuning>`` section; with no notes the prompts are unchanged.
 
+    *kb_overview* is the stored KB outline (phase 31, one PK lookup per
+    turn): when non-empty, both prompts carry the ``<knowledge_base>``
+    section (between ``<relevance>`` and ``<tuning>``) and
+    ``kb_chars`` records the outline's length; with no outline the
+    prompts are byte-identical to the pre-phase text and ``kb_chars``
+    is 0.
+
     ``summary_hits`` (phase 30) counts the hit chunks with
     ``is_summary`` whose parent document is among the selected
     top-N documents — both the HIGH and the LOW branch record it.
     """
     steering = list(notes or [])
+    kb_text = (kb_overview or "").strip()
+    kb_chars = len(kb_text)
     best_cosine = max((c.cosine for c in chunks), default=0.0)
     fts_hits = sum(1 for c in chunks if c.fts_hit)
     docs = select_documents(chunks, n=settings.top_n_docs)
@@ -148,22 +173,24 @@ def plan_turn(
             best_cosine,
             fts_hits,
             False,
-            build_high_prompt(docs, notes=steering),
+            build_high_prompt(docs, notes=steering, kb_overview=kb_text),
             docs,
             [],
             len(steering),
             summary_hits,
+            kb_chars,
         )
     titles = weak_hit_titles(chunks)
     return TurnPlan(
         best_cosine,
         fts_hits,
         True,
-        build_deflect_prompt(titles, notes=steering),
+        build_deflect_prompt(titles, notes=steering, kb_overview=kb_text),
         docs,
         derive_suggestions(titles, settings.suggestions),
         len(steering),
         summary_hits,
+        kb_chars,
     )
 
 
@@ -216,8 +243,11 @@ async def chat(
         settings = get_settings()
         try:
             steering_notes = load_steering_notes(db)
+            # KB overview (phase 31): one indexed PK lookup per turn — the
+            # outline is generated at import time, never per chat turn.
+            kb_overview = load_kb_overview(db)
             chunks = retrieve(db, request.message, question_vec)
-            plan = plan_turn(chunks, settings, notes=steering_notes)
+            plan = plan_turn(chunks, settings, notes=steering_notes, kb_overview=kb_overview)
         except Exception:  # noqa: BLE001 — DB failure mid-turn
             logger.exception(
                 "chat: retrieval failed question=%r total_ms=%d",
@@ -284,13 +314,14 @@ async def chat(
 
         logger.info(
             "question=%r embed_ms=%d top_score=%.3f fts_hits=%d summary_hits=%d tuning=%d "
-            "threshold=%.2f deflected=%s sources=%r thinking_chars=%d total_ms=%d",
+            "kb_chars=%d threshold=%.2f deflected=%s sources=%r thinking_chars=%d total_ms=%d",
             request.message,
             embed_ms,
             plan.top_score,
             plan.fts_hits,
             plan.summary_hits,
             plan.tuning_count,
+            plan.kb_chars,
             settings.relevance_threshold,
             plan.deflected,
             source_paths,

app/config.py

@@ -76,6 +76,17 @@ class Settings(BaseSettings):
     #: and the shared ``[…truncated…]`` marker is appended (see
     #: ``app.rag.summarizer``).
     summary_max_chars: int = 12_000
+    #: Char budget for the ``<knowledge_base>`` section of the system prompt
+    #: (phase 31: lite-generated KB overview, ``app.rag.overview``). The
+    #: newest-fitting prefix of the stored outline is kept and the overflow
+    #: is replaced by the shared ``[…truncated…]`` marker (phase 15
+    #: convention — ``app.rag.prompts``).
+    kb_overview_max_chars: int = 4_000
+    #: Cap on the document list (source/path/title/first summary line per
+    #: row) sent to the ``lite`` overview model in one call (phase 31,
+    #: ``app.rag.overview``). Overflow is cut at the cap and the shared
+    #: ``[…truncated…]`` marker is appended (summarizer convention).
+    overview_input_max_chars: int = 40_000
 
     # --- Hybrid retrieval (A7, revised 2026-08-21) ---
     # cosine top-N ∪ Postgres FTS top-N, fused with Reciprocal Rank Fusion

app/models.py

@@ -10,6 +10,9 @@ Data model — see ``.agent/PLAN.md`` §Data Model:
                       the deflection decision, and latency.
 * ``steering_notes`` — owner tuning notes injected into the system prompt
                       of every chat turn (phase 15, ``<tuning>`` section).
+* ``kb_overview``    — single-row lite-generated outline of the KB's basic
+                      categories, injected as the ``<knowledge_base>``
+                      section of every chat turn (phase 31).
 """
 from __future__ import annotations
 
@@ -109,3 +112,24 @@ class SteeringNote(Base):
     id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
     note: Mapped[str] = mapped_column(Text)  # trimmed, 1–2000 chars (API-enforced)
     created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
+
+
+class KbOverview(Base):
+    """Single-row, lite-generated outline of the knowledge base (phase 31).
+
+    Exactly one row (``id = 1``, enforced by the migration 0005 server
+    defaults) holds a plain-text outline of the KB's basic categories,
+    generated by the aipi ``lite`` model whenever an import changes the KB.
+    Chat turns only read this row (one indexed PK lookup) and inject it into
+    the system prompt of every turn as the ``<knowledge_base>`` section — an
+    empty row means the section is absent and the prompt stays
+    byte-identical to the pre-phase text (phase 15 convention).
+    """
+
+    __tablename__ = "kb_overview"
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True, server_default="1")
+    content: Mapped[str] = mapped_column(Text, server_default="")  # the outline text
+    updated_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), server_default=func.now()
+    )

app/rag/overview.py

@@ -0,0 +1,202 @@
+"""KB overview generator (phase 31, task 02).
+
+Builds the ``KB_OVERVIEW_MODE`` prompt from the document catalogue
+(source, path, title, first summary line per document), calls the aipi
+``lite`` model through the one-shot ``LLMClient.chat`` (phase 30, same
+model as document summaries — no new model management), and stores the
+plain-text outline in the single ``kb_overview`` row (migration 0005).
+
+Chat turns never generate an overview — they read the stored row
+(:func:`load_kb_overview`, one indexed PK lookup) and inject it into the
+system prompt of every turn as the ``<knowledge_base>`` section
+(phase 31, task 03). Generation happens at import time (phase 31, task
+04) and is **best-effort and change-gated**: callers invoke it only when
+an import added/updated at least one document (or no row exists yet),
+and a ``lite`` failure logs and leaves the previous outline intact — an
+old outline is better than none (phase locked decisions).
+
+The ``KB_OVERVIEW_MODE`` marker follows the ``SUMMARY_MODE`` /
+``DEFLECT_MODE`` convention: the deterministic E2E mock LLM keys on it
+in the system prompt (wired in phase 31, task 05).
+"""
+from __future__ import annotations
+
+import logging
+from collections.abc import Sequence
+from datetime import UTC, datetime
+from typing import Protocol
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from app.config import Settings, get_settings
+from app.db import SessionLocal
+from app.models import Document, KbOverview
+from app.rag.llm import LLMError
+from app.rag.retriever import TRUNCATION_MARKER
+
+logger = logging.getLogger("app.rag.overview")
+
+#: System-prompt marker for KB overview generation — the E2E mock LLM keys
+#: on it (same convention as ``SUMMARY_MODE`` / ``DEFLECT_MODE``).
+KB_OVERVIEW_MODE = "KB_OVERVIEW_MODE"
+
+#: Locked instruction for the ``lite`` model (phase 31): the overview is
+#: the agent's *a priori* picture of the knowledge base, so it must be a
+#: compact plain-text outline of basic categories and topics — strictly
+#: grounded in the document list, nothing invented.
+OVERVIEW_INSTRUCTION = (
+    "From the document list below, write a compact plain-text outline of "
+    "the basic categories and topics this knowledge base covers. Group by "
+    "source where useful, use `-` bullet lines, at most ~1500 characters, "
+    "no markdown headings, and no topics not present in the list."
+)
+
+#: Full system prompt: marker first (the mock's key), then the instruction.
+SYSTEM_PROMPT = f"{KB_OVERVIEW_MODE}: {OVERVIEW_INSTRUCTION}"
+
+
+class OverviewLLM(Protocol):
+    """The one-shot chat surface the overview generator needs.
+
+    :class:`app.rag.llm.LLMClient` satisfies it; unit tests pass a
+    duck-typed fake (``chat`` + ``settings``) instead — same pattern as
+    the summarizer's ``SummaryLLM`` protocol.
+    """
+
+    settings: Settings
+
+    async def chat(
+        self, messages: list[dict[str, str]], model: str | None = None
+    ) -> str: ...
+
+
+def _first_summary_line(summary: str | None) -> str:
+    """First line of a stored summary, stripped; ``''`` when absent.
+
+    The stored summary (phase 30) ends in the code-appended
+    ``Source: …`` pointer line, so its first line is the model-written
+    lead sentence — the best one-line picture of the document for the
+    outline. Blank/whitespace-only summaries yield ``''`` as well.
+    """
+    if not summary:
+        return ""
+    for line in summary.splitlines():
+        stripped = line.strip()
+        if stripped:
+            return stripped
+    return ""
+
+
+def build_overview_prompt(
+    rows: Sequence[tuple[str, str, str, str | None]], max_chars: int | None = None
+) -> tuple[str, str]:
+    """The ``(system, user)`` message pair for one overview call.
+
+    Each row is ``(source, path, title, summary)``.
+
+    * ``system`` — :data:`SYSTEM_PROMPT`: the ``KB_OVERVIEW_MODE`` marker
+      + the locked instruction.
+    * ``user`` — one line per document,
+      ``source — path — title — {first line of summary}``, joined with
+      newlines. The summary field is omitted when the document has no
+      summary (markdown docs and the fail-soft path — no dangling
+      dash). The list is capped at *max_chars* (default
+      ``BOR_OVERVIEW_INPUT_MAX_CHARS``): overflow is cut exactly at the
+      cap and the shared ``[…truncated…]`` marker is appended on its own
+      line, so the model never sees more than the cap and the cut is
+      visible (summarizer convention, phase 30).
+    """
+    limit = max_chars if max_chars is not None else get_settings().overview_input_max_chars
+    lines: list[str] = []
+    for source, path, title, summary in rows:
+        line = f"{source} — {path} — {title}"
+        first = _first_summary_line(summary)
+        if first:
+            line += f" — {first}"
+        lines.append(line)
+    user = "\n".join(lines)
+    if len(user) > limit:
+        user = user[:limit] + "\n" + TRUNCATION_MARKER
+    return SYSTEM_PROMPT, user
+
+
+def load_kb_overview(db: Session) -> str:
+    """The stored KB outline for prompt injection (phase 31, task 03).
+
+    One indexed PK lookup (the single row, ``id = 1``) returning
+    ``content`` trimmed. Returns ``""`` when the row is missing or
+    empty — the chat path then omits the ``<knowledge_base>`` section
+    entirely, keeping the prompt byte-identical to the pre-phase text
+    (phase 15 convention).
+    """
+    row = db.get(KbOverview, 1)
+    if row is None:
+        return ""
+    return (row.content or "").strip()
+
+
+async def regenerate_overview(
+    llm: OverviewLLM, session: Session | None = None
+) -> bool:
+    """Generate a fresh KB outline with the ``lite`` model and upsert it.
+
+    Best-effort and change-gated by the caller (phase locked decisions):
+    the import script (phase 31, task 04) and the admin sync (phase 32)
+    invoke this only when an import added/updated at least one document
+    (or no row exists yet). *session* may be supplied (the import
+    script's own session, tests); a private one is opened and closed
+    otherwise — same convention as ``import_sources``.
+
+    * Zero documents → the existing row (if any) is left untouched and
+      ``False`` is returned — no KB, no outline, no wasted model call.
+    * Happy path → the model's text is upserted into the single row
+      (``id = 1``, fresh UTC ``updated_at``), committed, and
+      ``overview: regenerated docs=… chars=…`` is logged; returns
+      ``True``. Idempotent by construction: one row, always ``id = 1``.
+    * :class:`LLMError` → ``overview: regeneration failed — …`` is
+      logged and ``False`` is returned with the previous row intact —
+      an old outline is better than none.
+    """
+    owns_session = session is None
+    if session is None:
+        session = SessionLocal()
+    try:
+        result = session.execute(
+            select(Document.source, Document.path, Document.title, Document.summary)
+            .order_by(Document.source, Document.path)
+        ).all()
+        rows: list[tuple[str, str, str, str | None]] = [
+            (source, path, title, summary)
+            for source, path, title, summary in result
+        ]
+        if not rows:
+            logger.info(
+                "overview: no documents in the knowledge base — "
+                "leaving any existing outline untouched"
+            )
+            return False
+
+        system, user = build_overview_prompt(rows)
+        try:
+            text = await llm.chat(
+                [{"role": "system", "content": system}, {"role": "user", "content": user}],
+                model=llm.settings.llm_summary_model,
+            )
+        except LLMError as e:
+            logger.error("overview: regeneration failed — %s", e)
+            return False
+
+        now = datetime.now(UTC)
+        row = session.get(KbOverview, 1)
+        if row is None:
+            session.add(KbOverview(id=1, content=text, updated_at=now))
+        else:
+            row.content = text
+            row.updated_at = now
+        session.commit()
+        logger.info("overview: regenerated docs=%d chars=%d", len(rows), len(text))
+        return True
+    finally:
+        if owns_session:
+            session.close()

app/rag/prompts.py

@@ -15,6 +15,14 @@ Steering (phase 15): when the owner has stored tuning notes, both modes
 carry a ``<tuning>`` section between ``<relevance>…</relevance>`` and the
 mode body. With zero notes the prompt is byte-identical to the
 pre-steering text.
+
+KB overview (phase 31): when the single ``kb_overview`` row holds a
+lite-generated outline of the knowledge base, both modes carry a
+``<knowledge_base>`` section between ``<relevance>…</relevance>`` and
+the ``<tuning>`` section (order: ``<relevance>`` →
+``<knowledge_base>`` → ``<tuning>`` → mode body) — the agent knows
+roughly what the KB contains before retrieval. With an empty row the
+prompt is byte-identical to the pre-phase text.
 """
 from __future__ import annotations
 
@@ -51,6 +59,13 @@ _STEERING_INTRO = (
     "Where these instructions conflict with the defaults above, follow the owner:\n"
 )
 
+#: One-line intro of the ``<knowledge_base>`` section (phase 31): the
+#: lite-generated outline is the agent's a-priori picture of the KB.
+_KB_INTRO = (
+    "The basic categories of everything in this knowledge base "
+    "(generated at import time):\n"
+)
+
 
 def _base(relevance: str) -> str:
     if relevance not in ("HIGH", "LOW"):
@@ -95,9 +110,57 @@ def build_steering_section(notes: Sequence[str], max_chars: int | None = None) -
     return ""
 
 
-def build_high_prompt(documents: Sequence[Document], notes: Sequence[str] | None = None) -> str:
-    """Grounded turn: locked persona (+ steering) + full texts of the top
-    documents."""
+def build_kb_section(overview: str, max_chars: int | None = None) -> str:
+    """The ``<knowledge_base>`` section of the system prompt (phase 31).
+
+    * No outline (or only whitespace) → ``""`` — callers then build the
+      prompt exactly as before, so a no-overview prompt is byte-identical
+      to the pre-phase text (phase 15 convention).
+    * Otherwise: the intro line + the stored outline, capped at
+      *max_chars* (default ``BOR_KB_OVERVIEW_MAX_CHARS``). When the budget
+      cannot hold the whole outline, the longest-fitting prefix is kept
+      and the overflow is replaced by the shared ``[…truncated…]`` marker
+      on its own line — the exact :func:`build_steering_section` pattern,
+      including its pathological-budget handling (never exceed the cap;
+      bare marker when even one outline character does not fit).
+    """
+    text = str(overview or "").strip()
+    if not text:
+        return ""
+    limit = max_chars if max_chars is not None else get_settings().kb_overview_max_chars
+    if limit <= 0:
+        return ""
+
+    def render(cut: int) -> str:
+        lines = [text[:cut]]
+        if cut < len(text):
+            lines.append(TRUNCATION_MARKER)
+        return f"<knowledge_base>\n{_KB_INTRO}" + "\n".join(lines) + "\n</knowledge_base>"
+
+    for cut in range(len(text), 0, -1):
+        rendered = render(cut)
+        if len(rendered) <= limit:
+            return rendered
+    # Pathological budget: not even one outline character fits. The
+    # section must still respect the cap — the bare marker when it fits,
+    # else none (steering precedent, phase 15).
+    if len(TRUNCATION_MARKER) <= limit:
+        return TRUNCATION_MARKER
+    return ""
+
+
+def build_high_prompt(
+    documents: Sequence[Document],
+    notes: Sequence[str] | None = None,
+    kb_overview: str | None = None,
+) -> str:
+    """Grounded turn: locked persona (+ steering, + KB overview) + full
+texts of the top documents.
+
+    Section order (phase 31): ``<relevance>`` → ``<knowledge_base>`` →
+    ``<tuning>`` → ``<documents>``; empty steering/overview omit their
+    section, keeping the prompt byte-identical to the pre-phase text.
+    """
     blocks = [
         f'<document source="{doc.source}" path="{doc.path}" title="{doc.title}">\n'
         f"{doc.content}\n"
@@ -107,21 +170,34 @@ def build_high_prompt(documents: Sequence[Document], notes: Sequence[str] | None
     body = "\n\n".join(blocks) if blocks else (
         "(no documents matched — do not invent specifics)"
     )
-    section = build_steering_section(notes or [])
     prompt = _base("HIGH")
-    if section:
-        prompt += "\n" + section
+    for part in (build_kb_section(kb_overview or ""), build_steering_section(notes or [])):
+        if part:
+            prompt += "\n" + part
     return prompt + "\n<documents>\n" + body + "\n</documents>"
 
 
-def build_deflect_prompt(titles: Sequence[str], notes: Sequence[str] | None = None) -> str:
-    """Deflection turn: weak-hit titles only (no document content)."""
+def build_deflect_prompt(
+    titles: Sequence[str],
+    notes: Sequence[str] | None = None,
+    kb_overview: str | None = None,
+) -> str:
+    """Deflection turn: weak-hit titles only (no document content).
+
+    Section order (phase 31): ``<relevance>`` → ``<knowledge_base>`` →
+    ``<tuning>`` → ``DEFLECT_MODE`` body; empty steering/overview omit
+    their section, keeping the prompt byte-identical to the pre-phase text.
+    """
     weak = "\n".join(f"- {t}" for t in titles) if titles else "(nothing close at all)"
-    section = build_steering_section(notes or [])
-    mid = f"\n{section}\n" if section else "\n"
+    mid = "\n".join(
+        part
+        for part in (build_kb_section(kb_overview or ""), build_steering_section(notes or []))
+        if part
+    )
+    gap = f"\n{mid}\n" if mid else "\n"
     return (
         _base("LOW")
-        + mid
+        + gap
         + "DEFLECT_MODE: retrieval was weak — the titles below are the closest "
         "your notes come to the question. They are titles only; do not pretend "
         "they answer it. Use them to propose 2-3 alternative questions.\n"