feat(rag): lite-generated KB overview in the system prompt — stored single row, regenerated on import, <knowledge_base> section in HIGH+LOW prompts

app/rag/prompts.py

@@ -15,6 +15,14 @@ Steering (phase 15): when the owner has stored tuning notes, both modes
 carry a ``<tuning>`` section between ``<relevance>…</relevance>`` and the
 mode body. With zero notes the prompt is byte-identical to the
 pre-steering text.
+
+KB overview (phase 31): when the single ``kb_overview`` row holds a
+lite-generated outline of the knowledge base, both modes carry a
+``<knowledge_base>`` section between ``<relevance>…</relevance>`` and
+the ``<tuning>`` section (order: ``<relevance>`` →
+``<knowledge_base>`` → ``<tuning>`` → mode body) — the agent knows
+roughly what the KB contains before retrieval. With an empty row the
+prompt is byte-identical to the pre-phase text.
 """
 from __future__ import annotations
 
@@ -51,6 +59,13 @@ _STEERING_INTRO = (
     "Where these instructions conflict with the defaults above, follow the owner:\n"
 )
 
+#: One-line intro of the ``<knowledge_base>`` section (phase 31): the
+#: lite-generated outline is the agent's a-priori picture of the KB.
+_KB_INTRO = (
+    "The basic categories of everything in this knowledge base "
+    "(generated at import time):\n"
+)
+
 
 def _base(relevance: str) -> str:
     if relevance not in ("HIGH", "LOW"):
@@ -95,9 +110,57 @@ def build_steering_section(notes: Sequence[str], max_chars: int | None = None) -
     return ""
 
 
-def build_high_prompt(documents: Sequence[Document], notes: Sequence[str] | None = None) -> str:
-    """Grounded turn: locked persona (+ steering) + full texts of the top
-    documents."""
+def build_kb_section(overview: str, max_chars: int | None = None) -> str:
+    """The ``<knowledge_base>`` section of the system prompt (phase 31).
+
+    * No outline (or only whitespace) → ``""`` — callers then build the
+      prompt exactly as before, so a no-overview prompt is byte-identical
+      to the pre-phase text (phase 15 convention).
+    * Otherwise: the intro line + the stored outline, capped at
+      *max_chars* (default ``BOR_KB_OVERVIEW_MAX_CHARS``). When the budget
+      cannot hold the whole outline, the longest-fitting prefix is kept
+      and the overflow is replaced by the shared ``[…truncated…]`` marker
+      on its own line — the exact :func:`build_steering_section` pattern,
+      including its pathological-budget handling (never exceed the cap;
+      bare marker when even one outline character does not fit).
+    """
+    text = str(overview or "").strip()
+    if not text:
+        return ""
+    limit = max_chars if max_chars is not None else get_settings().kb_overview_max_chars
+    if limit <= 0:
+        return ""
+
+    def render(cut: int) -> str:
+        lines = [text[:cut]]
+        if cut < len(text):
+            lines.append(TRUNCATION_MARKER)
+        return f"<knowledge_base>\n{_KB_INTRO}" + "\n".join(lines) + "\n</knowledge_base>"
+
+    for cut in range(len(text), 0, -1):
+        rendered = render(cut)
+        if len(rendered) <= limit:
+            return rendered
+    # Pathological budget: not even one outline character fits. The
+    # section must still respect the cap — the bare marker when it fits,
+    # else none (steering precedent, phase 15).
+    if len(TRUNCATION_MARKER) <= limit:
+        return TRUNCATION_MARKER
+    return ""
+
+
+def build_high_prompt(
+    documents: Sequence[Document],
+    notes: Sequence[str] | None = None,
+    kb_overview: str | None = None,
+) -> str:
+    """Grounded turn: locked persona (+ steering, + KB overview) + full
+texts of the top documents.
+
+    Section order (phase 31): ``<relevance>`` → ``<knowledge_base>`` →
+    ``<tuning>`` → ``<documents>``; empty steering/overview omit their
+    section, keeping the prompt byte-identical to the pre-phase text.
+    """
     blocks = [
         f'<document source="{doc.source}" path="{doc.path}" title="{doc.title}">\n'
         f"{doc.content}\n"
@@ -107,21 +170,34 @@ def build_high_prompt(documents: Sequence[Document], notes: Sequence[str] | None
     body = "\n\n".join(blocks) if blocks else (
         "(no documents matched — do not invent specifics)"
     )
-    section = build_steering_section(notes or [])
     prompt = _base("HIGH")
-    if section:
-        prompt += "\n" + section
+    for part in (build_kb_section(kb_overview or ""), build_steering_section(notes or [])):
+        if part:
+            prompt += "\n" + part
     return prompt + "\n<documents>\n" + body + "\n</documents>"
 
 
-def build_deflect_prompt(titles: Sequence[str], notes: Sequence[str] | None = None) -> str:
-    """Deflection turn: weak-hit titles only (no document content)."""
+def build_deflect_prompt(
+    titles: Sequence[str],
+    notes: Sequence[str] | None = None,
+    kb_overview: str | None = None,
+) -> str:
+    """Deflection turn: weak-hit titles only (no document content).
+
+    Section order (phase 31): ``<relevance>`` → ``<knowledge_base>`` →
+    ``<tuning>`` → ``DEFLECT_MODE`` body; empty steering/overview omit
+    their section, keeping the prompt byte-identical to the pre-phase text.
+    """
     weak = "\n".join(f"- {t}" for t in titles) if titles else "(nothing close at all)"
-    section = build_steering_section(notes or [])
-    mid = f"\n{section}\n" if section else "\n"
+    mid = "\n".join(
+        part
+        for part in (build_kb_section(kb_overview or ""), build_steering_section(notes or []))
+        if part
+    )
+    gap = f"\n{mid}\n" if mid else "\n"
     return (
         _base("LOW")
-        + mid
+        + gap
         + "DEFLECT_MODE: retrieval was weak — the titles below are the closest "
         "your notes come to the question. They are titles only; do not pretend "
         "they answer it. Use them to propose 2-3 alternative questions.\n"