feat(rag): lite-generated KB overview in the system prompt — stored single row, regenerated on import, <knowledge_base> section in HIGH+LOW prompts

tests/unit/test_chat_gate.py

@@ -19,13 +19,16 @@ from fastapi.testclient import TestClient
 from app.api import chat as chat_api
 from app.config import Settings
 from app.main import app as fastapi_app
-from app.models import Document, QueryLog
+from app.models import Document, KbOverview, QueryLog
 from app.rag.llm import StreamPiece
 from app.rag.retriever import RetrievedChunk, weak_hit_titles
 from app.rag.suggestions import MAX_SUGGESTIONS, derive_suggestions
 
 ANSWER = "I haven't done anything like that — try one of these instead!"
 
+#: A small KB outline standing in for the lite-generated one (phase 31).
+KB_OVERVIEW = "- Homelab\n  - Kubernetes (k3s)\n- Deployments\n  - Borg backups"
+
 
 def _settings(threshold: float = 0.30) -> Settings:
     return Settings(
@@ -236,6 +239,87 @@ def test_no_summary_chunks_yields_zero_summary_hits() -> None:
     assert plan_low.summary_hits == 0
 
 
+# ---------- KB overview (phase 31: <knowledge_base> section + kb_chars) ----------
+
+
+def test_plan_turn_high_injects_kb_overview() -> None:
+    """HIGH branch: the stored outline lands in the prompt between
+    ``<relevance>`` and ``<tuning>`` (or the ``<documents>`` body with no
+    notes), and ``kb_chars`` records the outline's length."""
+    doc = _doc("Kubernetes Homelab Cluster", "TALOS_DOC_CONTENT")
+    plan = chat_api.plan_turn(
+        [_chunk(doc, 0.90)], _settings(threshold=0.30), kb_overview=KB_OVERVIEW
+    )
+    assert plan.deflected is False
+    assert plan.kb_chars == len(KB_OVERVIEW)
+    prompt = plan.system_prompt
+    assert "<knowledge_base>" in prompt
+    assert KB_OVERVIEW in prompt
+    i_rel = prompt.index("<relevance>HIGH</relevance>")
+    i_kb = prompt.index("<knowledge_base>")
+    i_docs = prompt.index("<documents>")
+    assert i_rel < i_kb < i_docs
+
+
+def test_plan_turn_high_kb_section_ordered_before_tuning() -> None:
+    """Both sections present: ``<relevance>`` → ``<knowledge_base>`` →
+    ``<tuning>`` → ``<documents>`` (the locked phase-31 order)."""
+    doc = _doc("Kubernetes Homelab Cluster", "TALOS_DOC_CONTENT")
+    plan = chat_api.plan_turn(
+        [_chunk(doc, 0.90)],
+        _settings(threshold=0.30),
+        notes=["be concise"],
+        kb_overview=KB_OVERVIEW,
+    )
+    prompt = plan.system_prompt
+    i_rel = prompt.index("<relevance>HIGH</relevance>")
+    i_kb = prompt.index("<knowledge_base>")
+    i_kb_close = prompt.index("</knowledge_base>")
+    i_tuning = prompt.index("<tuning>")
+    i_docs = prompt.index("<documents>")
+    assert i_rel < i_kb < i_kb_close < i_tuning < i_docs
+    assert plan.kb_chars == len(KB_OVERVIEW)
+
+
+def test_plan_turn_low_injects_kb_overview() -> None:
+    """LOW (deflected) branch: the outline is injected there too, ahead
+    of the DEFLECT_MODE body, and document content stays excluded."""
+    doc = _doc("Kubernetes Homelab Cluster", "TALOS_DOC_NEVER_SENT")
+    plan = chat_api.plan_turn(
+        [_chunk(doc, 0.10)], _settings(threshold=0.30), kb_overview=KB_OVERVIEW
+    )
+    assert plan.deflected is True
+    assert plan.kb_chars == len(KB_OVERVIEW)
+    prompt = plan.system_prompt
+    assert "<knowledge_base>" in prompt
+    assert KB_OVERVIEW in prompt
+    i_rel = prompt.index("<relevance>LOW</relevance>")
+    i_kb = prompt.index("<knowledge_base>")
+    i_mode = prompt.index("DEFLECT_MODE")
+    assert i_rel < i_kb < i_mode
+    assert "TALOS_DOC_NEVER_SENT" not in prompt  # titles only, still
+
+
+def test_plan_turn_empty_overview_keeps_prompt_and_zero_kb_chars() -> None:
+    """No outline (None/empty/blank) → ``kb_chars == 0`` and a prompt
+    byte-identical to the no-overview build in both branches."""
+    high_doc = _doc("Kubernetes Homelab Cluster", "TALOS_DOC_CONTENT")
+    low_doc = _doc("Backup Strategy", "BACKUP_DOC_CONTENT")
+    for chunks, kb in (
+        ([_chunk(high_doc, 0.90)], None),
+        ([_chunk(high_doc, 0.90)], ""),
+        ([_chunk(high_doc, 0.90)], "   \n\t "),
+        ([_chunk(low_doc, 0.10)], None),
+        ([_chunk(low_doc, 0.10)], ""),
+    ):
+        plan = chat_api.plan_turn(chunks, _settings(threshold=0.30), kb_overview=kb)
+        baseline = chat_api.plan_turn(chunks, _settings(threshold=0.30))
+        assert plan.kb_chars == 0
+        assert baseline.kb_chars == 0
+        assert plan.system_prompt == baseline.system_prompt  # byte-identical
+        assert "<knowledge_base>" not in plan.system_prompt
+
+
 # ---------- prompt content (LOW vs HIGH) ----------
 
 
@@ -360,12 +444,15 @@ class _FakeSession:
     """Stands in for the DB session: records the QueryLog row it is given.
 
     ``scalars`` always yields no steering notes (phase 15) so the chat
-    turn's ``load_steering_notes`` call stays a no-op here.
+    turn's ``load_steering_notes`` call stays a no-op here, and ``get``
+    returns the single ``kb_overview`` row when one is configured
+    (phase 31) — ``None`` by default, i.e. no stored outline.
     """
 
-    def __init__(self) -> None:
+    def __init__(self, kb_overview: str = "") -> None:
         self.added: list[Any] = []
         self.commits = 0
+        self.kb_overview = kb_overview
 
     def add(self, obj: Any) -> None:
         self.added.append(obj)
@@ -376,6 +463,11 @@ class _FakeSession:
     def scalars(self, _stmt: Any) -> _FakeSteeringResult:
         return _FakeSteeringResult()
 
+    def get(self, model: Any, pk: Any) -> Any:
+        if model is KbOverview and self.kb_overview:
+            return KbOverview(id=1, content=self.kb_overview)
+        return None
+
 
 @pytest.fixture()
 def gate_env(monkeypatch: pytest.MonkeyPatch) -> Iterator[tuple[_FakeSession, _CannedLLM]]:
@@ -479,3 +571,55 @@ def test_endpoint_score_at_threshold_answers(
     assert isinstance(row, QueryLog)
     assert row.deflected is False
     assert row.top_score == pytest.approx(0.30)
+
+
+# ---------- endpoint: KB overview row (phase 31) ----------
+
+
+def test_endpoint_stored_kb_row_injected_into_system_prompt(
+    client: TestClient,
+    gate_env: tuple[_FakeSession, _CannedLLM],
+    monkeypatch: pytest.MonkeyPatch,
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    """A non-empty ``kb_overview`` row (read via one PK lookup) reaches
+    the LLM's system prompt in both modes, and the per-turn log line
+    records ``kb_chars=N`` (PLAN §9)."""
+    session, llm = gate_env
+    doc = _doc("Kubernetes Homelab Cluster", "TALOS_DOC_SENT")
+    monkeypatch.setattr(chat_api, "retrieve", _fake_retriever([_chunk(doc, 0.30)]))
+
+    session.kb_overview = f"  {KB_OVERVIEW}  "  # the loader trims it
+    with caplog.at_level("INFO", logger="app.chat"):
+        _ask(client, "How is my Kubernetes cluster set up?")
+
+    (system, _user) = llm.seen[0][0], llm.seen[0][1]
+    assert "<knowledge_base>" in system["content"]
+    assert KB_OVERVIEW in system["content"]
+    assert system["content"].index("<relevance>HIGH</relevance>") < system["content"].index(
+        "<knowledge_base>"
+    ) < system["content"].index("<documents>")
+    log_lines = [r.getMessage() for r in caplog.records if "question=" in r.getMessage()]
+    assert log_lines and f"kb_chars={len(KB_OVERVIEW)}" in log_lines[-1]
+
+
+def test_endpoint_no_kb_row_prompt_unchanged(
+    client: TestClient,
+    gate_env: tuple[_FakeSession, _CannedLLM],
+    monkeypatch: pytest.MonkeyPatch,
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    """No ``kb_overview`` row → the section is absent (byte-identical to
+    the pre-phase prompt) and the log line records ``kb_chars=0``."""
+    session, llm = gate_env
+    assert session.kb_overview == ""  # fixture default: no stored row
+    doc = _doc("Kubernetes Homelab Cluster", "TALOS_DOC_SENT")
+    monkeypatch.setattr(chat_api, "retrieve", _fake_retriever([_chunk(doc, 0.30)]))
+
+    with caplog.at_level("INFO", logger="app.chat"):
+        _ask(client, "How is my Kubernetes cluster set up?")
+
+    (system, _user) = llm.seen[0][0], llm.seen[0][1]
+    assert "<knowledge_base>" not in system["content"]
+    log_lines = [r.getMessage() for r in caplog.records if "question=" in r.getMessage()]
+    assert log_lines and "kb_chars=0" in log_lines[-1]

tests/unit/test_overview.py

@@ -0,0 +1,387 @@
+"""Unit: KB overview generator (phase 31, task 02).
+
+The prompt tests are pure (no DB): ``KB_OVERVIEW_MODE`` system prompt,
+per-document user lines (source/path/title/first summary line), and the
+input cap with the shared ``[…truncated…]`` marker. The loader and
+regenerator tests run against the local compose Postgres (preferred —
+real single-row upsert), skipping with clear instructions when the stack
+is not up — same pattern as ``test_importer.py``.
+"""
+from __future__ import annotations
+
+import asyncio
+import logging
+import uuid
+from datetime import UTC, datetime
+
+import pytest
+from sqlalchemy import text
+from sqlalchemy.orm import Session
+
+from app.config import Settings, get_settings
+from app.models import Document, KbOverview
+from app.rag.llm import LLMError
+from app.rag.overview import (
+    KB_OVERVIEW_MODE,
+    OVERVIEW_INSTRUCTION,
+    SYSTEM_PROMPT,
+    build_overview_prompt,
+    load_kb_overview,
+    regenerate_overview,
+)
+from app.rag.retriever import TRUNCATION_MARKER
+
+REPLY = "- Homelab\n  - Kubernetes (k3s)\n- Deployments\n  - Borg backups"
+
+
+class _FakeLLM:
+    """Duck-typed stand-in for ``LLMClient`` (``chat`` + ``settings``).
+
+    Records the messages and the ``model`` kwarg it was called with;
+    returns a canned reply or raises (e.g. :class:`LLMError`).
+    """
+
+    def __init__(self, reply: str = REPLY, fail: Exception | None = None) -> None:
+        self._reply = reply
+        self._fail = fail
+        self.settings = Settings(_env_file=None)  # pyright: ignore[reportCallIssue]
+        self.messages: list[dict[str, str]] = []
+        self.model: str | None = None
+        self.calls = 0
+
+    async def chat(
+        self, messages: list[dict[str, str]], model: str | None = None
+    ) -> str:
+        self.calls += 1
+        self.messages = list(messages)
+        self.model = model
+        if self._fail is not None:
+            raise self._fail
+        return self._reply
+
+
+# ---------- build_overview_prompt: system ----------
+
+
+def test_system_prompt_has_marker_and_locked_instruction() -> None:
+    assert SYSTEM_PROMPT.startswith(KB_OVERVIEW_MODE)
+    assert OVERVIEW_INSTRUCTION in SYSTEM_PROMPT
+    for fragment in (
+        "compact plain-text outline",
+        "basic categories and topics",
+        "Group by source",
+        "use `-` bullet lines",
+        "~1500 characters",
+        "no markdown headings",
+        "no topics not present in the list",
+    ):
+        assert fragment in SYSTEM_PROMPT
+    system, _ = build_overview_prompt([])
+    assert system == SYSTEM_PROMPT
+    assert KB_OVERVIEW_MODE in system  # the marker the E2E mock keys on
+
+
+# ---------- build_overview_prompt: user lines ----------
+
+
+def test_user_lines_carry_source_path_title_and_first_summary_line() -> None:
+    rows = [
+        ("Homelab", "kubernetes/k3s.md", "K3s Cluster", None),
+        (
+            "Deployments",
+            "backups/borg.yaml",
+            "Borg Backups",
+            "Backups run nightly via the borg schedule.\nSource: Deployments/backups/borg.yaml",
+        ),
+    ]
+    system, user = build_overview_prompt(rows)
+    assert system == SYSTEM_PROMPT
+    assert user == (
+        "Homelab — kubernetes/k3s.md — K3s Cluster\n"
+        "Deployments — backups/borg.yaml — Borg Backups — "
+        "Backups run nightly via the borg schedule."
+    )
+
+
+def test_user_line_omits_summary_field_when_absent() -> None:
+    _, user = build_overview_prompt([("Homelab", "a.md", "Title A", None)])
+    assert user == "Homelab — a.md — Title A"
+    assert not user.endswith(" — ")  # no dangling dash
+
+
+def test_user_line_omits_summary_field_when_blank() -> None:
+    _, user = build_overview_prompt([("Homelab", "a.md", "Title A", "  \n\t ")])
+    assert user == "Homelab — a.md — Title A"
+
+
+def test_user_line_uses_only_first_summary_line() -> None:
+    """Multi-line summaries contribute their first line only (the
+    model-written lead sentence; the ``Source: …`` pointer is last)."""
+    _, user = build_overview_prompt(
+        [
+            (
+                "Homelab",
+                "a.yaml",
+                "Title A",
+                "First line.\nSecond line.\nSource: Homelab/a.yaml",
+            )
+        ]
+    )
+    assert user == "Homelab — a.yaml — Title A — First line."
+    assert "Second line" not in user
+    assert "Source:" not in user
+
+
+def test_zero_rows_yield_empty_user() -> None:
+    _, user = build_overview_prompt([])
+    assert user == ""
+
+
+# ---------- build_overview_prompt: input cap ----------
+
+
+def test_user_prompt_truncated_with_marker_when_over_custom_cap() -> None:
+    rows = [("S", f"p{i}.md", f"T{i}", None) for i in range(10)]
+    _, full = build_overview_prompt(rows, max_chars=10_000)
+    cap = 25
+    _, user = build_overview_prompt(rows, max_chars=cap)
+    assert user == full[:cap] + "\n" + TRUNCATION_MARKER
+    assert user.endswith(TRUNCATION_MARKER)
+    assert len(user) > cap  # the marker makes the cut visible past the cap
+
+
+def test_user_prompt_at_exact_cap_not_truncated() -> None:
+    rows = [("S", "p.md", "T", None)]  # "S — p.md — T" = 12 chars
+    _, user = build_overview_prompt(rows, max_chars=12)
+    assert user == "S — p.md — T"
+    assert TRUNCATION_MARKER not in user
+
+
+def test_user_prompt_truncated_at_default_cap() -> None:
+    """No explicit cap → ``BOR_OVERVIEW_INPUT_MAX_CHARS`` (read from the
+    live settings, so the test holds for any configured value)."""
+    cap = get_settings().overview_input_max_chars
+    rows = [("S", f"p{i}.md", "T", None) for i in range(3_000)]
+    _, user = build_overview_prompt(rows)
+    assert user.endswith(TRUNCATION_MARKER)
+    body = user.removesuffix("\n" + TRUNCATION_MARKER)
+    assert len(body) == cap  # cut exactly at the cap, marker on its own line
+    assert "p2999.md" not in body  # the overflow never reaches the model
+
+
+# ---------- load_kb_overview ----------
+
+
+def test_load_kb_overview_no_row_empty(db: Session) -> None:
+    db.execute(text("DELETE FROM kb_overview"))
+    db.commit()
+    assert load_kb_overview(db) == ""
+
+
+def test_load_kb_overview_returns_trimmed_content(db: Session) -> None:
+    db.execute(text("DELETE FROM kb_overview"))
+    db.add(KbOverview(id=1, content=f"  {REPLY}  \n"))
+    db.commit()
+    assert load_kb_overview(db) == REPLY
+
+
+def test_load_kb_overview_whitespace_only_row_empty(db: Session) -> None:
+    db.execute(text("DELETE FROM kb_overview"))
+    db.add(KbOverview(id=1, content="  \n\t "))
+    db.commit()
+    assert load_kb_overview(db) == ""
+
+
+# ---------- regenerate_overview ----------
+
+
+def _add_doc(
+    db: Session, source: str, path: str, title: str, summary: str | None = None
+) -> Document:
+    doc = Document(
+        id=uuid.uuid4(),
+        source=source,
+        path=path,
+        full_path=f"/tmp/{path}",
+        title=title,
+        content="body",
+        content_hash="0" * 64,
+        summary=summary,
+    )
+    db.add(doc)
+    db.commit()
+    return doc
+
+
+def _truncate_documents(db: Session) -> None:
+    db.execute(text("TRUNCATE chunks, documents"))
+    db.commit()
+
+
+def _overview_row(db: Session) -> KbOverview | None:
+    return db.get(KbOverview, 1)
+
+
+@pytest.fixture()
+def clean_overview(db: Session):
+    db.execute(text("DELETE FROM kb_overview"))
+    db.commit()
+    yield
+    db.execute(text("DELETE FROM kb_overview"))
+    db.commit()
+
+
+def test_regenerate_happy_path_creates_single_row_id_1(
+    db: Session, clean_overview, caplog: pytest.LogCaptureFixture
+) -> None:
+    _truncate_documents(db)
+    try:
+        _add_doc(db, "Homelab", "b.md", "B")
+        _add_doc(db, "Homelab", "a.md", "A", "A summary.\nSource: Homelab/a.md")
+        llm = _FakeLLM()
+        with caplog.at_level(logging.INFO, logger="app.rag.overview"):
+            ok = asyncio.run(regenerate_overview(llm, db))
+            assert ok is True
+            assert llm.calls == 1
+            row = db.get(KbOverview, 1)
+            assert row is not None, "the single row must be upserted"
+            assert row.id == 1
+            assert row.content == REPLY
+            assert row.updated_at is not None
+            age = datetime.now(UTC) - row.updated_at
+            assert age.total_seconds() < 300, "updated_at must be a fresh UTC timestamp"
+            assert f"overview: regenerated docs=2 chars={len(REPLY)}" in caplog.text
+    finally:
+        _truncate_documents(db)
+
+
+def test_regenerate_updates_existing_row_in_place(db: Session, clean_overview) -> None:
+    _truncate_documents(db)
+    past = datetime(2020, 1, 1, 12, 0, 0, tzinfo=UTC)
+    db.add(KbOverview(id=1, content="old outline", updated_at=past))
+    db.commit()
+    try:
+        _add_doc(db, "Homelab", "a.md", "A")
+        ok = asyncio.run(regenerate_overview(_FakeLLM(), db))
+        assert ok is True
+        count = db.scalar(text("SELECT count(*) FROM kb_overview"))
+        assert count == 1, "still exactly one row after the re-generation"
+        row = db.get(KbOverview, 1)
+        assert row is not None
+        assert row.content == REPLY  # replaced, not appended
+        assert row.updated_at is not None and row.updated_at > past
+    finally:
+        _truncate_documents(db)
+
+
+def test_regenerate_orders_documents_by_source_path(db: Session, clean_overview) -> None:
+    _truncate_documents(db)
+    try:
+        # Inserted out of order — the model must see a deterministic order.
+        _add_doc(db, "Deployments", "z.md", "Z")
+        _add_doc(db, "Homelab", "b.md", "B")
+        _add_doc(db, "Homelab", "a.md", "A")
+        llm = _FakeLLM()
+        assert asyncio.run(regenerate_overview(llm, db)) is True
+    finally:
+        _truncate_documents(db)
+
+    user = next(m["content"] for m in llm.messages if m["role"] == "user")
+    assert user == (
+        "Deployments — z.md — Z\nHomelab — a.md — A\nHomelab — b.md — B"
+    )
+
+
+def test_regenerate_calls_the_configured_summary_model(
+    db: Session, clean_overview
+) -> None:
+    _truncate_documents(db)
+    try:
+        _add_doc(db, "Homelab", "a.md", "A")
+        llm = _FakeLLM()
+        assert asyncio.run(regenerate_overview(llm, db)) is True
+    finally:
+        _truncate_documents(db)
+
+    assert llm.model == llm.settings.llm_summary_model  # the ``lite`` default
+    assert llm.model == "lite"
+    assert [m["role"] for m in llm.messages] == ["system", "user"]
+    assert KB_OVERVIEW_MODE in llm.messages[0]["content"]
+    assert "A" in llm.messages[1]["content"]
+
+
+def test_regenerate_zero_docs_leaves_existing_row_untouched(
+    db: Session, clean_overview
+) -> None:
+    db.execute(text("DELETE FROM kb_overview"))
+    db.add(KbOverview(id=1, content="old outline"))
+    db.commit()
+    _truncate_documents(db)
+    llm = _FakeLLM()
+    ok = asyncio.run(regenerate_overview(llm, db))
+    assert ok is False
+    assert llm.calls == 0, "no KB → no wasted lite-model call"
+    row = db.get(KbOverview, 1)
+    assert row is not None and row.content == "old outline"  # untouched
+
+
+def test_regenerate_zero_docs_without_row_creates_nothing(db: Session, clean_overview) -> None:
+    _truncate_documents(db)
+    llm = _FakeLLM()
+    ok = asyncio.run(regenerate_overview(llm, db))
+    assert ok is False
+    assert llm.calls == 0
+    assert _overview_row(db) is None, "no row must be invented for an empty KB"
+
+
+def test_regenerate_llm_error_leaves_previous_row_intact(
+    db: Session, clean_overview, caplog: pytest.LogCaptureFixture
+) -> None:
+    db.execute(text("DELETE FROM kb_overview"))
+    db.add(KbOverview(id=1, content="old outline"))
+    db.commit()
+    _truncate_documents(db)
+    try:
+        _add_doc(db, "Homelab", "a.md", "A")
+        llm = _FakeLLM(fail=LLMError("simulated lite-model failure"))
+        with caplog.at_level(logging.ERROR, logger="app.rag.overview"):
+            ok = asyncio.run(regenerate_overview(llm, db))
+            assert ok is False, "fail-soft: the import must not be dragged down"
+            row = db.get(KbOverview, 1)
+            assert row is not None and row.content == "old outline", (
+                "the previous outline stays — an old outline is better than none"
+            )
+            assert "overview: regeneration failed —" in caplog.text
+            assert "simulated lite-model failure" in caplog.text
+    finally:
+        _truncate_documents(db)
+
+
+def test_regenerate_llm_error_without_row_creates_nothing(
+    db: Session, clean_overview
+) -> None:
+    _truncate_documents(db)
+    try:
+        _add_doc(db, "Homelab", "a.md", "A")
+        llm = _FakeLLM(fail=LLMError("simulated lite-model failure"))
+        ok = asyncio.run(regenerate_overview(llm, db))
+    finally:
+        _truncate_documents(db)
+
+    assert ok is False
+    assert _overview_row(db) is None
+
+
+def test_regenerate_opens_own_session_when_none(db: Session, clean_overview) -> None:
+    """``session=None`` → a private ``SessionLocal`` session is opened,
+    committed, and closed (the import-script call path)."""
+    _truncate_documents(db)
+    try:
+        _add_doc(db, "Homelab", "a.md", "A")
+        ok = asyncio.run(regenerate_overview(_FakeLLM()))
+    finally:
+        _truncate_documents(db)
+
+    assert ok is True
+    row = db.get(KbOverview, 1)  # fresh lookup via the test session
+    assert row is not None and row.content == REPLY

tests/unit/test_prompts.py

@@ -1,12 +1,34 @@
-"""Unit: locked persona prompt builder (PLAN §6 verbatim + both modes)."""
+"""Unit: locked persona prompt builder (PLAN §6 verbatim + both modes).
+
+Also covers the phase-31 ``<knowledge_base>`` section (the stored,
+lite-generated KB outline): its own builder contract (empty → ``""``,
+char budget + ``[…truncated…]`` marker, pathological budgets), its
+placement between ``<relevance>`` and ``<tuning>`` in both modes, and
+the byte-identical-when-absent convention (phase 15 precedent).
+"""
 from __future__ import annotations
 
 import uuid
 
 import pytest
 
+from app.config import Settings
 from app.models import Document
-from app.rag.prompts import PERSONA, _base, build_deflect_prompt, build_high_prompt
+from app.rag.prompts import (
+    PERSONA,
+    _base,
+    build_deflect_prompt,
+    build_high_prompt,
+    build_kb_section,
+    build_steering_section,
+)
+from app.rag.retriever import TRUNCATION_MARKER
+
+#: A small multi-line outline standing in for the lite-generated one.
+OVERVIEW = "- Homelab\n  - Kubernetes (k3s)\n- Deployments\n  - Borg backups"
+
+#: The locked one-line intro of the ``<knowledge_base>`` section.
+KB_INTRO = "The basic categories of everything in this knowledge base (generated at import time):"
 
 
 def _doc(path: str, content: str, title: str) -> Document:
@@ -116,3 +138,195 @@ def test_zero_note_prompt_is_byte_identical_to_pre_steering() -> None:
 def test_relevance_placeholder_rejected_for_garbage() -> None:
     with pytest.raises(ValueError, match="HIGH or LOW"):
         _base("MEDIUM")
+
+
+# ---------- <knowledge_base> section (phase 31) ----------
+
+
+def test_kb_section_empty_when_no_overview() -> None:
+    assert build_kb_section("") == ""
+    assert build_kb_section("   \n\t ") == ""
+
+
+def test_kb_section_format_intro_and_content() -> None:
+    assert build_kb_section(OVERVIEW) == (
+        f"<knowledge_base>\n{KB_INTRO}\n{OVERVIEW}\n</knowledge_base>"
+    )
+
+
+def test_kb_section_trims_overview_edges() -> None:
+    assert build_kb_section(f"  {OVERVIEW}  \n") == build_kb_section(OVERVIEW)
+
+
+def test_kb_section_fits_budget_exactly_no_marker() -> None:
+    exact = f"<knowledge_base>\n{KB_INTRO}\n{OVERVIEW}\n</knowledge_base>"
+    section = build_kb_section(OVERVIEW, max_chars=len(exact))
+    assert TRUNCATION_MARKER not in section
+    assert section == exact
+
+
+def test_kb_section_over_budget_capped_with_marker() -> None:
+    text = "- " + "x" * 500
+    # The section frame alone is 121 chars, so the cap must clear it for
+    # any outline prefix to fit (pathological budgets are tested below).
+    cap = 200
+    section = build_kb_section(text, max_chars=cap)
+    assert len(section) <= cap  # the budget is never exceeded
+    assert TRUNCATION_MARKER in section
+    assert section.startswith(f"<knowledge_base>\n{KB_INTRO}\n-")
+    assert section.endswith(f"{TRUNCATION_MARKER}\n</knowledge_base>")
+    # The body is the kept prefix + the marker on its own line, and the
+    # kept part must be a true prefix of the outline (longest-fitting).
+    body = section.removeprefix(f"<knowledge_base>\n{KB_INTRO}\n").removesuffix(
+        "\n</knowledge_base>"
+    )
+    kept, marker = body.rsplit("\n", 1)
+    assert marker == TRUNCATION_MARKER
+    assert kept.startswith("- ")
+    assert text.startswith(kept), "the kept part must be a prefix of the outline"
+    # And it is the longest such prefix: one more char would not fit.
+    assert len(section) > cap - 2, "the cut must sit as close to the cap as possible"
+
+
+def test_kb_section_default_budget_from_settings(monkeypatch: pytest.MonkeyPatch) -> None:
+    from app.rag import prompts as prompts_mod
+
+    monkeypatch.setattr(
+        prompts_mod, "get_settings", lambda: Settings(_env_file=None)  # pyright: ignore[reportCallIssue]
+    )
+    text = "y" * 9_000  # > the 4 000-char default
+    section = build_kb_section(text)
+    assert TRUNCATION_MARKER in section
+    assert len(section) <= 4_000
+
+
+def test_kb_section_nonpositive_budget_is_empty() -> None:
+    assert build_kb_section(OVERVIEW, max_chars=0) == ""
+    assert build_kb_section(OVERVIEW, max_chars=-10) == ""
+
+
+def test_kb_section_tiny_budget_never_exceeds_cap() -> None:
+    # Pathological budget (steering precedent, phase 15): the section must
+    # never exceed the cap — bare marker when it fits, no section at all
+    # when even that doesn't.
+    assert build_kb_section("a" * 500, max_chars=10) == ""  # marker (13) > 10
+    fits_marker = build_kb_section("a" * 500, max_chars=len(TRUNCATION_MARKER))
+    assert fits_marker == TRUNCATION_MARKER
+
+
+# ---------- <knowledge_base> placement (both modes) ----------
+
+
+def test_no_overview_prompt_is_byte_identical_to_pre_phase() -> None:
+    """Phase 31 contract: with no KB overview (None, empty, or blank)
+    every prompt is exactly what it was before the ``<knowledge_base>``
+    section existed — with or without steering notes."""
+    doc = _doc("kubernetes.md", "Talos Linux on three nodes.", "Kubernetes Homelab Cluster")
+    block = (
+        '<document source="Homelab" path="kubernetes.md" title="Kubernetes Homelab Cluster">\n'
+        "Talos Linux on three nodes.\n"
+        "</document>"
+    )
+    docs_block = "\n<documents>\n" + block + "\n</documents>"
+    high_plain = _base("HIGH") + docs_block
+    high_steered = _base("HIGH") + "\n" + build_steering_section(["be concise"]) + docs_block
+    low_plain = (
+        _base("LOW")
+        + "\nDEFLECT_MODE: retrieval was weak — the titles below are the closest "
+        "your notes come to the question. They are titles only; do not pretend "
+        "they answer it. Use them to propose 2-3 alternative questions.\n"
+        + "- T1\n- T2"
+    )
+    low_steered = (
+        _base("LOW")
+        + "\n"
+        + build_steering_section(["be concise"])
+        + "\nDEFLECT_MODE: retrieval was weak — the titles below are the closest "
+        "your notes come to the question. They are titles only; do not pretend "
+        "they answer it. Use them to propose 2-3 alternative questions.\n"
+        + "- T1\n- T2"
+    )
+    for kb in (None, "", "   \n\t "):
+        assert build_high_prompt([doc], kb_overview=kb) == high_plain
+        assert build_high_prompt([doc], notes=["be concise"], kb_overview=kb) == high_steered
+        assert build_deflect_prompt(["T1", "T2"], kb_overview=kb) == low_plain
+        assert build_deflect_prompt(
+            ["T1", "T2"], notes=["be concise"], kb_overview=kb
+        ) == low_steered
+        assert "<knowledge_base>" not in build_high_prompt(
+            [doc], notes=["be concise"], kb_overview=kb
+        )
+        assert "<knowledge_base>" not in build_deflect_prompt(
+            ["T1"], notes=["be concise"], kb_overview=kb
+        )
+
+
+def test_high_prompt_kb_section_ordered_between_relevance_and_tuning() -> None:
+    doc = _doc("kubernetes.md", "TALOS_DOC_CONTENT", "Kubernetes Homelab Cluster")
+    prompt = build_high_prompt([doc], notes=["be concise"], kb_overview=OVERVIEW)
+    i_rel = prompt.index("<relevance>HIGH</relevance>")
+    i_kb_open = prompt.index("<knowledge_base>")
+    i_kb_close = prompt.index("</knowledge_base>")
+    i_tuning = prompt.index("<tuning>")
+    i_docs = prompt.index("<documents>")
+    assert i_rel < i_kb_open < i_kb_close < i_tuning < i_docs
+    assert KB_INTRO in prompt
+    assert OVERVIEW in prompt  # outline intact within the section
+    assert "1. be concise" in prompt  # steering still there
+    assert "TALOS_DOC_CONTENT" in prompt  # documents still full
+
+
+def test_high_prompt_kb_section_without_steering() -> None:
+    doc = _doc("kubernetes.md", "TALOS_DOC_CONTENT", "Kubernetes Homelab Cluster")
+    prompt = build_high_prompt([doc], kb_overview=OVERVIEW)
+    i_rel = prompt.index("<relevance>HIGH</relevance>")
+    i_kb_close = prompt.index("</knowledge_base>")
+    i_docs = prompt.index("<documents>")
+    assert i_rel < i_kb_close < i_docs
+    assert "<tuning>" not in prompt  # no notes → no steering section
+    assert build_kb_section(OVERVIEW) in prompt
+
+
+def test_deflect_prompt_kb_section_ordered_between_relevance_and_tuning() -> None:
+    prompt = build_deflect_prompt(
+        ["Title A", "Title B"], notes=["be concise"], kb_overview=OVERVIEW
+    )
+    i_rel = prompt.index("<relevance>LOW</relevance>")
+    i_kb_open = prompt.index("<knowledge_base>")
+    i_kb_close = prompt.index("</knowledge_base>")
+    i_tuning = prompt.index("<tuning>")
+    i_mode = prompt.index("DEFLECT_MODE")
+    assert i_rel < i_kb_open < i_kb_close < i_tuning < i_mode
+    assert KB_INTRO in prompt
+    assert OVERVIEW in prompt
+    assert "1. be concise" in prompt
+    assert "- Title A" in prompt  # weak-hit titles still carried
+
+
+def test_prompt_kb_section_over_settings_budget_capped_with_marker(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """Prompt-level budget: an overview longer than
+    ``kb_overview_max_chars`` is capped with the shared marker — in both
+    modes."""
+    from app.rag import prompts as prompts_mod
+
+    monkeypatch.setattr(
+        prompts_mod,
+        "get_settings",
+        # 200 > the 121-char section frame, so a prefix + marker can fit.
+        lambda: Settings(_env_file=None, kb_overview_max_chars=200),  # pyright: ignore[reportCallIssue]
+    )
+    text = "- " + "z" * 500
+    doc = _doc("kubernetes.md", "TALOS_DOC_CONTENT", "Kubernetes Homelab Cluster")
+    for prompt in (
+        build_high_prompt([doc], kb_overview=text),
+        build_deflect_prompt(["Title A"], kb_overview=text),
+    ):
+        assert TRUNCATION_MARKER in prompt
+        assert prompt.index("<knowledge_base>") < prompt.index(TRUNCATION_MARKER)
+        # The capped section (open tag through close tag) fits the budget.
+        section = prompt[prompt.index("<knowledge_base>") :]
+        close = section.index("</knowledge_base>")
+        section = section[: close + len("</knowledge_base>")]
+        assert len(section) <= 200