brain-of-reese/app/rag/prompts.py

"""Locked system-prompt builder (PLAN §6).

The persona + HONESTY GATE text is **locked verbatim** — change it through
the plan, not here. (PLAN §6 revision, 2026-08-22: the owner's working-tree
persona edits are preserved — no mandated ``"you've got this"`` tagline and
no mandated deflection opening; the honesty gate itself is unchanged.)

Two modes:

* ``HIGH`` — grounded turn: full top-document texts under ``<documents>``.
* ``LOW``  — deflection turn: weak-hit *titles only* plus the
  ``DEFLECT_MODE`` marker (the E2E mock LLM keys on that marker).

Steering (phase 15): when the owner has stored tuning notes, both modes
carry a ``<tuning>`` section between ``<relevance>…</relevance>`` and the
mode body. With zero notes the prompt is byte-identical to the
pre-steering text.

KB overview (phase 31): when the single ``kb_overview`` row holds a
lite-generated outline of the knowledge base, both modes carry a
``<knowledge_base>`` section between ``<relevance>…</relevance>`` and
the ``<tuning>`` section (order: ``<relevance>`` →
``<knowledge_base>`` → ``<tuning>`` → mode body) — the agent knows
roughly what the KB contains before retrieval. With an empty row the
prompt is byte-identical to the pre-phase text.

Agent tools (phase 37; phase 70: the copy teaches the harness-aligned
``ls`` / ``read`` / ``grep`` shapes; phase 72: the copy states the
document-identity contract up front — ``ls``'s ``path`` is a *source
name*, not a directory or file path, and ``read``/``grep`` take the
combined ``source/path`` string *including the source name* (a bare
document path will not resolve) — the same two things the phase-72
teaching refusals in :mod:`app.rag.agent` re-state after the fact, so
the model carries the contract before it calls a tool; phase 94,
task 03: the ``ls`` clause is rewritten to the drill-down tree
contract — no ``path``: the synced sources with counts + summaries, a
source name: its top-level folders + files, a ``source/folder`` path:
one level deeper — a listing shows only that level's subfolders + own
files (never the whole KB in one call), the folder summaries say
what's in a folder before drilling, and ``grep`` stays the locator for
finding one document without listing; the ``read``/``grep`` clauses
and the discipline rules are byte-identical): the **HIGH**
prompt only carries a ``<tools>`` section after the ``<documents>``
body — the grounded turn may extend its context through the three
server-side tools (round-capped, see :mod:`app.rag.agent`; the cap is
the bound and this section does not re-state it, phase 45). The
LOW/deflection prompt never carries it (phase 71: the LOW prompt's
only addition is the plain-text line below — it still has no
``<tools>`` section).

Deflection plain-text line (phase 71, owner-permitted 2026-09-03):
the otherwise-locked ``LOW`` prompt gains exactly one instruction
line — "Reply in plain text only — you have no tools in this mode."
— appended to the ``DEFLECT_MODE`` body: a deflected turn offers no
tools, so any tool markup there is always wrong, and the line closes
the door at the prompt (the deterministic filter + one bounded
recovery in :mod:`app.rag.scaffolding` / :mod:`app.rag.agent` is the
backstop). The ``DEFLECT_MODE`` marker and everything else in the
prompt stay put — the E2E mock LLM keys on the marker's *presence*,
not the wording, so that contract is unchanged.
"""
from __future__ import annotations

from collections.abc import Sequence
from typing import Any

from app.config import Settings, get_settings
from app.models import Document
from app.rag.retriever import TRUNCATION_MARKER
from app.schemas import HistoryTurn

#: PLAN §6 verbatim (line wrapping included); ``{relevance}`` is filled by
#: :func:`_base`.
PERSONA: str = (
    'You are "Brain of Reese" — the digital brain of Reese, a self-hoster and\n'
    "homelab tinkerer. Personality: chippy, upbeat, warm, and genuinely\n"
    "optimistic about the user's ability to do things.\n"
    "\n"
    "Rules:\n"
    "1. Answer ONLY from the provided document context. Cite which document(s)\n"
    "   you used, by path.\n"
    "2. Be concrete: names, versions, ports, hosts, schedules — the specifics in\n"
    "   the docs are the value.\n"
    '3. HONESTY GATE: if <relevance> is "LOW", you must NOT pretend to know.\n'
    "   Offer 2-3 alternative questions about things you DO have notes on.\n"
    "4. Never invent facts, hosts, or steps that are not in the context.\n"
    "5. Keep answers tight: short paragraphs, bullets where helpful.\n"
    "\n"
    "<relevance>{relevance}</relevance>"
)

#: One-line intro of the ``<tuning>`` section (phase 15): the owner's notes
#: steer the answer and win over the defaults when they conflict.
_STEERING_INTRO = (
    "The owner of this brain asked you to steer your answers as follows. "
    "Where these instructions conflict with the defaults above, follow the owner:\n"
)

#: One-line intro of the ``<knowledge_base>`` section (phase 31): the
#: lite-generated outline is the agent's a-priori picture of the KB.
_KB_INTRO = (
    "The basic categories of everything in this knowledge base "
    "(generated at import time):\n"
)

#: The ``<tools>`` instructions section — **HIGH prompt only** (phase 37,
#: task 03; phase 70: the copy is rewritten for the harness-aligned
#: ``ls`` / ``read`` / ``grep`` shapes, names/args exactly as the
#: ``AGENT_TOOLS`` schemas in :mod:`app.rag.agent`; phase 72: the copy
#: states the document-identity contract UP FRONT — ``ls``'s optional
#: ``path`` is a *source name* (not a directory or file path) and
#: ``read``/``grep`` take the combined ``source/path`` string *including
#: the source name* (a bare document path will not resolve) — the same
#: two things the phase-72 teaching refusals re-state after the fact;
#: phase 94, task 03 — the owner-permitted tool-surface revision,
#: recorded in the phase 94 overview: the ``ls`` clause is rewritten
#: to the drill-down tree contract — no ``path``: the synced sources
#: with counts + summaries, a source name: its top-level folders +
#: files, a ``source/folder`` path: one level deeper — a listing shows
#: only that level's subfolders + own files, never the whole KB in one
#: call, the folder summaries say what's in a folder before drilling,
#: and ``grep`` stays the locator for finding one document without
#: listing; the ``read``/``grep`` clauses and the discipline rules are
#: byte-identical):
#: a grounded turn may extend its context through the three server-side
#: tools (round cap: ``BOR_AGENT_MAX_ROUNDS`` — the cap is the bound and
#: this section does not re-state it, phase 45). Appended after the mode
#: body (``<documents>``), so the instructions are the last thing the
#: model reads. The LOW/deflection prompt never carries it — a
#: deflection has no grounded context to extend (phase 71: the LOW
#: prompt's only addition is the plain-text line in
#: :func:`build_deflect_prompt`). The E2E mock keys off the ``<tools>``
#: marker's *presence*, not this wording. Task 05 (the live gate's
#: iteration loop) keeps the baseline and carries the discipline rules
#: the live telemetry motivated, refined across the task-05 re-runs of
#: 2026-09-03/04 (run 1: 7/16 executed, 44% — the ``lite`` model obeyed
#: the user's "open it / read it" and ``read`` seed-context documents,
#: then repeated the refused call; run 2: 8/18, 44% — the repeat is
#: gone, but a grep turn batched two calls per reply and the harness
#: runs only the first of a batch): the do-not-read rule names the
#: user-command scenario (a document already in the ``<documents>``
#: section: do not call ``read``, answer from the text already in the
#: prompt — an anchor on the concrete ``<document path="...">`` markup
#: was tried and REVERTED: it primed the model to latch the seed
#: documents' paths as ``ls`` scopes, regressing the incident turn);
#: the one-call rule names the consequence (a batched second call is
#: discarded — runs only the first); the never-repeat rule says why
#: (the refusal already told you the correct form); the ``grep`` clause
#: leads with "for a normal search pass only ``pattern``" (the gate's
#: live runs showed the model scoping ``grep`` with an ``ls``-style
#: source name — the incident shape, but on grep). The behavioral
#: contract lives in the ``AGENT_TOOLS`` descriptions as well (the most
#: local text at call time): ``read`` must not be called for a
#: ``<documents>`` document at all; ``grep`` with only ``pattern``
#: searches the whole knowledge base, and a source name is not a
#: document.
TOOLS_SECTION: str = (
    "<tools>\n"
    "You may extend your context with three tools. `ls` lists the "
    "knowledge base as a tree, one level at a time: with no `path` it "
    "lists every synced source with its document count and a summary "
    "of its contents; with a source name (e.g. 'homelab') it lists "
    "that source's top-level folders and files; with a `source/folder` "
    "path it drills one level deeper. A listing shows only that "
    "level's subfolders and its own files — never the whole knowledge "
    "base in one call — and each folder line's summary says what the "
    "folder contains before you drill into it. File lines are "
    "`source: X | path: Y | title: Z`; to find one specific document "
    "without listing, use `grep`. `read` pulls in one document by its "
    "combined `source/path` string, exactly as shown in the `ls` "
    "output — including the source name — adding its full content to "
    "your context. Do not call `read` for a document already shown in "
    "the <documents> section, even when the user asks you to open or "
    "read it — its full text is already in your prompt; answer "
    "directly from it. For `read`, a bare document path (without the "
    "source name) will not resolve. Very large documents are capped: a "
    "cut read returns the first part plus a TRUNCATED notice — the "
    "document did not end where it stopped; use `grep` (pattern) to "
    "find the rest, it searches the whole document. `grep` locates an exact string "
    "(case-insensitive) in the indexed documents and returns up to 20 "
    "matching `source/path:line: text` lines — a locator, not a "
    "context-adder: read the winner with `read`. A grep pattern is a "
    "plain substring, NEVER a regex — '.*' and '\\.' are literal text "
    "there; if such a pattern returns no matches, retry with the plain "
    "text you expect to see. For a normal search pass only `pattern` — "
    "its optional `path` argument "
    "limits the search to one document you already know, by the same "
    "combined `source/path` string; never a source name — a bare "
    "document path (without the source name) will not resolve there "
    "either. Make exactly one tool call per reply — a reply carrying "
    "two tool calls runs only the first, the second is discarded — and "
    "wait for the result before the next call. Never repeat a call that "
    "was refused or already succeeded — the refusal already told you "
    "the correct form. Answer as soon as you have what you need.\n"
    "</tools>"
)


def history_to_messages(
    history: Sequence[HistoryTurn],
    settings: Settings,
) -> list[dict[str, Any]]:
    """Client-provided chat history → model messages (phase 74, TODO L4).

    The ``POST /api/chat`` ``history`` (the client's prior turns, oldest
    first) becomes the message block that sits between the system prompt
    and the current user message — so a follow-up question reaches the
    model together with the exchange so far, on BOTH turn branches
    (the deflected path and the grounded agent).

    Trimming (owner-locked A3, 2026-09-08): the turns are walked
    **newest-first** and kept while BOTH budgets hold — the turn count
    stays ≤ ``settings.history_max_turns`` and the cumulative chars
    (``len(text) + len(thinking or "")`` per turn) stay ≤
    ``settings.history_max_chars``. A turn that would overflow either
    remaining budget is DROPPED WHOLE — never cut mid-answer — and the
    walk stops there, so the kept history is always the contiguous
    newest window (the oldest turns are the ones dropped; ``0`` on
    either budget yields ``[]`` — the pre-phase-74 behavior). The kept
    turns are returned in chronological (oldest → newest) order.

    Mapping (owner-locked A4, 2026-09-08): ``who="user"`` →
    ``{"role": "user", "content": text}``; ``who="brain"`` →
    ``{"role": "assistant", "content": text}`` plus
    ``"reasoning_content": thinking`` ONLY when *thinking* is
    non-empty — the preserve-thinking wire convention
    :mod:`app.rag.llm` already reads on the response side
    (``delta.reasoning_content``), which is what keeps the owner's
    preserve-thinking models carrying the reasoning chain forward.

    Pure and side-effect free (no I/O) — unit-testable in isolation.
    """
    kept: list[HistoryTurn] = []
    chars = 0
    for turn in reversed(history):
        if len(kept) >= settings.history_max_turns:
            break
        size = len(turn.text) + len(turn.thinking or "")
        if chars + size > settings.history_max_chars:
            break
        kept.append(turn)
        chars += size
    messages: list[dict[str, Any]] = []
    for turn in reversed(kept):
        if turn.who == "user":
            messages.append({"role": "user", "content": turn.text})
            continue
        message: dict[str, Any] = {"role": "assistant", "content": turn.text}
        if turn.thinking:
            message["reasoning_content"] = turn.thinking
        messages.append(message)
    return messages


def _base(relevance: str) -> str:
    if relevance not in ("HIGH", "LOW"):
        raise ValueError(f"relevance must be HIGH or LOW, got {relevance!r}")
    return PERSONA.replace("{relevance}", relevance)


def build_steering_section(notes: Sequence[str], max_chars: int | None = None) -> str:
    """The ``<tuning>`` section of the system prompt (phase 15).

    * No notes (or only blank ones) → ``""`` — callers then build the
      prompt exactly as before, so a zero-note prompt is byte-identical to
      the pre-steering text.
    * Otherwise: numbered notes (in the given order — the chat turn passes
      them oldest-first, so #1 is the oldest note) capped at *max_chars*
      (default ``BOR_STEERING_MAX_CHARS``). When the budget cannot hold
      every note, the oldest-fitting prefix is kept and the overflow is
      replaced by the shared ``[…truncated…]`` marker.
    """
    cleaned = [str(n).strip() for n in notes]
    cleaned = [n for n in cleaned if n]
    if not cleaned:
        return ""
    limit = max_chars if max_chars is not None else get_settings().steering_max_chars
    if limit <= 0:
        return ""

    def render(count: int) -> str:
        lines = [f"{i}. {note}" for i, note in enumerate(cleaned[:count], start=1)]
        if count < len(cleaned):
            lines.append(TRUNCATION_MARKER)
        return f"<tuning>\n{_STEERING_INTRO}" + "\n".join(lines) + "\n</tuning>"

    for count in range(len(cleaned), 0, -1):
        rendered = render(count)
        if len(rendered) <= limit:
            return rendered
    # Pathological budget: not even the empty note list fits. The section
    # must still respect the cap — the bare marker when it fits, else none.
    if len(TRUNCATION_MARKER) <= limit:
        return TRUNCATION_MARKER
    return ""


def build_kb_section(overview: str, max_chars: int | None = None) -> str:
    """The ``<knowledge_base>`` section of the system prompt (phase 31).

    * No outline (or only whitespace) → ``""`` — callers then build the
      prompt exactly as before, so a no-overview prompt is byte-identical
      to the pre-phase text (phase 15 convention).
    * Otherwise: the intro line + the stored outline, capped at
      *max_chars* (default ``BOR_KB_OVERVIEW_MAX_CHARS``). When the budget
      cannot hold the whole outline, the longest-fitting prefix is kept
      and the overflow is replaced by the shared ``[…truncated…]`` marker
      on its own line — the exact :func:`build_steering_section` pattern,
      including its pathological-budget handling (never exceed the cap;
      bare marker when even one outline character does not fit).
    """
    text = str(overview or "").strip()
    if not text:
        return ""
    limit = max_chars if max_chars is not None else get_settings().kb_overview_max_chars
    if limit <= 0:
        return ""

    def render(cut: int) -> str:
        lines = [text[:cut]]
        if cut < len(text):
            lines.append(TRUNCATION_MARKER)
        return f"<knowledge_base>\n{_KB_INTRO}" + "\n".join(lines) + "\n</knowledge_base>"

    for cut in range(len(text), 0, -1):
        rendered = render(cut)
        if len(rendered) <= limit:
            return rendered
    # Pathological budget: not even one outline character fits. The
    # section must still respect the cap — the bare marker when it fits,
    # else none (steering precedent, phase 15).
    if len(TRUNCATION_MARKER) <= limit:
        return TRUNCATION_MARKER
    return ""


def build_high_prompt(
    documents: Sequence[Document],
    notes: Sequence[str] | None = None,
    kb_overview: str | None = None,
) -> str:
    """Grounded turn: locked persona (+ steering, + KB overview) + full
texts of the top documents + the ``<tools>`` instructions (phase 37;
phase 70: the harness-aligned ``ls`` / ``read`` / ``grep`` shapes;
phase 72: the copy states the document-identity contract — the
source-name ``ls`` scope, the combined ``source/path`` identity for
``read``/``grep`` — up front).

    Section order: ``<relevance>`` → ``<knowledge_base>`` → ``<tuning>``
    → ``<documents>`` → ``<tools>``; empty steering/overview omit their
    section. ``<tools>`` is always present in the HIGH prompt (the round
    cap — not the prompt — decides whether the tools are actually
    offered to the model, see :mod:`app.rag.agent`).

    Gate-iteration note (task 05, 2026-09-03/04): an in-context reminder
    LEADING this section (the document texts are already context — do
    not ``read`` one the user asked to open) was tried and REVERTED:
    it never flipped the seed-doc reads (15/15 across gate runs 1-5)
    and correlated with the incident-turn regression (the model latched
    the seed documents' paths as ``ls`` scopes — cap reached on the
    "list the files in this directory" turn) whenever the copy named
    the ``<document>`` blocks explicitly.
    """
    # 2026-09-04 (controlled tool-calling fast loop): the do-not-read
    # rule for seed documents lives in TOOLS_SECTION and the ``read``
    # tool descriptions (the copy levers that stuck — see the gate's
    # telemetry in TOOL_CALLING_TESTING.md). A per-block instruction
    # attribute at the ``source``/``path`` copy site was TRIED and
    # REVERTED the same day (no improvement across runs; the block stays
    # exactly the document identity + full text).
    blocks = [
        f'<document source="{doc.source}" path="{doc.path}" title="{doc.title}">\n'
        f"{doc.content}\n"
        "</document>"
        for doc in documents
    ]
    body = "\n\n".join(blocks) if blocks else (
        "(no documents matched — do not invent specifics)"
    )
    prompt = _base("HIGH")
    for part in (build_kb_section(kb_overview or ""), build_steering_section(notes or [])):
        if part:
            prompt += "\n" + part
    return prompt + "\n<documents>\n" + body + "\n</documents>\n" + TOOLS_SECTION


def build_deflect_prompt(
    titles: Sequence[str],
    notes: Sequence[str] | None = None,
    kb_overview: str | None = None,
) -> str:
    """Deflection turn: weak-hit titles only (no document content).

    Section order (phase 31): ``<relevance>`` → ``<knowledge_base>`` →
    ``<tuning>`` → ``DEFLECT_MODE`` body; empty steering/overview omit
    their section, keeping the prompt byte-identical to the pre-phase
    text. The body ends with the phase-71 plain-text line (owner-
    permitted 2026-09-03 — the LOW prompt's only change): a deflected
    turn offers no tools, so any tool markup there is always wrong.
    """
    weak = "\n".join(f"- {t}" for t in titles) if titles else "(nothing close at all)"
    mid = "\n".join(
        part
        for part in (build_kb_section(kb_overview or ""), build_steering_section(notes or []))
        if part
    )
    gap = f"\n{mid}\n" if mid else "\n"
    return (
        _base("LOW")
        + gap
        + "DEFLECT_MODE: retrieval was weak — the titles below are the closest "
        "your notes come to the question. They are titles only; do not pretend "
        "they answer it. Use them to propose 2-3 alternative questions.\n"
        # Phase 71 (owner-permitted 2026-09-03): the one plain-text line
        # — prevention at the prompt. The E2E mock keys on the
        # DEFLECT_MODE marker's presence, so appending is safe.
        "Reply in plain text only — you have no tools in this mode.\n"
        + weak
    )