feat(rag): steering notes — tune how Brain answers, stored in Postgres and injected into every system prompt

This commit is contained in:
2026-08-22 16:44:42 -04:00
parent 19df7df99d
commit fc0d9a2d5c
19 changed files with 1589 additions and 34 deletions
+1
View File
@@ -22,6 +22,7 @@ BOR_TOP_N_DOCS=2
BOR_RELEVANCE_THRESHOLD=0.62 # answer when best cosine >= this OR an FTS hit; else honest deflection
BOR_MAX_CONTEXT_CHARS=24000 # cap on total document text sent to the LLM
BOR_MAX_OUTPUT_TOKENS=32768 # max answer length in tokens (answers must not be cut off)
BOR_STEERING_MAX_CHARS=8000 # char budget for the <tuning> (steering notes) prompt section
BOR_CHUNK_TARGET_CHARS=2000
BOR_CHUNK_OVERLAP_CHARS=200
BOR_EMBED_BATCH_SIZE=16
+29
View File
@@ -85,6 +85,34 @@ uv run uvicorn app.main:app --reload
- **Sources** (`/sources.html`) — the indexed document list; the *Path*
column links each document to the viewer in a new tab.
## Tuning your answers
If an answer isn't quite right — too chatty, wrong assumption, missing
context — **tune** Brain right there:
1. Press **“Tune”** in the meta row under any completed answer (deflected
ones included).
2. Type a short instruction (1–2000 chars), e.g. *“be more concise”* or
*“assume I'm on NixOS”*, and **Save**.
The note is stored in Postgres (`steering_notes`) and read into the
**system prompt of every subsequent chat turn** as a `<tuning>` section
(numbered, oldest first, capped at `BOR_STEERING_MAX_CHARS` chars —
default 8000, overflow marked `[…truncated…]`). With no stored notes the
prompt is byte-identical to the un-tuned one, so tuning is opt-in per
note.
List or remove notes at any time from the **“Tuning”** button in the chat
header (count badge, newest-first, per-note delete). The API is stateless
JSON if you prefer curl:
```bash
curl -s localhost:8000/api/steering # list (newest first)
curl -s -X POST localhost:8000/api/steering \
-H 'Content-Type: application/json' -d '{"note": "be more concise"}'
curl -s -X DELETE localhost:8000/api/steering/<note-id> # remove
```
## Updating the documents
**This is the workflow you'll use most.** The knowledge base is refreshed by
@@ -270,6 +298,7 @@ served locally (no CDN), `BOR_ENVIRONMENT=production`.
| `BOR_RRF_K` | `60` | RRF damping constant (`1/(k + rank)`) |
| `BOR_IMPORT_EXTENSIONS` | `md,markdown,txt,yaml,yml,json,py` | csv of importable formats (may only narrow the A9 set) |
| `BOR_MAX_CONTEXT_CHARS` | `24000` | cap on total document text sent to the LLM |
| `BOR_STEERING_MAX_CHARS` | `8000` | char budget for the `<tuning>` (steering notes) prompt section |
| `BOR_SUGGESTIONS` | built-in list | JSON list of onboarding chips |
| `DEBUGPY` | `0` | `1` ⇒ attach-on-demand debugpy on `DEBUGPY_PORT` (default 5678) |
| `BOR_LOG_LEVEL` | `INFO` | app log level |
+39
View File
@@ -0,0 +1,39 @@
"""steering notes: owner tuning notes injected into every system prompt
Revision ID: 0003
Revises: 0002
Create Date: 2026-08-22
Phase 15 (steering-notes story): the owner can "tune" how Brain answers
from the chat UI. Notes live in ``steering_notes`` and are read into the
system prompt of every subsequent chat turn (``<tuning>`` section).
"""
from __future__ import annotations
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
from alembic import op
revision = "0003"
down_revision = "0002"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"steering_notes",
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
sa.Column("note", sa.Text(), nullable=False),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
server_default=sa.func.now(),
nullable=False,
),
)
def downgrade() -> None:
op.drop_table("steering_notes")
+35 -7
View File
@@ -16,6 +16,11 @@ gets a grounded answer. Deflection mode carries weak-hit *titles only*
(never document content) plus deterministic "Maybe try" chips, and the
``done`` event / ``query_log`` row record ``deflected=true``, the weak
score and the ``fts_hits`` count.
Steering (phase 15): the owner's stored tuning notes are loaded per turn
(oldest first) and injected into the system prompt as a ``<tuning>``
section — both the HIGH and the LOW prompt carry it. The per-turn log
line records ``tuning=N`` (the number of injected notes).
"""
from __future__ import annotations
@@ -30,6 +35,7 @@ from fastapi import APIRouter, Depends
from fastapi.responses import JSONResponse, StreamingResponse
from sqlalchemy.orm import Session
from app.api.steering import load_steering_notes
from app.config import Settings, get_settings
from app.db import db_available, get_db
from app.models import Document, QueryLog
@@ -71,9 +77,14 @@ class TurnPlan:
system_prompt: str
docs: list[Document] # cited sources (weak hits when deflected)
suggestions: list[str] # "Maybe try" chips (deflected turns only)
tuning_count: int = 0 # steering notes injected into the system prompt
def plan_turn(chunks: Sequence[RetrievedChunk], settings: Settings) -> TurnPlan:
def plan_turn(
chunks: Sequence[RetrievedChunk],
settings: Settings,
notes: Sequence[str] | None = None,
) -> TurnPlan:
"""Apply the honesty gate (A8, revised) and assemble prompt + context.
* **HIGH (grounded)** when ``best_cosine >= threshold`` **or**
@@ -88,22 +99,36 @@ def plan_turn(chunks: Sequence[RetrievedChunk], settings: Settings) -> TurnPlan:
``top_score`` (stored in ``query_log``) is the best cosine, so the
gate input is always a pure vector-similarity number; the lexical
signal is recorded separately as ``fts_hits``.
*notes* are the owner's steering notes (phase 15, oldest first):
when non-empty, both the HIGH and the LOW prompt carry the
``<tuning>`` section; with no notes the prompts are unchanged.
"""
steering = list(notes or [])
best_cosine = max((c.cosine for c in chunks), default=0.0)
fts_hits = sum(1 for c in chunks if c.fts_hit)
if best_cosine >= settings.relevance_threshold or fts_hits > 0:
docs = select_documents(
chunks, n=settings.top_n_docs, max_chars=settings.max_context_chars
)
return TurnPlan(best_cosine, fts_hits, False, build_high_prompt(docs), docs, [])
return TurnPlan(
best_cosine,
fts_hits,
False,
build_high_prompt(docs, notes=steering),
docs,
[],
len(steering),
)
titles = weak_hit_titles(chunks)
return TurnPlan(
best_cosine,
fts_hits,
True,
build_deflect_prompt(titles),
build_deflect_prompt(titles, notes=steering),
select_documents(chunks, n=settings.top_n_docs, max_chars=settings.max_context_chars),
derive_suggestions(titles, settings.suggestions),
len(steering),
)
@@ -150,12 +175,14 @@ async def chat(
return
embed_ms = int((time.monotonic() - t0) * 1000)
# 2. Retrieve top-K chunks, then the honesty gate (A8) picks the
# HIGH (grounded) or LOW (deflected) prompt + context.
# 2. Retrieve top-K chunks, load the owner's steering notes
# (phase 15), then the honesty gate (A8) picks the HIGH
# (grounded) or LOW (deflected) prompt + context.
settings = get_settings()
try:
steering_notes = load_steering_notes(db)
chunks = retrieve(db, request.message, question_vec)
plan = plan_turn(chunks, settings)
plan = plan_turn(chunks, settings, notes=steering_notes)
except Exception: # noqa: BLE001 — DB failure mid-turn
logger.exception(
"chat: retrieval failed question=%r total_ms=%d",
@@ -211,12 +238,13 @@ async def chat(
logger.exception("chat: failed to write query_log question=%r", request.message)
logger.info(
"question=%r embed_ms=%d top_score=%.3f fts_hits=%d threshold=%.2f "
"question=%r embed_ms=%d top_score=%.3f fts_hits=%d tuning=%d threshold=%.2f "
"deflected=%s sources=%r total_ms=%d",
request.message,
embed_ms,
plan.top_score,
plan.fts_hits,
plan.tuning_count,
settings.relevance_threshold,
plan.deflected,
source_paths,
+74
View File
@@ -0,0 +1,74 @@
"""Steering notes API — tune how Brain answers (phase 15, story
``steering-notes``).
Stateless CRUD under ``/api/steering`` (A10): notes are owner instructions
stored in Postgres (``steering_notes``) and read into the system prompt of
**every** chat turn as the ``<tuning>`` section (see
:func:`app.rag.prompts.build_steering_section` and
:func:`app.api.chat.chat`).
"""
from __future__ import annotations
import uuid
from fastapi import APIRouter, Depends, HTTPException, Response
from sqlalchemy import select
from sqlalchemy.orm import Session
from app.db import get_db
from app.models import SteeringNote
from app.schemas import SteeringNote as SteeringNoteOut
from app.schemas import SteeringNoteIn, SteeringNoteList
router = APIRouter(prefix="/steering", tags=["steering"])
def load_steering_notes(db: Session) -> list[str]:
"""All steering notes, oldest first (the order they are numbered in the
``<tuning>`` prompt section). Used by the chat turn (``app.api.chat``)."""
rows = db.scalars(
select(SteeringNote).order_by(SteeringNote.created_at.asc(), SteeringNote.id.asc())
).all()
return [row.note for row in rows]
@router.get("", response_model=SteeringNoteList)
def list_steering_notes(
db: Session = Depends(get_db), # noqa: B008
) -> SteeringNoteList:
"""All notes, newest first (the UI panel's display order)."""
rows = db.scalars(
select(SteeringNote).order_by(SteeringNote.created_at.desc(), SteeringNote.id.desc())
).all()
return SteeringNoteList(
notes=[
SteeringNoteOut(id=row.id, note=row.note, created_at=row.created_at) for row in rows
]
)
@router.post("", response_model=SteeringNoteOut, status_code=201)
def create_steering_note(
payload: SteeringNoteIn,
db: Session = Depends(get_db), # noqa: B008
) -> SteeringNoteOut:
"""Store one tuning instruction (trimmed, 1–2000 chars — 422 otherwise)."""
row = SteeringNote(note=payload.note)
db.add(row)
db.commit()
db.refresh(row)
return SteeringNoteOut(id=row.id, note=row.note, created_at=row.created_at)
@router.delete("/{note_id}", status_code=204)
def delete_steering_note(
note_id: uuid.UUID,
db: Session = Depends(get_db), # noqa: B008
) -> Response:
"""Remove a note; 404 when the id is unknown."""
row = db.get(SteeringNote, note_id)
if row is None:
raise HTTPException(status_code=404, detail="steering note not found")
db.delete(row)
db.commit()
return Response(status_code=204)
+4
View File
@@ -58,6 +58,10 @@ class Settings(BaseSettings):
chunk_target_chars: int = 2_000
chunk_overlap_chars: int = 200
embed_batch_size: int = 16
#: Total char budget for the ``<tuning>`` section of the system prompt
#: (phase 15, steering notes). The newest-fitting notes are kept and the
#: overflow is replaced by the ``[…truncated…]`` marker.
steering_max_chars: int = 8_000
# --- Hybrid retrieval (A7, revised 2026-08-21) ---
# cosine top-N ∪ Postgres FTS top-N, fused with Reciprocal Rank Fusion
+2
View File
@@ -16,6 +16,7 @@ from fastapi.staticfiles import StaticFiles
from app.api.chat import router as chat_router
from app.api.docs import router as docs_router
from app.api.health import router as health_router
from app.api.steering import router as steering_router
from app.api.suggestions import router as suggestions_router
from app.config import get_settings
from app.core.debugging import configure_debugging
@@ -36,6 +37,7 @@ def create_app() -> FastAPI:
app.include_router(suggestions_router, prefix="/api")
app.include_router(docs_router, prefix="/api")
app.include_router(chat_router, prefix="/api")
app.include_router(steering_router, prefix="/api")
static_dir = Path(settings.static_dir).resolve()
if static_dir.is_dir():
+19 -2
View File
@@ -6,8 +6,10 @@ Data model — see ``.agent/PLAN.md`` §Data Model:
* ``chunks`` — retrieval units; each chunk points at its parent document
via ``document_id``. This is how an embedding maps back to
a document path (the "feed the whole document" requirement).
* ``query_log`` — observability: every question, its retrieval score, the
deflection decision, and latency.
* ``query_log`` — observability: every question, its retrieval score,
the deflection decision, and latency.
* ``steering_notes`` — owner tuning notes injected into the system prompt
of every chat turn (phase 15, ``<tuning>`` section).
"""
from __future__ import annotations
@@ -82,3 +84,18 @@ class QueryLog(Base):
sources: Mapped[str] = mapped_column(Text, default="") # comma-joined source paths
latency_ms: Mapped[int] = mapped_column(Integer, default=0)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
class SteeringNote(Base):
"""One owner tuning instruction (phase 15).
Notes are read into the system prompt of **every** chat turn as the
``<tuning>`` section (oldest first, char-budgeted — see
:func:`app.rag.prompts.build_steering_section`).
"""
__tablename__ = "steering_notes"
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
note: Mapped[str] = mapped_column(Text) # trimmed, 1–2000 chars (API-enforced)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
+71 -9
View File
@@ -1,24 +1,35 @@
"""Locked system-prompt builder (PLAN §6).
The persona + HONESTY GATE text is **locked verbatim** — change it through
the plan, not here. Two modes:
the plan, not here. (PLAN §6 revision, 2026-08-22: the owner's working-tree
persona edits are preserved — no mandated ``"you've got this"`` tagline and
no mandated deflection opening; the honesty gate itself is unchanged.)
Two modes:
* ``HIGH`` — grounded turn: full top-document texts under ``<documents>``.
* ``LOW`` — deflection turn: weak-hit *titles only* plus the
``DEFLECT_MODE`` marker (the E2E mock LLM keys on that marker).
Steering (phase 15): when the owner has stored tuning notes, both modes
carry a ``<tuning>`` section between ``<relevance>…</relevance>`` and the
mode body. With zero notes the prompt is byte-identical to the
pre-steering text.
"""
from __future__ import annotations
from collections.abc import Sequence
from app.config import get_settings
from app.models import Document
from app.rag.retriever import TRUNCATION_MARKER
#: PLAN §6 verbatim (line wrapping included); ``{relevance}`` is filled by
#: :func:`_base`.
PERSONA: str = (
'You are "Brain of Reese" — the digital brain of Reese, a self-hoster and\n'
"homelab tinkerer. Personality: chippy, upbeat, warm, and genuinely\n"
'optimistic about the user\'s ability to do things ("you\'ve got this").\n'
"optimistic about the user's ability to do things.\n"
"\n"
"Rules:\n"
"1. Answer ONLY from the provided document context. Cite which document(s)\n"
@@ -26,14 +37,20 @@ PERSONA: str = (
"2. Be concrete: names, versions, ports, hosts, schedules — the specifics in\n"
" the docs are the value.\n"
'3. HONESTY GATE: if <relevance> is "LOW", you must NOT pretend to know.\n'
' Start your answer with a variant of: "I haven\'t done anything like that."\n'
" Then offer 2-3 alternative questions about things you DO have notes on.\n"
" Offer 2-3 alternative questions about things you DO have notes on.\n"
"4. Never invent facts, hosts, or steps that are not in the context.\n"
"5. Keep answers tight: short paragraphs, bullets where helpful.\n"
"\n"
"<relevance>{relevance}</relevance>"
)
#: One-line intro of the ``<tuning>`` section (phase 15): the owner's notes
#: steer the answer and win over the defaults when they conflict.
_STEERING_INTRO = (
"The owner of this brain asked you to steer your answers as follows. "
"Where these instructions conflict with the defaults above, follow the owner:\n"
)
def _base(relevance: str) -> str:
if relevance not in ("HIGH", "LOW"):
@@ -41,8 +58,46 @@ def _base(relevance: str) -> str:
return PERSONA.replace("{relevance}", relevance)
def build_high_prompt(documents: Sequence[Document]) -> str:
"""Grounded turn: locked persona + full texts of the top documents."""
def build_steering_section(notes: Sequence[str], max_chars: int | None = None) -> str:
"""The ``<tuning>`` section of the system prompt (phase 15).
* No notes (or only blank ones) → ``""`` — callers then build the
prompt exactly as before, so a zero-note prompt is byte-identical to
the pre-steering text.
* Otherwise: numbered notes (in the given order — the chat turn passes
them oldest-first, so #1 is the oldest note) capped at *max_chars*
(default ``BOR_STEERING_MAX_CHARS``). When the budget cannot hold
every note, the oldest-fitting prefix is kept and the overflow is
replaced by the shared ``[…truncated…]`` marker.
"""
cleaned = [str(n).strip() for n in notes]
cleaned = [n for n in cleaned if n]
if not cleaned:
return ""
limit = max_chars if max_chars is not None else get_settings().steering_max_chars
if limit <= 0:
return ""
def render(count: int) -> str:
lines = [f"{i}. {note}" for i, note in enumerate(cleaned[:count], start=1)]
if count < len(cleaned):
lines.append(TRUNCATION_MARKER)
return f"<tuning>\n{_STEERING_INTRO}" + "\n".join(lines) + "\n</tuning>"
for count in range(len(cleaned), 0, -1):
rendered = render(count)
if len(rendered) <= limit:
return rendered
# Pathological budget: not even the empty note list fits. The section
# must still respect the cap — the bare marker when it fits, else none.
if len(TRUNCATION_MARKER) <= limit:
return TRUNCATION_MARKER
return ""
def build_high_prompt(documents: Sequence[Document], notes: Sequence[str] | None = None) -> str:
"""Grounded turn: locked persona (+ steering) + full texts of the top
documents."""
blocks = [
f'<document source="{doc.source}" path="{doc.path}" title="{doc.title}">\n'
f"{doc.content}\n"
@@ -52,15 +107,22 @@ def build_high_prompt(documents: Sequence[Document]) -> str:
body = "\n\n".join(blocks) if blocks else (
"(no documents matched — do not invent specifics)"
)
return _base("HIGH") + "\n<documents>\n" + body + "\n</documents>"
section = build_steering_section(notes or [])
prompt = _base("HIGH")
if section:
prompt += "\n" + section
return prompt + "\n<documents>\n" + body + "\n</documents>"
def build_deflect_prompt(titles: Sequence[str]) -> str:
def build_deflect_prompt(titles: Sequence[str], notes: Sequence[str] | None = None) -> str:
"""Deflection turn: weak-hit titles only (no document content)."""
weak = "\n".join(f"- {t}" for t in titles) if titles else "(nothing close at all)"
section = build_steering_section(notes or [])
mid = f"\n{section}\n" if section else "\n"
return (
_base("LOW")
+ "\nDEFLECT_MODE: retrieval was weak — the titles below are the closest "
+ mid
+ "DEFLECT_MODE: retrieval was weak — the titles below are the closest "
"your notes come to the question. They are titles only; do not pretend "
"they answer it. Use them to propose 2-3 alternative questions.\n"
+ weak
+34 -1
View File
@@ -1,7 +1,10 @@
"""Pydantic request/response schemas (API contract)."""
from __future__ import annotations
from pydantic import BaseModel, Field
import uuid
from datetime import datetime
from pydantic import BaseModel, Field, field_validator
class HealthResponse(BaseModel):
@@ -73,3 +76,33 @@ class DocContent(BaseModel):
content: str
indexed_at: str
chunks: int
class SteeringNoteIn(BaseModel):
"""``POST /api/steering`` body: one tuning instruction (phase 15).
The note is trimmed *before* the length constraints run, so a
whitespace-only body is a 422 and a 2000-char note with surrounding
spaces still passes.
"""
note: str = Field(min_length=1, max_length=2000)
@field_validator("note", mode="before")
@classmethod
def _trim_note(cls, v: object) -> object:
return v.strip() if isinstance(v, str) else v
class SteeringNote(BaseModel):
"""One stored steering note (API shape — ISO-8601 ``created_at``)."""
id: uuid.UUID
note: str
created_at: datetime
class SteeringNoteList(BaseModel):
"""``GET /api/steering`` response: all notes, newest first."""
notes: list[SteeringNote]
+212 -1
View File
@@ -28,6 +28,16 @@
* "New chat" (#new-chat-btn) clears the key + the list back to the empty
* state.
*
* Steering notes (phase 15) let the owner tune how Brain answers: a
* "Tune" button under every completed brain bubble (deflected included)
* opens an inline form → POST /api/steering → the note is stored in
* Postgres and injected into the system prompt of every subsequent turn
* (the <tuning> section). Notes are listed newest-first in the header
* "Tuning" panel (#steering-panel), where each can be deleted. Note text
* is always rendered with textContent (XSS-safe), save/delete are
* announced through a polite live region (#steering-announcer), and the
* panel + count badge update on every change.
*
* All DOM ids match frontend/index.html.
*/
@@ -90,6 +100,203 @@ export function documentUrl(source, path, back = "/") {
return url;
}
/* ---------- steering notes (phase 15) ----------
*
* The owner's tuning notes steer every future answer: they live in
* Postgres (stateless API, A10) and the chat turn reads them into the
* system prompt. UI contract: Tune button → inline form → save →
* confirmation (or inline error, form kept); the header panel lists the
* notes (newest first) with per-note delete.
*/
const steeringToggle = document.querySelector("#steering-toggle");
const steeringCount = document.querySelector("#steering-count");
const steeringPanel = document.querySelector("#steering-panel");
const steeringList = document.querySelector("#steering-list");
const steeringEmpty = document.querySelector("#steering-empty");
const steeringAnnouncer = document.querySelector("#steering-announcer");
const TUNE_ICON =
'<svg aria-hidden="true" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.7" stroke-linecap="round"><path d="M4 7h10M18 7h2M4 17h4M12 17h8"/><circle cx="15.5" cy="7" r="2.2"/><circle cx="9.5" cy="17" r="2.2"/></svg>';
let tuneSeq = 0; // unique ids for one open tune form's inputs
function announceSteering(message) {
if (steeringAnnouncer) steeringAnnouncer.textContent = message;
}
/* "Tune" button in the meta row of a completed brain bubble. Reuses the
sources' .msg-meta row when it exists (role=list → the button joins as
a listitem so ARIA stays valid); otherwise creates a plain meta row. */
function appendTuneButton(wrap) {
const body = wrap.querySelector(".msg-body");
if (!body) return;
let meta = body.querySelector(".msg-meta");
if (!meta) {
meta = document.createElement("div");
meta.className = "msg-meta";
body.appendChild(meta);
}
if (meta.querySelector(".tune-btn")) return; // one per bubble
const btn = document.createElement("button");
btn.type = "button";
btn.className = "tune-btn";
if (meta.getAttribute("role") === "list") btn.setAttribute("role", "listitem");
btn.innerHTML = TUNE_ICON + "<span>Tune</span>";
btn.addEventListener("click", () => openTuneForm(wrap, btn));
meta.appendChild(btn);
}
/* Inline tuning form under the bubble: labeled textarea (maxlength 2000)
+ Save / Cancel. Success replaces the form with the .tune-saved status
(role=status); failure keeps the form and shows an inline error
(role=alert) — the note is never lost on a failed save. */
function openTuneForm(wrap, toggleBtn) {
document.querySelectorAll(".tune-form").forEach((f) => f.remove()); // one at a time
const body = wrap.querySelector(".msg-body");
if (!body) return;
tuneSeq += 1;
const inputId = `tune-input-${tuneSeq}`;
const form = document.createElement("form");
form.className = "tune-form";
form.noValidate = true;
form.innerHTML =
`<label for="${inputId}">Tuning note — how should Brain answer from now on?</label>` +
`<textarea id="${inputId}" name="note" rows="2" maxlength="2000"
placeholder="e.g. be more concise — or: assume I'm on NixOS"></textarea>`;
const actions = document.createElement("div");
actions.className = "tune-form-actions";
const saveBtn = document.createElement("button");
saveBtn.type = "submit";
saveBtn.className = "tune-save";
saveBtn.textContent = "Save";
const cancelBtn = document.createElement("button");
cancelBtn.type = "button";
cancelBtn.className = "tune-cancel";
cancelBtn.textContent = "Cancel";
actions.append(saveBtn, cancelBtn);
form.appendChild(actions);
const status = document.createElement("p");
status.className = "tune-error";
status.setAttribute("role", "alert");
status.hidden = true;
form.appendChild(status);
form.addEventListener("submit", async (e) => {
e.preventDefault();
saveBtn.disabled = true;
status.hidden = true;
try {
const r = await fetch("/api/steering", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ note: form.querySelector("textarea").value }),
});
if (!r.ok) {
let detail = "Could not save the note — try again.";
try {
const data = await r.json();
if (Array.isArray(data.detail) && data.detail[0] && data.detail[0].msg) {
detail = String(data.detail[0].msg);
} else if (typeof data.detail === "string" && data.detail) {
detail = data.detail;
}
} catch { /* non-JSON error body */ }
status.textContent = detail;
status.hidden = false;
saveBtn.disabled = false;
return; // form kept on failure — the instruction survives
}
const saved = document.createElement("p");
saved.className = "tune-saved";
saved.setAttribute("role", "status");
saved.textContent = "Saved — future answers will follow this.";
form.replaceWith(saved);
announceSteering("Tuning note saved. Future answers will follow it.");
await loadSteering(); // panel + count badge update
} catch {
status.textContent = "Could not save the note — is the app reachable?";
status.hidden = false;
saveBtn.disabled = false;
}
});
cancelBtn.addEventListener("click", () => {
form.remove();
toggleBtn.focus();
});
body.appendChild(form);
form.querySelector("textarea").focus();
}
/* Panel: newest-first list (textContent — XSS-safe), per-note delete,
empty text, and the header count badge. */
async function loadSteering() {
let notes = [];
try {
const r = await fetch("/api/steering");
if (r.ok) notes = (await r.json()).notes || [];
} catch { /* API unreachable: keep the last rendered list */ }
renderSteeringPanel(notes);
return notes;
}
function renderSteeringPanel(notes) {
if (!steeringList) return;
steeringList.textContent = "";
for (const n of notes) {
const li = document.createElement("li");
li.className = "steering-note";
const text = document.createElement("span");
text.className = "steering-note-text";
text.textContent = n.note; // rendered as text, never as HTML
li.appendChild(text);
const del = document.createElement("button");
del.type = "button";
del.className = "steering-delete";
del.setAttribute("aria-label", `Delete tuning note: ${n.note}`);
del.innerHTML =
'<svg aria-hidden="true" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.8" stroke-linecap="round"><path d="M5 7h14M10 7V5h4v2M8.5 7l.7 12h5.6l.7-12"/></svg>';
del.addEventListener("click", () => deleteSteeringNote(n.id, del));
li.appendChild(del);
steeringList.appendChild(li);
}
if (steeringEmpty) steeringEmpty.hidden = notes.length > 0;
if (steeringCount) steeringCount.textContent = String(notes.length);
}
async function deleteSteeringNote(id, btn) {
btn.disabled = true;
try {
const r = await fetch(`/api/steering/${encodeURIComponent(id)}`, { method: "DELETE" });
if (r.status === 404) {
announceSteering("That note was already removed.");
await loadSteering();
return;
}
if (!r.ok) {
announceSteering("Could not delete the note — try again.");
btn.disabled = false;
return;
}
await loadSteering();
announceSteering("Tuning note deleted.");
} catch {
announceSteering("Could not delete the note — is the app reachable?");
btn.disabled = false;
}
}
function setSteeringPanel(open) {
if (!steeringPanel || !steeringToggle) return;
steeringPanel.hidden = !open;
steeringToggle.setAttribute("aria-expanded", open ? "true" : "false");
}
if (steeringToggle && steeringPanel) {
steeringToggle.addEventListener("click", () => {
setSteeringPanel(steeringPanel.hidden);
if (!steeringPanel.hidden) loadSteering(); // refresh when (re)opened
});
}
/* ---------- avatar glyphs (phase 08: emoji-free chrome) ----------
* Inline SVG as string constants so the message renderer and the typing
* indicator share exactly the same marks. currentColor lets the CSS theme
@@ -426,6 +633,7 @@ function renderStoredMessage(m) {
appendMaybeTry(wrap, m.suggestions);
}
appendSources(wrap, m.sources);
appendTuneButton(wrap); // restored brain answers are tunable too
}
/* On load: re-render the stored conversation (markdown, source chips,
@@ -545,6 +753,7 @@ async function handleSend(e) {
appendMaybeTry(wrap, ev.suggestions);
}
appendSources(wrap, ev.sources);
appendTuneButton(wrap); // every completed brain bubble is tunable
// Persistence save point 2: the answer lands only when the turn is
// complete (raw text + the done metadata).
rememberBrainTurn(acc, {
@@ -558,7 +767,8 @@ async function handleSend(e) {
});
if (!aborted && !wrap) {
const fallback = "Hmm — that came back empty. Ask me again?";
addMessage("brain", fallback);
const fwrap = addMessage("brain", fallback);
appendTuneButton(fwrap);
rememberBrainTurn(fallback, {}); // persist what the user actually saw
}
} catch (err) {
@@ -592,3 +802,4 @@ composer.addEventListener("submit", handleSend);
restoreConversation(); // phase 14: the conversation comes back as left
loadSuggestions();
loadHealth();
loadSteering(); // phase 15: tuning notes (panel + count badge)
+220
View File
@@ -241,6 +241,48 @@ body::after {
whole control below 640px. */
.new-chat-btn svg { width: 16px; height: 16px; display: none; }
/* "Tuning" toggle (phase 15): ghost pill like New chat + a mono count
badge (brand-ink on brand-soft ≈6.9:1). The label is visually-hidden
(not removed) below 640px so the accessible name keeps the word.
≥44px touch target at every width. */
.steering-toggle {
display: inline-flex;
align-items: center;
justify-content: center;
gap: 0.4rem;
min-height: 44px;
padding: 0.5rem 0.9rem;
border-radius: 999px;
border: 1px solid var(--line);
background: transparent;
color: var(--ink-soft);
font: inherit;
font-weight: 600;
font-size: 0.95rem;
white-space: nowrap;
cursor: pointer;
}
.steering-toggle:hover, .steering-toggle[aria-expanded="true"] {
background: var(--brand-soft);
color: var(--brand-ink);
}
.steering-toggle svg { width: 16px; height: 16px; display: block; }
.steering-count {
font-family: var(--mono);
font-size: 0.78rem;
font-weight: 700;
min-width: 1.35rem;
text-align: center;
padding: 0.05rem 0.4rem;
border-radius: 999px;
background: var(--brand-soft);
color: var(--brand-ink);
}
.steering-toggle[aria-expanded="true"] .steering-count {
background: var(--brand);
color: var(--bg); /* dark ink on brand: 5.2:1 */
}
/* ---------- Main frame ---------- */
.app-main {
flex: 1;
@@ -381,6 +423,170 @@ body::after {
max-width: 100%;
}
/* ---------- Steering notes (phase 15) ---------- */
/* "Tune" button in the meta row of every completed brain bubble: ghost
pill, ≥44px, right-aligned after the source chips. ink-soft on surface
≈6.9:1; hover pair brand-ink/brand-soft ≈6.9:1. */
.tune-btn {
display: inline-flex;
align-items: center;
justify-content: center;
gap: 0.35rem;
min-height: 44px;
margin-left: auto;
padding: 0.35rem 0.8rem;
border-radius: 999px;
border: 1px solid var(--line);
background: transparent;
color: var(--ink-soft);
font: inherit;
font-weight: 600;
font-size: 0.82rem;
white-space: nowrap;
cursor: pointer;
}
.tune-btn svg { width: 14px; height: 14px; display: block; }
.tune-btn:hover { background: var(--brand-soft); color: var(--brand-ink); }
/* Inline tuning form under the bubble: labeled textarea + Save/Cancel
(both ≥44px). Save = brand button (dark ink 5.2:1), Cancel = ghost. */
.tune-form {
display: flex;
flex-direction: column;
gap: 0.5rem;
background: var(--surface);
border: 1px solid var(--brand-soft);
border-radius: var(--radius-sm);
padding: 0.75rem 0.85rem;
}
.tune-form label {
font-size: 0.85rem;
font-weight: 600;
color: var(--ink-soft);
}
.tune-form textarea {
font: inherit;
font-size: 0.9rem;
color: var(--ink);
background: #0d1120;
border: 1px solid var(--line);
border-radius: var(--radius-sm);
padding: 0.5rem 0.6rem;
resize: vertical;
min-height: 2.6rem;
}
.tune-form-actions { display: flex; gap: 0.5rem; }
.tune-save {
display: inline-flex;
align-items: center;
justify-content: center;
min-height: 44px;
padding: 0.4rem 1.1rem;
border: 0;
border-radius: var(--radius-sm);
background: var(--brand);
color: var(--bg);
font: inherit;
font-weight: 700;
cursor: pointer;
}
.tune-save:hover:not(:disabled) { background: #7d88f5; }
.tune-save:disabled { opacity: 0.6; cursor: wait; }
.tune-cancel {
display: inline-flex;
align-items: center;
justify-content: center;
min-height: 44px;
padding: 0.4rem 1rem;
border-radius: var(--radius-sm);
border: 1px solid var(--line);
background: transparent;
color: var(--ink-soft);
font: inherit;
font-weight: 600;
cursor: pointer;
}
.tune-cancel:hover { background: var(--brand-soft); color: var(--brand-ink); }
/* Save confirmation (role=status): ok pair ≈10.6:1. */
.tune-saved {
margin: 0.2rem 0 0 0.25rem;
background: var(--ok-bg);
color: var(--ok-ink);
border: 1px solid rgb(110 231 168 / 0.35);
border-radius: var(--radius-sm);
padding: 0.45rem 0.8rem;
font-size: 0.85rem;
font-weight: 600;
}
/* Inline save failure (role=alert): err pair ≈9.1:1 — form is kept. */
.tune-error {
margin: 0 0 0 0.25rem;
background: var(--err-bg);
color: var(--err-ink);
border: 1px solid var(--err-line);
border-radius: var(--radius-sm);
padding: 0.45rem 0.8rem;
font-size: 0.85rem;
font-weight: 600;
}
/* Header panel above the messages: notes newest-first, per-note delete,
designed empty state. */
.steering-panel {
background: var(--surface);
border: 1px solid var(--brand-soft);
border-radius: var(--radius);
box-shadow: var(--shadow);
padding: 0.9rem 1.1rem 1rem;
}
.steering-panel-head { display: flex; flex-wrap: wrap; align-items: baseline; gap: 0.15rem 0.6rem; }
.steering-panel-title { margin: 0; font-size: 1rem; font-weight: 700; color: var(--ink); }
.steering-panel-sub { margin: 0; font-size: 0.82rem; color: var(--ink-soft); }
.steering-list {
list-style: none;
margin: 0.65rem 0 0;
padding: 0;
display: flex;
flex-direction: column;
gap: 0.4rem;
}
.steering-note {
display: flex;
align-items: stretch;
gap: 0.6rem;
background: #0d1120;
border: 1px solid var(--line);
border-radius: var(--radius-sm);
padding: 0.35rem 0.4rem 0.35rem 0.8rem;
}
.steering-note-text {
color: var(--ink);
font-size: 0.9rem;
line-height: 1.45;
/* Notes may be multi-line instructions — keep line breaks as typed. */
white-space: pre-wrap;
overflow-wrap: anywhere;
flex: 1;
min-width: 0;
}
.steering-delete {
display: inline-flex;
align-items: center;
justify-content: center;
min-height: 44px;
min-width: 44px;
flex: 0 0 auto;
border: 1px solid var(--line);
border-radius: var(--radius-sm);
background: transparent;
color: var(--ink-soft);
cursor: pointer;
}
.steering-delete svg { width: 16px; height: 16px; display: block; }
.steering-delete:hover:not(:disabled) { background: var(--err-bg); color: var(--err-ink); border-color: var(--err-line); }
.steering-delete:disabled { opacity: 0.5; cursor: wait; }
.steering-empty { margin: 0.65rem 0 0; color: var(--ink-soft); font-size: 0.88rem; }
/* typing indicator */
.typing { display: inline-flex; gap: 5px; padding: 0.9rem 1rem; }
.typing span {
@@ -799,6 +1005,20 @@ body::after {
.new-chat-btn { padding: 0.4rem 0.55rem; }
.new-chat-label { display: none; }
.new-chat-btn svg { display: block; }
.steering-toggle { padding: 0.4rem 0.55rem; }
/* Visually hidden, NOT display:none — the accessible name keeps the
word "Tuning" next to the count badge. */
.steering-label {
position: absolute !important;
width: 1px; height: 1px;
margin: -1px; padding: 0;
overflow: hidden;
clip: rect(0 0 0 0);
white-space: nowrap;
border: 0;
}
.steering-note { padding: 0.3rem 0.3rem 0.3rem 0.7rem; }
.tune-btn { min-height: 44px; }
.msg-body { max-width: 92%; }
.empty-state { padding: 1.75rem 1.1rem; margin-top: 0.25rem; }
.empty-state-title { font-size: 1.25rem; }
+20
View File
@@ -21,6 +21,14 @@
<a href="/" class="nav-link is-active" aria-current="page">Chat</a>
<a href="/sources.html" class="nav-link">Sources</a>
</nav>
<!-- Phase 15: open the tuning-notes panel (stored in Postgres, read
into every system prompt) — chat page only. -->
<button type="button" class="steering-toggle" id="steering-toggle"
aria-expanded="false" aria-controls="steering-panel">
<svg aria-hidden="true" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.7" stroke-linecap="round"><path d="M4 7h10M18 7h2M4 17h4M12 17h8"/><circle cx="15.5" cy="7" r="2.2"/><circle cx="9.5" cy="17" r="2.2"/></svg>
<span class="steering-label">Tuning</span>
<span class="steering-count" id="steering-count">0</span>
</button>
<!-- Phase 14: reset the local (localStorage) conversation — chat page only. -->
<button type="button" class="new-chat-btn" id="new-chat-btn" aria-label="New chat">
<svg aria-hidden="true" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.7" stroke-linecap="round"><path d="M12 5v14M5 12h14"/></svg>
@@ -36,6 +44,18 @@
<span id="kb-banner-text"></span>
</div>
<!-- Phase 15: tuning-notes panel (stored notes, newest first). -->
<section class="steering-panel" id="steering-panel" role="region"
aria-label="Tuning notes" hidden>
<div class="steering-panel-head">
<h2 class="steering-panel-title">Tuning notes</h2>
<p class="steering-panel-sub">Every note below steers all future answers.</p>
</div>
<ul class="steering-list" id="steering-list"></ul>
<p class="steering-empty" id="steering-empty">No tuning notes yet — press “Tune” under any answer to add one.</p>
</section>
<p class="visually-hidden" id="steering-announcer" role="status" aria-live="polite" aria-atomic="true"></p>
<section class="messages" id="messages" aria-live="polite" aria-label="Conversation with Brain of Reese">
<div class="empty-state" id="empty-state">
<div class="empty-state-glyph" aria-hidden="true">
+36 -4
View File
@@ -15,6 +15,9 @@ Implements just enough of the aipi surface:
- otherwise -> upbeat answer quoting the provided document context
- user message containing ``pretend to think slowly`` -> 3s warm-up delay
(used by the loading-feedback story).
- system prompt containing ``<tuning>`` (phase 15, steering notes) ->
the composed answer ends with `` (tuning: <first note line>)`` —
makes prompt injection observable in the UI deterministically.
``max_tokens`` is honored deterministically (token ≈ whitespace word),
like a real endpoint: an answer longer than the cap is truncated. This
@@ -89,25 +92,54 @@ def long_answer() -> str:
return "\n".join(lines)
#: First numbered note line of a ``<tuning>`` section (phase 15).
_TUNING_BLOCK_RE = re.compile(r"<tuning>\n(.*?)\n</tuning>", re.S)
_NOTE_LINE_RE = re.compile(r"^\d+\.\s*(.+)$")
def first_tuning_note(system: str) -> str | None:
"""The first steering note in the system prompt, or ``None``.
The prompt numbers notes 1..N oldest-first (see
``app.rag.prompts.build_steering_section``); the mock echoes the first
one into its answer so prompt injection is observable in the UI.
"""
block = _TUNING_BLOCK_RE.search(system)
if not block:
return None
for line in block.group(1).splitlines():
m = _NOTE_LINE_RE.match(line.strip())
if m:
return m.group(1).strip()
return None
def compose_answer(body: dict[str, Any]) -> str:
system = _system(body)
user = _user(body)
if LONG_ANSWER_TRIGGER in user.lower():
return long_answer()
if "DEFLECT_MODE" in system:
return (
answer = long_answer()
elif "DEFLECT_MODE" in system:
answer = (
"Ah — I haven't done anything like that, so I don't want to make stuff up! "
"You're thinking bigger than my notes for a second. Try asking about "
"kubernetes, backups, or deploying a new service — I know those inside out. "
"You've got this!"
)
else:
ctx = _context(body)
snippet = ctx[:220].replace("\n", " ").strip()
return (
answer = (
f"Great question — you've absolutely got this! Here's what my notes say about "
f"“{user.strip()[:80]}”: {snippet}… That's the gist from the docs; happy to "
"dig into any of it. (Deterministic mock answer for E2E.)"
)
# Steering (phase 15): when the system prompt carries <tuning>, the
# answer ends with the first note — deterministically observable.
note = first_tuning_note(system)
if note:
answer = f"{answer} (tuning: {note})"
return answer
@app.post("/__shutdown__")
+293
View File
@@ -0,0 +1,293 @@
"""Phase 15 E2E (Playwright): tune how Brain answers (steering notes).
Story: ``.agent/user_stories/steering-notes.md``
Run in isolation (DB must be up: ``podman compose up -d db``):
uv run pytest tests/e2e/test_steering.py -v --no-cov
The steering loop: "Tune" under a completed answer → short instruction →
stored in Postgres (``steering_notes``) → injected into the system prompt
of every subsequent turn as the ``<tuning>`` section. The mock LLM
echoes the first tuning note into its answer
(`` (tuning: <first note line>)``), so prompt injection is observable in
the UI deterministically. Notes are listed newest-first in the header
"Tuning" panel, where each can be deleted.
Test → story mapping (Playwright Mapping Rule):
1. ``test_tune_under_answer_persists_and_steers``
2. ``test_delete_note_stops_steering``
3. ``test_note_rendered_as_text_xss_safe``
4. ``test_tuning_panel_a11y``
"""
from __future__ import annotations
import asyncio
from pathlib import Path
from threading import Thread
from typing import Any
from playwright.sync_api import Page, expect
from sqlalchemy import select, text
from app.config import Settings
from app.db import SessionLocal
from app.models import SteeringNote
from app.rag.importer import ImportSummary, import_sources
from app.rag.llm import LLMClient
REPO = Path(__file__).resolve().parents[2]
FIXTURES = REPO / "tests" / "fixtures" / "docs"
QUESTION = "How is my Kubernetes cluster set up?"
MOCK_ANSWER_MARKER = "Deterministic mock answer for E2E"
NOTE = "STEEER-MARKER be concise"
XSS_NOTE = "<script>window.__xss = true; alert('xss')</script>"
#: index.html ships exactly two classic/module script tags.
BASE_SCRIPT_COUNT = 2
async def _import_fixtures(mock_port: int) -> ImportSummary:
kwargs: dict[str, Any] = {"_env_file": None, "llm_base_url": f"http://127.0.0.1:{mock_port}/v1"}
settings = Settings(**kwargs) # pyright: ignore[reportCallIssue]
return await import_sources([FIXTURES], LLMClient(settings))
def _run_in_thread(coro: Any) -> Any:
"""Run a coroutine on a worker thread.
Playwright's sync API keeps an asyncio loop running on the test thread,
so ``asyncio.run`` cannot be called directly from a test body.
"""
box: dict[str, Any] = {}
def runner() -> None:
try:
box["value"] = asyncio.run(coro)
except BaseException as e: # noqa: BLE001 — re-raised on the test thread
box["error"] = e
t = Thread(target=runner)
t.start()
t.join()
if "error" in box:
raise box["error"]
return box["value"]
def _reset_db(mock_port: int, seed: bool) -> ImportSummary | None:
"""Truncate the KB (and query log + steering notes), re-import fixtures."""
with SessionLocal() as db:
db.execute(text("TRUNCATE chunks, documents, query_log, steering_notes"))
db.commit()
if not seed:
return None
return _run_in_thread(_import_fixtures(mock_port))
def _ask(page: Page, question: str) -> None:
"""Send one turn and wait until the grounded answer has fully landed."""
page.fill("#message-input", question)
page.click("#send-btn")
expect(page.locator(".msg.user .bubble").last).to_contain_text(question)
expect(page.locator(".msg.brain .bubble").last).to_contain_text(
MOCK_ANSWER_MARKER, timeout=30_000
)
expect(page.locator("#send-btn")).to_be_enabled()
expect(page.locator("#send-label")).to_have_text("Send")
def _tune_and_save(page: Page, note: str) -> None:
"""Tune the last completed brain bubble and save *note*."""
tune = page.locator(".msg.brain .tune-btn").last
expect(tune).to_be_visible()
tune.click()
form = page.locator(".msg.brain .tune-form").last
expect(form).to_be_visible()
form.locator("textarea").fill(note)
form.locator(".tune-save").click()
saved = page.locator(".msg.brain .tune-saved").last
expect(saved).to_contain_text("Saved — future answers will follow this.", timeout=15_000)
def _open_panel(page: Page) -> None:
page.click("#steering-toggle")
expect(page.locator("#steering-panel")).to_be_visible()
# ---------------------------------------------------------------------------
# 1. Tune under an answer → persisted → next answer carries the note
# ---------------------------------------------------------------------------
def test_tune_under_answer_persists_and_steers(
page: Page, app_url: str, mock_llm: int, db_ready: None
) -> None:
summary = _reset_db(mock_llm, seed=True)
assert summary is not None and summary.added == 8 # A9 formats
page.set_default_timeout(30_000)
page.goto(app_url)
_ask(page, QUESTION)
# The Tune control: ghost button in the answer's meta row, ≥44px.
tune = page.locator(".msg.brain .tune-btn").last
expect(tune).to_have_count(1)
expect(tune).to_have_attribute("type", "button")
box = tune.bounding_box()
assert box is not None and box["height"] >= 44
_tune_and_save(page, NOTE)
# Persisted in Postgres.
with SessionLocal() as db:
rows = db.scalars(select(SteeringNote)).all()
assert [r.note for r in rows] == [NOTE]
# The header panel shows the note with an updated count badge.
_open_panel(page)
expect(page.locator("#steering-count")).to_have_text("1")
expect(page.locator("#steering-list .steering-note")).to_have_count(1)
expect(page.locator("#steering-list .steering-note-text")).to_have_text(NOTE)
page.click("#steering-toggle") # close again
# The NEXT answer carries the note — it reached the system prompt.
_ask(page, QUESTION)
bubble = page.locator(".msg.brain .bubble").last
expect(bubble).to_contain_text(f"(tuning: {NOTE})")
# ---------------------------------------------------------------------------
# 2. Delete from the panel → count 0 → steering stops
# ---------------------------------------------------------------------------
def test_delete_note_stops_steering(
page: Page, app_url: str, mock_llm: int, db_ready: None
) -> None:
_reset_db(mock_llm, seed=True)
page.set_default_timeout(30_000)
page.goto(app_url)
_ask(page, QUESTION)
_tune_and_save(page, NOTE)
# Steering is live: one more answer carries the marker.
_ask(page, QUESTION)
expect(page.locator(".msg.brain .bubble").last).to_contain_text(f"(tuning: {NOTE})")
# Delete the note from the panel.
_open_panel(page)
expect(page.locator("#steering-count")).to_have_text("1")
page.locator("#steering-list .steering-delete").click()
expect(page.locator("#steering-list .steering-note")).to_have_count(0)
expect(page.locator("#steering-count")).to_have_text("0")
expect(page.locator("#steering-empty")).to_be_visible()
expect(page.locator("#steering-announcer")).to_contain_text("deleted")
with SessionLocal() as db:
assert db.scalars(select(SteeringNote)).all() == []
# The next answer no longer carries the marker.
_ask(page, QUESTION)
bubble = page.locator(".msg.brain .bubble").last
expect(bubble).to_contain_text(MOCK_ANSWER_MARKER)
expect(bubble).not_to_contain_text("STEEER-MARKER")
# ---------------------------------------------------------------------------
# 3. Notes render as text (XSS-safe)
# ---------------------------------------------------------------------------
def test_note_rendered_as_text_xss_safe(
page: Page, app_url: str, mock_llm: int, db_ready: None
) -> None:
_reset_db(mock_llm, seed=True)
page.set_default_timeout(30_000)
page.goto(app_url)
dialogs: list[str] = []
def _handle_dialog(d) -> None:
dialogs.append(d.message)
d.dismiss()
page.on("dialog", _handle_dialog)
_ask(page, QUESTION)
_tune_and_save(page, XSS_NOTE)
# Panel: the payload is visible as LITERAL text…
_open_panel(page)
expect(page.locator("#steering-list .steering-note-text")).to_have_text(XSS_NOTE)
# …never as an executed element: no script tag anywhere, no dialog.
assert page.locator("#steering-panel script").count() == 0
expect(page.locator("script")).to_have_count(BASE_SCRIPT_COUNT)
assert dialogs == [], f"the note must never execute as script: {dialogs}"
assert page.evaluate("() => window.__xss === undefined") is True
# ---------------------------------------------------------------------------
# 4. Tuning panel accessibility
# ---------------------------------------------------------------------------
def test_tuning_panel_a11y(page: Page, app_url: str, db_ready: None) -> None:
_reset_db(mock_port=0, seed=False) # no KB seeding needed for the panel a11y
page.set_default_timeout(30_000)
page.goto(app_url)
toggle = page.locator("#steering-toggle")
panel = page.locator("#steering-panel")
announcer = page.locator("#steering-announcer")
# Initial: closed, correctly wired, polite live region present.
expect(toggle).to_have_attribute("aria-expanded", "false")
expect(toggle).to_have_attribute("aria-controls", "steering-panel")
expect(panel).to_have_attribute("role", "region")
assert "Tuning notes" in (panel.get_attribute("aria-label") or "")
expect(panel).to_be_hidden()
assert announcer.get_attribute("role") == "status"
assert announcer.get_attribute("aria-live") == "polite"
# Accessible name comes from its visible text (icon is aria-hidden).
assert "Tuning" in toggle.inner_text()
# Open: expanded + the designed empty state.
toggle.click()
expect(toggle).to_have_attribute("aria-expanded", "true")
expect(panel).to_be_visible()
expect(page.locator("#steering-empty")).to_be_visible()
expect(page.locator("#steering-count")).to_have_text("0")
# Add a note (API), then re-open the panel to refresh it.
page.evaluate(
"""async () => {
const r = await fetch('/api/steering', {
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({note: 'a11y note one'}),
});
if (!r.ok) throw new Error('steering POST failed: ' + r.status);
}"""
)
toggle.click() # close
toggle.click() # re-open (refreshes the list)
note_item = page.locator("#steering-list .steering-note")
expect(note_item).to_have_count(1)
expect(note_item.locator(".steering-note-text")).to_have_text("a11y note one")
# The per-note delete is a real, labeled button (≥44px target).
delete = page.locator("#steering-list .steering-delete")
expect(delete).to_have_attribute("type", "button")
assert (delete.get_attribute("aria-label") or "").startswith("Delete tuning note:")
box = delete.bounding_box()
assert box is not None and box["height"] >= 44
# Delete: list empties, count updates, the live region announces it.
delete.click()
expect(page.locator("#steering-list .steering-note")).to_have_count(0)
expect(page.locator("#steering-count")).to_have_text("0")
expect(announcer).to_contain_text("deleted")
# And the toggle closes cleanly again.
toggle.click()
expect(toggle).to_have_attribute("aria-expanded", "false")
expect(panel).to_be_hidden()
+253
View File
@@ -0,0 +1,253 @@
"""Integration: steering notes (phase 15) — CRUD + system-prompt injection.
Real Postgres (``podman compose up -d db``); the chat path reuses the
deterministic fake LLM from ``test_chat_api`` (token-overlap embeddings),
so the stored note's journey — API → Postgres → ``<tuning>`` section of
the captured system prompt — is verified end-to-end without a network.
Requires: podman compose up -d db
"""
from __future__ import annotations
import asyncio
import logging
import uuid
from collections.abc import Iterator
from datetime import UTC, datetime, timedelta
from pathlib import Path
import pytest
from fastapi.testclient import TestClient
from sqlalchemy import select, text
from test_chat_api import FakeRagLLM, _stream_chat
from app.api import chat as chat_api
from app.main import app as fastapi_app
from app.models import SteeringNote
from app.rag.importer import import_sources
FIXTURES = Path(__file__).resolve().parents[1] / "fixtures" / "docs"
QUESTION = "How is my Kubernetes cluster set up?"
OFF_TOPIC = "How do I bake sourdough bread?"
NOTE = "STEEER-MARKER be concise"
@pytest.fixture(autouse=True)
def clean_steering(db) -> Iterator[None]:
"""Steering notes + query log are global state: reset around every test."""
db.execute(text("TRUNCATE steering_notes, query_log"))
db.commit()
yield
db.execute(text("TRUNCATE steering_notes, query_log"))
db.commit()
@pytest.fixture()
def seeded_kb(db) -> Iterator[FakeRagLLM]:
"""Fresh Postgres with the fixture docs imported (real pipeline)."""
db.execute(text("TRUNCATE chunks, documents, query_log, steering_notes"))
db.commit()
llm = FakeRagLLM()
summary = asyncio.run(import_sources([FIXTURES], llm, session=db))
assert summary.added == 8 # A9 formats; .hidden/ skipped
yield llm
db.execute(text("TRUNCATE chunks, documents, query_log, steering_notes"))
db.commit()
def _turn_log_lines(caplog: pytest.LogCaptureFixture) -> list[str]:
"""The per-turn ``question=…`` log lines (PLAN §9) from this test."""
return [r.getMessage() for r in caplog.records if "question=" in r.getMessage()]
# ---------- CRUD ----------
def test_create_note_returns_201_and_stores_trimmed(client: TestClient, db) -> None:
r = client.post("/api/steering", json={"note": f" {NOTE} "})
assert r.status_code == 201
body = r.json()
assert body["note"] == NOTE # trimmed before storage
uuid.UUID(body["id"]) # valid UUID
assert body["created_at"]
rows = db.scalars(select(SteeringNote)).all()
assert [row.note for row in rows] == [NOTE]
def test_list_notes_empty(client: TestClient) -> None:
r = client.get("/api/steering")
assert r.status_code == 200
assert r.json() == {"notes": []}
def test_list_notes_newest_first(client: TestClient, db) -> None:
base = datetime.now(UTC)
db.add_all(
[
SteeringNote(note="oldest", created_at=base),
SteeringNote(note="newest", created_at=base + timedelta(hours=2)),
SteeringNote(note="middle", created_at=base + timedelta(hours=1)),
]
)
db.commit()
r = client.get("/api/steering")
assert r.status_code == 200
body = r.json()
assert [n["note"] for n in body["notes"]] == ["newest", "middle", "oldest"]
for n in body["notes"]:
assert set(n) == {"id", "note", "created_at"}
uuid.UUID(n["id"])
def test_delete_note_returns_204_and_removes(client: TestClient, db) -> None:
created = client.post("/api/steering", json={"note": NOTE}).json()
assert client.delete(f"/api/steering/{created['id']}").status_code == 204
assert client.get("/api/steering").json() == {"notes": []}
assert db.scalars(select(SteeringNote)).all() == []
def test_delete_unknown_note_returns_404(client: TestClient) -> None:
r = client.delete(f"/api/steering/{uuid.uuid4()}")
assert r.status_code == 404
assert "not found" in r.json()["detail"]
def test_delete_invalid_id_returns_422(client: TestClient) -> None:
assert client.delete("/api/steering/not-a-uuid").status_code == 422
def test_create_rejects_empty_and_blank_notes(client: TestClient) -> None:
assert client.post("/api/steering", json={"note": ""}).status_code == 422
assert client.post("/api/steering", json={"note": " \t\n "}).status_code == 422
assert client.get("/api/steering").json() == {"notes": []}
def test_create_enforces_2000_char_limit(client: TestClient) -> None:
assert client.post("/api/steering", json={"note": "x" * 2001}).status_code == 422
r = client.post("/api/steering", json={"note": "x" * 2000})
assert r.status_code == 201
assert len(r.json()["note"]) == 2000
# ---------- chat turn: note reaches the system prompt ----------
def test_chat_turn_high_mode_receives_note_in_system_prompt(
client: TestClient, seeded_kb: FakeRagLLM, caplog: pytest.LogCaptureFixture
) -> None:
client.post("/api/steering", json={"note": NOTE})
caplog.set_level(logging.INFO, logger="app.chat")
fastapi_app.dependency_overrides[chat_api.get_llm] = lambda: seeded_kb
try:
_, _, frames = _stream_chat(client, QUESTION)
finally:
fastapi_app.dependency_overrides.clear()
assert frames[-1]["type"] == "done"
assert frames[-1]["deflected"] is False
# The LLM received the HIGH prompt with the <tuning> section.
(system, user) = seeded_kb.seen_messages[0][0], seeded_kb.seen_messages[0][1]
assert user["content"] == QUESTION
assert "<relevance>HIGH</relevance>" in system["content"]
assert "<tuning>" in system["content"]
assert f"1. {NOTE}" in system["content"]
assert "<documents>" in system["content"]
# The section sits between the relevance marker and the documents.
assert (
system["content"].index("<relevance>HIGH</relevance>")
< system["content"].index("<tuning>")
< system["content"].index("</tuning>")
< system["content"].index("<documents>")
)
# The per-turn log line records tuning=N (PLAN §9).
lines = _turn_log_lines(caplog)
assert lines and "tuning=1" in lines[-1]
def test_chat_turn_low_mode_receives_note_in_system_prompt(
client: TestClient, seeded_kb: FakeRagLLM, caplog: pytest.LogCaptureFixture
) -> None:
client.post("/api/steering", json={"note": NOTE})
caplog.set_level(logging.INFO, logger="app.chat")
fastapi_app.dependency_overrides[chat_api.get_llm] = lambda: seeded_kb
try:
_, _, frames = _stream_chat(client, OFF_TOPIC)
finally:
fastapi_app.dependency_overrides.clear()
assert frames[-1]["deflected"] is True
(system, _user) = seeded_kb.seen_messages[0][0], seeded_kb.seen_messages[0][1]
assert "<relevance>LOW</relevance>" in system["content"]
assert "DEFLECT_MODE" in system["content"]
assert "<tuning>" in system["content"]
assert f"1. {NOTE}" in system["content"]
lines = _turn_log_lines(caplog)
assert lines and "tuning=1" in lines[-1]
def test_chat_turn_without_notes_has_no_tuning_section(
client: TestClient, seeded_kb: FakeRagLLM, caplog: pytest.LogCaptureFixture
) -> None:
caplog.set_level(logging.INFO, logger="app.chat")
fastapi_app.dependency_overrides[chat_api.get_llm] = lambda: seeded_kb
try:
_stream_chat(client, QUESTION)
finally:
fastapi_app.dependency_overrides.clear()
(system, _user) = seeded_kb.seen_messages[0][0], seeded_kb.seen_messages[0][1]
assert "<tuning>" not in system["content"]
lines = _turn_log_lines(caplog)
assert lines and "tuning=0" in lines[-1]
def test_chat_turn_numbers_notes_oldest_first(client: TestClient, db, seeded_kb) -> None:
base = datetime.now(UTC)
db.add_all(
[
SteeringNote(note="older note", created_at=base),
SteeringNote(note="newer note", created_at=base + timedelta(hours=1)),
]
)
db.commit()
fastapi_app.dependency_overrides[chat_api.get_llm] = lambda: seeded_kb
try:
_stream_chat(client, QUESTION)
finally:
fastapi_app.dependency_overrides.clear()
(system, _user) = seeded_kb.seen_messages[0][0], seeded_kb.seen_messages[0][1]
assert "<tuning>" in system["content"]
assert "1. older note" in system["content"]
assert "2. newer note" in system["content"]
assert system["content"].index("1. older note") < system["content"].index("2. newer note")
def test_multiple_turns_keep_reading_notes(client: TestClient, seeded_kb) -> None:
"""The note steers EVERY subsequent turn, not just the next one."""
client.post("/api/steering", json={"note": NOTE})
fastapi_app.dependency_overrides[chat_api.get_llm] = lambda: seeded_kb
try:
_stream_chat(client, QUESTION)
_stream_chat(client, QUESTION)
assert len(seeded_kb.seen_messages) == 2
for messages in seeded_kb.seen_messages:
assert f"1. {NOTE}" in messages[0]["content"]
# Delete → the following turn is clean again.
note_id = client.get("/api/steering").json()["notes"][0]["id"]
assert client.delete(f"/api/steering/{note_id}").status_code == 204
_stream_chat(client, QUESTION)
assert len(seeded_kb.seen_messages) == 3
assert "<tuning>" not in seeded_kb.seen_messages[-1][0]["content"]
finally:
fastapi_app.dependency_overrides.clear()
+15 -1
View File
@@ -279,8 +279,19 @@ class _CannedLLM:
yield self.answer[i : i + 12]
class _FakeSteeringResult:
"""Empty steering-note result (no stored notes in these unit tests)."""
def all(self) -> list[Any]:
return []
class _FakeSession:
"""Stands in for the DB session: records the QueryLog row it is given."""
"""Stands in for the DB session: records the QueryLog row it is given.
``scalars`` always yields no steering notes (phase 15) so the chat
turn's ``load_steering_notes`` call stays a no-op here.
"""
def __init__(self) -> None:
self.added: list[Any] = []
@@ -292,6 +303,9 @@ class _FakeSession:
def commit(self) -> None:
self.commits += 1
def scalars(self, _stmt: Any) -> _FakeSteeringResult:
return _FakeSteeringResult()
@pytest.fixture()
def gate_env(monkeypatch: pytest.MonkeyPatch) -> Iterator[tuple[_FakeSession, _CannedLLM]]:
+33 -3
View File
@@ -22,21 +22,30 @@ def _doc(path: str, content: str, title: str) -> Document:
def test_persona_rules_present_verbatim() -> None:
# Aligned to the owner's working-tree persona edits (PLAN §6 revision,
# 2026-08-22): no "you've got this" tagline, no mandated deflection
# opening. The honesty gate itself (rule 3) is unchanged.
for fragment in (
'You are "Brain of Reese" — the digital brain of Reese, a self-hoster and',
'optimistic about the user\'s ability to do things ("you\'ve got this")',
"optimistic about the user's ability to do things",
"Answer ONLY from the provided document context. Cite which document(s)",
"you used, by path.",
"Be concrete: names, versions, ports, hosts, schedules",
'HONESTY GATE: if <relevance> is "LOW", you must NOT pretend to know',
'Start your answer with a variant of: "I haven\'t done anything like that."',
"Then offer 2-3 alternative questions about things you DO have notes on.",
"Offer 2-3 alternative questions about things you DO have notes on.",
"Never invent facts, hosts, or steps that are not in the context.",
"Keep answers tight: short paragraphs, bullets where helpful.",
):
assert fragment in PERSONA
def test_persona_owner_edits_are_preserved() -> None:
"""PLAN §6 revision (2026-08-22): the removed elements must stay out."""
assert 'you\'ve got this' not in PERSONA # tagline removed by the owner
assert "Start your answer with a variant of" not in PERSONA # no mandated opening
assert "HONESTY GATE" in PERSONA # the gate itself is intact
def test_high_prompt_carries_relevance_marker_and_full_documents() -> None:
doc = _doc("kubernetes.md", "Talos Linux on three nodes.", "Kubernetes Homelab Cluster")
prompt = build_high_prompt([doc])
@@ -83,6 +92,27 @@ def test_low_prompt_with_no_titles() -> None:
assert "nothing close at all" in build_deflect_prompt([])
def test_zero_note_prompt_is_byte_identical_to_pre_steering() -> None:
"""Phase 15 contract: with no steering notes the prompt is exactly what
it was before the <tuning> section existed."""
doc = _doc("kubernetes.md", "Talos Linux on three nodes.", "Kubernetes Homelab Cluster")
block = (
'<document source="Homelab" path="kubernetes.md" title="Kubernetes Homelab Cluster">\n'
"Talos Linux on three nodes.\n"
"</document>"
)
assert build_high_prompt([doc]) == _base("HIGH") + "\n<documents>\n" + block + "\n</documents>"
assert build_deflect_prompt(["T1", "T2"]) == (
_base("LOW")
+ "\nDEFLECT_MODE: retrieval was weak — the titles below are the closest "
"your notes come to the question. They are titles only; do not pretend "
"they answer it. Use them to propose 2-3 alternative questions.\n"
+ "- T1\n- T2"
)
assert "<tuning>" not in build_high_prompt([doc])
assert "<tuning>" not in build_deflect_prompt([])
def test_relevance_placeholder_rejected_for_garbage() -> None:
with pytest.raises(ValueError, match="HIGH or LOW"):
_base("MEDIUM")
+193
View File
@@ -0,0 +1,193 @@
"""Unit: steering notes (phase 15) — the <tuning> prompt section.
Pure logic, no Postgres and no network: :func:`build_steering_section`
(empty/one/many/budget-truncation), its placement in the HIGH and LOW
prompts, and the ``plan_turn`` wiring (notes → prompt + ``tuning_count``).
"""
from __future__ import annotations
import uuid
import pytest
from app.api import chat as chat_api
from app.config import Settings
from app.models import Document
from app.rag.prompts import build_deflect_prompt, build_high_prompt, build_steering_section
from app.rag.retriever import TRUNCATION_MARKER, RetrievedChunk
def _settings() -> Settings:
return Settings(_env_file=None, relevance_threshold=0.30) # pyright: ignore[reportCallIssue]
def _doc(title: str, content: str) -> Document:
return Document(
id=uuid.uuid4(),
source="Homelab",
path=f"{title.lower().replace(' ', '-')}.md",
full_path="/tmp/doc.md",
title=title,
content=content,
content_hash="0" * 64,
)
def _chunk(doc: Document, score: float) -> RetrievedChunk:
return RetrievedChunk(
chunk_id=uuid.uuid4(),
position=0,
content=doc.content[:32],
score=score,
document=doc,
cosine=score,
)
# ---------- build_steering_section ----------
def test_steering_section_empty_when_no_notes() -> None:
assert build_steering_section([]) == ""
def test_steering_section_empty_when_notes_are_blank() -> None:
assert build_steering_section(["", " ", "\n\t"]) == ""
def test_steering_section_single_note_numbered() -> None:
section = build_steering_section(["be more concise"])
assert section.startswith("<tuning>\n")
assert section.endswith("\n</tuning>")
assert "1. be more concise" in section
assert TRUNCATION_MARKER not in section
def test_steering_section_trims_note_edges() -> None:
section = build_steering_section([" be more concise "])
assert "1. be more concise" in section
assert "1. be more concise" not in section
def test_steering_section_many_notes_numbered_in_order() -> None:
section = build_steering_section(["alpha", "beta", "gamma"])
assert "1. alpha" in section
assert "2. beta" in section
assert "3. gamma" in section
assert section.index("1. alpha") < section.index("2. beta") < section.index("3. gamma")
assert TRUNCATION_MARKER not in section
def test_steering_section_budget_truncation_keeps_oldest_prefix_and_marker() -> None:
# Each note is 300 chars; with a 600-char budget only note 1 fits, so
# the oldest-fitting prefix is kept and the overflow is marked.
notes = [f"note-{i} " + "x" * (300 - len(f"note-{i} ")) for i in range(3)]
section = build_steering_section(notes, max_chars=600)
assert TRUNCATION_MARKER in section
assert len(section) <= 600
assert "1. note-0" in section
assert "note-1" not in section
assert "note-2" not in section
# The marker comes last, after the kept notes.
assert section.index("1. note-0") < section.index(TRUNCATION_MARKER)
def test_steering_section_fits_budget_exactly_when_all_notes_fit() -> None:
section = build_steering_section(["a", "b", "c"], max_chars=10_000)
assert TRUNCATION_MARKER not in section
assert len(section) <= 10_000
def test_steering_section_default_budget_from_settings(monkeypatch: pytest.MonkeyPatch) -> None:
from app.rag import prompts as prompts_mod
monkeypatch.setattr(
prompts_mod, "get_settings", lambda: Settings(_env_file=None) # pyright: ignore[reportCallIssue]
)
# 5 notes of 2000 chars (the API max) = 10k+ chars > the 8000 default.
notes = [f"note-{i} " + "y" * (2000 - len(f"note-{i} ")) for i in range(5)]
section = build_steering_section(notes)
assert TRUNCATION_MARKER in section
assert len(section) <= 8_000
def test_steering_section_nonpositive_budget_is_empty() -> None:
assert build_steering_section(["be concise"], max_chars=0) == ""
assert build_steering_section(["be concise"], max_chars=-10) == ""
def test_steering_section_tiny_budget_never_exceeds_cap() -> None:
# Pathological budget: the section must never exceed the cap — bare
# marker when it fits, no section at all when even that doesn't.
assert len(build_steering_section(["a" * 500], max_chars=10)) <= 10
fits_marker = build_steering_section(["a" * 500], max_chars=len(TRUNCATION_MARKER))
assert fits_marker == TRUNCATION_MARKER
# ---------- prompt placement (both modes) ----------
def test_high_prompt_steering_sits_between_relevance_and_documents() -> None:
doc = _doc("Kubernetes Homelab Cluster", "TALOS_DOC_CONTENT")
prompt = build_high_prompt([doc], notes=["be concise"])
i_rel = prompt.index("<relevance>HIGH</relevance>")
i_open = prompt.index("<tuning>")
i_close = prompt.index("</tuning>")
i_docs = prompt.index("<documents>")
assert i_rel < i_open < i_close < i_docs
assert "1. be concise" in prompt
assert "TALOS_DOC_CONTENT" in prompt # documents still full
def test_deflect_prompt_steering_sits_between_relevance_and_deflect_mode() -> None:
prompt = build_deflect_prompt(["Title A", "Title B"], notes=["be concise", "cite paths"])
i_rel = prompt.index("<relevance>LOW</relevance>")
i_open = prompt.index("<tuning>")
i_close = prompt.index("</tuning>")
i_mode = prompt.index("DEFLECT_MODE")
assert i_rel < i_open < i_close < i_mode
assert "1. be concise" in prompt
assert "2. cite paths" in prompt
assert "- Title A" in prompt # weak-hit titles still carried
assert "DEFLECT_MODE" in prompt
# ---------- plan_turn wiring (gate + steering, fake retriever rows) ----------
def test_plan_turn_high_mode_injects_notes() -> None:
doc = _doc("Kubernetes Homelab Cluster", "TALOS_DOC_SENT")
plan = chat_api.plan_turn(
[_chunk(doc, 0.90)], _settings(), notes=["be concise", "assume NixOS"]
)
assert plan.deflected is False
assert plan.tuning_count == 2
assert "<tuning>" in plan.system_prompt
assert "1. be concise" in plan.system_prompt
assert "2. assume NixOS" in plan.system_prompt
assert "<relevance>HIGH</relevance>" in plan.system_prompt
assert "TALOS_DOC_SENT" in plan.system_prompt
def test_plan_turn_low_mode_injects_notes() -> None:
doc = _doc("Kubernetes Homelab Cluster", "TALOS_DOC_NEVER_SENT")
plan = chat_api.plan_turn(
[_chunk(doc, 0.10)], _settings(), notes=["be concise"]
)
assert plan.deflected is True
assert plan.tuning_count == 1
assert "DEFLECT_MODE" in plan.system_prompt
assert "<tuning>" in plan.system_prompt
assert "1. be concise" in plan.system_prompt
assert "TALOS_DOC_NEVER_SENT" not in plan.system_prompt # titles only, still
def test_plan_turn_without_notes_has_no_tuning_section() -> None:
doc = _doc("Kubernetes Homelab Cluster", "TALOS_DOC_SENT")
for chunks in (
[_chunk(doc, 0.90)], # HIGH
[_chunk(doc, 0.10)], # LOW
):
plan = chat_api.plan_turn(chunks, _settings())
assert plan.tuning_count == 0
assert "<tuning>" not in plan.system_prompt