feat(rag): steering notes — tune how Brain answers, stored in Postgres and injected into every system prompt
This commit is contained in:
+42
-10
@@ -15,6 +15,9 @@ Implements just enough of the aipi surface:
|
||||
- otherwise -> upbeat answer quoting the provided document context
|
||||
- user message containing ``pretend to think slowly`` -> 3s warm-up delay
|
||||
(used by the loading-feedback story).
|
||||
- system prompt containing ``<tuning>`` (phase 15, steering notes) ->
|
||||
the composed answer ends with `` (tuning: <first note line>)`` —
|
||||
makes prompt injection observable in the UI deterministically.
|
||||
|
||||
``max_tokens`` is honored deterministically (token ≈ whitespace word),
|
||||
like a real endpoint: an answer longer than the cap is truncated. This
|
||||
@@ -89,25 +92,54 @@ def long_answer() -> str:
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
#: First numbered note line of a ``<tuning>`` section (phase 15).
|
||||
_TUNING_BLOCK_RE = re.compile(r"<tuning>\n(.*?)\n</tuning>", re.S)
|
||||
_NOTE_LINE_RE = re.compile(r"^\d+\.\s*(.+)$")
|
||||
|
||||
|
||||
def first_tuning_note(system: str) -> str | None:
|
||||
"""The first steering note in the system prompt, or ``None``.
|
||||
|
||||
The prompt numbers notes 1..N oldest-first (see
|
||||
``app.rag.prompts.build_steering_section``); the mock echoes the first
|
||||
one into its answer so prompt injection is observable in the UI.
|
||||
"""
|
||||
block = _TUNING_BLOCK_RE.search(system)
|
||||
if not block:
|
||||
return None
|
||||
for line in block.group(1).splitlines():
|
||||
m = _NOTE_LINE_RE.match(line.strip())
|
||||
if m:
|
||||
return m.group(1).strip()
|
||||
return None
|
||||
|
||||
|
||||
def compose_answer(body: dict[str, Any]) -> str:
|
||||
system = _system(body)
|
||||
user = _user(body)
|
||||
if LONG_ANSWER_TRIGGER in user.lower():
|
||||
return long_answer()
|
||||
if "DEFLECT_MODE" in system:
|
||||
return (
|
||||
answer = long_answer()
|
||||
elif "DEFLECT_MODE" in system:
|
||||
answer = (
|
||||
"Ah — I haven't done anything like that, so I don't want to make stuff up! "
|
||||
"You're thinking bigger than my notes for a second. Try asking about "
|
||||
"kubernetes, backups, or deploying a new service — I know those inside out. "
|
||||
"You've got this!"
|
||||
)
|
||||
ctx = _context(body)
|
||||
snippet = ctx[:220].replace("\n", " ").strip()
|
||||
return (
|
||||
f"Great question — you've absolutely got this! Here's what my notes say about "
|
||||
f"“{user.strip()[:80]}”: {snippet}… That's the gist from the docs; happy to "
|
||||
"dig into any of it. (Deterministic mock answer for E2E.)"
|
||||
)
|
||||
else:
|
||||
ctx = _context(body)
|
||||
snippet = ctx[:220].replace("\n", " ").strip()
|
||||
answer = (
|
||||
f"Great question — you've absolutely got this! Here's what my notes say about "
|
||||
f"“{user.strip()[:80]}”: {snippet}… That's the gist from the docs; happy to "
|
||||
"dig into any of it. (Deterministic mock answer for E2E.)"
|
||||
)
|
||||
# Steering (phase 15): when the system prompt carries <tuning>, the
|
||||
# answer ends with the first note — deterministically observable.
|
||||
note = first_tuning_note(system)
|
||||
if note:
|
||||
answer = f"{answer} (tuning: {note})"
|
||||
return answer
|
||||
|
||||
|
||||
@app.post("/__shutdown__")
|
||||
|
||||
@@ -0,0 +1,293 @@
|
||||
"""Phase 15 E2E (Playwright): tune how Brain answers (steering notes).
|
||||
|
||||
Story: ``.agent/user_stories/steering-notes.md``
|
||||
Run in isolation (DB must be up: ``podman compose up -d db``):
|
||||
|
||||
uv run pytest tests/e2e/test_steering.py -v --no-cov
|
||||
|
||||
The steering loop: "Tune" under a completed answer → short instruction →
|
||||
stored in Postgres (``steering_notes``) → injected into the system prompt
|
||||
of every subsequent turn as the ``<tuning>`` section. The mock LLM
|
||||
echoes the first tuning note into its answer
|
||||
(`` (tuning: <first note line>)``), so prompt injection is observable in
|
||||
the UI deterministically. Notes are listed newest-first in the header
|
||||
"Tuning" panel, where each can be deleted.
|
||||
|
||||
Test → story mapping (Playwright Mapping Rule):
|
||||
1. ``test_tune_under_answer_persists_and_steers``
|
||||
2. ``test_delete_note_stops_steering``
|
||||
3. ``test_note_rendered_as_text_xss_safe``
|
||||
4. ``test_tuning_panel_a11y``
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
from threading import Thread
|
||||
from typing import Any
|
||||
|
||||
from playwright.sync_api import Page, expect
|
||||
from sqlalchemy import select, text
|
||||
|
||||
from app.config import Settings
|
||||
from app.db import SessionLocal
|
||||
from app.models import SteeringNote
|
||||
from app.rag.importer import ImportSummary, import_sources
|
||||
from app.rag.llm import LLMClient
|
||||
|
||||
REPO = Path(__file__).resolve().parents[2]
|
||||
FIXTURES = REPO / "tests" / "fixtures" / "docs"
|
||||
QUESTION = "How is my Kubernetes cluster set up?"
|
||||
MOCK_ANSWER_MARKER = "Deterministic mock answer for E2E"
|
||||
NOTE = "STEEER-MARKER be concise"
|
||||
XSS_NOTE = "<script>window.__xss = true; alert('xss')</script>"
|
||||
#: index.html ships exactly two classic/module script tags.
|
||||
BASE_SCRIPT_COUNT = 2
|
||||
|
||||
|
||||
async def _import_fixtures(mock_port: int) -> ImportSummary:
|
||||
kwargs: dict[str, Any] = {"_env_file": None, "llm_base_url": f"http://127.0.0.1:{mock_port}/v1"}
|
||||
settings = Settings(**kwargs) # pyright: ignore[reportCallIssue]
|
||||
return await import_sources([FIXTURES], LLMClient(settings))
|
||||
|
||||
|
||||
def _run_in_thread(coro: Any) -> Any:
|
||||
"""Run a coroutine on a worker thread.
|
||||
|
||||
Playwright's sync API keeps an asyncio loop running on the test thread,
|
||||
so ``asyncio.run`` cannot be called directly from a test body.
|
||||
"""
|
||||
box: dict[str, Any] = {}
|
||||
|
||||
def runner() -> None:
|
||||
try:
|
||||
box["value"] = asyncio.run(coro)
|
||||
except BaseException as e: # noqa: BLE001 — re-raised on the test thread
|
||||
box["error"] = e
|
||||
|
||||
t = Thread(target=runner)
|
||||
t.start()
|
||||
t.join()
|
||||
if "error" in box:
|
||||
raise box["error"]
|
||||
return box["value"]
|
||||
|
||||
|
||||
def _reset_db(mock_port: int, seed: bool) -> ImportSummary | None:
|
||||
"""Truncate the KB (and query log + steering notes), re-import fixtures."""
|
||||
with SessionLocal() as db:
|
||||
db.execute(text("TRUNCATE chunks, documents, query_log, steering_notes"))
|
||||
db.commit()
|
||||
if not seed:
|
||||
return None
|
||||
return _run_in_thread(_import_fixtures(mock_port))
|
||||
|
||||
|
||||
def _ask(page: Page, question: str) -> None:
|
||||
"""Send one turn and wait until the grounded answer has fully landed."""
|
||||
page.fill("#message-input", question)
|
||||
page.click("#send-btn")
|
||||
expect(page.locator(".msg.user .bubble").last).to_contain_text(question)
|
||||
expect(page.locator(".msg.brain .bubble").last).to_contain_text(
|
||||
MOCK_ANSWER_MARKER, timeout=30_000
|
||||
)
|
||||
expect(page.locator("#send-btn")).to_be_enabled()
|
||||
expect(page.locator("#send-label")).to_have_text("Send")
|
||||
|
||||
|
||||
def _tune_and_save(page: Page, note: str) -> None:
|
||||
"""Tune the last completed brain bubble and save *note*."""
|
||||
tune = page.locator(".msg.brain .tune-btn").last
|
||||
expect(tune).to_be_visible()
|
||||
tune.click()
|
||||
form = page.locator(".msg.brain .tune-form").last
|
||||
expect(form).to_be_visible()
|
||||
form.locator("textarea").fill(note)
|
||||
form.locator(".tune-save").click()
|
||||
saved = page.locator(".msg.brain .tune-saved").last
|
||||
expect(saved).to_contain_text("Saved — future answers will follow this.", timeout=15_000)
|
||||
|
||||
|
||||
def _open_panel(page: Page) -> None:
|
||||
page.click("#steering-toggle")
|
||||
expect(page.locator("#steering-panel")).to_be_visible()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 1. Tune under an answer → persisted → next answer carries the note
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_tune_under_answer_persists_and_steers(
|
||||
page: Page, app_url: str, mock_llm: int, db_ready: None
|
||||
) -> None:
|
||||
summary = _reset_db(mock_llm, seed=True)
|
||||
assert summary is not None and summary.added == 8 # A9 formats
|
||||
page.set_default_timeout(30_000)
|
||||
page.goto(app_url)
|
||||
_ask(page, QUESTION)
|
||||
|
||||
# The Tune control: ghost button in the answer's meta row, ≥44px.
|
||||
tune = page.locator(".msg.brain .tune-btn").last
|
||||
expect(tune).to_have_count(1)
|
||||
expect(tune).to_have_attribute("type", "button")
|
||||
box = tune.bounding_box()
|
||||
assert box is not None and box["height"] >= 44
|
||||
|
||||
_tune_and_save(page, NOTE)
|
||||
|
||||
# Persisted in Postgres.
|
||||
with SessionLocal() as db:
|
||||
rows = db.scalars(select(SteeringNote)).all()
|
||||
assert [r.note for r in rows] == [NOTE]
|
||||
|
||||
# The header panel shows the note with an updated count badge.
|
||||
_open_panel(page)
|
||||
expect(page.locator("#steering-count")).to_have_text("1")
|
||||
expect(page.locator("#steering-list .steering-note")).to_have_count(1)
|
||||
expect(page.locator("#steering-list .steering-note-text")).to_have_text(NOTE)
|
||||
page.click("#steering-toggle") # close again
|
||||
|
||||
# The NEXT answer carries the note — it reached the system prompt.
|
||||
_ask(page, QUESTION)
|
||||
bubble = page.locator(".msg.brain .bubble").last
|
||||
expect(bubble).to_contain_text(f"(tuning: {NOTE})")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 2. Delete from the panel → count 0 → steering stops
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_delete_note_stops_steering(
|
||||
page: Page, app_url: str, mock_llm: int, db_ready: None
|
||||
) -> None:
|
||||
_reset_db(mock_llm, seed=True)
|
||||
page.set_default_timeout(30_000)
|
||||
page.goto(app_url)
|
||||
_ask(page, QUESTION)
|
||||
_tune_and_save(page, NOTE)
|
||||
|
||||
# Steering is live: one more answer carries the marker.
|
||||
_ask(page, QUESTION)
|
||||
expect(page.locator(".msg.brain .bubble").last).to_contain_text(f"(tuning: {NOTE})")
|
||||
|
||||
# Delete the note from the panel.
|
||||
_open_panel(page)
|
||||
expect(page.locator("#steering-count")).to_have_text("1")
|
||||
page.locator("#steering-list .steering-delete").click()
|
||||
expect(page.locator("#steering-list .steering-note")).to_have_count(0)
|
||||
expect(page.locator("#steering-count")).to_have_text("0")
|
||||
expect(page.locator("#steering-empty")).to_be_visible()
|
||||
expect(page.locator("#steering-announcer")).to_contain_text("deleted")
|
||||
|
||||
with SessionLocal() as db:
|
||||
assert db.scalars(select(SteeringNote)).all() == []
|
||||
|
||||
# The next answer no longer carries the marker.
|
||||
_ask(page, QUESTION)
|
||||
bubble = page.locator(".msg.brain .bubble").last
|
||||
expect(bubble).to_contain_text(MOCK_ANSWER_MARKER)
|
||||
expect(bubble).not_to_contain_text("STEEER-MARKER")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 3. Notes render as text (XSS-safe)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_note_rendered_as_text_xss_safe(
|
||||
page: Page, app_url: str, mock_llm: int, db_ready: None
|
||||
) -> None:
|
||||
_reset_db(mock_llm, seed=True)
|
||||
page.set_default_timeout(30_000)
|
||||
page.goto(app_url)
|
||||
|
||||
dialogs: list[str] = []
|
||||
|
||||
def _handle_dialog(d) -> None:
|
||||
dialogs.append(d.message)
|
||||
d.dismiss()
|
||||
|
||||
page.on("dialog", _handle_dialog)
|
||||
|
||||
_ask(page, QUESTION)
|
||||
_tune_and_save(page, XSS_NOTE)
|
||||
|
||||
# Panel: the payload is visible as LITERAL text…
|
||||
_open_panel(page)
|
||||
expect(page.locator("#steering-list .steering-note-text")).to_have_text(XSS_NOTE)
|
||||
|
||||
# …never as an executed element: no script tag anywhere, no dialog.
|
||||
assert page.locator("#steering-panel script").count() == 0
|
||||
expect(page.locator("script")).to_have_count(BASE_SCRIPT_COUNT)
|
||||
assert dialogs == [], f"the note must never execute as script: {dialogs}"
|
||||
assert page.evaluate("() => window.__xss === undefined") is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 4. Tuning panel accessibility
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_tuning_panel_a11y(page: Page, app_url: str, db_ready: None) -> None:
|
||||
_reset_db(mock_port=0, seed=False) # no KB seeding needed for the panel a11y
|
||||
page.set_default_timeout(30_000)
|
||||
page.goto(app_url)
|
||||
|
||||
toggle = page.locator("#steering-toggle")
|
||||
panel = page.locator("#steering-panel")
|
||||
announcer = page.locator("#steering-announcer")
|
||||
|
||||
# Initial: closed, correctly wired, polite live region present.
|
||||
expect(toggle).to_have_attribute("aria-expanded", "false")
|
||||
expect(toggle).to_have_attribute("aria-controls", "steering-panel")
|
||||
expect(panel).to_have_attribute("role", "region")
|
||||
assert "Tuning notes" in (panel.get_attribute("aria-label") or "")
|
||||
expect(panel).to_be_hidden()
|
||||
assert announcer.get_attribute("role") == "status"
|
||||
assert announcer.get_attribute("aria-live") == "polite"
|
||||
# Accessible name comes from its visible text (icon is aria-hidden).
|
||||
assert "Tuning" in toggle.inner_text()
|
||||
|
||||
# Open: expanded + the designed empty state.
|
||||
toggle.click()
|
||||
expect(toggle).to_have_attribute("aria-expanded", "true")
|
||||
expect(panel).to_be_visible()
|
||||
expect(page.locator("#steering-empty")).to_be_visible()
|
||||
expect(page.locator("#steering-count")).to_have_text("0")
|
||||
|
||||
# Add a note (API), then re-open the panel to refresh it.
|
||||
page.evaluate(
|
||||
"""async () => {
|
||||
const r = await fetch('/api/steering', {
|
||||
method: 'POST',
|
||||
headers: {'Content-Type': 'application/json'},
|
||||
body: JSON.stringify({note: 'a11y note one'}),
|
||||
});
|
||||
if (!r.ok) throw new Error('steering POST failed: ' + r.status);
|
||||
}"""
|
||||
)
|
||||
toggle.click() # close
|
||||
toggle.click() # re-open (refreshes the list)
|
||||
note_item = page.locator("#steering-list .steering-note")
|
||||
expect(note_item).to_have_count(1)
|
||||
expect(note_item.locator(".steering-note-text")).to_have_text("a11y note one")
|
||||
|
||||
# The per-note delete is a real, labeled button (≥44px target).
|
||||
delete = page.locator("#steering-list .steering-delete")
|
||||
expect(delete).to_have_attribute("type", "button")
|
||||
assert (delete.get_attribute("aria-label") or "").startswith("Delete tuning note:")
|
||||
box = delete.bounding_box()
|
||||
assert box is not None and box["height"] >= 44
|
||||
|
||||
# Delete: list empties, count updates, the live region announces it.
|
||||
delete.click()
|
||||
expect(page.locator("#steering-list .steering-note")).to_have_count(0)
|
||||
expect(page.locator("#steering-count")).to_have_text("0")
|
||||
expect(announcer).to_contain_text("deleted")
|
||||
|
||||
# And the toggle closes cleanly again.
|
||||
toggle.click()
|
||||
expect(toggle).to_have_attribute("aria-expanded", "false")
|
||||
expect(panel).to_be_hidden()
|
||||
@@ -0,0 +1,253 @@
|
||||
"""Integration: steering notes (phase 15) — CRUD + system-prompt injection.
|
||||
|
||||
Real Postgres (``podman compose up -d db``); the chat path reuses the
|
||||
deterministic fake LLM from ``test_chat_api`` (token-overlap embeddings),
|
||||
so the stored note's journey — API → Postgres → ``<tuning>`` section of
|
||||
the captured system prompt — is verified end-to-end without a network.
|
||||
|
||||
Requires: podman compose up -d db
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import uuid
|
||||
from collections.abc import Iterator
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlalchemy import select, text
|
||||
from test_chat_api import FakeRagLLM, _stream_chat
|
||||
|
||||
from app.api import chat as chat_api
|
||||
from app.main import app as fastapi_app
|
||||
from app.models import SteeringNote
|
||||
from app.rag.importer import import_sources
|
||||
|
||||
FIXTURES = Path(__file__).resolve().parents[1] / "fixtures" / "docs"
|
||||
QUESTION = "How is my Kubernetes cluster set up?"
|
||||
OFF_TOPIC = "How do I bake sourdough bread?"
|
||||
NOTE = "STEEER-MARKER be concise"
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def clean_steering(db) -> Iterator[None]:
|
||||
"""Steering notes + query log are global state: reset around every test."""
|
||||
db.execute(text("TRUNCATE steering_notes, query_log"))
|
||||
db.commit()
|
||||
yield
|
||||
db.execute(text("TRUNCATE steering_notes, query_log"))
|
||||
db.commit()
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def seeded_kb(db) -> Iterator[FakeRagLLM]:
|
||||
"""Fresh Postgres with the fixture docs imported (real pipeline)."""
|
||||
db.execute(text("TRUNCATE chunks, documents, query_log, steering_notes"))
|
||||
db.commit()
|
||||
llm = FakeRagLLM()
|
||||
summary = asyncio.run(import_sources([FIXTURES], llm, session=db))
|
||||
assert summary.added == 8 # A9 formats; .hidden/ skipped
|
||||
yield llm
|
||||
db.execute(text("TRUNCATE chunks, documents, query_log, steering_notes"))
|
||||
db.commit()
|
||||
|
||||
|
||||
def _turn_log_lines(caplog: pytest.LogCaptureFixture) -> list[str]:
|
||||
"""The per-turn ``question=…`` log lines (PLAN §9) from this test."""
|
||||
return [r.getMessage() for r in caplog.records if "question=" in r.getMessage()]
|
||||
|
||||
|
||||
# ---------- CRUD ----------
|
||||
|
||||
|
||||
def test_create_note_returns_201_and_stores_trimmed(client: TestClient, db) -> None:
|
||||
r = client.post("/api/steering", json={"note": f" {NOTE} "})
|
||||
assert r.status_code == 201
|
||||
body = r.json()
|
||||
assert body["note"] == NOTE # trimmed before storage
|
||||
uuid.UUID(body["id"]) # valid UUID
|
||||
assert body["created_at"]
|
||||
rows = db.scalars(select(SteeringNote)).all()
|
||||
assert [row.note for row in rows] == [NOTE]
|
||||
|
||||
|
||||
def test_list_notes_empty(client: TestClient) -> None:
|
||||
r = client.get("/api/steering")
|
||||
assert r.status_code == 200
|
||||
assert r.json() == {"notes": []}
|
||||
|
||||
|
||||
def test_list_notes_newest_first(client: TestClient, db) -> None:
|
||||
base = datetime.now(UTC)
|
||||
db.add_all(
|
||||
[
|
||||
SteeringNote(note="oldest", created_at=base),
|
||||
SteeringNote(note="newest", created_at=base + timedelta(hours=2)),
|
||||
SteeringNote(note="middle", created_at=base + timedelta(hours=1)),
|
||||
]
|
||||
)
|
||||
db.commit()
|
||||
|
||||
r = client.get("/api/steering")
|
||||
assert r.status_code == 200
|
||||
body = r.json()
|
||||
assert [n["note"] for n in body["notes"]] == ["newest", "middle", "oldest"]
|
||||
for n in body["notes"]:
|
||||
assert set(n) == {"id", "note", "created_at"}
|
||||
uuid.UUID(n["id"])
|
||||
|
||||
|
||||
def test_delete_note_returns_204_and_removes(client: TestClient, db) -> None:
|
||||
created = client.post("/api/steering", json={"note": NOTE}).json()
|
||||
|
||||
assert client.delete(f"/api/steering/{created['id']}").status_code == 204
|
||||
assert client.get("/api/steering").json() == {"notes": []}
|
||||
assert db.scalars(select(SteeringNote)).all() == []
|
||||
|
||||
|
||||
def test_delete_unknown_note_returns_404(client: TestClient) -> None:
|
||||
r = client.delete(f"/api/steering/{uuid.uuid4()}")
|
||||
assert r.status_code == 404
|
||||
assert "not found" in r.json()["detail"]
|
||||
|
||||
|
||||
def test_delete_invalid_id_returns_422(client: TestClient) -> None:
|
||||
assert client.delete("/api/steering/not-a-uuid").status_code == 422
|
||||
|
||||
|
||||
def test_create_rejects_empty_and_blank_notes(client: TestClient) -> None:
|
||||
assert client.post("/api/steering", json={"note": ""}).status_code == 422
|
||||
assert client.post("/api/steering", json={"note": " \t\n "}).status_code == 422
|
||||
assert client.get("/api/steering").json() == {"notes": []}
|
||||
|
||||
|
||||
def test_create_enforces_2000_char_limit(client: TestClient) -> None:
|
||||
assert client.post("/api/steering", json={"note": "x" * 2001}).status_code == 422
|
||||
r = client.post("/api/steering", json={"note": "x" * 2000})
|
||||
assert r.status_code == 201
|
||||
assert len(r.json()["note"]) == 2000
|
||||
|
||||
|
||||
# ---------- chat turn: note reaches the system prompt ----------
|
||||
|
||||
|
||||
def test_chat_turn_high_mode_receives_note_in_system_prompt(
|
||||
client: TestClient, seeded_kb: FakeRagLLM, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
client.post("/api/steering", json={"note": NOTE})
|
||||
caplog.set_level(logging.INFO, logger="app.chat")
|
||||
|
||||
fastapi_app.dependency_overrides[chat_api.get_llm] = lambda: seeded_kb
|
||||
try:
|
||||
_, _, frames = _stream_chat(client, QUESTION)
|
||||
finally:
|
||||
fastapi_app.dependency_overrides.clear()
|
||||
|
||||
assert frames[-1]["type"] == "done"
|
||||
assert frames[-1]["deflected"] is False
|
||||
|
||||
# The LLM received the HIGH prompt with the <tuning> section.
|
||||
(system, user) = seeded_kb.seen_messages[0][0], seeded_kb.seen_messages[0][1]
|
||||
assert user["content"] == QUESTION
|
||||
assert "<relevance>HIGH</relevance>" in system["content"]
|
||||
assert "<tuning>" in system["content"]
|
||||
assert f"1. {NOTE}" in system["content"]
|
||||
assert "<documents>" in system["content"]
|
||||
# The section sits between the relevance marker and the documents.
|
||||
assert (
|
||||
system["content"].index("<relevance>HIGH</relevance>")
|
||||
< system["content"].index("<tuning>")
|
||||
< system["content"].index("</tuning>")
|
||||
< system["content"].index("<documents>")
|
||||
)
|
||||
|
||||
# The per-turn log line records tuning=N (PLAN §9).
|
||||
lines = _turn_log_lines(caplog)
|
||||
assert lines and "tuning=1" in lines[-1]
|
||||
|
||||
|
||||
def test_chat_turn_low_mode_receives_note_in_system_prompt(
|
||||
client: TestClient, seeded_kb: FakeRagLLM, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
client.post("/api/steering", json={"note": NOTE})
|
||||
caplog.set_level(logging.INFO, logger="app.chat")
|
||||
|
||||
fastapi_app.dependency_overrides[chat_api.get_llm] = lambda: seeded_kb
|
||||
try:
|
||||
_, _, frames = _stream_chat(client, OFF_TOPIC)
|
||||
finally:
|
||||
fastapi_app.dependency_overrides.clear()
|
||||
|
||||
assert frames[-1]["deflected"] is True
|
||||
(system, _user) = seeded_kb.seen_messages[0][0], seeded_kb.seen_messages[0][1]
|
||||
assert "<relevance>LOW</relevance>" in system["content"]
|
||||
assert "DEFLECT_MODE" in system["content"]
|
||||
assert "<tuning>" in system["content"]
|
||||
assert f"1. {NOTE}" in system["content"]
|
||||
lines = _turn_log_lines(caplog)
|
||||
assert lines and "tuning=1" in lines[-1]
|
||||
|
||||
|
||||
def test_chat_turn_without_notes_has_no_tuning_section(
|
||||
client: TestClient, seeded_kb: FakeRagLLM, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
caplog.set_level(logging.INFO, logger="app.chat")
|
||||
|
||||
fastapi_app.dependency_overrides[chat_api.get_llm] = lambda: seeded_kb
|
||||
try:
|
||||
_stream_chat(client, QUESTION)
|
||||
finally:
|
||||
fastapi_app.dependency_overrides.clear()
|
||||
|
||||
(system, _user) = seeded_kb.seen_messages[0][0], seeded_kb.seen_messages[0][1]
|
||||
assert "<tuning>" not in system["content"]
|
||||
lines = _turn_log_lines(caplog)
|
||||
assert lines and "tuning=0" in lines[-1]
|
||||
|
||||
|
||||
def test_chat_turn_numbers_notes_oldest_first(client: TestClient, db, seeded_kb) -> None:
|
||||
base = datetime.now(UTC)
|
||||
db.add_all(
|
||||
[
|
||||
SteeringNote(note="older note", created_at=base),
|
||||
SteeringNote(note="newer note", created_at=base + timedelta(hours=1)),
|
||||
]
|
||||
)
|
||||
db.commit()
|
||||
|
||||
fastapi_app.dependency_overrides[chat_api.get_llm] = lambda: seeded_kb
|
||||
try:
|
||||
_stream_chat(client, QUESTION)
|
||||
finally:
|
||||
fastapi_app.dependency_overrides.clear()
|
||||
|
||||
(system, _user) = seeded_kb.seen_messages[0][0], seeded_kb.seen_messages[0][1]
|
||||
assert "<tuning>" in system["content"]
|
||||
assert "1. older note" in system["content"]
|
||||
assert "2. newer note" in system["content"]
|
||||
assert system["content"].index("1. older note") < system["content"].index("2. newer note")
|
||||
|
||||
|
||||
def test_multiple_turns_keep_reading_notes(client: TestClient, seeded_kb) -> None:
|
||||
"""The note steers EVERY subsequent turn, not just the next one."""
|
||||
client.post("/api/steering", json={"note": NOTE})
|
||||
|
||||
fastapi_app.dependency_overrides[chat_api.get_llm] = lambda: seeded_kb
|
||||
try:
|
||||
_stream_chat(client, QUESTION)
|
||||
_stream_chat(client, QUESTION)
|
||||
assert len(seeded_kb.seen_messages) == 2
|
||||
for messages in seeded_kb.seen_messages:
|
||||
assert f"1. {NOTE}" in messages[0]["content"]
|
||||
|
||||
# Delete → the following turn is clean again.
|
||||
note_id = client.get("/api/steering").json()["notes"][0]["id"]
|
||||
assert client.delete(f"/api/steering/{note_id}").status_code == 204
|
||||
_stream_chat(client, QUESTION)
|
||||
assert len(seeded_kb.seen_messages) == 3
|
||||
assert "<tuning>" not in seeded_kb.seen_messages[-1][0]["content"]
|
||||
finally:
|
||||
fastapi_app.dependency_overrides.clear()
|
||||
@@ -279,8 +279,19 @@ class _CannedLLM:
|
||||
yield self.answer[i : i + 12]
|
||||
|
||||
|
||||
class _FakeSteeringResult:
|
||||
"""Empty steering-note result (no stored notes in these unit tests)."""
|
||||
|
||||
def all(self) -> list[Any]:
|
||||
return []
|
||||
|
||||
|
||||
class _FakeSession:
|
||||
"""Stands in for the DB session: records the QueryLog row it is given."""
|
||||
"""Stands in for the DB session: records the QueryLog row it is given.
|
||||
|
||||
``scalars`` always yields no steering notes (phase 15) so the chat
|
||||
turn's ``load_steering_notes`` call stays a no-op here.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.added: list[Any] = []
|
||||
@@ -292,6 +303,9 @@ class _FakeSession:
|
||||
def commit(self) -> None:
|
||||
self.commits += 1
|
||||
|
||||
def scalars(self, _stmt: Any) -> _FakeSteeringResult:
|
||||
return _FakeSteeringResult()
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def gate_env(monkeypatch: pytest.MonkeyPatch) -> Iterator[tuple[_FakeSession, _CannedLLM]]:
|
||||
|
||||
@@ -22,21 +22,30 @@ def _doc(path: str, content: str, title: str) -> Document:
|
||||
|
||||
|
||||
def test_persona_rules_present_verbatim() -> None:
|
||||
# Aligned to the owner's working-tree persona edits (PLAN §6 revision,
|
||||
# 2026-08-22): no "you've got this" tagline, no mandated deflection
|
||||
# opening. The honesty gate itself (rule 3) is unchanged.
|
||||
for fragment in (
|
||||
'You are "Brain of Reese" — the digital brain of Reese, a self-hoster and',
|
||||
'optimistic about the user\'s ability to do things ("you\'ve got this")',
|
||||
"optimistic about the user's ability to do things",
|
||||
"Answer ONLY from the provided document context. Cite which document(s)",
|
||||
"you used, by path.",
|
||||
"Be concrete: names, versions, ports, hosts, schedules",
|
||||
'HONESTY GATE: if <relevance> is "LOW", you must NOT pretend to know',
|
||||
'Start your answer with a variant of: "I haven\'t done anything like that."',
|
||||
"Then offer 2-3 alternative questions about things you DO have notes on.",
|
||||
"Offer 2-3 alternative questions about things you DO have notes on.",
|
||||
"Never invent facts, hosts, or steps that are not in the context.",
|
||||
"Keep answers tight: short paragraphs, bullets where helpful.",
|
||||
):
|
||||
assert fragment in PERSONA
|
||||
|
||||
|
||||
def test_persona_owner_edits_are_preserved() -> None:
|
||||
"""PLAN §6 revision (2026-08-22): the removed elements must stay out."""
|
||||
assert 'you\'ve got this' not in PERSONA # tagline removed by the owner
|
||||
assert "Start your answer with a variant of" not in PERSONA # no mandated opening
|
||||
assert "HONESTY GATE" in PERSONA # the gate itself is intact
|
||||
|
||||
|
||||
def test_high_prompt_carries_relevance_marker_and_full_documents() -> None:
|
||||
doc = _doc("kubernetes.md", "Talos Linux on three nodes.", "Kubernetes Homelab Cluster")
|
||||
prompt = build_high_prompt([doc])
|
||||
@@ -83,6 +92,27 @@ def test_low_prompt_with_no_titles() -> None:
|
||||
assert "nothing close at all" in build_deflect_prompt([])
|
||||
|
||||
|
||||
def test_zero_note_prompt_is_byte_identical_to_pre_steering() -> None:
|
||||
"""Phase 15 contract: with no steering notes the prompt is exactly what
|
||||
it was before the <tuning> section existed."""
|
||||
doc = _doc("kubernetes.md", "Talos Linux on three nodes.", "Kubernetes Homelab Cluster")
|
||||
block = (
|
||||
'<document source="Homelab" path="kubernetes.md" title="Kubernetes Homelab Cluster">\n'
|
||||
"Talos Linux on three nodes.\n"
|
||||
"</document>"
|
||||
)
|
||||
assert build_high_prompt([doc]) == _base("HIGH") + "\n<documents>\n" + block + "\n</documents>"
|
||||
assert build_deflect_prompt(["T1", "T2"]) == (
|
||||
_base("LOW")
|
||||
+ "\nDEFLECT_MODE: retrieval was weak — the titles below are the closest "
|
||||
"your notes come to the question. They are titles only; do not pretend "
|
||||
"they answer it. Use them to propose 2-3 alternative questions.\n"
|
||||
+ "- T1\n- T2"
|
||||
)
|
||||
assert "<tuning>" not in build_high_prompt([doc])
|
||||
assert "<tuning>" not in build_deflect_prompt([])
|
||||
|
||||
|
||||
def test_relevance_placeholder_rejected_for_garbage() -> None:
|
||||
with pytest.raises(ValueError, match="HIGH or LOW"):
|
||||
_base("MEDIUM")
|
||||
|
||||
@@ -0,0 +1,193 @@
|
||||
"""Unit: steering notes (phase 15) — the <tuning> prompt section.
|
||||
|
||||
Pure logic, no Postgres and no network: :func:`build_steering_section`
|
||||
(empty/one/many/budget-truncation), its placement in the HIGH and LOW
|
||||
prompts, and the ``plan_turn`` wiring (notes → prompt + ``tuning_count``).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
|
||||
from app.api import chat as chat_api
|
||||
from app.config import Settings
|
||||
from app.models import Document
|
||||
from app.rag.prompts import build_deflect_prompt, build_high_prompt, build_steering_section
|
||||
from app.rag.retriever import TRUNCATION_MARKER, RetrievedChunk
|
||||
|
||||
|
||||
def _settings() -> Settings:
|
||||
return Settings(_env_file=None, relevance_threshold=0.30) # pyright: ignore[reportCallIssue]
|
||||
|
||||
|
||||
def _doc(title: str, content: str) -> Document:
|
||||
return Document(
|
||||
id=uuid.uuid4(),
|
||||
source="Homelab",
|
||||
path=f"{title.lower().replace(' ', '-')}.md",
|
||||
full_path="/tmp/doc.md",
|
||||
title=title,
|
||||
content=content,
|
||||
content_hash="0" * 64,
|
||||
)
|
||||
|
||||
|
||||
def _chunk(doc: Document, score: float) -> RetrievedChunk:
|
||||
return RetrievedChunk(
|
||||
chunk_id=uuid.uuid4(),
|
||||
position=0,
|
||||
content=doc.content[:32],
|
||||
score=score,
|
||||
document=doc,
|
||||
cosine=score,
|
||||
)
|
||||
|
||||
|
||||
# ---------- build_steering_section ----------
|
||||
|
||||
|
||||
def test_steering_section_empty_when_no_notes() -> None:
|
||||
assert build_steering_section([]) == ""
|
||||
|
||||
|
||||
def test_steering_section_empty_when_notes_are_blank() -> None:
|
||||
assert build_steering_section(["", " ", "\n\t"]) == ""
|
||||
|
||||
|
||||
def test_steering_section_single_note_numbered() -> None:
|
||||
section = build_steering_section(["be more concise"])
|
||||
assert section.startswith("<tuning>\n")
|
||||
assert section.endswith("\n</tuning>")
|
||||
assert "1. be more concise" in section
|
||||
assert TRUNCATION_MARKER not in section
|
||||
|
||||
|
||||
def test_steering_section_trims_note_edges() -> None:
|
||||
section = build_steering_section([" be more concise "])
|
||||
assert "1. be more concise" in section
|
||||
assert "1. be more concise" not in section
|
||||
|
||||
|
||||
def test_steering_section_many_notes_numbered_in_order() -> None:
|
||||
section = build_steering_section(["alpha", "beta", "gamma"])
|
||||
assert "1. alpha" in section
|
||||
assert "2. beta" in section
|
||||
assert "3. gamma" in section
|
||||
assert section.index("1. alpha") < section.index("2. beta") < section.index("3. gamma")
|
||||
assert TRUNCATION_MARKER not in section
|
||||
|
||||
|
||||
def test_steering_section_budget_truncation_keeps_oldest_prefix_and_marker() -> None:
|
||||
# Each note is 300 chars; with a 600-char budget only note 1 fits, so
|
||||
# the oldest-fitting prefix is kept and the overflow is marked.
|
||||
notes = [f"note-{i} " + "x" * (300 - len(f"note-{i} ")) for i in range(3)]
|
||||
section = build_steering_section(notes, max_chars=600)
|
||||
assert TRUNCATION_MARKER in section
|
||||
assert len(section) <= 600
|
||||
assert "1. note-0" in section
|
||||
assert "note-1" not in section
|
||||
assert "note-2" not in section
|
||||
# The marker comes last, after the kept notes.
|
||||
assert section.index("1. note-0") < section.index(TRUNCATION_MARKER)
|
||||
|
||||
|
||||
def test_steering_section_fits_budget_exactly_when_all_notes_fit() -> None:
|
||||
section = build_steering_section(["a", "b", "c"], max_chars=10_000)
|
||||
assert TRUNCATION_MARKER not in section
|
||||
assert len(section) <= 10_000
|
||||
|
||||
|
||||
def test_steering_section_default_budget_from_settings(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from app.rag import prompts as prompts_mod
|
||||
|
||||
monkeypatch.setattr(
|
||||
prompts_mod, "get_settings", lambda: Settings(_env_file=None) # pyright: ignore[reportCallIssue]
|
||||
)
|
||||
# 5 notes of 2000 chars (the API max) = 10k+ chars > the 8000 default.
|
||||
notes = [f"note-{i} " + "y" * (2000 - len(f"note-{i} ")) for i in range(5)]
|
||||
section = build_steering_section(notes)
|
||||
assert TRUNCATION_MARKER in section
|
||||
assert len(section) <= 8_000
|
||||
|
||||
|
||||
def test_steering_section_nonpositive_budget_is_empty() -> None:
|
||||
assert build_steering_section(["be concise"], max_chars=0) == ""
|
||||
assert build_steering_section(["be concise"], max_chars=-10) == ""
|
||||
|
||||
|
||||
def test_steering_section_tiny_budget_never_exceeds_cap() -> None:
|
||||
# Pathological budget: the section must never exceed the cap — bare
|
||||
# marker when it fits, no section at all when even that doesn't.
|
||||
assert len(build_steering_section(["a" * 500], max_chars=10)) <= 10
|
||||
fits_marker = build_steering_section(["a" * 500], max_chars=len(TRUNCATION_MARKER))
|
||||
assert fits_marker == TRUNCATION_MARKER
|
||||
|
||||
|
||||
# ---------- prompt placement (both modes) ----------
|
||||
|
||||
|
||||
def test_high_prompt_steering_sits_between_relevance_and_documents() -> None:
|
||||
doc = _doc("Kubernetes Homelab Cluster", "TALOS_DOC_CONTENT")
|
||||
prompt = build_high_prompt([doc], notes=["be concise"])
|
||||
i_rel = prompt.index("<relevance>HIGH</relevance>")
|
||||
i_open = prompt.index("<tuning>")
|
||||
i_close = prompt.index("</tuning>")
|
||||
i_docs = prompt.index("<documents>")
|
||||
assert i_rel < i_open < i_close < i_docs
|
||||
assert "1. be concise" in prompt
|
||||
assert "TALOS_DOC_CONTENT" in prompt # documents still full
|
||||
|
||||
|
||||
def test_deflect_prompt_steering_sits_between_relevance_and_deflect_mode() -> None:
|
||||
prompt = build_deflect_prompt(["Title A", "Title B"], notes=["be concise", "cite paths"])
|
||||
i_rel = prompt.index("<relevance>LOW</relevance>")
|
||||
i_open = prompt.index("<tuning>")
|
||||
i_close = prompt.index("</tuning>")
|
||||
i_mode = prompt.index("DEFLECT_MODE")
|
||||
assert i_rel < i_open < i_close < i_mode
|
||||
assert "1. be concise" in prompt
|
||||
assert "2. cite paths" in prompt
|
||||
assert "- Title A" in prompt # weak-hit titles still carried
|
||||
assert "DEFLECT_MODE" in prompt
|
||||
|
||||
|
||||
# ---------- plan_turn wiring (gate + steering, fake retriever rows) ----------
|
||||
|
||||
|
||||
def test_plan_turn_high_mode_injects_notes() -> None:
|
||||
doc = _doc("Kubernetes Homelab Cluster", "TALOS_DOC_SENT")
|
||||
plan = chat_api.plan_turn(
|
||||
[_chunk(doc, 0.90)], _settings(), notes=["be concise", "assume NixOS"]
|
||||
)
|
||||
assert plan.deflected is False
|
||||
assert plan.tuning_count == 2
|
||||
assert "<tuning>" in plan.system_prompt
|
||||
assert "1. be concise" in plan.system_prompt
|
||||
assert "2. assume NixOS" in plan.system_prompt
|
||||
assert "<relevance>HIGH</relevance>" in plan.system_prompt
|
||||
assert "TALOS_DOC_SENT" in plan.system_prompt
|
||||
|
||||
|
||||
def test_plan_turn_low_mode_injects_notes() -> None:
|
||||
doc = _doc("Kubernetes Homelab Cluster", "TALOS_DOC_NEVER_SENT")
|
||||
plan = chat_api.plan_turn(
|
||||
[_chunk(doc, 0.10)], _settings(), notes=["be concise"]
|
||||
)
|
||||
assert plan.deflected is True
|
||||
assert plan.tuning_count == 1
|
||||
assert "DEFLECT_MODE" in plan.system_prompt
|
||||
assert "<tuning>" in plan.system_prompt
|
||||
assert "1. be concise" in plan.system_prompt
|
||||
assert "TALOS_DOC_NEVER_SENT" not in plan.system_prompt # titles only, still
|
||||
|
||||
|
||||
def test_plan_turn_without_notes_has_no_tuning_section() -> None:
|
||||
doc = _doc("Kubernetes Homelab Cluster", "TALOS_DOC_SENT")
|
||||
for chunks in (
|
||||
[_chunk(doc, 0.90)], # HIGH
|
||||
[_chunk(doc, 0.10)], # LOW
|
||||
):
|
||||
plan = chat_api.plan_turn(chunks, _settings())
|
||||
assert plan.tuning_count == 0
|
||||
assert "<tuning>" not in plan.system_prompt
|
||||
Reference in New Issue
Block a user