feat(rag): steering notes — tune how Brain answers, stored in Postgres and injected into every system prompt
This commit is contained in:
@@ -22,6 +22,7 @@ BOR_TOP_N_DOCS=2
|
|||||||
BOR_RELEVANCE_THRESHOLD=0.62 # answer when best cosine >= this OR an FTS hit; else honest deflection
|
BOR_RELEVANCE_THRESHOLD=0.62 # answer when best cosine >= this OR an FTS hit; else honest deflection
|
||||||
BOR_MAX_CONTEXT_CHARS=24000 # cap on total document text sent to the LLM
|
BOR_MAX_CONTEXT_CHARS=24000 # cap on total document text sent to the LLM
|
||||||
BOR_MAX_OUTPUT_TOKENS=32768 # max answer length in tokens (answers must not be cut off)
|
BOR_MAX_OUTPUT_TOKENS=32768 # max answer length in tokens (answers must not be cut off)
|
||||||
|
BOR_STEERING_MAX_CHARS=8000 # char budget for the <tuning> (steering notes) prompt section
|
||||||
BOR_CHUNK_TARGET_CHARS=2000
|
BOR_CHUNK_TARGET_CHARS=2000
|
||||||
BOR_CHUNK_OVERLAP_CHARS=200
|
BOR_CHUNK_OVERLAP_CHARS=200
|
||||||
BOR_EMBED_BATCH_SIZE=16
|
BOR_EMBED_BATCH_SIZE=16
|
||||||
|
|||||||
@@ -85,6 +85,34 @@ uv run uvicorn app.main:app --reload
|
|||||||
- **Sources** (`/sources.html`) — the indexed document list; the *Path*
|
- **Sources** (`/sources.html`) — the indexed document list; the *Path*
|
||||||
column links each document to the viewer in a new tab.
|
column links each document to the viewer in a new tab.
|
||||||
|
|
||||||
|
## Tuning your answers
|
||||||
|
|
||||||
|
If an answer isn't quite right — too chatty, wrong assumption, missing
|
||||||
|
context — **tune** Brain right there:
|
||||||
|
|
||||||
|
1. Press **“Tune”** in the meta row under any completed answer (deflected
|
||||||
|
ones included).
|
||||||
|
2. Type a short instruction (1–2000 chars), e.g. *“be more concise”* or
|
||||||
|
*“assume I'm on NixOS”*, and **Save**.
|
||||||
|
|
||||||
|
The note is stored in Postgres (`steering_notes`) and read into the
|
||||||
|
**system prompt of every subsequent chat turn** as a `<tuning>` section
|
||||||
|
(numbered, oldest first, capped at `BOR_STEERING_MAX_CHARS` chars —
|
||||||
|
default 8000, overflow marked `[…truncated…]`). With no stored notes the
|
||||||
|
prompt is byte-identical to the un-tuned one, so tuning is opt-in per
|
||||||
|
note.
|
||||||
|
|
||||||
|
List or remove notes at any time from the **“Tuning”** button in the chat
|
||||||
|
header (count badge, newest-first, per-note delete). The API is stateless
|
||||||
|
JSON if you prefer curl:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl -s localhost:8000/api/steering # list (newest first)
|
||||||
|
curl -s -X POST localhost:8000/api/steering \
|
||||||
|
-H 'Content-Type: application/json' -d '{"note": "be more concise"}'
|
||||||
|
curl -s -X DELETE localhost:8000/api/steering/<note-id> # remove
|
||||||
|
```
|
||||||
|
|
||||||
## Updating the documents
|
## Updating the documents
|
||||||
|
|
||||||
**This is the workflow you'll use most.** The knowledge base is refreshed by
|
**This is the workflow you'll use most.** The knowledge base is refreshed by
|
||||||
@@ -270,6 +298,7 @@ served locally (no CDN), `BOR_ENVIRONMENT=production`.
|
|||||||
| `BOR_RRF_K` | `60` | RRF damping constant (`1/(k + rank)`) |
|
| `BOR_RRF_K` | `60` | RRF damping constant (`1/(k + rank)`) |
|
||||||
| `BOR_IMPORT_EXTENSIONS` | `md,markdown,txt,yaml,yml,json,py` | csv of importable formats (may only narrow the A9 set) |
|
| `BOR_IMPORT_EXTENSIONS` | `md,markdown,txt,yaml,yml,json,py` | csv of importable formats (may only narrow the A9 set) |
|
||||||
| `BOR_MAX_CONTEXT_CHARS` | `24000` | cap on total document text sent to the LLM |
|
| `BOR_MAX_CONTEXT_CHARS` | `24000` | cap on total document text sent to the LLM |
|
||||||
|
| `BOR_STEERING_MAX_CHARS` | `8000` | char budget for the `<tuning>` (steering notes) prompt section |
|
||||||
| `BOR_SUGGESTIONS` | built-in list | JSON list of onboarding chips |
|
| `BOR_SUGGESTIONS` | built-in list | JSON list of onboarding chips |
|
||||||
| `DEBUGPY` | `0` | `1` ⇒ attach-on-demand debugpy on `DEBUGPY_PORT` (default 5678) |
|
| `DEBUGPY` | `0` | `1` ⇒ attach-on-demand debugpy on `DEBUGPY_PORT` (default 5678) |
|
||||||
| `BOR_LOG_LEVEL` | `INFO` | app log level |
|
| `BOR_LOG_LEVEL` | `INFO` | app log level |
|
||||||
|
|||||||
@@ -0,0 +1,39 @@
|
|||||||
|
"""steering notes: owner tuning notes injected into every system prompt
|
||||||
|
|
||||||
|
Revision ID: 0003
|
||||||
|
Revises: 0002
|
||||||
|
Create Date: 2026-08-22
|
||||||
|
|
||||||
|
Phase 15 (steering-notes story): the owner can "tune" how Brain answers
|
||||||
|
from the chat UI. Notes live in ``steering_notes`` and are read into the
|
||||||
|
system prompt of every subsequent chat turn (``<tuning>`` section).
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from sqlalchemy.dialects import postgresql
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision = "0003"
|
||||||
|
down_revision = "0002"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"steering_notes",
|
||||||
|
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
||||||
|
sa.Column("note", sa.Text(), nullable=False),
|
||||||
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.func.now(),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_table("steering_notes")
|
||||||
+35
-7
@@ -16,6 +16,11 @@ gets a grounded answer. Deflection mode carries weak-hit *titles only*
|
|||||||
(never document content) plus deterministic "Maybe try" chips, and the
|
(never document content) plus deterministic "Maybe try" chips, and the
|
||||||
``done`` event / ``query_log`` row record ``deflected=true``, the weak
|
``done`` event / ``query_log`` row record ``deflected=true``, the weak
|
||||||
score and the ``fts_hits`` count.
|
score and the ``fts_hits`` count.
|
||||||
|
|
||||||
|
Steering (phase 15): the owner's stored tuning notes are loaded per turn
|
||||||
|
(oldest first) and injected into the system prompt as a ``<tuning>``
|
||||||
|
section — both the HIGH and the LOW prompt carry it. The per-turn log
|
||||||
|
line records ``tuning=N`` (the number of injected notes).
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
@@ -30,6 +35,7 @@ from fastapi import APIRouter, Depends
|
|||||||
from fastapi.responses import JSONResponse, StreamingResponse
|
from fastapi.responses import JSONResponse, StreamingResponse
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.api.steering import load_steering_notes
|
||||||
from app.config import Settings, get_settings
|
from app.config import Settings, get_settings
|
||||||
from app.db import db_available, get_db
|
from app.db import db_available, get_db
|
||||||
from app.models import Document, QueryLog
|
from app.models import Document, QueryLog
|
||||||
@@ -71,9 +77,14 @@ class TurnPlan:
|
|||||||
system_prompt: str
|
system_prompt: str
|
||||||
docs: list[Document] # cited sources (weak hits when deflected)
|
docs: list[Document] # cited sources (weak hits when deflected)
|
||||||
suggestions: list[str] # "Maybe try" chips (deflected turns only)
|
suggestions: list[str] # "Maybe try" chips (deflected turns only)
|
||||||
|
tuning_count: int = 0 # steering notes injected into the system prompt
|
||||||
|
|
||||||
|
|
||||||
def plan_turn(chunks: Sequence[RetrievedChunk], settings: Settings) -> TurnPlan:
|
def plan_turn(
|
||||||
|
chunks: Sequence[RetrievedChunk],
|
||||||
|
settings: Settings,
|
||||||
|
notes: Sequence[str] | None = None,
|
||||||
|
) -> TurnPlan:
|
||||||
"""Apply the honesty gate (A8, revised) and assemble prompt + context.
|
"""Apply the honesty gate (A8, revised) and assemble prompt + context.
|
||||||
|
|
||||||
* **HIGH (grounded)** when ``best_cosine >= threshold`` **or**
|
* **HIGH (grounded)** when ``best_cosine >= threshold`` **or**
|
||||||
@@ -88,22 +99,36 @@ def plan_turn(chunks: Sequence[RetrievedChunk], settings: Settings) -> TurnPlan:
|
|||||||
``top_score`` (stored in ``query_log``) is the best cosine, so the
|
``top_score`` (stored in ``query_log``) is the best cosine, so the
|
||||||
gate input is always a pure vector-similarity number; the lexical
|
gate input is always a pure vector-similarity number; the lexical
|
||||||
signal is recorded separately as ``fts_hits``.
|
signal is recorded separately as ``fts_hits``.
|
||||||
|
|
||||||
|
*notes* are the owner's steering notes (phase 15, oldest first):
|
||||||
|
when non-empty, both the HIGH and the LOW prompt carry the
|
||||||
|
``<tuning>`` section; with no notes the prompts are unchanged.
|
||||||
"""
|
"""
|
||||||
|
steering = list(notes or [])
|
||||||
best_cosine = max((c.cosine for c in chunks), default=0.0)
|
best_cosine = max((c.cosine for c in chunks), default=0.0)
|
||||||
fts_hits = sum(1 for c in chunks if c.fts_hit)
|
fts_hits = sum(1 for c in chunks if c.fts_hit)
|
||||||
if best_cosine >= settings.relevance_threshold or fts_hits > 0:
|
if best_cosine >= settings.relevance_threshold or fts_hits > 0:
|
||||||
docs = select_documents(
|
docs = select_documents(
|
||||||
chunks, n=settings.top_n_docs, max_chars=settings.max_context_chars
|
chunks, n=settings.top_n_docs, max_chars=settings.max_context_chars
|
||||||
)
|
)
|
||||||
return TurnPlan(best_cosine, fts_hits, False, build_high_prompt(docs), docs, [])
|
return TurnPlan(
|
||||||
|
best_cosine,
|
||||||
|
fts_hits,
|
||||||
|
False,
|
||||||
|
build_high_prompt(docs, notes=steering),
|
||||||
|
docs,
|
||||||
|
[],
|
||||||
|
len(steering),
|
||||||
|
)
|
||||||
titles = weak_hit_titles(chunks)
|
titles = weak_hit_titles(chunks)
|
||||||
return TurnPlan(
|
return TurnPlan(
|
||||||
best_cosine,
|
best_cosine,
|
||||||
fts_hits,
|
fts_hits,
|
||||||
True,
|
True,
|
||||||
build_deflect_prompt(titles),
|
build_deflect_prompt(titles, notes=steering),
|
||||||
select_documents(chunks, n=settings.top_n_docs, max_chars=settings.max_context_chars),
|
select_documents(chunks, n=settings.top_n_docs, max_chars=settings.max_context_chars),
|
||||||
derive_suggestions(titles, settings.suggestions),
|
derive_suggestions(titles, settings.suggestions),
|
||||||
|
len(steering),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -150,12 +175,14 @@ async def chat(
|
|||||||
return
|
return
|
||||||
embed_ms = int((time.monotonic() - t0) * 1000)
|
embed_ms = int((time.monotonic() - t0) * 1000)
|
||||||
|
|
||||||
# 2. Retrieve top-K chunks, then the honesty gate (A8) picks the
|
# 2. Retrieve top-K chunks, load the owner's steering notes
|
||||||
# HIGH (grounded) or LOW (deflected) prompt + context.
|
# (phase 15), then the honesty gate (A8) picks the HIGH
|
||||||
|
# (grounded) or LOW (deflected) prompt + context.
|
||||||
settings = get_settings()
|
settings = get_settings()
|
||||||
try:
|
try:
|
||||||
|
steering_notes = load_steering_notes(db)
|
||||||
chunks = retrieve(db, request.message, question_vec)
|
chunks = retrieve(db, request.message, question_vec)
|
||||||
plan = plan_turn(chunks, settings)
|
plan = plan_turn(chunks, settings, notes=steering_notes)
|
||||||
except Exception: # noqa: BLE001 — DB failure mid-turn
|
except Exception: # noqa: BLE001 — DB failure mid-turn
|
||||||
logger.exception(
|
logger.exception(
|
||||||
"chat: retrieval failed question=%r total_ms=%d",
|
"chat: retrieval failed question=%r total_ms=%d",
|
||||||
@@ -211,12 +238,13 @@ async def chat(
|
|||||||
logger.exception("chat: failed to write query_log question=%r", request.message)
|
logger.exception("chat: failed to write query_log question=%r", request.message)
|
||||||
|
|
||||||
logger.info(
|
logger.info(
|
||||||
"question=%r embed_ms=%d top_score=%.3f fts_hits=%d threshold=%.2f "
|
"question=%r embed_ms=%d top_score=%.3f fts_hits=%d tuning=%d threshold=%.2f "
|
||||||
"deflected=%s sources=%r total_ms=%d",
|
"deflected=%s sources=%r total_ms=%d",
|
||||||
request.message,
|
request.message,
|
||||||
embed_ms,
|
embed_ms,
|
||||||
plan.top_score,
|
plan.top_score,
|
||||||
plan.fts_hits,
|
plan.fts_hits,
|
||||||
|
plan.tuning_count,
|
||||||
settings.relevance_threshold,
|
settings.relevance_threshold,
|
||||||
plan.deflected,
|
plan.deflected,
|
||||||
source_paths,
|
source_paths,
|
||||||
|
|||||||
@@ -0,0 +1,74 @@
|
|||||||
|
"""Steering notes API — tune how Brain answers (phase 15, story
|
||||||
|
``steering-notes``).
|
||||||
|
|
||||||
|
Stateless CRUD under ``/api/steering`` (A10): notes are owner instructions
|
||||||
|
stored in Postgres (``steering_notes``) and read into the system prompt of
|
||||||
|
**every** chat turn as the ``<tuning>`` section (see
|
||||||
|
:func:`app.rag.prompts.build_steering_section` and
|
||||||
|
:func:`app.api.chat.chat`).
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends, HTTPException, Response
|
||||||
|
from sqlalchemy import select
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.db import get_db
|
||||||
|
from app.models import SteeringNote
|
||||||
|
from app.schemas import SteeringNote as SteeringNoteOut
|
||||||
|
from app.schemas import SteeringNoteIn, SteeringNoteList
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/steering", tags=["steering"])
|
||||||
|
|
||||||
|
|
||||||
|
def load_steering_notes(db: Session) -> list[str]:
|
||||||
|
"""All steering notes, oldest first (the order they are numbered in the
|
||||||
|
``<tuning>`` prompt section). Used by the chat turn (``app.api.chat``)."""
|
||||||
|
rows = db.scalars(
|
||||||
|
select(SteeringNote).order_by(SteeringNote.created_at.asc(), SteeringNote.id.asc())
|
||||||
|
).all()
|
||||||
|
return [row.note for row in rows]
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("", response_model=SteeringNoteList)
|
||||||
|
def list_steering_notes(
|
||||||
|
db: Session = Depends(get_db), # noqa: B008
|
||||||
|
) -> SteeringNoteList:
|
||||||
|
"""All notes, newest first (the UI panel's display order)."""
|
||||||
|
rows = db.scalars(
|
||||||
|
select(SteeringNote).order_by(SteeringNote.created_at.desc(), SteeringNote.id.desc())
|
||||||
|
).all()
|
||||||
|
return SteeringNoteList(
|
||||||
|
notes=[
|
||||||
|
SteeringNoteOut(id=row.id, note=row.note, created_at=row.created_at) for row in rows
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("", response_model=SteeringNoteOut, status_code=201)
|
||||||
|
def create_steering_note(
|
||||||
|
payload: SteeringNoteIn,
|
||||||
|
db: Session = Depends(get_db), # noqa: B008
|
||||||
|
) -> SteeringNoteOut:
|
||||||
|
"""Store one tuning instruction (trimmed, 1–2000 chars — 422 otherwise)."""
|
||||||
|
row = SteeringNote(note=payload.note)
|
||||||
|
db.add(row)
|
||||||
|
db.commit()
|
||||||
|
db.refresh(row)
|
||||||
|
return SteeringNoteOut(id=row.id, note=row.note, created_at=row.created_at)
|
||||||
|
|
||||||
|
|
||||||
|
@router.delete("/{note_id}", status_code=204)
|
||||||
|
def delete_steering_note(
|
||||||
|
note_id: uuid.UUID,
|
||||||
|
db: Session = Depends(get_db), # noqa: B008
|
||||||
|
) -> Response:
|
||||||
|
"""Remove a note; 404 when the id is unknown."""
|
||||||
|
row = db.get(SteeringNote, note_id)
|
||||||
|
if row is None:
|
||||||
|
raise HTTPException(status_code=404, detail="steering note not found")
|
||||||
|
db.delete(row)
|
||||||
|
db.commit()
|
||||||
|
return Response(status_code=204)
|
||||||
@@ -58,6 +58,10 @@ class Settings(BaseSettings):
|
|||||||
chunk_target_chars: int = 2_000
|
chunk_target_chars: int = 2_000
|
||||||
chunk_overlap_chars: int = 200
|
chunk_overlap_chars: int = 200
|
||||||
embed_batch_size: int = 16
|
embed_batch_size: int = 16
|
||||||
|
#: Total char budget for the ``<tuning>`` section of the system prompt
|
||||||
|
#: (phase 15, steering notes). The newest-fitting notes are kept and the
|
||||||
|
#: overflow is replaced by the ``[…truncated…]`` marker.
|
||||||
|
steering_max_chars: int = 8_000
|
||||||
|
|
||||||
# --- Hybrid retrieval (A7, revised 2026-08-21) ---
|
# --- Hybrid retrieval (A7, revised 2026-08-21) ---
|
||||||
# cosine top-N ∪ Postgres FTS top-N, fused with Reciprocal Rank Fusion
|
# cosine top-N ∪ Postgres FTS top-N, fused with Reciprocal Rank Fusion
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ from fastapi.staticfiles import StaticFiles
|
|||||||
from app.api.chat import router as chat_router
|
from app.api.chat import router as chat_router
|
||||||
from app.api.docs import router as docs_router
|
from app.api.docs import router as docs_router
|
||||||
from app.api.health import router as health_router
|
from app.api.health import router as health_router
|
||||||
|
from app.api.steering import router as steering_router
|
||||||
from app.api.suggestions import router as suggestions_router
|
from app.api.suggestions import router as suggestions_router
|
||||||
from app.config import get_settings
|
from app.config import get_settings
|
||||||
from app.core.debugging import configure_debugging
|
from app.core.debugging import configure_debugging
|
||||||
@@ -36,6 +37,7 @@ def create_app() -> FastAPI:
|
|||||||
app.include_router(suggestions_router, prefix="/api")
|
app.include_router(suggestions_router, prefix="/api")
|
||||||
app.include_router(docs_router, prefix="/api")
|
app.include_router(docs_router, prefix="/api")
|
||||||
app.include_router(chat_router, prefix="/api")
|
app.include_router(chat_router, prefix="/api")
|
||||||
|
app.include_router(steering_router, prefix="/api")
|
||||||
|
|
||||||
static_dir = Path(settings.static_dir).resolve()
|
static_dir = Path(settings.static_dir).resolve()
|
||||||
if static_dir.is_dir():
|
if static_dir.is_dir():
|
||||||
|
|||||||
+19
-2
@@ -6,8 +6,10 @@ Data model — see ``.agent/PLAN.md`` §Data Model:
|
|||||||
* ``chunks`` — retrieval units; each chunk points at its parent document
|
* ``chunks`` — retrieval units; each chunk points at its parent document
|
||||||
via ``document_id``. This is how an embedding maps back to
|
via ``document_id``. This is how an embedding maps back to
|
||||||
a document path (the "feed the whole document" requirement).
|
a document path (the "feed the whole document" requirement).
|
||||||
* ``query_log`` — observability: every question, its retrieval score, the
|
* ``query_log`` — observability: every question, its retrieval score,
|
||||||
deflection decision, and latency.
|
the deflection decision, and latency.
|
||||||
|
* ``steering_notes`` — owner tuning notes injected into the system prompt
|
||||||
|
of every chat turn (phase 15, ``<tuning>`` section).
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
@@ -82,3 +84,18 @@ class QueryLog(Base):
|
|||||||
sources: Mapped[str] = mapped_column(Text, default="") # comma-joined source paths
|
sources: Mapped[str] = mapped_column(Text, default="") # comma-joined source paths
|
||||||
latency_ms: Mapped[int] = mapped_column(Integer, default=0)
|
latency_ms: Mapped[int] = mapped_column(Integer, default=0)
|
||||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||||
|
|
||||||
|
|
||||||
|
class SteeringNote(Base):
|
||||||
|
"""One owner tuning instruction (phase 15).
|
||||||
|
|
||||||
|
Notes are read into the system prompt of **every** chat turn as the
|
||||||
|
``<tuning>`` section (oldest first, char-budgeted — see
|
||||||
|
:func:`app.rag.prompts.build_steering_section`).
|
||||||
|
"""
|
||||||
|
|
||||||
|
__tablename__ = "steering_notes"
|
||||||
|
|
||||||
|
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||||
|
note: Mapped[str] = mapped_column(Text) # trimmed, 1–2000 chars (API-enforced)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||||
|
|||||||
+71
-9
@@ -1,24 +1,35 @@
|
|||||||
"""Locked system-prompt builder (PLAN §6).
|
"""Locked system-prompt builder (PLAN §6).
|
||||||
|
|
||||||
The persona + HONESTY GATE text is **locked verbatim** — change it through
|
The persona + HONESTY GATE text is **locked verbatim** — change it through
|
||||||
the plan, not here. Two modes:
|
the plan, not here. (PLAN §6 revision, 2026-08-22: the owner's working-tree
|
||||||
|
persona edits are preserved — no mandated ``"you've got this"`` tagline and
|
||||||
|
no mandated deflection opening; the honesty gate itself is unchanged.)
|
||||||
|
|
||||||
|
Two modes:
|
||||||
|
|
||||||
* ``HIGH`` — grounded turn: full top-document texts under ``<documents>``.
|
* ``HIGH`` — grounded turn: full top-document texts under ``<documents>``.
|
||||||
* ``LOW`` — deflection turn: weak-hit *titles only* plus the
|
* ``LOW`` — deflection turn: weak-hit *titles only* plus the
|
||||||
``DEFLECT_MODE`` marker (the E2E mock LLM keys on that marker).
|
``DEFLECT_MODE`` marker (the E2E mock LLM keys on that marker).
|
||||||
|
|
||||||
|
Steering (phase 15): when the owner has stored tuning notes, both modes
|
||||||
|
carry a ``<tuning>`` section between ``<relevance>…</relevance>`` and the
|
||||||
|
mode body. With zero notes the prompt is byte-identical to the
|
||||||
|
pre-steering text.
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from collections.abc import Sequence
|
from collections.abc import Sequence
|
||||||
|
|
||||||
|
from app.config import get_settings
|
||||||
from app.models import Document
|
from app.models import Document
|
||||||
|
from app.rag.retriever import TRUNCATION_MARKER
|
||||||
|
|
||||||
#: PLAN §6 verbatim (line wrapping included); ``{relevance}`` is filled by
|
#: PLAN §6 verbatim (line wrapping included); ``{relevance}`` is filled by
|
||||||
#: :func:`_base`.
|
#: :func:`_base`.
|
||||||
PERSONA: str = (
|
PERSONA: str = (
|
||||||
'You are "Brain of Reese" — the digital brain of Reese, a self-hoster and\n'
|
'You are "Brain of Reese" — the digital brain of Reese, a self-hoster and\n'
|
||||||
"homelab tinkerer. Personality: chippy, upbeat, warm, and genuinely\n"
|
"homelab tinkerer. Personality: chippy, upbeat, warm, and genuinely\n"
|
||||||
'optimistic about the user\'s ability to do things ("you\'ve got this").\n'
|
"optimistic about the user's ability to do things.\n"
|
||||||
"\n"
|
"\n"
|
||||||
"Rules:\n"
|
"Rules:\n"
|
||||||
"1. Answer ONLY from the provided document context. Cite which document(s)\n"
|
"1. Answer ONLY from the provided document context. Cite which document(s)\n"
|
||||||
@@ -26,14 +37,20 @@ PERSONA: str = (
|
|||||||
"2. Be concrete: names, versions, ports, hosts, schedules — the specifics in\n"
|
"2. Be concrete: names, versions, ports, hosts, schedules — the specifics in\n"
|
||||||
" the docs are the value.\n"
|
" the docs are the value.\n"
|
||||||
'3. HONESTY GATE: if <relevance> is "LOW", you must NOT pretend to know.\n'
|
'3. HONESTY GATE: if <relevance> is "LOW", you must NOT pretend to know.\n'
|
||||||
' Start your answer with a variant of: "I haven\'t done anything like that."\n'
|
" Offer 2-3 alternative questions about things you DO have notes on.\n"
|
||||||
" Then offer 2-3 alternative questions about things you DO have notes on.\n"
|
|
||||||
"4. Never invent facts, hosts, or steps that are not in the context.\n"
|
"4. Never invent facts, hosts, or steps that are not in the context.\n"
|
||||||
"5. Keep answers tight: short paragraphs, bullets where helpful.\n"
|
"5. Keep answers tight: short paragraphs, bullets where helpful.\n"
|
||||||
"\n"
|
"\n"
|
||||||
"<relevance>{relevance}</relevance>"
|
"<relevance>{relevance}</relevance>"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
#: One-line intro of the ``<tuning>`` section (phase 15): the owner's notes
|
||||||
|
#: steer the answer and win over the defaults when they conflict.
|
||||||
|
_STEERING_INTRO = (
|
||||||
|
"The owner of this brain asked you to steer your answers as follows. "
|
||||||
|
"Where these instructions conflict with the defaults above, follow the owner:\n"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _base(relevance: str) -> str:
|
def _base(relevance: str) -> str:
|
||||||
if relevance not in ("HIGH", "LOW"):
|
if relevance not in ("HIGH", "LOW"):
|
||||||
@@ -41,8 +58,46 @@ def _base(relevance: str) -> str:
|
|||||||
return PERSONA.replace("{relevance}", relevance)
|
return PERSONA.replace("{relevance}", relevance)
|
||||||
|
|
||||||
|
|
||||||
def build_high_prompt(documents: Sequence[Document]) -> str:
|
def build_steering_section(notes: Sequence[str], max_chars: int | None = None) -> str:
|
||||||
"""Grounded turn: locked persona + full texts of the top documents."""
|
"""The ``<tuning>`` section of the system prompt (phase 15).
|
||||||
|
|
||||||
|
* No notes (or only blank ones) → ``""`` — callers then build the
|
||||||
|
prompt exactly as before, so a zero-note prompt is byte-identical to
|
||||||
|
the pre-steering text.
|
||||||
|
* Otherwise: numbered notes (in the given order — the chat turn passes
|
||||||
|
them oldest-first, so #1 is the oldest note) capped at *max_chars*
|
||||||
|
(default ``BOR_STEERING_MAX_CHARS``). When the budget cannot hold
|
||||||
|
every note, the oldest-fitting prefix is kept and the overflow is
|
||||||
|
replaced by the shared ``[…truncated…]`` marker.
|
||||||
|
"""
|
||||||
|
cleaned = [str(n).strip() for n in notes]
|
||||||
|
cleaned = [n for n in cleaned if n]
|
||||||
|
if not cleaned:
|
||||||
|
return ""
|
||||||
|
limit = max_chars if max_chars is not None else get_settings().steering_max_chars
|
||||||
|
if limit <= 0:
|
||||||
|
return ""
|
||||||
|
|
||||||
|
def render(count: int) -> str:
|
||||||
|
lines = [f"{i}. {note}" for i, note in enumerate(cleaned[:count], start=1)]
|
||||||
|
if count < len(cleaned):
|
||||||
|
lines.append(TRUNCATION_MARKER)
|
||||||
|
return f"<tuning>\n{_STEERING_INTRO}" + "\n".join(lines) + "\n</tuning>"
|
||||||
|
|
||||||
|
for count in range(len(cleaned), 0, -1):
|
||||||
|
rendered = render(count)
|
||||||
|
if len(rendered) <= limit:
|
||||||
|
return rendered
|
||||||
|
# Pathological budget: not even the empty note list fits. The section
|
||||||
|
# must still respect the cap — the bare marker when it fits, else none.
|
||||||
|
if len(TRUNCATION_MARKER) <= limit:
|
||||||
|
return TRUNCATION_MARKER
|
||||||
|
return ""
|
||||||
|
|
||||||
|
|
||||||
|
def build_high_prompt(documents: Sequence[Document], notes: Sequence[str] | None = None) -> str:
|
||||||
|
"""Grounded turn: locked persona (+ steering) + full texts of the top
|
||||||
|
documents."""
|
||||||
blocks = [
|
blocks = [
|
||||||
f'<document source="{doc.source}" path="{doc.path}" title="{doc.title}">\n'
|
f'<document source="{doc.source}" path="{doc.path}" title="{doc.title}">\n'
|
||||||
f"{doc.content}\n"
|
f"{doc.content}\n"
|
||||||
@@ -52,15 +107,22 @@ def build_high_prompt(documents: Sequence[Document]) -> str:
|
|||||||
body = "\n\n".join(blocks) if blocks else (
|
body = "\n\n".join(blocks) if blocks else (
|
||||||
"(no documents matched — do not invent specifics)"
|
"(no documents matched — do not invent specifics)"
|
||||||
)
|
)
|
||||||
return _base("HIGH") + "\n<documents>\n" + body + "\n</documents>"
|
section = build_steering_section(notes or [])
|
||||||
|
prompt = _base("HIGH")
|
||||||
|
if section:
|
||||||
|
prompt += "\n" + section
|
||||||
|
return prompt + "\n<documents>\n" + body + "\n</documents>"
|
||||||
|
|
||||||
|
|
||||||
def build_deflect_prompt(titles: Sequence[str]) -> str:
|
def build_deflect_prompt(titles: Sequence[str], notes: Sequence[str] | None = None) -> str:
|
||||||
"""Deflection turn: weak-hit titles only (no document content)."""
|
"""Deflection turn: weak-hit titles only (no document content)."""
|
||||||
weak = "\n".join(f"- {t}" for t in titles) if titles else "(nothing close at all)"
|
weak = "\n".join(f"- {t}" for t in titles) if titles else "(nothing close at all)"
|
||||||
|
section = build_steering_section(notes or [])
|
||||||
|
mid = f"\n{section}\n" if section else "\n"
|
||||||
return (
|
return (
|
||||||
_base("LOW")
|
_base("LOW")
|
||||||
+ "\nDEFLECT_MODE: retrieval was weak — the titles below are the closest "
|
+ mid
|
||||||
|
+ "DEFLECT_MODE: retrieval was weak — the titles below are the closest "
|
||||||
"your notes come to the question. They are titles only; do not pretend "
|
"your notes come to the question. They are titles only; do not pretend "
|
||||||
"they answer it. Use them to propose 2-3 alternative questions.\n"
|
"they answer it. Use them to propose 2-3 alternative questions.\n"
|
||||||
+ weak
|
+ weak
|
||||||
|
|||||||
+34
-1
@@ -1,7 +1,10 @@
|
|||||||
"""Pydantic request/response schemas (API contract)."""
|
"""Pydantic request/response schemas (API contract)."""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from pydantic import BaseModel, Field
|
import uuid
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from pydantic import BaseModel, Field, field_validator
|
||||||
|
|
||||||
|
|
||||||
class HealthResponse(BaseModel):
|
class HealthResponse(BaseModel):
|
||||||
@@ -73,3 +76,33 @@ class DocContent(BaseModel):
|
|||||||
content: str
|
content: str
|
||||||
indexed_at: str
|
indexed_at: str
|
||||||
chunks: int
|
chunks: int
|
||||||
|
|
||||||
|
|
||||||
|
class SteeringNoteIn(BaseModel):
|
||||||
|
"""``POST /api/steering`` body: one tuning instruction (phase 15).
|
||||||
|
|
||||||
|
The note is trimmed *before* the length constraints run, so a
|
||||||
|
whitespace-only body is a 422 and a 2000-char note with surrounding
|
||||||
|
spaces still passes.
|
||||||
|
"""
|
||||||
|
|
||||||
|
note: str = Field(min_length=1, max_length=2000)
|
||||||
|
|
||||||
|
@field_validator("note", mode="before")
|
||||||
|
@classmethod
|
||||||
|
def _trim_note(cls, v: object) -> object:
|
||||||
|
return v.strip() if isinstance(v, str) else v
|
||||||
|
|
||||||
|
|
||||||
|
class SteeringNote(BaseModel):
|
||||||
|
"""One stored steering note (API shape — ISO-8601 ``created_at``)."""
|
||||||
|
|
||||||
|
id: uuid.UUID
|
||||||
|
note: str
|
||||||
|
created_at: datetime
|
||||||
|
|
||||||
|
|
||||||
|
class SteeringNoteList(BaseModel):
|
||||||
|
"""``GET /api/steering`` response: all notes, newest first."""
|
||||||
|
|
||||||
|
notes: list[SteeringNote]
|
||||||
|
|||||||
+212
-1
@@ -28,6 +28,16 @@
|
|||||||
* "New chat" (#new-chat-btn) clears the key + the list back to the empty
|
* "New chat" (#new-chat-btn) clears the key + the list back to the empty
|
||||||
* state.
|
* state.
|
||||||
*
|
*
|
||||||
|
* Steering notes (phase 15) let the owner tune how Brain answers: a
|
||||||
|
* "Tune" button under every completed brain bubble (deflected included)
|
||||||
|
* opens an inline form → POST /api/steering → the note is stored in
|
||||||
|
* Postgres and injected into the system prompt of every subsequent turn
|
||||||
|
* (the <tuning> section). Notes are listed newest-first in the header
|
||||||
|
* "Tuning" panel (#steering-panel), where each can be deleted. Note text
|
||||||
|
* is always rendered with textContent (XSS-safe), save/delete are
|
||||||
|
* announced through a polite live region (#steering-announcer), and the
|
||||||
|
* panel + count badge update on every change.
|
||||||
|
*
|
||||||
* All DOM ids match frontend/index.html.
|
* All DOM ids match frontend/index.html.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
@@ -90,6 +100,203 @@ export function documentUrl(source, path, back = "/") {
|
|||||||
return url;
|
return url;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* ---------- steering notes (phase 15) ----------
|
||||||
|
*
|
||||||
|
* The owner's tuning notes steer every future answer: they live in
|
||||||
|
* Postgres (stateless API, A10) and the chat turn reads them into the
|
||||||
|
* system prompt. UI contract: Tune button → inline form → save →
|
||||||
|
* confirmation (or inline error, form kept); the header panel lists the
|
||||||
|
* notes (newest first) with per-note delete.
|
||||||
|
*/
|
||||||
|
const steeringToggle = document.querySelector("#steering-toggle");
|
||||||
|
const steeringCount = document.querySelector("#steering-count");
|
||||||
|
const steeringPanel = document.querySelector("#steering-panel");
|
||||||
|
const steeringList = document.querySelector("#steering-list");
|
||||||
|
const steeringEmpty = document.querySelector("#steering-empty");
|
||||||
|
const steeringAnnouncer = document.querySelector("#steering-announcer");
|
||||||
|
|
||||||
|
const TUNE_ICON =
|
||||||
|
'<svg aria-hidden="true" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.7" stroke-linecap="round"><path d="M4 7h10M18 7h2M4 17h4M12 17h8"/><circle cx="15.5" cy="7" r="2.2"/><circle cx="9.5" cy="17" r="2.2"/></svg>';
|
||||||
|
|
||||||
|
let tuneSeq = 0; // unique ids for one open tune form's inputs
|
||||||
|
|
||||||
|
function announceSteering(message) {
|
||||||
|
if (steeringAnnouncer) steeringAnnouncer.textContent = message;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* "Tune" button in the meta row of a completed brain bubble. Reuses the
|
||||||
|
sources' .msg-meta row when it exists (role=list → the button joins as
|
||||||
|
a listitem so ARIA stays valid); otherwise creates a plain meta row. */
|
||||||
|
function appendTuneButton(wrap) {
|
||||||
|
const body = wrap.querySelector(".msg-body");
|
||||||
|
if (!body) return;
|
||||||
|
let meta = body.querySelector(".msg-meta");
|
||||||
|
if (!meta) {
|
||||||
|
meta = document.createElement("div");
|
||||||
|
meta.className = "msg-meta";
|
||||||
|
body.appendChild(meta);
|
||||||
|
}
|
||||||
|
if (meta.querySelector(".tune-btn")) return; // one per bubble
|
||||||
|
const btn = document.createElement("button");
|
||||||
|
btn.type = "button";
|
||||||
|
btn.className = "tune-btn";
|
||||||
|
if (meta.getAttribute("role") === "list") btn.setAttribute("role", "listitem");
|
||||||
|
btn.innerHTML = TUNE_ICON + "<span>Tune</span>";
|
||||||
|
btn.addEventListener("click", () => openTuneForm(wrap, btn));
|
||||||
|
meta.appendChild(btn);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Inline tuning form under the bubble: labeled textarea (maxlength 2000)
|
||||||
|
+ Save / Cancel. Success replaces the form with the .tune-saved status
|
||||||
|
(role=status); failure keeps the form and shows an inline error
|
||||||
|
(role=alert) — the note is never lost on a failed save. */
|
||||||
|
function openTuneForm(wrap, toggleBtn) {
|
||||||
|
document.querySelectorAll(".tune-form").forEach((f) => f.remove()); // one at a time
|
||||||
|
const body = wrap.querySelector(".msg-body");
|
||||||
|
if (!body) return;
|
||||||
|
tuneSeq += 1;
|
||||||
|
const inputId = `tune-input-${tuneSeq}`;
|
||||||
|
const form = document.createElement("form");
|
||||||
|
form.className = "tune-form";
|
||||||
|
form.noValidate = true;
|
||||||
|
form.innerHTML =
|
||||||
|
`<label for="${inputId}">Tuning note — how should Brain answer from now on?</label>` +
|
||||||
|
`<textarea id="${inputId}" name="note" rows="2" maxlength="2000"
|
||||||
|
placeholder="e.g. be more concise — or: assume I'm on NixOS"></textarea>`;
|
||||||
|
const actions = document.createElement("div");
|
||||||
|
actions.className = "tune-form-actions";
|
||||||
|
const saveBtn = document.createElement("button");
|
||||||
|
saveBtn.type = "submit";
|
||||||
|
saveBtn.className = "tune-save";
|
||||||
|
saveBtn.textContent = "Save";
|
||||||
|
const cancelBtn = document.createElement("button");
|
||||||
|
cancelBtn.type = "button";
|
||||||
|
cancelBtn.className = "tune-cancel";
|
||||||
|
cancelBtn.textContent = "Cancel";
|
||||||
|
actions.append(saveBtn, cancelBtn);
|
||||||
|
form.appendChild(actions);
|
||||||
|
const status = document.createElement("p");
|
||||||
|
status.className = "tune-error";
|
||||||
|
status.setAttribute("role", "alert");
|
||||||
|
status.hidden = true;
|
||||||
|
form.appendChild(status);
|
||||||
|
|
||||||
|
form.addEventListener("submit", async (e) => {
|
||||||
|
e.preventDefault();
|
||||||
|
saveBtn.disabled = true;
|
||||||
|
status.hidden = true;
|
||||||
|
try {
|
||||||
|
const r = await fetch("/api/steering", {
|
||||||
|
method: "POST",
|
||||||
|
headers: { "Content-Type": "application/json" },
|
||||||
|
body: JSON.stringify({ note: form.querySelector("textarea").value }),
|
||||||
|
});
|
||||||
|
if (!r.ok) {
|
||||||
|
let detail = "Could not save the note — try again.";
|
||||||
|
try {
|
||||||
|
const data = await r.json();
|
||||||
|
if (Array.isArray(data.detail) && data.detail[0] && data.detail[0].msg) {
|
||||||
|
detail = String(data.detail[0].msg);
|
||||||
|
} else if (typeof data.detail === "string" && data.detail) {
|
||||||
|
detail = data.detail;
|
||||||
|
}
|
||||||
|
} catch { /* non-JSON error body */ }
|
||||||
|
status.textContent = detail;
|
||||||
|
status.hidden = false;
|
||||||
|
saveBtn.disabled = false;
|
||||||
|
return; // form kept on failure — the instruction survives
|
||||||
|
}
|
||||||
|
const saved = document.createElement("p");
|
||||||
|
saved.className = "tune-saved";
|
||||||
|
saved.setAttribute("role", "status");
|
||||||
|
saved.textContent = "Saved — future answers will follow this.";
|
||||||
|
form.replaceWith(saved);
|
||||||
|
announceSteering("Tuning note saved. Future answers will follow it.");
|
||||||
|
await loadSteering(); // panel + count badge update
|
||||||
|
} catch {
|
||||||
|
status.textContent = "Could not save the note — is the app reachable?";
|
||||||
|
status.hidden = false;
|
||||||
|
saveBtn.disabled = false;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
cancelBtn.addEventListener("click", () => {
|
||||||
|
form.remove();
|
||||||
|
toggleBtn.focus();
|
||||||
|
});
|
||||||
|
body.appendChild(form);
|
||||||
|
form.querySelector("textarea").focus();
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Panel: newest-first list (textContent — XSS-safe), per-note delete,
|
||||||
|
empty text, and the header count badge. */
|
||||||
|
async function loadSteering() {
|
||||||
|
let notes = [];
|
||||||
|
try {
|
||||||
|
const r = await fetch("/api/steering");
|
||||||
|
if (r.ok) notes = (await r.json()).notes || [];
|
||||||
|
} catch { /* API unreachable: keep the last rendered list */ }
|
||||||
|
renderSteeringPanel(notes);
|
||||||
|
return notes;
|
||||||
|
}
|
||||||
|
|
||||||
|
function renderSteeringPanel(notes) {
|
||||||
|
if (!steeringList) return;
|
||||||
|
steeringList.textContent = "";
|
||||||
|
for (const n of notes) {
|
||||||
|
const li = document.createElement("li");
|
||||||
|
li.className = "steering-note";
|
||||||
|
const text = document.createElement("span");
|
||||||
|
text.className = "steering-note-text";
|
||||||
|
text.textContent = n.note; // rendered as text, never as HTML
|
||||||
|
li.appendChild(text);
|
||||||
|
const del = document.createElement("button");
|
||||||
|
del.type = "button";
|
||||||
|
del.className = "steering-delete";
|
||||||
|
del.setAttribute("aria-label", `Delete tuning note: ${n.note}`);
|
||||||
|
del.innerHTML =
|
||||||
|
'<svg aria-hidden="true" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.8" stroke-linecap="round"><path d="M5 7h14M10 7V5h4v2M8.5 7l.7 12h5.6l.7-12"/></svg>';
|
||||||
|
del.addEventListener("click", () => deleteSteeringNote(n.id, del));
|
||||||
|
li.appendChild(del);
|
||||||
|
steeringList.appendChild(li);
|
||||||
|
}
|
||||||
|
if (steeringEmpty) steeringEmpty.hidden = notes.length > 0;
|
||||||
|
if (steeringCount) steeringCount.textContent = String(notes.length);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function deleteSteeringNote(id, btn) {
|
||||||
|
btn.disabled = true;
|
||||||
|
try {
|
||||||
|
const r = await fetch(`/api/steering/${encodeURIComponent(id)}`, { method: "DELETE" });
|
||||||
|
if (r.status === 404) {
|
||||||
|
announceSteering("That note was already removed.");
|
||||||
|
await loadSteering();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!r.ok) {
|
||||||
|
announceSteering("Could not delete the note — try again.");
|
||||||
|
btn.disabled = false;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
await loadSteering();
|
||||||
|
announceSteering("Tuning note deleted.");
|
||||||
|
} catch {
|
||||||
|
announceSteering("Could not delete the note — is the app reachable?");
|
||||||
|
btn.disabled = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function setSteeringPanel(open) {
|
||||||
|
if (!steeringPanel || !steeringToggle) return;
|
||||||
|
steeringPanel.hidden = !open;
|
||||||
|
steeringToggle.setAttribute("aria-expanded", open ? "true" : "false");
|
||||||
|
}
|
||||||
|
if (steeringToggle && steeringPanel) {
|
||||||
|
steeringToggle.addEventListener("click", () => {
|
||||||
|
setSteeringPanel(steeringPanel.hidden);
|
||||||
|
if (!steeringPanel.hidden) loadSteering(); // refresh when (re)opened
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
/* ---------- avatar glyphs (phase 08: emoji-free chrome) ----------
|
/* ---------- avatar glyphs (phase 08: emoji-free chrome) ----------
|
||||||
* Inline SVG as string constants so the message renderer and the typing
|
* Inline SVG as string constants so the message renderer and the typing
|
||||||
* indicator share exactly the same marks. currentColor lets the CSS theme
|
* indicator share exactly the same marks. currentColor lets the CSS theme
|
||||||
@@ -426,6 +633,7 @@ function renderStoredMessage(m) {
|
|||||||
appendMaybeTry(wrap, m.suggestions);
|
appendMaybeTry(wrap, m.suggestions);
|
||||||
}
|
}
|
||||||
appendSources(wrap, m.sources);
|
appendSources(wrap, m.sources);
|
||||||
|
appendTuneButton(wrap); // restored brain answers are tunable too
|
||||||
}
|
}
|
||||||
|
|
||||||
/* On load: re-render the stored conversation (markdown, source chips,
|
/* On load: re-render the stored conversation (markdown, source chips,
|
||||||
@@ -545,6 +753,7 @@ async function handleSend(e) {
|
|||||||
appendMaybeTry(wrap, ev.suggestions);
|
appendMaybeTry(wrap, ev.suggestions);
|
||||||
}
|
}
|
||||||
appendSources(wrap, ev.sources);
|
appendSources(wrap, ev.sources);
|
||||||
|
appendTuneButton(wrap); // every completed brain bubble is tunable
|
||||||
// Persistence save point 2: the answer lands only when the turn is
|
// Persistence save point 2: the answer lands only when the turn is
|
||||||
// complete (raw text + the done metadata).
|
// complete (raw text + the done metadata).
|
||||||
rememberBrainTurn(acc, {
|
rememberBrainTurn(acc, {
|
||||||
@@ -558,7 +767,8 @@ async function handleSend(e) {
|
|||||||
});
|
});
|
||||||
if (!aborted && !wrap) {
|
if (!aborted && !wrap) {
|
||||||
const fallback = "Hmm — that came back empty. Ask me again?";
|
const fallback = "Hmm — that came back empty. Ask me again?";
|
||||||
addMessage("brain", fallback);
|
const fwrap = addMessage("brain", fallback);
|
||||||
|
appendTuneButton(fwrap);
|
||||||
rememberBrainTurn(fallback, {}); // persist what the user actually saw
|
rememberBrainTurn(fallback, {}); // persist what the user actually saw
|
||||||
}
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
@@ -592,3 +802,4 @@ composer.addEventListener("submit", handleSend);
|
|||||||
restoreConversation(); // phase 14: the conversation comes back as left
|
restoreConversation(); // phase 14: the conversation comes back as left
|
||||||
loadSuggestions();
|
loadSuggestions();
|
||||||
loadHealth();
|
loadHealth();
|
||||||
|
loadSteering(); // phase 15: tuning notes (panel + count badge)
|
||||||
|
|||||||
@@ -241,6 +241,48 @@ body::after {
|
|||||||
whole control below 640px. */
|
whole control below 640px. */
|
||||||
.new-chat-btn svg { width: 16px; height: 16px; display: none; }
|
.new-chat-btn svg { width: 16px; height: 16px; display: none; }
|
||||||
|
|
||||||
|
/* "Tuning" toggle (phase 15): ghost pill like New chat + a mono count
|
||||||
|
badge (brand-ink on brand-soft ≈6.9:1). The label is visually-hidden
|
||||||
|
(not removed) below 640px so the accessible name keeps the word.
|
||||||
|
≥44px touch target at every width. */
|
||||||
|
.steering-toggle {
|
||||||
|
display: inline-flex;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: center;
|
||||||
|
gap: 0.4rem;
|
||||||
|
min-height: 44px;
|
||||||
|
padding: 0.5rem 0.9rem;
|
||||||
|
border-radius: 999px;
|
||||||
|
border: 1px solid var(--line);
|
||||||
|
background: transparent;
|
||||||
|
color: var(--ink-soft);
|
||||||
|
font: inherit;
|
||||||
|
font-weight: 600;
|
||||||
|
font-size: 0.95rem;
|
||||||
|
white-space: nowrap;
|
||||||
|
cursor: pointer;
|
||||||
|
}
|
||||||
|
.steering-toggle:hover, .steering-toggle[aria-expanded="true"] {
|
||||||
|
background: var(--brand-soft);
|
||||||
|
color: var(--brand-ink);
|
||||||
|
}
|
||||||
|
.steering-toggle svg { width: 16px; height: 16px; display: block; }
|
||||||
|
.steering-count {
|
||||||
|
font-family: var(--mono);
|
||||||
|
font-size: 0.78rem;
|
||||||
|
font-weight: 700;
|
||||||
|
min-width: 1.35rem;
|
||||||
|
text-align: center;
|
||||||
|
padding: 0.05rem 0.4rem;
|
||||||
|
border-radius: 999px;
|
||||||
|
background: var(--brand-soft);
|
||||||
|
color: var(--brand-ink);
|
||||||
|
}
|
||||||
|
.steering-toggle[aria-expanded="true"] .steering-count {
|
||||||
|
background: var(--brand);
|
||||||
|
color: var(--bg); /* dark ink on brand: 5.2:1 */
|
||||||
|
}
|
||||||
|
|
||||||
/* ---------- Main frame ---------- */
|
/* ---------- Main frame ---------- */
|
||||||
.app-main {
|
.app-main {
|
||||||
flex: 1;
|
flex: 1;
|
||||||
@@ -381,6 +423,170 @@ body::after {
|
|||||||
max-width: 100%;
|
max-width: 100%;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* ---------- Steering notes (phase 15) ---------- */
|
||||||
|
/* "Tune" button in the meta row of every completed brain bubble: ghost
|
||||||
|
pill, ≥44px, right-aligned after the source chips. ink-soft on surface
|
||||||
|
≈6.9:1; hover pair brand-ink/brand-soft ≈6.9:1. */
|
||||||
|
.tune-btn {
|
||||||
|
display: inline-flex;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: center;
|
||||||
|
gap: 0.35rem;
|
||||||
|
min-height: 44px;
|
||||||
|
margin-left: auto;
|
||||||
|
padding: 0.35rem 0.8rem;
|
||||||
|
border-radius: 999px;
|
||||||
|
border: 1px solid var(--line);
|
||||||
|
background: transparent;
|
||||||
|
color: var(--ink-soft);
|
||||||
|
font: inherit;
|
||||||
|
font-weight: 600;
|
||||||
|
font-size: 0.82rem;
|
||||||
|
white-space: nowrap;
|
||||||
|
cursor: pointer;
|
||||||
|
}
|
||||||
|
.tune-btn svg { width: 14px; height: 14px; display: block; }
|
||||||
|
.tune-btn:hover { background: var(--brand-soft); color: var(--brand-ink); }
|
||||||
|
|
||||||
|
/* Inline tuning form under the bubble: labeled textarea + Save/Cancel
|
||||||
|
(both ≥44px). Save = brand button (dark ink 5.2:1), Cancel = ghost. */
|
||||||
|
.tune-form {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 0.5rem;
|
||||||
|
background: var(--surface);
|
||||||
|
border: 1px solid var(--brand-soft);
|
||||||
|
border-radius: var(--radius-sm);
|
||||||
|
padding: 0.75rem 0.85rem;
|
||||||
|
}
|
||||||
|
.tune-form label {
|
||||||
|
font-size: 0.85rem;
|
||||||
|
font-weight: 600;
|
||||||
|
color: var(--ink-soft);
|
||||||
|
}
|
||||||
|
.tune-form textarea {
|
||||||
|
font: inherit;
|
||||||
|
font-size: 0.9rem;
|
||||||
|
color: var(--ink);
|
||||||
|
background: #0d1120;
|
||||||
|
border: 1px solid var(--line);
|
||||||
|
border-radius: var(--radius-sm);
|
||||||
|
padding: 0.5rem 0.6rem;
|
||||||
|
resize: vertical;
|
||||||
|
min-height: 2.6rem;
|
||||||
|
}
|
||||||
|
.tune-form-actions { display: flex; gap: 0.5rem; }
|
||||||
|
.tune-save {
|
||||||
|
display: inline-flex;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: center;
|
||||||
|
min-height: 44px;
|
||||||
|
padding: 0.4rem 1.1rem;
|
||||||
|
border: 0;
|
||||||
|
border-radius: var(--radius-sm);
|
||||||
|
background: var(--brand);
|
||||||
|
color: var(--bg);
|
||||||
|
font: inherit;
|
||||||
|
font-weight: 700;
|
||||||
|
cursor: pointer;
|
||||||
|
}
|
||||||
|
.tune-save:hover:not(:disabled) { background: #7d88f5; }
|
||||||
|
.tune-save:disabled { opacity: 0.6; cursor: wait; }
|
||||||
|
.tune-cancel {
|
||||||
|
display: inline-flex;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: center;
|
||||||
|
min-height: 44px;
|
||||||
|
padding: 0.4rem 1rem;
|
||||||
|
border-radius: var(--radius-sm);
|
||||||
|
border: 1px solid var(--line);
|
||||||
|
background: transparent;
|
||||||
|
color: var(--ink-soft);
|
||||||
|
font: inherit;
|
||||||
|
font-weight: 600;
|
||||||
|
cursor: pointer;
|
||||||
|
}
|
||||||
|
.tune-cancel:hover { background: var(--brand-soft); color: var(--brand-ink); }
|
||||||
|
/* Save confirmation (role=status): ok pair ≈10.6:1. */
|
||||||
|
.tune-saved {
|
||||||
|
margin: 0.2rem 0 0 0.25rem;
|
||||||
|
background: var(--ok-bg);
|
||||||
|
color: var(--ok-ink);
|
||||||
|
border: 1px solid rgb(110 231 168 / 0.35);
|
||||||
|
border-radius: var(--radius-sm);
|
||||||
|
padding: 0.45rem 0.8rem;
|
||||||
|
font-size: 0.85rem;
|
||||||
|
font-weight: 600;
|
||||||
|
}
|
||||||
|
/* Inline save failure (role=alert): err pair ≈9.1:1 — form is kept. */
|
||||||
|
.tune-error {
|
||||||
|
margin: 0 0 0 0.25rem;
|
||||||
|
background: var(--err-bg);
|
||||||
|
color: var(--err-ink);
|
||||||
|
border: 1px solid var(--err-line);
|
||||||
|
border-radius: var(--radius-sm);
|
||||||
|
padding: 0.45rem 0.8rem;
|
||||||
|
font-size: 0.85rem;
|
||||||
|
font-weight: 600;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Header panel above the messages: notes newest-first, per-note delete,
|
||||||
|
designed empty state. */
|
||||||
|
.steering-panel {
|
||||||
|
background: var(--surface);
|
||||||
|
border: 1px solid var(--brand-soft);
|
||||||
|
border-radius: var(--radius);
|
||||||
|
box-shadow: var(--shadow);
|
||||||
|
padding: 0.9rem 1.1rem 1rem;
|
||||||
|
}
|
||||||
|
.steering-panel-head { display: flex; flex-wrap: wrap; align-items: baseline; gap: 0.15rem 0.6rem; }
|
||||||
|
.steering-panel-title { margin: 0; font-size: 1rem; font-weight: 700; color: var(--ink); }
|
||||||
|
.steering-panel-sub { margin: 0; font-size: 0.82rem; color: var(--ink-soft); }
|
||||||
|
.steering-list {
|
||||||
|
list-style: none;
|
||||||
|
margin: 0.65rem 0 0;
|
||||||
|
padding: 0;
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 0.4rem;
|
||||||
|
}
|
||||||
|
.steering-note {
|
||||||
|
display: flex;
|
||||||
|
align-items: stretch;
|
||||||
|
gap: 0.6rem;
|
||||||
|
background: #0d1120;
|
||||||
|
border: 1px solid var(--line);
|
||||||
|
border-radius: var(--radius-sm);
|
||||||
|
padding: 0.35rem 0.4rem 0.35rem 0.8rem;
|
||||||
|
}
|
||||||
|
.steering-note-text {
|
||||||
|
color: var(--ink);
|
||||||
|
font-size: 0.9rem;
|
||||||
|
line-height: 1.45;
|
||||||
|
/* Notes may be multi-line instructions — keep line breaks as typed. */
|
||||||
|
white-space: pre-wrap;
|
||||||
|
overflow-wrap: anywhere;
|
||||||
|
flex: 1;
|
||||||
|
min-width: 0;
|
||||||
|
}
|
||||||
|
.steering-delete {
|
||||||
|
display: inline-flex;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: center;
|
||||||
|
min-height: 44px;
|
||||||
|
min-width: 44px;
|
||||||
|
flex: 0 0 auto;
|
||||||
|
border: 1px solid var(--line);
|
||||||
|
border-radius: var(--radius-sm);
|
||||||
|
background: transparent;
|
||||||
|
color: var(--ink-soft);
|
||||||
|
cursor: pointer;
|
||||||
|
}
|
||||||
|
.steering-delete svg { width: 16px; height: 16px; display: block; }
|
||||||
|
.steering-delete:hover:not(:disabled) { background: var(--err-bg); color: var(--err-ink); border-color: var(--err-line); }
|
||||||
|
.steering-delete:disabled { opacity: 0.5; cursor: wait; }
|
||||||
|
.steering-empty { margin: 0.65rem 0 0; color: var(--ink-soft); font-size: 0.88rem; }
|
||||||
|
|
||||||
/* typing indicator */
|
/* typing indicator */
|
||||||
.typing { display: inline-flex; gap: 5px; padding: 0.9rem 1rem; }
|
.typing { display: inline-flex; gap: 5px; padding: 0.9rem 1rem; }
|
||||||
.typing span {
|
.typing span {
|
||||||
@@ -799,6 +1005,20 @@ body::after {
|
|||||||
.new-chat-btn { padding: 0.4rem 0.55rem; }
|
.new-chat-btn { padding: 0.4rem 0.55rem; }
|
||||||
.new-chat-label { display: none; }
|
.new-chat-label { display: none; }
|
||||||
.new-chat-btn svg { display: block; }
|
.new-chat-btn svg { display: block; }
|
||||||
|
.steering-toggle { padding: 0.4rem 0.55rem; }
|
||||||
|
/* Visually hidden, NOT display:none — the accessible name keeps the
|
||||||
|
word "Tuning" next to the count badge. */
|
||||||
|
.steering-label {
|
||||||
|
position: absolute !important;
|
||||||
|
width: 1px; height: 1px;
|
||||||
|
margin: -1px; padding: 0;
|
||||||
|
overflow: hidden;
|
||||||
|
clip: rect(0 0 0 0);
|
||||||
|
white-space: nowrap;
|
||||||
|
border: 0;
|
||||||
|
}
|
||||||
|
.steering-note { padding: 0.3rem 0.3rem 0.3rem 0.7rem; }
|
||||||
|
.tune-btn { min-height: 44px; }
|
||||||
.msg-body { max-width: 92%; }
|
.msg-body { max-width: 92%; }
|
||||||
.empty-state { padding: 1.75rem 1.1rem; margin-top: 0.25rem; }
|
.empty-state { padding: 1.75rem 1.1rem; margin-top: 0.25rem; }
|
||||||
.empty-state-title { font-size: 1.25rem; }
|
.empty-state-title { font-size: 1.25rem; }
|
||||||
|
|||||||
@@ -21,6 +21,14 @@
|
|||||||
<a href="/" class="nav-link is-active" aria-current="page">Chat</a>
|
<a href="/" class="nav-link is-active" aria-current="page">Chat</a>
|
||||||
<a href="/sources.html" class="nav-link">Sources</a>
|
<a href="/sources.html" class="nav-link">Sources</a>
|
||||||
</nav>
|
</nav>
|
||||||
|
<!-- Phase 15: open the tuning-notes panel (stored in Postgres, read
|
||||||
|
into every system prompt) — chat page only. -->
|
||||||
|
<button type="button" class="steering-toggle" id="steering-toggle"
|
||||||
|
aria-expanded="false" aria-controls="steering-panel">
|
||||||
|
<svg aria-hidden="true" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.7" stroke-linecap="round"><path d="M4 7h10M18 7h2M4 17h4M12 17h8"/><circle cx="15.5" cy="7" r="2.2"/><circle cx="9.5" cy="17" r="2.2"/></svg>
|
||||||
|
<span class="steering-label">Tuning</span>
|
||||||
|
<span class="steering-count" id="steering-count">0</span>
|
||||||
|
</button>
|
||||||
<!-- Phase 14: reset the local (localStorage) conversation — chat page only. -->
|
<!-- Phase 14: reset the local (localStorage) conversation — chat page only. -->
|
||||||
<button type="button" class="new-chat-btn" id="new-chat-btn" aria-label="New chat">
|
<button type="button" class="new-chat-btn" id="new-chat-btn" aria-label="New chat">
|
||||||
<svg aria-hidden="true" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.7" stroke-linecap="round"><path d="M12 5v14M5 12h14"/></svg>
|
<svg aria-hidden="true" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.7" stroke-linecap="round"><path d="M12 5v14M5 12h14"/></svg>
|
||||||
@@ -36,6 +44,18 @@
|
|||||||
<span id="kb-banner-text"></span>
|
<span id="kb-banner-text"></span>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<!-- Phase 15: tuning-notes panel (stored notes, newest first). -->
|
||||||
|
<section class="steering-panel" id="steering-panel" role="region"
|
||||||
|
aria-label="Tuning notes" hidden>
|
||||||
|
<div class="steering-panel-head">
|
||||||
|
<h2 class="steering-panel-title">Tuning notes</h2>
|
||||||
|
<p class="steering-panel-sub">Every note below steers all future answers.</p>
|
||||||
|
</div>
|
||||||
|
<ul class="steering-list" id="steering-list"></ul>
|
||||||
|
<p class="steering-empty" id="steering-empty">No tuning notes yet — press “Tune” under any answer to add one.</p>
|
||||||
|
</section>
|
||||||
|
<p class="visually-hidden" id="steering-announcer" role="status" aria-live="polite" aria-atomic="true"></p>
|
||||||
|
|
||||||
<section class="messages" id="messages" aria-live="polite" aria-label="Conversation with Brain of Reese">
|
<section class="messages" id="messages" aria-live="polite" aria-label="Conversation with Brain of Reese">
|
||||||
<div class="empty-state" id="empty-state">
|
<div class="empty-state" id="empty-state">
|
||||||
<div class="empty-state-glyph" aria-hidden="true">
|
<div class="empty-state-glyph" aria-hidden="true">
|
||||||
|
|||||||
+36
-4
@@ -15,6 +15,9 @@ Implements just enough of the aipi surface:
|
|||||||
- otherwise -> upbeat answer quoting the provided document context
|
- otherwise -> upbeat answer quoting the provided document context
|
||||||
- user message containing ``pretend to think slowly`` -> 3s warm-up delay
|
- user message containing ``pretend to think slowly`` -> 3s warm-up delay
|
||||||
(used by the loading-feedback story).
|
(used by the loading-feedback story).
|
||||||
|
- system prompt containing ``<tuning>`` (phase 15, steering notes) ->
|
||||||
|
the composed answer ends with `` (tuning: <first note line>)`` —
|
||||||
|
makes prompt injection observable in the UI deterministically.
|
||||||
|
|
||||||
``max_tokens`` is honored deterministically (token ≈ whitespace word),
|
``max_tokens`` is honored deterministically (token ≈ whitespace word),
|
||||||
like a real endpoint: an answer longer than the cap is truncated. This
|
like a real endpoint: an answer longer than the cap is truncated. This
|
||||||
@@ -89,25 +92,54 @@ def long_answer() -> str:
|
|||||||
return "\n".join(lines)
|
return "\n".join(lines)
|
||||||
|
|
||||||
|
|
||||||
|
#: First numbered note line of a ``<tuning>`` section (phase 15).
|
||||||
|
_TUNING_BLOCK_RE = re.compile(r"<tuning>\n(.*?)\n</tuning>", re.S)
|
||||||
|
_NOTE_LINE_RE = re.compile(r"^\d+\.\s*(.+)$")
|
||||||
|
|
||||||
|
|
||||||
|
def first_tuning_note(system: str) -> str | None:
|
||||||
|
"""The first steering note in the system prompt, or ``None``.
|
||||||
|
|
||||||
|
The prompt numbers notes 1..N oldest-first (see
|
||||||
|
``app.rag.prompts.build_steering_section``); the mock echoes the first
|
||||||
|
one into its answer so prompt injection is observable in the UI.
|
||||||
|
"""
|
||||||
|
block = _TUNING_BLOCK_RE.search(system)
|
||||||
|
if not block:
|
||||||
|
return None
|
||||||
|
for line in block.group(1).splitlines():
|
||||||
|
m = _NOTE_LINE_RE.match(line.strip())
|
||||||
|
if m:
|
||||||
|
return m.group(1).strip()
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def compose_answer(body: dict[str, Any]) -> str:
|
def compose_answer(body: dict[str, Any]) -> str:
|
||||||
system = _system(body)
|
system = _system(body)
|
||||||
user = _user(body)
|
user = _user(body)
|
||||||
if LONG_ANSWER_TRIGGER in user.lower():
|
if LONG_ANSWER_TRIGGER in user.lower():
|
||||||
return long_answer()
|
answer = long_answer()
|
||||||
if "DEFLECT_MODE" in system:
|
elif "DEFLECT_MODE" in system:
|
||||||
return (
|
answer = (
|
||||||
"Ah — I haven't done anything like that, so I don't want to make stuff up! "
|
"Ah — I haven't done anything like that, so I don't want to make stuff up! "
|
||||||
"You're thinking bigger than my notes for a second. Try asking about "
|
"You're thinking bigger than my notes for a second. Try asking about "
|
||||||
"kubernetes, backups, or deploying a new service — I know those inside out. "
|
"kubernetes, backups, or deploying a new service — I know those inside out. "
|
||||||
"You've got this!"
|
"You've got this!"
|
||||||
)
|
)
|
||||||
|
else:
|
||||||
ctx = _context(body)
|
ctx = _context(body)
|
||||||
snippet = ctx[:220].replace("\n", " ").strip()
|
snippet = ctx[:220].replace("\n", " ").strip()
|
||||||
return (
|
answer = (
|
||||||
f"Great question — you've absolutely got this! Here's what my notes say about "
|
f"Great question — you've absolutely got this! Here's what my notes say about "
|
||||||
f"“{user.strip()[:80]}”: {snippet}… That's the gist from the docs; happy to "
|
f"“{user.strip()[:80]}”: {snippet}… That's the gist from the docs; happy to "
|
||||||
"dig into any of it. (Deterministic mock answer for E2E.)"
|
"dig into any of it. (Deterministic mock answer for E2E.)"
|
||||||
)
|
)
|
||||||
|
# Steering (phase 15): when the system prompt carries <tuning>, the
|
||||||
|
# answer ends with the first note — deterministically observable.
|
||||||
|
note = first_tuning_note(system)
|
||||||
|
if note:
|
||||||
|
answer = f"{answer} (tuning: {note})"
|
||||||
|
return answer
|
||||||
|
|
||||||
|
|
||||||
@app.post("/__shutdown__")
|
@app.post("/__shutdown__")
|
||||||
|
|||||||
@@ -0,0 +1,293 @@
|
|||||||
|
"""Phase 15 E2E (Playwright): tune how Brain answers (steering notes).
|
||||||
|
|
||||||
|
Story: ``.agent/user_stories/steering-notes.md``
|
||||||
|
Run in isolation (DB must be up: ``podman compose up -d db``):
|
||||||
|
|
||||||
|
uv run pytest tests/e2e/test_steering.py -v --no-cov
|
||||||
|
|
||||||
|
The steering loop: "Tune" under a completed answer → short instruction →
|
||||||
|
stored in Postgres (``steering_notes``) → injected into the system prompt
|
||||||
|
of every subsequent turn as the ``<tuning>`` section. The mock LLM
|
||||||
|
echoes the first tuning note into its answer
|
||||||
|
(`` (tuning: <first note line>)``), so prompt injection is observable in
|
||||||
|
the UI deterministically. Notes are listed newest-first in the header
|
||||||
|
"Tuning" panel, where each can be deleted.
|
||||||
|
|
||||||
|
Test → story mapping (Playwright Mapping Rule):
|
||||||
|
1. ``test_tune_under_answer_persists_and_steers``
|
||||||
|
2. ``test_delete_note_stops_steering``
|
||||||
|
3. ``test_note_rendered_as_text_xss_safe``
|
||||||
|
4. ``test_tuning_panel_a11y``
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
from pathlib import Path
|
||||||
|
from threading import Thread
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from playwright.sync_api import Page, expect
|
||||||
|
from sqlalchemy import select, text
|
||||||
|
|
||||||
|
from app.config import Settings
|
||||||
|
from app.db import SessionLocal
|
||||||
|
from app.models import SteeringNote
|
||||||
|
from app.rag.importer import ImportSummary, import_sources
|
||||||
|
from app.rag.llm import LLMClient
|
||||||
|
|
||||||
|
REPO = Path(__file__).resolve().parents[2]
|
||||||
|
FIXTURES = REPO / "tests" / "fixtures" / "docs"
|
||||||
|
QUESTION = "How is my Kubernetes cluster set up?"
|
||||||
|
MOCK_ANSWER_MARKER = "Deterministic mock answer for E2E"
|
||||||
|
NOTE = "STEEER-MARKER be concise"
|
||||||
|
XSS_NOTE = "<script>window.__xss = true; alert('xss')</script>"
|
||||||
|
#: index.html ships exactly two classic/module script tags.
|
||||||
|
BASE_SCRIPT_COUNT = 2
|
||||||
|
|
||||||
|
|
||||||
|
async def _import_fixtures(mock_port: int) -> ImportSummary:
|
||||||
|
kwargs: dict[str, Any] = {"_env_file": None, "llm_base_url": f"http://127.0.0.1:{mock_port}/v1"}
|
||||||
|
settings = Settings(**kwargs) # pyright: ignore[reportCallIssue]
|
||||||
|
return await import_sources([FIXTURES], LLMClient(settings))
|
||||||
|
|
||||||
|
|
||||||
|
def _run_in_thread(coro: Any) -> Any:
|
||||||
|
"""Run a coroutine on a worker thread.
|
||||||
|
|
||||||
|
Playwright's sync API keeps an asyncio loop running on the test thread,
|
||||||
|
so ``asyncio.run`` cannot be called directly from a test body.
|
||||||
|
"""
|
||||||
|
box: dict[str, Any] = {}
|
||||||
|
|
||||||
|
def runner() -> None:
|
||||||
|
try:
|
||||||
|
box["value"] = asyncio.run(coro)
|
||||||
|
except BaseException as e: # noqa: BLE001 — re-raised on the test thread
|
||||||
|
box["error"] = e
|
||||||
|
|
||||||
|
t = Thread(target=runner)
|
||||||
|
t.start()
|
||||||
|
t.join()
|
||||||
|
if "error" in box:
|
||||||
|
raise box["error"]
|
||||||
|
return box["value"]
|
||||||
|
|
||||||
|
|
||||||
|
def _reset_db(mock_port: int, seed: bool) -> ImportSummary | None:
|
||||||
|
"""Truncate the KB (and query log + steering notes), re-import fixtures."""
|
||||||
|
with SessionLocal() as db:
|
||||||
|
db.execute(text("TRUNCATE chunks, documents, query_log, steering_notes"))
|
||||||
|
db.commit()
|
||||||
|
if not seed:
|
||||||
|
return None
|
||||||
|
return _run_in_thread(_import_fixtures(mock_port))
|
||||||
|
|
||||||
|
|
||||||
|
def _ask(page: Page, question: str) -> None:
|
||||||
|
"""Send one turn and wait until the grounded answer has fully landed."""
|
||||||
|
page.fill("#message-input", question)
|
||||||
|
page.click("#send-btn")
|
||||||
|
expect(page.locator(".msg.user .bubble").last).to_contain_text(question)
|
||||||
|
expect(page.locator(".msg.brain .bubble").last).to_contain_text(
|
||||||
|
MOCK_ANSWER_MARKER, timeout=30_000
|
||||||
|
)
|
||||||
|
expect(page.locator("#send-btn")).to_be_enabled()
|
||||||
|
expect(page.locator("#send-label")).to_have_text("Send")
|
||||||
|
|
||||||
|
|
||||||
|
def _tune_and_save(page: Page, note: str) -> None:
|
||||||
|
"""Tune the last completed brain bubble and save *note*."""
|
||||||
|
tune = page.locator(".msg.brain .tune-btn").last
|
||||||
|
expect(tune).to_be_visible()
|
||||||
|
tune.click()
|
||||||
|
form = page.locator(".msg.brain .tune-form").last
|
||||||
|
expect(form).to_be_visible()
|
||||||
|
form.locator("textarea").fill(note)
|
||||||
|
form.locator(".tune-save").click()
|
||||||
|
saved = page.locator(".msg.brain .tune-saved").last
|
||||||
|
expect(saved).to_contain_text("Saved — future answers will follow this.", timeout=15_000)
|
||||||
|
|
||||||
|
|
||||||
|
def _open_panel(page: Page) -> None:
|
||||||
|
page.click("#steering-toggle")
|
||||||
|
expect(page.locator("#steering-panel")).to_be_visible()
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 1. Tune under an answer → persisted → next answer carries the note
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_tune_under_answer_persists_and_steers(
|
||||||
|
page: Page, app_url: str, mock_llm: int, db_ready: None
|
||||||
|
) -> None:
|
||||||
|
summary = _reset_db(mock_llm, seed=True)
|
||||||
|
assert summary is not None and summary.added == 8 # A9 formats
|
||||||
|
page.set_default_timeout(30_000)
|
||||||
|
page.goto(app_url)
|
||||||
|
_ask(page, QUESTION)
|
||||||
|
|
||||||
|
# The Tune control: ghost button in the answer's meta row, ≥44px.
|
||||||
|
tune = page.locator(".msg.brain .tune-btn").last
|
||||||
|
expect(tune).to_have_count(1)
|
||||||
|
expect(tune).to_have_attribute("type", "button")
|
||||||
|
box = tune.bounding_box()
|
||||||
|
assert box is not None and box["height"] >= 44
|
||||||
|
|
||||||
|
_tune_and_save(page, NOTE)
|
||||||
|
|
||||||
|
# Persisted in Postgres.
|
||||||
|
with SessionLocal() as db:
|
||||||
|
rows = db.scalars(select(SteeringNote)).all()
|
||||||
|
assert [r.note for r in rows] == [NOTE]
|
||||||
|
|
||||||
|
# The header panel shows the note with an updated count badge.
|
||||||
|
_open_panel(page)
|
||||||
|
expect(page.locator("#steering-count")).to_have_text("1")
|
||||||
|
expect(page.locator("#steering-list .steering-note")).to_have_count(1)
|
||||||
|
expect(page.locator("#steering-list .steering-note-text")).to_have_text(NOTE)
|
||||||
|
page.click("#steering-toggle") # close again
|
||||||
|
|
||||||
|
# The NEXT answer carries the note — it reached the system prompt.
|
||||||
|
_ask(page, QUESTION)
|
||||||
|
bubble = page.locator(".msg.brain .bubble").last
|
||||||
|
expect(bubble).to_contain_text(f"(tuning: {NOTE})")
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 2. Delete from the panel → count 0 → steering stops
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_delete_note_stops_steering(
|
||||||
|
page: Page, app_url: str, mock_llm: int, db_ready: None
|
||||||
|
) -> None:
|
||||||
|
_reset_db(mock_llm, seed=True)
|
||||||
|
page.set_default_timeout(30_000)
|
||||||
|
page.goto(app_url)
|
||||||
|
_ask(page, QUESTION)
|
||||||
|
_tune_and_save(page, NOTE)
|
||||||
|
|
||||||
|
# Steering is live: one more answer carries the marker.
|
||||||
|
_ask(page, QUESTION)
|
||||||
|
expect(page.locator(".msg.brain .bubble").last).to_contain_text(f"(tuning: {NOTE})")
|
||||||
|
|
||||||
|
# Delete the note from the panel.
|
||||||
|
_open_panel(page)
|
||||||
|
expect(page.locator("#steering-count")).to_have_text("1")
|
||||||
|
page.locator("#steering-list .steering-delete").click()
|
||||||
|
expect(page.locator("#steering-list .steering-note")).to_have_count(0)
|
||||||
|
expect(page.locator("#steering-count")).to_have_text("0")
|
||||||
|
expect(page.locator("#steering-empty")).to_be_visible()
|
||||||
|
expect(page.locator("#steering-announcer")).to_contain_text("deleted")
|
||||||
|
|
||||||
|
with SessionLocal() as db:
|
||||||
|
assert db.scalars(select(SteeringNote)).all() == []
|
||||||
|
|
||||||
|
# The next answer no longer carries the marker.
|
||||||
|
_ask(page, QUESTION)
|
||||||
|
bubble = page.locator(".msg.brain .bubble").last
|
||||||
|
expect(bubble).to_contain_text(MOCK_ANSWER_MARKER)
|
||||||
|
expect(bubble).not_to_contain_text("STEEER-MARKER")
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 3. Notes render as text (XSS-safe)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_note_rendered_as_text_xss_safe(
|
||||||
|
page: Page, app_url: str, mock_llm: int, db_ready: None
|
||||||
|
) -> None:
|
||||||
|
_reset_db(mock_llm, seed=True)
|
||||||
|
page.set_default_timeout(30_000)
|
||||||
|
page.goto(app_url)
|
||||||
|
|
||||||
|
dialogs: list[str] = []
|
||||||
|
|
||||||
|
def _handle_dialog(d) -> None:
|
||||||
|
dialogs.append(d.message)
|
||||||
|
d.dismiss()
|
||||||
|
|
||||||
|
page.on("dialog", _handle_dialog)
|
||||||
|
|
||||||
|
_ask(page, QUESTION)
|
||||||
|
_tune_and_save(page, XSS_NOTE)
|
||||||
|
|
||||||
|
# Panel: the payload is visible as LITERAL text…
|
||||||
|
_open_panel(page)
|
||||||
|
expect(page.locator("#steering-list .steering-note-text")).to_have_text(XSS_NOTE)
|
||||||
|
|
||||||
|
# …never as an executed element: no script tag anywhere, no dialog.
|
||||||
|
assert page.locator("#steering-panel script").count() == 0
|
||||||
|
expect(page.locator("script")).to_have_count(BASE_SCRIPT_COUNT)
|
||||||
|
assert dialogs == [], f"the note must never execute as script: {dialogs}"
|
||||||
|
assert page.evaluate("() => window.__xss === undefined") is True
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 4. Tuning panel accessibility
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_tuning_panel_a11y(page: Page, app_url: str, db_ready: None) -> None:
|
||||||
|
_reset_db(mock_port=0, seed=False) # no KB seeding needed for the panel a11y
|
||||||
|
page.set_default_timeout(30_000)
|
||||||
|
page.goto(app_url)
|
||||||
|
|
||||||
|
toggle = page.locator("#steering-toggle")
|
||||||
|
panel = page.locator("#steering-panel")
|
||||||
|
announcer = page.locator("#steering-announcer")
|
||||||
|
|
||||||
|
# Initial: closed, correctly wired, polite live region present.
|
||||||
|
expect(toggle).to_have_attribute("aria-expanded", "false")
|
||||||
|
expect(toggle).to_have_attribute("aria-controls", "steering-panel")
|
||||||
|
expect(panel).to_have_attribute("role", "region")
|
||||||
|
assert "Tuning notes" in (panel.get_attribute("aria-label") or "")
|
||||||
|
expect(panel).to_be_hidden()
|
||||||
|
assert announcer.get_attribute("role") == "status"
|
||||||
|
assert announcer.get_attribute("aria-live") == "polite"
|
||||||
|
# Accessible name comes from its visible text (icon is aria-hidden).
|
||||||
|
assert "Tuning" in toggle.inner_text()
|
||||||
|
|
||||||
|
# Open: expanded + the designed empty state.
|
||||||
|
toggle.click()
|
||||||
|
expect(toggle).to_have_attribute("aria-expanded", "true")
|
||||||
|
expect(panel).to_be_visible()
|
||||||
|
expect(page.locator("#steering-empty")).to_be_visible()
|
||||||
|
expect(page.locator("#steering-count")).to_have_text("0")
|
||||||
|
|
||||||
|
# Add a note (API), then re-open the panel to refresh it.
|
||||||
|
page.evaluate(
|
||||||
|
"""async () => {
|
||||||
|
const r = await fetch('/api/steering', {
|
||||||
|
method: 'POST',
|
||||||
|
headers: {'Content-Type': 'application/json'},
|
||||||
|
body: JSON.stringify({note: 'a11y note one'}),
|
||||||
|
});
|
||||||
|
if (!r.ok) throw new Error('steering POST failed: ' + r.status);
|
||||||
|
}"""
|
||||||
|
)
|
||||||
|
toggle.click() # close
|
||||||
|
toggle.click() # re-open (refreshes the list)
|
||||||
|
note_item = page.locator("#steering-list .steering-note")
|
||||||
|
expect(note_item).to_have_count(1)
|
||||||
|
expect(note_item.locator(".steering-note-text")).to_have_text("a11y note one")
|
||||||
|
|
||||||
|
# The per-note delete is a real, labeled button (≥44px target).
|
||||||
|
delete = page.locator("#steering-list .steering-delete")
|
||||||
|
expect(delete).to_have_attribute("type", "button")
|
||||||
|
assert (delete.get_attribute("aria-label") or "").startswith("Delete tuning note:")
|
||||||
|
box = delete.bounding_box()
|
||||||
|
assert box is not None and box["height"] >= 44
|
||||||
|
|
||||||
|
# Delete: list empties, count updates, the live region announces it.
|
||||||
|
delete.click()
|
||||||
|
expect(page.locator("#steering-list .steering-note")).to_have_count(0)
|
||||||
|
expect(page.locator("#steering-count")).to_have_text("0")
|
||||||
|
expect(announcer).to_contain_text("deleted")
|
||||||
|
|
||||||
|
# And the toggle closes cleanly again.
|
||||||
|
toggle.click()
|
||||||
|
expect(toggle).to_have_attribute("aria-expanded", "false")
|
||||||
|
expect(panel).to_be_hidden()
|
||||||
@@ -0,0 +1,253 @@
|
|||||||
|
"""Integration: steering notes (phase 15) — CRUD + system-prompt injection.
|
||||||
|
|
||||||
|
Real Postgres (``podman compose up -d db``); the chat path reuses the
|
||||||
|
deterministic fake LLM from ``test_chat_api`` (token-overlap embeddings),
|
||||||
|
so the stored note's journey — API → Postgres → ``<tuning>`` section of
|
||||||
|
the captured system prompt — is verified end-to-end without a network.
|
||||||
|
|
||||||
|
Requires: podman compose up -d db
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import logging
|
||||||
|
import uuid
|
||||||
|
from collections.abc import Iterator
|
||||||
|
from datetime import UTC, datetime, timedelta
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from fastapi.testclient import TestClient
|
||||||
|
from sqlalchemy import select, text
|
||||||
|
from test_chat_api import FakeRagLLM, _stream_chat
|
||||||
|
|
||||||
|
from app.api import chat as chat_api
|
||||||
|
from app.main import app as fastapi_app
|
||||||
|
from app.models import SteeringNote
|
||||||
|
from app.rag.importer import import_sources
|
||||||
|
|
||||||
|
FIXTURES = Path(__file__).resolve().parents[1] / "fixtures" / "docs"
|
||||||
|
QUESTION = "How is my Kubernetes cluster set up?"
|
||||||
|
OFF_TOPIC = "How do I bake sourdough bread?"
|
||||||
|
NOTE = "STEEER-MARKER be concise"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def clean_steering(db) -> Iterator[None]:
|
||||||
|
"""Steering notes + query log are global state: reset around every test."""
|
||||||
|
db.execute(text("TRUNCATE steering_notes, query_log"))
|
||||||
|
db.commit()
|
||||||
|
yield
|
||||||
|
db.execute(text("TRUNCATE steering_notes, query_log"))
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture()
|
||||||
|
def seeded_kb(db) -> Iterator[FakeRagLLM]:
|
||||||
|
"""Fresh Postgres with the fixture docs imported (real pipeline)."""
|
||||||
|
db.execute(text("TRUNCATE chunks, documents, query_log, steering_notes"))
|
||||||
|
db.commit()
|
||||||
|
llm = FakeRagLLM()
|
||||||
|
summary = asyncio.run(import_sources([FIXTURES], llm, session=db))
|
||||||
|
assert summary.added == 8 # A9 formats; .hidden/ skipped
|
||||||
|
yield llm
|
||||||
|
db.execute(text("TRUNCATE chunks, documents, query_log, steering_notes"))
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
|
||||||
|
def _turn_log_lines(caplog: pytest.LogCaptureFixture) -> list[str]:
|
||||||
|
"""The per-turn ``question=…`` log lines (PLAN §9) from this test."""
|
||||||
|
return [r.getMessage() for r in caplog.records if "question=" in r.getMessage()]
|
||||||
|
|
||||||
|
|
||||||
|
# ---------- CRUD ----------
|
||||||
|
|
||||||
|
|
||||||
|
def test_create_note_returns_201_and_stores_trimmed(client: TestClient, db) -> None:
|
||||||
|
r = client.post("/api/steering", json={"note": f" {NOTE} "})
|
||||||
|
assert r.status_code == 201
|
||||||
|
body = r.json()
|
||||||
|
assert body["note"] == NOTE # trimmed before storage
|
||||||
|
uuid.UUID(body["id"]) # valid UUID
|
||||||
|
assert body["created_at"]
|
||||||
|
rows = db.scalars(select(SteeringNote)).all()
|
||||||
|
assert [row.note for row in rows] == [NOTE]
|
||||||
|
|
||||||
|
|
||||||
|
def test_list_notes_empty(client: TestClient) -> None:
|
||||||
|
r = client.get("/api/steering")
|
||||||
|
assert r.status_code == 200
|
||||||
|
assert r.json() == {"notes": []}
|
||||||
|
|
||||||
|
|
||||||
|
def test_list_notes_newest_first(client: TestClient, db) -> None:
|
||||||
|
base = datetime.now(UTC)
|
||||||
|
db.add_all(
|
||||||
|
[
|
||||||
|
SteeringNote(note="oldest", created_at=base),
|
||||||
|
SteeringNote(note="newest", created_at=base + timedelta(hours=2)),
|
||||||
|
SteeringNote(note="middle", created_at=base + timedelta(hours=1)),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
r = client.get("/api/steering")
|
||||||
|
assert r.status_code == 200
|
||||||
|
body = r.json()
|
||||||
|
assert [n["note"] for n in body["notes"]] == ["newest", "middle", "oldest"]
|
||||||
|
for n in body["notes"]:
|
||||||
|
assert set(n) == {"id", "note", "created_at"}
|
||||||
|
uuid.UUID(n["id"])
|
||||||
|
|
||||||
|
|
||||||
|
def test_delete_note_returns_204_and_removes(client: TestClient, db) -> None:
|
||||||
|
created = client.post("/api/steering", json={"note": NOTE}).json()
|
||||||
|
|
||||||
|
assert client.delete(f"/api/steering/{created['id']}").status_code == 204
|
||||||
|
assert client.get("/api/steering").json() == {"notes": []}
|
||||||
|
assert db.scalars(select(SteeringNote)).all() == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_delete_unknown_note_returns_404(client: TestClient) -> None:
|
||||||
|
r = client.delete(f"/api/steering/{uuid.uuid4()}")
|
||||||
|
assert r.status_code == 404
|
||||||
|
assert "not found" in r.json()["detail"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_delete_invalid_id_returns_422(client: TestClient) -> None:
|
||||||
|
assert client.delete("/api/steering/not-a-uuid").status_code == 422
|
||||||
|
|
||||||
|
|
||||||
|
def test_create_rejects_empty_and_blank_notes(client: TestClient) -> None:
|
||||||
|
assert client.post("/api/steering", json={"note": ""}).status_code == 422
|
||||||
|
assert client.post("/api/steering", json={"note": " \t\n "}).status_code == 422
|
||||||
|
assert client.get("/api/steering").json() == {"notes": []}
|
||||||
|
|
||||||
|
|
||||||
|
def test_create_enforces_2000_char_limit(client: TestClient) -> None:
|
||||||
|
assert client.post("/api/steering", json={"note": "x" * 2001}).status_code == 422
|
||||||
|
r = client.post("/api/steering", json={"note": "x" * 2000})
|
||||||
|
assert r.status_code == 201
|
||||||
|
assert len(r.json()["note"]) == 2000
|
||||||
|
|
||||||
|
|
||||||
|
# ---------- chat turn: note reaches the system prompt ----------
|
||||||
|
|
||||||
|
|
||||||
|
def test_chat_turn_high_mode_receives_note_in_system_prompt(
|
||||||
|
client: TestClient, seeded_kb: FakeRagLLM, caplog: pytest.LogCaptureFixture
|
||||||
|
) -> None:
|
||||||
|
client.post("/api/steering", json={"note": NOTE})
|
||||||
|
caplog.set_level(logging.INFO, logger="app.chat")
|
||||||
|
|
||||||
|
fastapi_app.dependency_overrides[chat_api.get_llm] = lambda: seeded_kb
|
||||||
|
try:
|
||||||
|
_, _, frames = _stream_chat(client, QUESTION)
|
||||||
|
finally:
|
||||||
|
fastapi_app.dependency_overrides.clear()
|
||||||
|
|
||||||
|
assert frames[-1]["type"] == "done"
|
||||||
|
assert frames[-1]["deflected"] is False
|
||||||
|
|
||||||
|
# The LLM received the HIGH prompt with the <tuning> section.
|
||||||
|
(system, user) = seeded_kb.seen_messages[0][0], seeded_kb.seen_messages[0][1]
|
||||||
|
assert user["content"] == QUESTION
|
||||||
|
assert "<relevance>HIGH</relevance>" in system["content"]
|
||||||
|
assert "<tuning>" in system["content"]
|
||||||
|
assert f"1. {NOTE}" in system["content"]
|
||||||
|
assert "<documents>" in system["content"]
|
||||||
|
# The section sits between the relevance marker and the documents.
|
||||||
|
assert (
|
||||||
|
system["content"].index("<relevance>HIGH</relevance>")
|
||||||
|
< system["content"].index("<tuning>")
|
||||||
|
< system["content"].index("</tuning>")
|
||||||
|
< system["content"].index("<documents>")
|
||||||
|
)
|
||||||
|
|
||||||
|
# The per-turn log line records tuning=N (PLAN §9).
|
||||||
|
lines = _turn_log_lines(caplog)
|
||||||
|
assert lines and "tuning=1" in lines[-1]
|
||||||
|
|
||||||
|
|
||||||
|
def test_chat_turn_low_mode_receives_note_in_system_prompt(
|
||||||
|
client: TestClient, seeded_kb: FakeRagLLM, caplog: pytest.LogCaptureFixture
|
||||||
|
) -> None:
|
||||||
|
client.post("/api/steering", json={"note": NOTE})
|
||||||
|
caplog.set_level(logging.INFO, logger="app.chat")
|
||||||
|
|
||||||
|
fastapi_app.dependency_overrides[chat_api.get_llm] = lambda: seeded_kb
|
||||||
|
try:
|
||||||
|
_, _, frames = _stream_chat(client, OFF_TOPIC)
|
||||||
|
finally:
|
||||||
|
fastapi_app.dependency_overrides.clear()
|
||||||
|
|
||||||
|
assert frames[-1]["deflected"] is True
|
||||||
|
(system, _user) = seeded_kb.seen_messages[0][0], seeded_kb.seen_messages[0][1]
|
||||||
|
assert "<relevance>LOW</relevance>" in system["content"]
|
||||||
|
assert "DEFLECT_MODE" in system["content"]
|
||||||
|
assert "<tuning>" in system["content"]
|
||||||
|
assert f"1. {NOTE}" in system["content"]
|
||||||
|
lines = _turn_log_lines(caplog)
|
||||||
|
assert lines and "tuning=1" in lines[-1]
|
||||||
|
|
||||||
|
|
||||||
|
def test_chat_turn_without_notes_has_no_tuning_section(
|
||||||
|
client: TestClient, seeded_kb: FakeRagLLM, caplog: pytest.LogCaptureFixture
|
||||||
|
) -> None:
|
||||||
|
caplog.set_level(logging.INFO, logger="app.chat")
|
||||||
|
|
||||||
|
fastapi_app.dependency_overrides[chat_api.get_llm] = lambda: seeded_kb
|
||||||
|
try:
|
||||||
|
_stream_chat(client, QUESTION)
|
||||||
|
finally:
|
||||||
|
fastapi_app.dependency_overrides.clear()
|
||||||
|
|
||||||
|
(system, _user) = seeded_kb.seen_messages[0][0], seeded_kb.seen_messages[0][1]
|
||||||
|
assert "<tuning>" not in system["content"]
|
||||||
|
lines = _turn_log_lines(caplog)
|
||||||
|
assert lines and "tuning=0" in lines[-1]
|
||||||
|
|
||||||
|
|
||||||
|
def test_chat_turn_numbers_notes_oldest_first(client: TestClient, db, seeded_kb) -> None:
|
||||||
|
base = datetime.now(UTC)
|
||||||
|
db.add_all(
|
||||||
|
[
|
||||||
|
SteeringNote(note="older note", created_at=base),
|
||||||
|
SteeringNote(note="newer note", created_at=base + timedelta(hours=1)),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
fastapi_app.dependency_overrides[chat_api.get_llm] = lambda: seeded_kb
|
||||||
|
try:
|
||||||
|
_stream_chat(client, QUESTION)
|
||||||
|
finally:
|
||||||
|
fastapi_app.dependency_overrides.clear()
|
||||||
|
|
||||||
|
(system, _user) = seeded_kb.seen_messages[0][0], seeded_kb.seen_messages[0][1]
|
||||||
|
assert "<tuning>" in system["content"]
|
||||||
|
assert "1. older note" in system["content"]
|
||||||
|
assert "2. newer note" in system["content"]
|
||||||
|
assert system["content"].index("1. older note") < system["content"].index("2. newer note")
|
||||||
|
|
||||||
|
|
||||||
|
def test_multiple_turns_keep_reading_notes(client: TestClient, seeded_kb) -> None:
|
||||||
|
"""The note steers EVERY subsequent turn, not just the next one."""
|
||||||
|
client.post("/api/steering", json={"note": NOTE})
|
||||||
|
|
||||||
|
fastapi_app.dependency_overrides[chat_api.get_llm] = lambda: seeded_kb
|
||||||
|
try:
|
||||||
|
_stream_chat(client, QUESTION)
|
||||||
|
_stream_chat(client, QUESTION)
|
||||||
|
assert len(seeded_kb.seen_messages) == 2
|
||||||
|
for messages in seeded_kb.seen_messages:
|
||||||
|
assert f"1. {NOTE}" in messages[0]["content"]
|
||||||
|
|
||||||
|
# Delete → the following turn is clean again.
|
||||||
|
note_id = client.get("/api/steering").json()["notes"][0]["id"]
|
||||||
|
assert client.delete(f"/api/steering/{note_id}").status_code == 204
|
||||||
|
_stream_chat(client, QUESTION)
|
||||||
|
assert len(seeded_kb.seen_messages) == 3
|
||||||
|
assert "<tuning>" not in seeded_kb.seen_messages[-1][0]["content"]
|
||||||
|
finally:
|
||||||
|
fastapi_app.dependency_overrides.clear()
|
||||||
@@ -279,8 +279,19 @@ class _CannedLLM:
|
|||||||
yield self.answer[i : i + 12]
|
yield self.answer[i : i + 12]
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeSteeringResult:
|
||||||
|
"""Empty steering-note result (no stored notes in these unit tests)."""
|
||||||
|
|
||||||
|
def all(self) -> list[Any]:
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
class _FakeSession:
|
class _FakeSession:
|
||||||
"""Stands in for the DB session: records the QueryLog row it is given."""
|
"""Stands in for the DB session: records the QueryLog row it is given.
|
||||||
|
|
||||||
|
``scalars`` always yields no steering notes (phase 15) so the chat
|
||||||
|
turn's ``load_steering_notes`` call stays a no-op here.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(self) -> None:
|
def __init__(self) -> None:
|
||||||
self.added: list[Any] = []
|
self.added: list[Any] = []
|
||||||
@@ -292,6 +303,9 @@ class _FakeSession:
|
|||||||
def commit(self) -> None:
|
def commit(self) -> None:
|
||||||
self.commits += 1
|
self.commits += 1
|
||||||
|
|
||||||
|
def scalars(self, _stmt: Any) -> _FakeSteeringResult:
|
||||||
|
return _FakeSteeringResult()
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture()
|
@pytest.fixture()
|
||||||
def gate_env(monkeypatch: pytest.MonkeyPatch) -> Iterator[tuple[_FakeSession, _CannedLLM]]:
|
def gate_env(monkeypatch: pytest.MonkeyPatch) -> Iterator[tuple[_FakeSession, _CannedLLM]]:
|
||||||
|
|||||||
@@ -22,21 +22,30 @@ def _doc(path: str, content: str, title: str) -> Document:
|
|||||||
|
|
||||||
|
|
||||||
def test_persona_rules_present_verbatim() -> None:
|
def test_persona_rules_present_verbatim() -> None:
|
||||||
|
# Aligned to the owner's working-tree persona edits (PLAN §6 revision,
|
||||||
|
# 2026-08-22): no "you've got this" tagline, no mandated deflection
|
||||||
|
# opening. The honesty gate itself (rule 3) is unchanged.
|
||||||
for fragment in (
|
for fragment in (
|
||||||
'You are "Brain of Reese" — the digital brain of Reese, a self-hoster and',
|
'You are "Brain of Reese" — the digital brain of Reese, a self-hoster and',
|
||||||
'optimistic about the user\'s ability to do things ("you\'ve got this")',
|
"optimistic about the user's ability to do things",
|
||||||
"Answer ONLY from the provided document context. Cite which document(s)",
|
"Answer ONLY from the provided document context. Cite which document(s)",
|
||||||
"you used, by path.",
|
"you used, by path.",
|
||||||
"Be concrete: names, versions, ports, hosts, schedules",
|
"Be concrete: names, versions, ports, hosts, schedules",
|
||||||
'HONESTY GATE: if <relevance> is "LOW", you must NOT pretend to know',
|
'HONESTY GATE: if <relevance> is "LOW", you must NOT pretend to know',
|
||||||
'Start your answer with a variant of: "I haven\'t done anything like that."',
|
"Offer 2-3 alternative questions about things you DO have notes on.",
|
||||||
"Then offer 2-3 alternative questions about things you DO have notes on.",
|
|
||||||
"Never invent facts, hosts, or steps that are not in the context.",
|
"Never invent facts, hosts, or steps that are not in the context.",
|
||||||
"Keep answers tight: short paragraphs, bullets where helpful.",
|
"Keep answers tight: short paragraphs, bullets where helpful.",
|
||||||
):
|
):
|
||||||
assert fragment in PERSONA
|
assert fragment in PERSONA
|
||||||
|
|
||||||
|
|
||||||
|
def test_persona_owner_edits_are_preserved() -> None:
|
||||||
|
"""PLAN §6 revision (2026-08-22): the removed elements must stay out."""
|
||||||
|
assert 'you\'ve got this' not in PERSONA # tagline removed by the owner
|
||||||
|
assert "Start your answer with a variant of" not in PERSONA # no mandated opening
|
||||||
|
assert "HONESTY GATE" in PERSONA # the gate itself is intact
|
||||||
|
|
||||||
|
|
||||||
def test_high_prompt_carries_relevance_marker_and_full_documents() -> None:
|
def test_high_prompt_carries_relevance_marker_and_full_documents() -> None:
|
||||||
doc = _doc("kubernetes.md", "Talos Linux on three nodes.", "Kubernetes Homelab Cluster")
|
doc = _doc("kubernetes.md", "Talos Linux on three nodes.", "Kubernetes Homelab Cluster")
|
||||||
prompt = build_high_prompt([doc])
|
prompt = build_high_prompt([doc])
|
||||||
@@ -83,6 +92,27 @@ def test_low_prompt_with_no_titles() -> None:
|
|||||||
assert "nothing close at all" in build_deflect_prompt([])
|
assert "nothing close at all" in build_deflect_prompt([])
|
||||||
|
|
||||||
|
|
||||||
|
def test_zero_note_prompt_is_byte_identical_to_pre_steering() -> None:
|
||||||
|
"""Phase 15 contract: with no steering notes the prompt is exactly what
|
||||||
|
it was before the <tuning> section existed."""
|
||||||
|
doc = _doc("kubernetes.md", "Talos Linux on three nodes.", "Kubernetes Homelab Cluster")
|
||||||
|
block = (
|
||||||
|
'<document source="Homelab" path="kubernetes.md" title="Kubernetes Homelab Cluster">\n'
|
||||||
|
"Talos Linux on three nodes.\n"
|
||||||
|
"</document>"
|
||||||
|
)
|
||||||
|
assert build_high_prompt([doc]) == _base("HIGH") + "\n<documents>\n" + block + "\n</documents>"
|
||||||
|
assert build_deflect_prompt(["T1", "T2"]) == (
|
||||||
|
_base("LOW")
|
||||||
|
+ "\nDEFLECT_MODE: retrieval was weak — the titles below are the closest "
|
||||||
|
"your notes come to the question. They are titles only; do not pretend "
|
||||||
|
"they answer it. Use them to propose 2-3 alternative questions.\n"
|
||||||
|
+ "- T1\n- T2"
|
||||||
|
)
|
||||||
|
assert "<tuning>" not in build_high_prompt([doc])
|
||||||
|
assert "<tuning>" not in build_deflect_prompt([])
|
||||||
|
|
||||||
|
|
||||||
def test_relevance_placeholder_rejected_for_garbage() -> None:
|
def test_relevance_placeholder_rejected_for_garbage() -> None:
|
||||||
with pytest.raises(ValueError, match="HIGH or LOW"):
|
with pytest.raises(ValueError, match="HIGH or LOW"):
|
||||||
_base("MEDIUM")
|
_base("MEDIUM")
|
||||||
|
|||||||
@@ -0,0 +1,193 @@
|
|||||||
|
"""Unit: steering notes (phase 15) — the <tuning> prompt section.
|
||||||
|
|
||||||
|
Pure logic, no Postgres and no network: :func:`build_steering_section`
|
||||||
|
(empty/one/many/budget-truncation), its placement in the HIGH and LOW
|
||||||
|
prompts, and the ``plan_turn`` wiring (notes → prompt + ``tuning_count``).
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.api import chat as chat_api
|
||||||
|
from app.config import Settings
|
||||||
|
from app.models import Document
|
||||||
|
from app.rag.prompts import build_deflect_prompt, build_high_prompt, build_steering_section
|
||||||
|
from app.rag.retriever import TRUNCATION_MARKER, RetrievedChunk
|
||||||
|
|
||||||
|
|
||||||
|
def _settings() -> Settings:
|
||||||
|
return Settings(_env_file=None, relevance_threshold=0.30) # pyright: ignore[reportCallIssue]
|
||||||
|
|
||||||
|
|
||||||
|
def _doc(title: str, content: str) -> Document:
|
||||||
|
return Document(
|
||||||
|
id=uuid.uuid4(),
|
||||||
|
source="Homelab",
|
||||||
|
path=f"{title.lower().replace(' ', '-')}.md",
|
||||||
|
full_path="/tmp/doc.md",
|
||||||
|
title=title,
|
||||||
|
content=content,
|
||||||
|
content_hash="0" * 64,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _chunk(doc: Document, score: float) -> RetrievedChunk:
|
||||||
|
return RetrievedChunk(
|
||||||
|
chunk_id=uuid.uuid4(),
|
||||||
|
position=0,
|
||||||
|
content=doc.content[:32],
|
||||||
|
score=score,
|
||||||
|
document=doc,
|
||||||
|
cosine=score,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------- build_steering_section ----------
|
||||||
|
|
||||||
|
|
||||||
|
def test_steering_section_empty_when_no_notes() -> None:
|
||||||
|
assert build_steering_section([]) == ""
|
||||||
|
|
||||||
|
|
||||||
|
def test_steering_section_empty_when_notes_are_blank() -> None:
|
||||||
|
assert build_steering_section(["", " ", "\n\t"]) == ""
|
||||||
|
|
||||||
|
|
||||||
|
def test_steering_section_single_note_numbered() -> None:
|
||||||
|
section = build_steering_section(["be more concise"])
|
||||||
|
assert section.startswith("<tuning>\n")
|
||||||
|
assert section.endswith("\n</tuning>")
|
||||||
|
assert "1. be more concise" in section
|
||||||
|
assert TRUNCATION_MARKER not in section
|
||||||
|
|
||||||
|
|
||||||
|
def test_steering_section_trims_note_edges() -> None:
|
||||||
|
section = build_steering_section([" be more concise "])
|
||||||
|
assert "1. be more concise" in section
|
||||||
|
assert "1. be more concise" not in section
|
||||||
|
|
||||||
|
|
||||||
|
def test_steering_section_many_notes_numbered_in_order() -> None:
|
||||||
|
section = build_steering_section(["alpha", "beta", "gamma"])
|
||||||
|
assert "1. alpha" in section
|
||||||
|
assert "2. beta" in section
|
||||||
|
assert "3. gamma" in section
|
||||||
|
assert section.index("1. alpha") < section.index("2. beta") < section.index("3. gamma")
|
||||||
|
assert TRUNCATION_MARKER not in section
|
||||||
|
|
||||||
|
|
||||||
|
def test_steering_section_budget_truncation_keeps_oldest_prefix_and_marker() -> None:
|
||||||
|
# Each note is 300 chars; with a 600-char budget only note 1 fits, so
|
||||||
|
# the oldest-fitting prefix is kept and the overflow is marked.
|
||||||
|
notes = [f"note-{i} " + "x" * (300 - len(f"note-{i} ")) for i in range(3)]
|
||||||
|
section = build_steering_section(notes, max_chars=600)
|
||||||
|
assert TRUNCATION_MARKER in section
|
||||||
|
assert len(section) <= 600
|
||||||
|
assert "1. note-0" in section
|
||||||
|
assert "note-1" not in section
|
||||||
|
assert "note-2" not in section
|
||||||
|
# The marker comes last, after the kept notes.
|
||||||
|
assert section.index("1. note-0") < section.index(TRUNCATION_MARKER)
|
||||||
|
|
||||||
|
|
||||||
|
def test_steering_section_fits_budget_exactly_when_all_notes_fit() -> None:
|
||||||
|
section = build_steering_section(["a", "b", "c"], max_chars=10_000)
|
||||||
|
assert TRUNCATION_MARKER not in section
|
||||||
|
assert len(section) <= 10_000
|
||||||
|
|
||||||
|
|
||||||
|
def test_steering_section_default_budget_from_settings(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
from app.rag import prompts as prompts_mod
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
prompts_mod, "get_settings", lambda: Settings(_env_file=None) # pyright: ignore[reportCallIssue]
|
||||||
|
)
|
||||||
|
# 5 notes of 2000 chars (the API max) = 10k+ chars > the 8000 default.
|
||||||
|
notes = [f"note-{i} " + "y" * (2000 - len(f"note-{i} ")) for i in range(5)]
|
||||||
|
section = build_steering_section(notes)
|
||||||
|
assert TRUNCATION_MARKER in section
|
||||||
|
assert len(section) <= 8_000
|
||||||
|
|
||||||
|
|
||||||
|
def test_steering_section_nonpositive_budget_is_empty() -> None:
|
||||||
|
assert build_steering_section(["be concise"], max_chars=0) == ""
|
||||||
|
assert build_steering_section(["be concise"], max_chars=-10) == ""
|
||||||
|
|
||||||
|
|
||||||
|
def test_steering_section_tiny_budget_never_exceeds_cap() -> None:
|
||||||
|
# Pathological budget: the section must never exceed the cap — bare
|
||||||
|
# marker when it fits, no section at all when even that doesn't.
|
||||||
|
assert len(build_steering_section(["a" * 500], max_chars=10)) <= 10
|
||||||
|
fits_marker = build_steering_section(["a" * 500], max_chars=len(TRUNCATION_MARKER))
|
||||||
|
assert fits_marker == TRUNCATION_MARKER
|
||||||
|
|
||||||
|
|
||||||
|
# ---------- prompt placement (both modes) ----------
|
||||||
|
|
||||||
|
|
||||||
|
def test_high_prompt_steering_sits_between_relevance_and_documents() -> None:
|
||||||
|
doc = _doc("Kubernetes Homelab Cluster", "TALOS_DOC_CONTENT")
|
||||||
|
prompt = build_high_prompt([doc], notes=["be concise"])
|
||||||
|
i_rel = prompt.index("<relevance>HIGH</relevance>")
|
||||||
|
i_open = prompt.index("<tuning>")
|
||||||
|
i_close = prompt.index("</tuning>")
|
||||||
|
i_docs = prompt.index("<documents>")
|
||||||
|
assert i_rel < i_open < i_close < i_docs
|
||||||
|
assert "1. be concise" in prompt
|
||||||
|
assert "TALOS_DOC_CONTENT" in prompt # documents still full
|
||||||
|
|
||||||
|
|
||||||
|
def test_deflect_prompt_steering_sits_between_relevance_and_deflect_mode() -> None:
|
||||||
|
prompt = build_deflect_prompt(["Title A", "Title B"], notes=["be concise", "cite paths"])
|
||||||
|
i_rel = prompt.index("<relevance>LOW</relevance>")
|
||||||
|
i_open = prompt.index("<tuning>")
|
||||||
|
i_close = prompt.index("</tuning>")
|
||||||
|
i_mode = prompt.index("DEFLECT_MODE")
|
||||||
|
assert i_rel < i_open < i_close < i_mode
|
||||||
|
assert "1. be concise" in prompt
|
||||||
|
assert "2. cite paths" in prompt
|
||||||
|
assert "- Title A" in prompt # weak-hit titles still carried
|
||||||
|
assert "DEFLECT_MODE" in prompt
|
||||||
|
|
||||||
|
|
||||||
|
# ---------- plan_turn wiring (gate + steering, fake retriever rows) ----------
|
||||||
|
|
||||||
|
|
||||||
|
def test_plan_turn_high_mode_injects_notes() -> None:
|
||||||
|
doc = _doc("Kubernetes Homelab Cluster", "TALOS_DOC_SENT")
|
||||||
|
plan = chat_api.plan_turn(
|
||||||
|
[_chunk(doc, 0.90)], _settings(), notes=["be concise", "assume NixOS"]
|
||||||
|
)
|
||||||
|
assert plan.deflected is False
|
||||||
|
assert plan.tuning_count == 2
|
||||||
|
assert "<tuning>" in plan.system_prompt
|
||||||
|
assert "1. be concise" in plan.system_prompt
|
||||||
|
assert "2. assume NixOS" in plan.system_prompt
|
||||||
|
assert "<relevance>HIGH</relevance>" in plan.system_prompt
|
||||||
|
assert "TALOS_DOC_SENT" in plan.system_prompt
|
||||||
|
|
||||||
|
|
||||||
|
def test_plan_turn_low_mode_injects_notes() -> None:
|
||||||
|
doc = _doc("Kubernetes Homelab Cluster", "TALOS_DOC_NEVER_SENT")
|
||||||
|
plan = chat_api.plan_turn(
|
||||||
|
[_chunk(doc, 0.10)], _settings(), notes=["be concise"]
|
||||||
|
)
|
||||||
|
assert plan.deflected is True
|
||||||
|
assert plan.tuning_count == 1
|
||||||
|
assert "DEFLECT_MODE" in plan.system_prompt
|
||||||
|
assert "<tuning>" in plan.system_prompt
|
||||||
|
assert "1. be concise" in plan.system_prompt
|
||||||
|
assert "TALOS_DOC_NEVER_SENT" not in plan.system_prompt # titles only, still
|
||||||
|
|
||||||
|
|
||||||
|
def test_plan_turn_without_notes_has_no_tuning_section() -> None:
|
||||||
|
doc = _doc("Kubernetes Homelab Cluster", "TALOS_DOC_SENT")
|
||||||
|
for chunks in (
|
||||||
|
[_chunk(doc, 0.90)], # HIGH
|
||||||
|
[_chunk(doc, 0.10)], # LOW
|
||||||
|
):
|
||||||
|
plan = chat_api.plan_turn(chunks, _settings())
|
||||||
|
assert plan.tuning_count == 0
|
||||||
|
assert "<tuning>" not in plan.system_prompt
|
||||||
Reference in New Issue
Block a user