diff --git a/.agents/phases/todo/113_source_chip_quality/00_phase.md b/.agents/phases/complete/113_source_chip_quality/00_phase.md similarity index 100% rename from .agents/phases/todo/113_source_chip_quality/00_phase.md rename to .agents/phases/complete/113_source_chip_quality/00_phase.md diff --git a/.agents/phases/todo/113_source_chip_quality/01_usefulness_bar_sources.md b/.agents/phases/complete/113_source_chip_quality/01_usefulness_bar_sources.md similarity index 100% rename from .agents/phases/todo/113_source_chip_quality/01_usefulness_bar_sources.md rename to .agents/phases/complete/113_source_chip_quality/01_usefulness_bar_sources.md diff --git a/.agents/phases/todo/113_source_chip_quality/02_secondary_related_docs_ui.md b/.agents/phases/complete/113_source_chip_quality/02_secondary_related_docs_ui.md similarity index 100% rename from .agents/phases/todo/113_source_chip_quality/02_secondary_related_docs_ui.md rename to .agents/phases/complete/113_source_chip_quality/02_secondary_related_docs_ui.md diff --git a/.agents/phases/todo/113_source_chip_quality/03_chip_filter_tests.md b/.agents/phases/complete/113_source_chip_quality/03_chip_filter_tests.md similarity index 100% rename from .agents/phases/todo/113_source_chip_quality/03_chip_filter_tests.md rename to .agents/phases/complete/113_source_chip_quality/03_chip_filter_tests.md diff --git a/.agents/reports/113_source_chip_quality/113_source_chip_quality__00_phase.a1.err b/.agents/reports/113_source_chip_quality/113_source_chip_quality__00_phase.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/113_source_chip_quality/113_source_chip_quality__00_phase.a1.md b/.agents/reports/113_source_chip_quality/113_source_chip_quality__00_phase.a1.md new file mode 100644 index 0000000..cce6b56 --- /dev/null +++ b/.agents/reports/113_source_chip_quality/113_source_chip_quality__00_phase.a1.md @@ -0,0 +1,18 @@ +All gates green — no defects found; this pass was verification only. + +**Phase 113 final verification pass — report** + +- Verified (no code changes needed): `select_documents_tiered` cited/related tiering + `select_documents` wrapper, `TurnPlan.related_docs`, `ChatDoneEvent.related` (additive, old payloads parse), `appendRelated` UI row (`.related-doc`, never `.source-chip`), done-frame + restore-path wiring, two settings with validators, `.env.example` entries +- `uv run pytest --cov=app --cov-report=term-missing` → 2422 passed, app/ coverage **99%** (>90% gate) +- `uv run pytest tests/e2e/test_source_chip_quality.py -v --no-cov` (isolated) → 2 passed +- Regression E2E `test_retrieval_quality.py` + `test_honest_deflection.py` + `test_chat_rag.py` + `test_sources_midstream_bug.py` → 17 passed +- `uv run ruff check . && uv run pyright` → clean (0 errors); `bash .agents/validate.sh` → "validation OK" + +Completion criteria: +1. Single-doc question → exactly one `.source-chip` (E2E): ✅ passed +2. Weak 2nd doc only in de-emphasized related row, never `.source-chip` (unit + E2E): ✅ passed +3. Deflected turn → zero citation chips, weak hits in related row: ✅ passed +4. Full suite green, coverage >90%, isolated E2E green, lint/types clean: ✅ passed +5. `--no-gpg-sign` commit + phase dir move: left to harness per pass rules (task files already in `complete/`) + +No deviations. Next pending phase: `114_embed_question_length`. diff --git a/.agents/reports/113_source_chip_quality/113_source_chip_quality__00_phase.a1.validate b/.agents/reports/113_source_chip_quality/113_source_chip_quality__00_phase.a1.validate new file mode 100644 index 0000000..fc53db6 --- /dev/null +++ b/.agents/reports/113_source_chip_quality/113_source_chip_quality__00_phase.a1.validate @@ -0,0 +1,102 @@ +........................................................................ [ 2%] +........................................................................ [ 5%] +........................................................................ [ 8%] +........................................................................ [ 11%] +........................................................................ [ 14%] +........................................................................ [ 17%] +........................................................................ [ 20%] +........................................................................ [ 23%] +........................................................................ [ 26%] +........................................................................ [ 29%] +........................................................................ [ 32%] +........................................................................ [ 35%] +........................................................................ [ 38%] +........................................................................ [ 41%] +........................................................................ [ 44%] +........................................................................ [ 47%] +........................................................................ [ 50%] +........................................................................ [ 53%] +........................................................................ [ 56%] +........................................................................ [ 59%] +........................................................................ [ 62%] +........................................................................ [ 65%] +........................................................................ [ 68%] +........................................................................ [ 71%] +........................................................................ [ 74%] +........................................................................ [ 77%] +........................................................................ [ 80%] +........................................................................ [ 83%] +........................................................................ [ 86%] +........................................................................ [ 89%] +........................................................................ [ 92%] +........................................................................ [ 95%] +........................................................................ [ 98%] +.............................................. [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain-of-reese/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.7-final-0 ________________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 217 1 99% +app/api/chats.py 110 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 94 0 100% +app/api/docs.py 156 1 99% +app/api/git_sources.py 232 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 139 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 203 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 20 0 100% +app/core/theming.py 38 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 66 0 100% +app/models.py 128 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 317 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 206 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 123 0 100% +app/rag/git_sources.py 14 0 100% +app/rag/importer.py 215 3 99% +app/rag/llm.py 243 1 99% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 88 0 100% +app/rag/retriever.py 192 3 98% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 24 0 100% +app/schemas.py 328 0 100% +-------------------------------------------------- +TOTAL 4126 16 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/113_source_chip_quality/113_source_chip_quality__01_usefulness_bar_sources.a1.err b/.agents/reports/113_source_chip_quality/113_source_chip_quality__01_usefulness_bar_sources.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/113_source_chip_quality/113_source_chip_quality__01_usefulness_bar_sources.a1.md b/.agents/reports/113_source_chip_quality/113_source_chip_quality__01_usefulness_bar_sources.a1.md new file mode 100644 index 0000000..19641fb --- /dev/null +++ b/.agents/reports/113_source_chip_quality/113_source_chip_quality__01_usefulness_bar_sources.a1.md @@ -0,0 +1,15 @@ +All gates green. Task complete. + +**Implemented (task 01 — usefulness bar):** +- `app/config.py`: `source_usefulness_floor=0.35` (`BOR_SOURCE_USEFULNESS_FLOOR`, 0≤floor≤threshold) + `related_max_docs=2` (`BOR_RELATED_MAX_DOCS`, ≥0) + `.env.example` entries +- `app/rag/retriever.py`: `select_documents_tiered()` — rank-ordered distinct docs, best-chunk-cosine tracked; cited = bar-clearers ≤n (ceiling), related = next ranked non-cited ≤cap; `select_documents` = thin wrapper (floor 0.0 = no bar, legacy byte-identical) +- `app/api/chat.py`: `TurnPlan.related_docs`; `plan_turn` tiers both HIGH/LOW branches (deflected weak hits → related); done frame carries `related` deduped against cited (incl. agent-read); query_log/log line keep the full retrieval (A3) +- `app/schemas.py`: `ChatDoneEvent.related: list[SourceRef] = []` (additive; old frames parse) +- Tests: retriever tiering table (unit), `plan_turn` tier pins, settings validators, integration done-frame/related/dedupe/cap + back-compat pins +- **Defect fixes (pre-existing work):** phase-112 e2e boot defect (e2e app couldn't start: threshold 0.30 vs code-default lexical floor 0.35 → validator failure) fixed via mock-calibrated `BOR_LEXICAL_SUPPORT_FLOOR=0.15`; `get_settings()` lru_cache leak in the kafkabridge integration test hardened; one unit assertion updated to the new tier semantics + +**Results:** `uv run pytest` → 2406 passed · `uv run pytest --cov=app` → TOTAL 99% (>90%) · `uv run ruff check .` → clean · `uv run pyright` → 0 errors · E2E in isolation (`--no-cov`): test_retrieval_quality 4✓, test_honest_deflection 4✓, test_chat_rag 3✓, test_sources_midstream_bug 6✓ (+4 more e2e suites spot-checked green) + +**Notable decisions:** bar on best hit-chunk cosine (A2); related = rank-order continuation never overlapping cited; e2e keyword test's original first-chip assertion was actually correct (mock-server summary digests rank differently than the in-process fake) — my initial edit was reverted; test-env floors mock-calibrated (unit/int 0.02, e2e 0.15). + +**Next pending task:** `02_secondary_related_docs_ui.md` (related-docs row in the chat UI). diff --git a/.agents/reports/113_source_chip_quality/113_source_chip_quality__01_usefulness_bar_sources.a1.validate b/.agents/reports/113_source_chip_quality/113_source_chip_quality__01_usefulness_bar_sources.a1.validate new file mode 100644 index 0000000..86b048b --- /dev/null +++ b/.agents/reports/113_source_chip_quality/113_source_chip_quality__01_usefulness_bar_sources.a1.validate @@ -0,0 +1,102 @@ +........................................................................ [ 2%] +........................................................................ [ 5%] +........................................................................ [ 8%] +........................................................................ [ 11%] +........................................................................ [ 14%] +........................................................................ [ 17%] +........................................................................ [ 20%] +........................................................................ [ 23%] +........................................................................ [ 26%] +........................................................................ [ 29%] +........................................................................ [ 32%] +........................................................................ [ 35%] +........................................................................ [ 38%] +........................................................................ [ 41%] +........................................................................ [ 44%] +........................................................................ [ 47%] +........................................................................ [ 50%] +........................................................................ [ 53%] +........................................................................ [ 56%] +........................................................................ [ 59%] +........................................................................ [ 62%] +........................................................................ [ 65%] +........................................................................ [ 68%] +........................................................................ [ 71%] +........................................................................ [ 74%] +........................................................................ [ 77%] +........................................................................ [ 80%] +........................................................................ [ 83%] +........................................................................ [ 86%] +........................................................................ [ 89%] +........................................................................ [ 92%] +........................................................................ [ 95%] +........................................................................ [ 98%] +.............................. [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain-of-reese/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.7-final-0 ________________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 217 1 99% +app/api/chats.py 110 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 94 0 100% +app/api/docs.py 156 1 99% +app/api/git_sources.py 232 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 139 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 203 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 20 0 100% +app/core/theming.py 38 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 66 0 100% +app/models.py 128 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 317 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 206 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 123 0 100% +app/rag/git_sources.py 14 0 100% +app/rag/importer.py 215 3 99% +app/rag/llm.py 243 1 99% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 88 0 100% +app/rag/retriever.py 192 3 98% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 24 0 100% +app/schemas.py 328 0 100% +-------------------------------------------------- +TOTAL 4126 16 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/113_source_chip_quality/113_source_chip_quality__02_secondary_related_docs_ui.a1.err b/.agents/reports/113_source_chip_quality/113_source_chip_quality__02_secondary_related_docs_ui.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/113_source_chip_quality/113_source_chip_quality__02_secondary_related_docs_ui.a1.md b/.agents/reports/113_source_chip_quality/113_source_chip_quality__02_secondary_related_docs_ui.a1.md new file mode 100644 index 0000000..42f27f9 --- /dev/null +++ b/.agents/reports/113_source_chip_quality/113_source_chip_quality__02_secondary_related_docs_ui.a1.md @@ -0,0 +1,13 @@ +All gates green. Final report: + +**Implemented (task 02 — related-docs row, never a citation chip):** +- `app.js`: new `appendRelated(wrap, related)` next to `appendSources` — early-return on empty; `.msg-meta.related-docs` row (`role="list"`, `aria-label="Nearby docs, in case"`), small-caps label, one `.related-doc` link per doc (NOT `.source-chip`) with the same `documentUrl(...)` href + left-click → `openDocumentModal`, full path in `title`/`aria-label` +- Done-frame handler: `appendRelated(wrap, ev.related)` appended **last** (after tune/save/retry claimers — so a deflected turn's meta buttons get their own row, never the related row) + `related` persisted in the save payload (house optional-meta pattern); restore path (`renderStoredMessage`) re-renders it — pre-phase records no-op gracefully +- `styles.css`: `.related-docs-label` / `.related-doc` (dashed border, transparent, 0.7rem mono, AA ink-soft 8.6:1) / flat hover (no bg swap); palette vars only (phase-92 zero-literal) +- `tests/unit/test_source_chip_quality.py` (new): 10 source-pin tests (task 02 work item 3; task 03 extends this file) + +**Results:** `uv run pytest` → 2416 passed · `--cov=app` → TOTAL 99% (>90%) · `uv run ruff check .` → clean · `uv run pyright` → 0 errors · E2E regressions in isolation: `test_retrieval_quality` 4✓, `test_honest_deflection` 4✓, `test_chat_rag` 3✓, `test_sources_midstream_bug` 6✓ + +**Decisions:** related row appends last so first-`.msg-meta` claimers can't join it (pinned); `related` only persisted when non-empty (`undefined` drops the key). No deviations; task 01's working-tree changes left intact. + +**Next pending task:** `03_chip_filter_tests.md` (four-observed-shapes unit table + E2E chip-count assertions). diff --git a/.agents/reports/113_source_chip_quality/113_source_chip_quality__02_secondary_related_docs_ui.a1.validate b/.agents/reports/113_source_chip_quality/113_source_chip_quality__02_secondary_related_docs_ui.a1.validate new file mode 100644 index 0000000..0c298bd --- /dev/null +++ b/.agents/reports/113_source_chip_quality/113_source_chip_quality__02_secondary_related_docs_ui.a1.validate @@ -0,0 +1,102 @@ +........................................................................ [ 2%] +........................................................................ [ 5%] +........................................................................ [ 8%] +........................................................................ [ 11%] +........................................................................ [ 14%] +........................................................................ [ 17%] +........................................................................ [ 20%] +........................................................................ [ 23%] +........................................................................ [ 26%] +........................................................................ [ 29%] +........................................................................ [ 32%] +........................................................................ [ 35%] +........................................................................ [ 38%] +........................................................................ [ 41%] +........................................................................ [ 44%] +........................................................................ [ 47%] +........................................................................ [ 50%] +........................................................................ [ 53%] +........................................................................ [ 56%] +........................................................................ [ 59%] +........................................................................ [ 62%] +........................................................................ [ 65%] +........................................................................ [ 68%] +........................................................................ [ 71%] +........................................................................ [ 74%] +........................................................................ [ 77%] +........................................................................ [ 80%] +........................................................................ [ 83%] +........................................................................ [ 86%] +........................................................................ [ 89%] +........................................................................ [ 92%] +........................................................................ [ 95%] +........................................................................ [ 98%] +........................................ [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain-of-reese/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.7-final-0 ________________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 217 1 99% +app/api/chats.py 110 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 94 0 100% +app/api/docs.py 156 1 99% +app/api/git_sources.py 232 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 139 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 203 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 20 0 100% +app/core/theming.py 38 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 66 0 100% +app/models.py 128 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 317 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 206 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 123 0 100% +app/rag/git_sources.py 14 0 100% +app/rag/importer.py 215 3 99% +app/rag/llm.py 243 1 99% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 88 0 100% +app/rag/retriever.py 192 3 98% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 24 0 100% +app/schemas.py 328 0 100% +-------------------------------------------------- +TOTAL 4126 16 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/113_source_chip_quality/113_source_chip_quality__03_chip_filter_tests.a1.err b/.agents/reports/113_source_chip_quality/113_source_chip_quality__03_chip_filter_tests.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/113_source_chip_quality/113_source_chip_quality__03_chip_filter_tests.a1.md b/.agents/reports/113_source_chip_quality/113_source_chip_quality__03_chip_filter_tests.a1.md new file mode 100644 index 0000000..564fcef --- /dev/null +++ b/.agents/reports/113_source_chip_quality/113_source_chip_quality__03_chip_filter_tests.a1.md @@ -0,0 +1,17 @@ +Task 03 complete — all gates green. + +**Implemented** +- `tests/unit/test_source_chip_quality.py` (extended): the four observed live shapes pinned under production calibration (0.62/0.35/0.35, top_n=2, cap=2) — shape 1 Mongolia (both docs weak: FTS hit uncorroborated stays LOW, `docs=[]`, related capped at 2), shape 2 validate.sh (1 cited + 1 related), shape 3 Nagraz/Byzin (same shape, different fixtures) at `plan_turn` level; shape 4 (meta/history) + shape 2 pinned on the actual done-frame wire via endpoint-level fakes (row-only deflected frame: `sources: []` + `related`; exactly-one-cited-ref frame); agent-read exemption (below-floor doc `read` via the tool joins `sources` last, excluded from `related`). +- `tests/e2e/test_source_chip_quality.py` (new): single-source question "What SSH aliases do I have?" → exactly one `.source-chip` (ssh_aliases.txt) + labeled `.related-docs` row with 2 `.related-doc` links, zero `.source-chip` in the row, full retrieval in `query_log` (A3); deflected "sourdough" question → zero `.source-chip`, weak hits row-only, `query_log` intact. + +**Results** +- `uv run pytest tests/e2e/test_source_chip_quality.py -v --no-cov` → 2 passed (in isolation) +- Regression E2E (`test_retrieval_quality`, `test_honest_deflection`, `test_chat_rag`, `test_sources_midstream_bug`) → 17 passed +- `uv run pytest --cov=app --cov-report=term-missing` → 2422 passed, `app/` coverage **99%** (>90%) +- `uv run ruff check . && uv run pyright` → clean (0 errors) + +**Notes** +- E2E tier shapes measured against the real pipeline (mock BoW embeddings, mock-calibrated floor 0.15) before pinning; fixture KB does produce the strong+weak two-tier shape live, so no docstring caveat was needed. +- No app code changes — task 03 is test-only; no defects found in prior tasks' work. + +**Next pending task:** none in phase 113 — this was its final task (phase dir moves to `complete/` by the pipeline gate). diff --git a/.agents/reports/113_source_chip_quality/113_source_chip_quality__03_chip_filter_tests.a1.validate b/.agents/reports/113_source_chip_quality/113_source_chip_quality__03_chip_filter_tests.a1.validate new file mode 100644 index 0000000..fc53db6 --- /dev/null +++ b/.agents/reports/113_source_chip_quality/113_source_chip_quality__03_chip_filter_tests.a1.validate @@ -0,0 +1,102 @@ +........................................................................ [ 2%] +........................................................................ [ 5%] +........................................................................ [ 8%] +........................................................................ [ 11%] +........................................................................ [ 14%] +........................................................................ [ 17%] +........................................................................ [ 20%] +........................................................................ [ 23%] +........................................................................ [ 26%] +........................................................................ [ 29%] +........................................................................ [ 32%] +........................................................................ [ 35%] +........................................................................ [ 38%] +........................................................................ [ 41%] +........................................................................ [ 44%] +........................................................................ [ 47%] +........................................................................ [ 50%] +........................................................................ [ 53%] +........................................................................ [ 56%] +........................................................................ [ 59%] +........................................................................ [ 62%] +........................................................................ [ 65%] +........................................................................ [ 68%] +........................................................................ [ 71%] +........................................................................ [ 74%] +........................................................................ [ 77%] +........................................................................ [ 80%] +........................................................................ [ 83%] +........................................................................ [ 86%] +........................................................................ [ 89%] +........................................................................ [ 92%] +........................................................................ [ 95%] +........................................................................ [ 98%] +.............................................. [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain-of-reese/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.7-final-0 ________________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 217 1 99% +app/api/chats.py 110 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 94 0 100% +app/api/docs.py 156 1 99% +app/api/git_sources.py 232 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 139 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 203 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 20 0 100% +app/core/theming.py 38 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 66 0 100% +app/models.py 128 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 317 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 206 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 123 0 100% +app/rag/git_sources.py 14 0 100% +app/rag/importer.py 215 3 99% +app/rag/llm.py 243 1 99% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 88 0 100% +app/rag/retriever.py 192 3 98% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 24 0 100% +app/schemas.py 328 0 100% +-------------------------------------------------- +TOTAL 4126 16 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.env.example b/.env.example index 6741ed3..6f2a195 100644 --- a/.env.example +++ b/.env.example @@ -33,9 +33,11 @@ BOR_EMBEDDING_DIM=768 # verified 2026-08 via scripts/llm_probe.py BOR_STREAM_THINKING=1 # stream the model's thinking as `thinking` SSE events (0 to suppress) # --- RAG tuning --- -BOR_TOP_N_DOCS=2 +BOR_TOP_N_DOCS=2 # CEILING for the cited tier, not a quota (phase 113): a single strong doc yields one citation BOR_RELEVANCE_THRESHOLD=0.62 # answer when best cosine >= this OR an FTS hit corroborated by cosine >= lexical_support_floor; else honest deflection # BOR_LEXICAL_SUPPORT_FLOOR=0.35 # cosine floor for FTS hits to flip HIGH (A8 revised 2026-09-14); 0 <= floor <= relevance_threshold +# BOR_SOURCE_USEFULNESS_FLOOR=0.35 # cosine bar for a citation slot (phase 113): a doc earns done.sources only when its best-chunk cosine clears it; 0 <= floor <= relevance_threshold; 0 = no bar +# BOR_RELATED_MAX_DOCS=2 # cap on the related-doc tier in the done frame (phase 113: the UI's de-emphasized "nearby docs" row, never a citation chip); 0 = no related docs BOR_MAX_OUTPUT_TOKENS=32768 # max answer length in tokens (answers must not be cut off) BOR_STEERING_MAX_CHARS=8000 # char budget for the (steering notes) prompt section BOR_SUMMARY_MAX_CHARS=12000 # cap on document content sent to the lite summary model (phase 30) diff --git a/app/api/chat.py b/app/api/chat.py index 2eff670..39c502a 100644 --- a/app/api/chat.py +++ b/app/api/chat.py @@ -164,7 +164,7 @@ import json import logging import time from collections.abc import AsyncIterator, Sequence -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Any from fastapi import APIRouter, Depends @@ -194,7 +194,7 @@ from app.rag.llm import ( ) from app.rag.overview import load_kb_overview from app.rag.prompts import build_deflect_prompt, build_high_prompt, history_to_messages -from app.rag.retriever import RetrievedChunk, retrieve, select_documents, weak_hit_titles +from app.rag.retriever import RetrievedChunk, retrieve, select_documents_tiered, weak_hit_titles from app.rag.scaffolding import ScaffoldingFilter # phase 71: the streaming filter from app.rag.suggestions import derive_suggestions from app.schemas import ( @@ -257,8 +257,9 @@ class TurnPlan: fts_hits: int # lexical (OR-tsquery) candidates matched deflected: bool system_prompt: str - docs: list[Document] # cited sources (weak hits when deflected) - suggestions: list[str] # "Maybe try" chips (deflected turns only) + docs: list[Document] # cited sources (phase 113: the bar-clearing tier) + related_docs: list[Document] = field(default_factory=list) # phase 113 + suggestions: list[str] = field(default_factory=list) # "Maybe try" chips (deflected turns only) tuning_count: int = 0 # steering notes injected into the system prompt #: Hit chunks with ``is_summary`` whose parent document made it into #: *docs* (phase 30; per-turn log line ``summary_hits=N``). @@ -306,13 +307,29 @@ def plan_turn( ``summary_hits`` (phase 30) counts the hit chunks with ``is_summary`` whose parent document is among the selected top-N documents — both the HIGH and the LOW branch record it. + + Phase 113 (the usefulness bar, LOCKED A2): retrieval documents are + tiered before either branch — ``docs`` (cited) are the distinct + parent documents whose best hit-chunk cosine clears + ``settings.source_usefulness_floor`` (at most ``top_n_docs`` — the + ceiling, never a quota); ``related_docs`` are the next ranked + documents (at most ``related_max_docs``) that did not earn a cited + slot. On a deflected turn the weak hits fall to ``related_docs`` + (the cited tier is usually empty — nothing below the bar earned a + citation slot); the LOW prompt itself is unchanged (weak-hit titles + only). """ steering = list(notes or []) kb_text = (kb_overview or "").strip() kb_chars = len(kb_text) best_cosine = max((c.cosine for c in chunks), default=0.0) fts_hits = sum(1 for c in chunks if c.fts_hit) - docs = select_documents(chunks, n=settings.top_n_docs) + docs, related_docs = select_documents_tiered( + chunks, + n=settings.top_n_docs, + floor=settings.source_usefulness_floor, + related_cap=settings.related_max_docs, + ) selected_ids = {d.id for d in docs} summary_hits = sum(1 for c in chunks if c.is_summary and c.document.id in selected_ids) lexical_supported = fts_hits > 0 and best_cosine >= settings.lexical_support_floor @@ -323,6 +340,7 @@ def plan_turn( False, build_high_prompt(docs, notes=steering, kb_overview=kb_text), docs, + related_docs, [], len(steering), summary_hits, @@ -335,6 +353,7 @@ def plan_turn( True, build_deflect_prompt(titles, notes=steering, kb_overview=kb_text), docs, + related_docs, derive_suggestions(titles, settings.suggestions), len(steering), summary_hits, @@ -747,24 +766,36 @@ async def chat( # 4. Durable record + required per-turn log line (PLAN §9). # Phase 37: the agent's read documents join the - # retrieval's — deduped by (source, path), order preserved. - # The combined list feeds query_log.sources and the log - # line (retrieval docs even on deflected turns — - # observability, the phase-113 A3 precedent). Phase 112: + # retrieval's — deduped by (source, path), order + # preserved (the read doc stays last). Phase 113: + # the retrieval now arrives in two tiers — the cited + # docs (``plan.docs``) and the related docs + # (``plan.related_docs``, the scored-but-below-the-bar + # documents). The DURABLE record keeps the full + # retrieval (LOCKED A3: query_log records retrieval, + # not citations — even on deflected turns, where the + # weak hits live in the related tier). Phase 112: # done.sources is the CITATION surface — it carries the - # combined list on grounded turns and [] on deflected - # ones (a deflected answer cites nothing; the weak hits - # stay in the durable record). A cancelled turn (the - # generator closed by the consumer) never reaches this - # step — no query_log row. + # cited docs + the agent-read docs on grounded turns + # and [] on deflected ones (a deflected answer cites + # nothing; the weak hits stay in the durable record). + # A cancelled turn (the generator closed by the + # consumer) never reaches this step — no query_log row. cited_docs: list[Document] = [] - seen: set[tuple[str, str]] = set() + cited_seen: set[tuple[str, str]] = set() for doc in [*plan.docs, *holder.read_docs]: + key = (doc.source, doc.path) + if key not in cited_seen: + cited_seen.add(key) + cited_docs.append(doc) + record_docs: list[Document] = [] + seen: set[tuple[str, str]] = set() + for doc in [*plan.docs, *plan.related_docs, *holder.read_docs]: key = (doc.source, doc.path) if key not in seen: seen.add(key) - cited_docs.append(doc) - source_paths = [f"{d.source}/{d.path}" for d in cited_docs] + record_docs.append(doc) + source_paths = [f"{d.source}/{d.path}" for d in record_docs] total_ms = int((time.monotonic() - started) * 1000) try: with SessionLocal() as log_db: @@ -810,18 +841,31 @@ async def chat( # as a citation; the weak hits are scored docs, not # citations). The retrieval stays durably recorded above # (query_log.sources + the log line — observability - # unchanged); the phase-113 related-doc tier is the home - # for the weak hits' visibility. + # unchanged). + # Phase 113 (A2/A4): done.related carries the related + # tier — the scored documents that did not clear the + # usefulness bar (deduped against the cited list, the + # same (source, path) pattern as cited_docs: an agent- + # read related doc is a citation, never a "nearby doc" + # — and capped by related_max_docs in the tiering). The + # UI renders it as the de-emphasized related-docs row, + # never a citation chip; old clients ignore the field. cited_refs: list[SourceRef] = [] if not plan.deflected: cited_refs = [ SourceRef(source=d.source, path=d.path, title=d.title) for d in cited_docs ] + related_refs = [ + SourceRef(source=d.source, path=d.path, title=d.title) + for d in plan.related_docs + if (d.source, d.path) not in cited_seen + ] yield sse_event( ChatDoneEvent( deflected=plan.deflected, sources=cited_refs, + related=related_refs, suggestions=plan.suggestions, ).model_dump() ) diff --git a/app/config.py b/app/config.py index c318e39..c0d58cb 100644 --- a/app/config.py +++ b/app/config.py @@ -126,6 +126,29 @@ class Settings(BaseSettings): # ``relevance_threshold`` (a floor above the threshold is a typo that # would make every FTS hit require a HIGH cosine anyway). lexical_support_floor: float = 0.35 + #: Usefulness bar for the citation slot (phase 113, LOCKED A2): a + #: retrieved document earns ``done.sources`` (the UI's citation chip) + #: only when the **cosine** of its best hit chunk clears this floor — + #: the vector signal must corroborate the citation, mirroring the A8 + #: honesty gate's ``lexical_support_floor``. Documents that scored but + #: stay below the bar are demoted to the secondary related-doc tier + #: (at most ``related_max_docs``). ``top_n_docs`` is the CEILING for + #: the cited tier, never a quota: a single strong document yields one + #: citation. Default 0.35 — the same bar as ``lexical_support_floor``; + #: tunable via ``BOR_SOURCE_USEFULNESS_FLOOR``. Must satisfy + #: ``0 <= source_usefulness_floor <= relevance_threshold`` (a floor + #: above the threshold would demote to the related tier documents the + #: gate itself calls grounded — the ``lexical_support_floor`` typo + #: guard). ``0`` disables the bar (every scored doc is citable — the + #: pre-phase behavior, the kill switch). + source_usefulness_floor: float = 0.35 + #: Cap on the secondary related-doc tier (phase 113, LOCKED A4): + #: documents that scored but did not clear ``source_usefulness_floor`` + #: ride the ``done`` frame's ``related`` list (the UI's de-emphasized + #: "nearby docs" row — never a citation chip). ``0`` = no related + #: docs at all (the kill switch); a negative value fails startup + #: loudly (the ``agent_max_rounds`` pattern). + related_max_docs: int = 2 #: Maximum output tokens a chat answer may use (owner instruction #: 2026-08-22: answers must run to their natural end — the old hard #: 700-token cap cut long answers off mid-sentence). @@ -327,6 +350,31 @@ class Settings(BaseSettings): ) return v + @field_validator("source_usefulness_floor") + @classmethod + def _source_usefulness_floor_bounds(cls, v: float, info: ValidationInfo) -> float: + """The usefulness bar must be in [0, relevance_threshold]. A value + above the relevance threshold would be a typo — it would demote to + the related tier documents the honesty gate itself calls grounded + (the ``lexical_support_floor`` typo guard, phase 113).""" + if v < 0: + raise ValueError("source_usefulness_floor must be >= 0") + threshold = info.data.get("relevance_threshold") + if isinstance(threshold, float) and v > threshold: + raise ValueError( + f"source_usefulness_floor ({v}) must be <= relevance_threshold ({threshold})" + ) + return v + + @field_validator("related_max_docs") + @classmethod + def _related_max_docs_non_negative(cls, v: int) -> int: + """``0`` is the no-related-docs kill switch — a negative cap is a + typo (the ``agent_max_rounds`` pattern, phase 113).""" + if v < 0: + raise ValueError("related_max_docs must be >= 0 (0 = no related docs)") + return v + @field_validator("import_extensions") @classmethod def _import_extensions_known(cls, v: str) -> str: diff --git a/app/rag/retriever.py b/app/rag/retriever.py index cbb339d..573101e 100644 --- a/app/rag/retriever.py +++ b/app/rag/retriever.py @@ -27,8 +27,13 @@ unchanged (same lists, same ``1/(k+rank)`` terms). * **Fusion** — Reciprocal Rank Fusion (``score = Σ 1/(k + rank)`` over the lists a chunk appears in; chunks hit by both lists get both terms). The - fused score ranks; :meth:`select_documents` and :func:`weak_hit_titles` - keep working off ``score``. + fused score ranks; :func:`select_documents` and :func:`weak_hit_titles` + keep working off ``score``. Phase 113: the document tiering + (:func:`select_documents_tiered`) keeps the same rank order and adds the + usefulness bar — a document earns the cited tier only when its best + hit-chunk cosine clears ``BOR_SOURCE_USEFULNESS_FLOOR``; the rest of the + ranked documents (up to ``BOR_RELATED_MAX_DOCS``) become the related + tier. The product requirement (LOCKED A7, revised 2026-08-24): the LLM receives the **entire relevant document**, not just the chunk — chunk hits map back @@ -560,6 +565,77 @@ def weak_hit_titles(chunks: Sequence[RetrievedChunk]) -> list[str]: return titles +def select_documents_tiered( + chunks: Sequence[RetrievedChunk], + n: int | None = None, + floor: float = 0.0, + related_cap: int = 0, +) -> tuple[list[Document], list[Document]]: + """Tier chunk hits into the cited and the related parent documents + (phase 113, LOCKED A2/A4 — the usefulness bar). + + Distinct parent documents are ranked exactly like :func:`select_documents` + (best fused score first — the same stable score-descending walk, so a + document's rank position is fixed by its FIRST seen chunk) and each + document's **best hit-chunk cosine** is tracked across all of its + chunks. The tiers are then cut in that rank order: + + * **cited** — the documents whose best-chunk cosine clears *floor*, + up to *n* (default ``BOR_TOP_N_DOCS``). The ceiling, never a quota: + a single strong document yields one cited document, and documents + whose cosine stays below the bar are skipped (the next-ranked + clearing document takes their slot — the bar filters, it does not + backfill). The bar is on the **cosine**, not the RRF fused score: + the fused ``score`` is a rank key, not a similarity, and a + lexical-only hit has cosine 0.0 — vector-unsupported by definition + (LOCKED A2, consistent with the A8 honesty gate). + * **related** — the next distinct documents in the same rank order + that are not already cited (any cosine, including 0.0 lexical-only + hits), up to *related_cap* (default 0). Never overlaps the cited + list. The done frame carries them in the secondary ``related`` + tier — the UI's de-emphasized "nearby docs" row, never a citation + chip (LOCKED A4). + + With ``floor=0.0`` (no bar — a zero floor admits every scored + document, so the legacy "any score, top-N" selection holds exactly) + and ``related_cap=0`` the tiering degenerates to the legacy behavior: + :func:`select_documents` is a thin wrapper on that. + + The returned rows carry the full document content, byte-identical — + a matched parent document is **never truncated** (A7 revised, owner + permission 2026-08-24). + """ + top_n = n if n is not None else get_settings().top_n_docs + no_bar = floor <= 0.0 + + order: list[Document] = [] + best_cosine: dict[uuid.UUID, float] = {} + for rc in sorted(chunks, key=lambda c: c.score, reverse=True): + doc = rc.document + if doc.id in best_cosine: + if rc.cosine > best_cosine[doc.id]: + best_cosine[doc.id] = rc.cosine + continue + best_cosine[doc.id] = rc.cosine + order.append(doc) + + cited: list[Document] = [] + for doc in order: + if len(cited) >= top_n: + break + if no_bar or best_cosine[doc.id] >= floor: + cited.append(doc) + + cited_ids = {doc.id for doc in cited} + related: list[Document] = [] + for doc in order: + if len(related) >= related_cap: + break + if doc.id not in cited_ids: + related.append(doc) + return cited, related + + def select_documents( chunks: Sequence[RetrievedChunk], n: int | None = None, @@ -572,14 +648,11 @@ def select_documents( permission 2026-08-24). There is deliberately no context budget: an oversized prompt must fail loudly through the ``LLMError`` → SSE ``error`` path, never arrive as silent partial context. - """ - top_n = n if n is not None else get_settings().top_n_docs - docs: list[Document] = [] - seen: set[uuid.UUID] = set() - for rc in sorted(chunks, key=lambda c: c.score, reverse=True): - if rc.document.id in seen: - continue - seen.add(rc.document.id) - docs.append(rc.document) - return docs[:top_n] + Phase 113: a thin wrapper on :func:`select_documents_tiered` — the + legacy "any score, top-N" behavior is the cited tier with a zero + floor (no bar) and an empty related tier, byte-identical for all + existing callers. + """ + cited, _ = select_documents_tiered(chunks, n, 0.0, 0) + return cited diff --git a/app/schemas.py b/app/schemas.py index b048545..2bbb5ca 100644 --- a/app/schemas.py +++ b/app/schemas.py @@ -183,11 +183,20 @@ class ChatToolResultEvent(BaseModel): class ChatDoneEvent(BaseModel): - """Final SSE event of a chat turn: metadata for the finished answer.""" + """Final SSE event of a chat turn: metadata for the finished answer. + + Phase 113: ``related`` — the secondary related-doc tier (documents + that scored but did not clear the usefulness bar, LOCKED A2/A4). The + UI renders it as the de-emphasized "nearby docs" row — never a + citation chip — while ``sources`` stays the citation surface. The + field is ADDITIVE: old clients ignore unknown fields (PLAN §4 house + contract) and old frames without it parse with the default ``[]``. + """ type: str = "done" deflected: bool sources: list[SourceRef] + related: list[SourceRef] = [] suggestions: list[str] = [] diff --git a/frontend/assets/app.js b/frontend/assets/app.js index 8fb4d7f..58882b2 100644 --- a/frontend/assets/app.js +++ b/frontend/assets/app.js @@ -1395,6 +1395,51 @@ function appendSources(wrap, sources) { } } +/* Phase 113 (task 02): the DE-EMPHASIZED related-docs row — the done + * frame's second tier (phase 113 task 01): documents that scored but + * did not clear the usefulness bar. It must never read as a citation: + * the links carry the .related-doc class (NOT .source-chip — the + * citation surface stays appendSources' alone) while behaving exactly + * like the chips — the same documentUrl href (the /document.html escape + * hatch) and the same left-click → same-page modal (phase 26). The + * labeled row ("Nearby docs, in case:") joins the bubble's .msg-meta + * family and stacks below the citation row in the .msg-body flex gap. + * app.js appends it LAST among the meta rows — after appendTuneButton / + * appendSaveAsDocButton / appendRetryButton claimed the FIRST + * .msg-meta row — so a deflected turn (no citation row) never lets a + * meta action join this row. Empty/absent input → no DOM at all + * (pre-phase saved chats carry no `related` — the row is simply absent). + */ +function appendRelated(wrap, related) { + if (!related || !related.length) return; + const body = wrap.querySelector(".msg-body"); + const row = document.createElement("div"); + row.className = "msg-meta related-docs"; + row.setAttribute("role", "list"); + row.setAttribute("aria-label", "Nearby docs, in case"); + const label = document.createElement("span"); + label.className = "related-docs-label"; + label.textContent = "Nearby docs, in case:"; + row.appendChild(label); + for (const s of related) { + const docLabel = `${s.source}/${s.path}`; + const link = document.createElement("a"); + link.className = "related-doc"; + link.setAttribute("role", "listitem"); + link.href = documentUrl(s.source, s.path, "/"); // back → the chat page + link.addEventListener("click", (e) => { + e.preventDefault(); // no new tab (phase 26) — the modal takes over + e.stopPropagation(); + openDocumentModal(s.source, s.path, link); + }); + link.textContent = docLabel; + link.title = docLabel; // full path as the native tooltip (chip pattern) + link.setAttribute("aria-label", docLabel); // the accessible name is the full path + row.appendChild(link); + } + body.appendChild(row); +} + /* "Maybe try:" chips under a deflected bubble (honesty gate, phase 04, shared component + one-tap submit, phase 05). The group is accessible (role=list + aria-label) and wraps cleanly at every width. */ @@ -1565,6 +1610,11 @@ function renderStoredMessage(m) { // — no button (the live stop path adds none either). if (!m.stopped) appendSaveAsDocButton(wrap, m.text); if (m.stopped) appendStoppedNote(wrap); // phase 48: the stop marker restores + // Phase 113 (task 02): the related tier restores with the bubble + // (LAST — after the meta-row claimers, exactly like the live done + // path). Pre-phase records carry no `related` → appendRelated + // no-ops and the row is simply absent (graceful). + appendRelated(wrap, m.related); lastBrainWrap = wrap; // phase 49: the LAST restored brain bubble wins } @@ -2433,6 +2483,15 @@ async function runTurn(text, { reask = false } = {}) { appendMaybeTry(wrap, ev.suggestions); } appendSources(wrap, ev.sources); + // Phase 113 (task 02): the cited tier stays the citation + // surface (the chips above); the related tier (scored docs + // under the usefulness bar — and the weak hits of a DEFLECTED + // turn, whose `ev.sources` is empty → zero chips) renders as + // the de-emphasized labeled row, appended LAST among the meta + // rows below (after the appendTuneButton/SaveAsDoc/Retry + // claimers took the first .msg-meta row, so an action never + // joins the related row — a deflected turn with related docs + // still gets its own meta row for the buttons). // Thinking-without-answer (reasoning can exhaust max_tokens): the // bubble gets the empty-answer fallback — what the user saw is // what gets persisted. @@ -2457,10 +2516,16 @@ async function runTurn(text, { reask = false } = {}) { tools: toolAcc.length ? toolAcc : undefined, deflected: !!ev.deflected, sources: ev.sources, + // Phase 113 (task 02): the related tier persists with the + // turn (undefined drops the key from the JSON — the house + // optional-meta pattern), so the restore path re-renders the + // row exactly as it looked live. + related: ev.related?.length ? ev.related : undefined, suggestions: ev.suggestions, }, leavePartialIndex); lastBrainWrap = wrap; // this bubble is now the last brain answer markLastRetryable(); // phase 49: the Retry button is last-bubble-only + appendRelated(wrap, ev.related); // phase 113 (task 02) — LAST meta row } else if (ev.type === "tool_result") { // Phase 95 (A15 extension, task 02): the truncation the LLM is // told about is told to the USER. One frame per truncated read, diff --git a/frontend/assets/styles.css b/frontend/assets/styles.css index f28c3de..260bca7 100644 --- a/frontend/assets/styles.css +++ b/frontend/assets/styles.css @@ -692,6 +692,51 @@ details.thinking .thinking-text ul { margin: 0 0 0.5rem; } } .source-chip:hover { background: var(--brand-soft); text-decoration: underline; } +/* Phase 113 (task 02): the related-docs row — the SECONDARY tier of + scored docs (phase 113 task 01's usefulness bar demotes the + sub-floor hits out of the citation surface; on a deflected turn the + weak hits live HERE and the chip row is absent). It must never read + as a citation: smaller mono links, DASHED border (the solid brand + pill is the citation look), transparent fill, muted --ink-soft text + — 8.6:1 on --bg (verified ≥4.5:1, WCAG 2.1 AA), palette variables + only (phase-92 zero-literal: no new hue, the monochrome theme grays + it automatically). The row joins .msg-meta (the .msg-body flex gap + stacks it below the citation row — no extra margin) but the meta + ACTIONS never do: app.js appends it last, after the first-row + claimers. :focus-visible via the global 3px outline rule. The hover + is deliberately flat — NO background swap (the chip's brand-soft + hover is the citation affordance): only the ink-soft → ink step-up + plus the underline. */ +.related-docs-label { + font-size: 0.68rem; + font-weight: 600; + text-transform: uppercase; + letter-spacing: 0.05em; + color: var(--ink-soft); +} +.related-doc { + display: inline-flex; + align-items: center; + font-family: var(--mono); + font-size: 0.7rem; + color: var(--ink-soft); + background: transparent; + border: 1px dashed var(--line); + border-radius: 999px; + padding: 0.12rem 0.55rem; + text-decoration: none; + max-width: 100%; + /* min-width: 0 so the nowrap pill ellipsizes (same as the chips) */ + min-width: 0; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} +.related-doc:hover { + color: var(--ink); /* ink on --bg 16.7:1 — the flat secondary hover */ + text-decoration: underline; +} + /* "Maybe try" chips under a deflected bubble (phase 04). Unlike the onboarding row (which scrolls horizontally on mobile), this group wraps at every width: the chips are the actionable follow-up, not decoration. diff --git a/tests/conftest.py b/tests/conftest.py index 50f7c5c..ebff90a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -17,6 +17,15 @@ from sqlalchemy.orm import Session os.environ.setdefault("BOR_RELEVANCE_THRESHOLD", "0.30") # A8 revised 2026-09-14: lexical_support_floor must be <= relevance_threshold. os.environ.setdefault("BOR_LEXICAL_SUPPORT_FLOOR", "0.15") +# Phase 113: the source usefulness bar (citation slot) is calibrated to +# the mock's compressed distribution, like the threshold above — the +# weakest GROUNDED fixture cosine in the in-process suites (the +# kafkabridge question's static-dns.json, ~0.055) must stay citable so +# the pre-phase citation pins hold, while the bar is ON so endpoint +# tests exercise the tier (tests that pin bar-specific behavior set a +# higher floor explicitly, the BOR_LEXICAL_SUPPORT_FLOOR pattern). +# Production default stays 0.35 (app/config.py, LOCKED A2). +os.environ.setdefault("BOR_SOURCE_USEFULNESS_FLOOR", "0.02") # Phase 16: single-admin auth is fail-loud — create_app() refuses to boot # without both vars, and app.main (imported below) builds the app at diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 6cf0f94..d8f7be1 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -98,6 +98,21 @@ def app_server(mock_llm: int) -> Iterator[str]: # The production default stays 0.62 (re-tuned against the real # `embed` model's 0.41–0.84 cosine range, PLAN A8). env["BOR_RELEVANCE_THRESHOLD"] = "0.30" + # Phase 112 (defect fix): the code-default lexical_support_floor (0.35) + # violates the startup validator against the mock-calibrated threshold + # (0.30) and the app under test refused to boot — pin the + # mock-calibrated value (tests/conftest.py pattern). It also pins the + # gate quadrant end-to-end: "Who composed Rhapsody in Blue?" (top + # ~0.124 with FTS hits) deflects while the corroborated-lexical + # kafkabridge question (top ~0.239) grounds. + env["BOR_LEXICAL_SUPPORT_FLOOR"] = "0.15" + # Phase 113: the source usefulness bar (citation slot), mock-calibrated + # (half the threshold, like the lexical floor): a doc earns a citation + # chip only when its best-chunk cosine clears it (e.g. the kafkabridge + # doc at ~0.20 clears; the rhapsody weak hits at ~0.12 or below do not + # and demote to the related tier). The production default stays 0.35 + # (app/config.py, LOCKED A2). + env["BOR_SOURCE_USEFULNESS_FLOOR"] = "0.15" # Phase 67 (LLM retry): the e2e pins the retry MECHANISM with instant # waits (BOR_LLM_RETRY_DELAY=0 — the 5 s default is unit-pinned via # tests/unit/test_config.py). BOR_LLM_RETRIES is forced to the code diff --git a/tests/e2e/test_source_chip_quality.py b/tests/e2e/test_source_chip_quality.py new file mode 100644 index 0000000..8e21645 --- /dev/null +++ b/tests/e2e/test_source_chip_quality.py @@ -0,0 +1,201 @@ +"""Phase 113 E2E (Playwright): the source-chip quality contract (TODO L5 + +L2c) — the usefulness bar + the de-emphasized related-docs row, as VISIBLE +chip counts. + +Run in isolation (DB must be up: ``podman compose up -d db``): + + uv run pytest tests/e2e/test_source_chip_quality.py -v --no-cov + +Acceptance (TODO L144–146): "for a single-document question, the turn +shows one citation chip"; a weak 2nd doc renders only in the +de-emphasized related row (``.related-doc`` links, NEVER ``.source-chip``); +a deflected turn shows zero citation chips (its weak hits, if any, live in +the related row). + +The fixture KB's tier shapes are deterministic under the E2E mock's +bag-of-words embeddings + the mock-calibrated bar (conftest: +``BOR_SOURCE_USEFULNESS_FLOOR=0.15``, half the 0.30 threshold — like the +lexical floor): + +* **single-source question** — "What SSH aliases do I have?": + ``ssh_aliases.txt`` is the ONLY doc whose best-chunk cosine clears the + bar (0.352 ≥ 0.15; grounded at 0.352 ≥ 0.30) → the done frame carries + exactly ONE cited ref; the two below-bar docs (gitlab-compose.yaml + 0.025, uptime_probe.py 0.113) ride the related tier. This IS the + strong+weak two-tier shape on the wire (the four OBSERVED live shapes + are unit-pinned at plan level in + ``tests/unit/test_source_chip_quality.py`` — the fixture KB reproduces + the same shape live, so no docstring caveat is needed). +* **deflected question** — "How do I bake sourdough bread?": best cosine + 0.109 < 0.30 and zero FTS hits → honest deflection; nothing clears the + bar → zero cited refs; the weak hits (new-service.md 0.109, + ssh_aliases.txt 0.050) ride the related row. +""" +from __future__ import annotations + +import asyncio +from pathlib import Path +from threading import Thread +from typing import Any + +from playwright.sync_api import Page, expect +from sqlalchemy import select, text + +from app.config import Settings, get_settings +from app.db import SessionLocal +from app.models import QueryLog +from app.rag.importer import ImportSummary, import_sources +from app.rag.llm import LLMClient +from e2e.auth_helpers import login + +REPO = Path(__file__).resolve().parents[2] +FIXTURES = REPO / "tests" / "fixtures" / "docs" +SINGLE_SOURCE_QUESTION = "What SSH aliases do I have?" +OFF_TOPIC = "How do I bake sourdough bread?" +MOCK_ANSWER_MARKER = "Deterministic mock answer for E2E" + + +async def _import_fixtures(mock_port: int) -> ImportSummary: + kwargs: dict[str, Any] = {"_env_file": None, "llm_base_url": f"http://127.0.0.1:{mock_port}/v1"} + settings = Settings(**kwargs) # pyright: ignore[reportCallIssue] + return await import_sources([FIXTURES], LLMClient(settings)) + + +def _run_in_thread(coro: Any) -> Any: + """Run a coroutine on a worker thread (Playwright owns the test loop).""" + box: dict[str, Any] = {} + + def runner() -> None: + try: + box["value"] = asyncio.run(coro) + except BaseException as e: # noqa: BLE001 — re-raised on the test thread + box["error"] = e + + t = Thread(target=runner) + t.start() + t.join() + if "error" in box: + raise box["error"] + return box["value"] + + +def _reset_db(mock_port: int, seed: bool = True) -> ImportSummary | None: + """Truncate the KB (and query log), then optionally re-import fixtures.""" + with SessionLocal() as db: + db.execute(text("TRUNCATE chunks, documents, query_log")) + db.commit() + if not seed: + return None + return _run_in_thread(_import_fixtures(mock_port)) + + +def _ask(page: Page, message: str) -> None: + page.fill("#message-input", message) + page.click("#send-btn") + + +def test_single_source_question_shows_exactly_one_citation_chip( + page: Page, app_url: str, mock_llm: int, db_ready: None +) -> None: + """The acceptance pin: a single-document question → the done bubble + carries EXACTLY ONE ``.source-chip`` (the bar-clearing doc) and the + below-bar docs render only in the de-emphasized ``.related-docs`` row + (``.related-doc`` links — never ``.source-chip``), labeled + "Nearby docs, in case:". The durable record keeps the FULL retrieval + (LOCKED A3).""" + _reset_db(mock_llm) + page.set_default_timeout(30_000) + login(page, app_url, next="/") # phase 79: chat is require_user-gated + + _ask(page, SINGLE_SOURCE_QUESTION) + expect(page.locator(".msg.user .bubble")).to_contain_text(SINGLE_SOURCE_QUESTION) + + bubble = page.locator(".msg.brain .bubble").first + bubble.wait_for(state="visible", timeout=30_000) + expect(bubble).to_contain_text(MOCK_ANSWER_MARKER, timeout=30_000) + # Grounded: no deflected bubble at all. + expect(page.locator(".msg.brain.is-deflected")).to_have_count(0) + + # EXACTLY ONE citation chip — the bar-clearing doc (the 2nd chip of + # the pre-phase turn demoted to the row; the acceptance criterion). + chip = page.locator(".msg.brain .source-chip") + expect(chip).to_have_count(1, timeout=30_000) + expect(chip.first).to_contain_text("ssh_aliases.txt") + + # The below-bar docs ride the related row: a labeled, de-emphasized + # list — one .related-doc link per doc (rank order, capped at + # related_max_docs = 2), never a .source-chip. + row = page.locator(".msg.brain .related-docs") + expect(row).to_have_count(1) + expect(row.first).to_have_attribute("aria-label", "Nearby docs, in case") + expect(row.first.locator(".related-docs-label")).to_have_text("Nearby docs, in case:") + links = page.locator(".msg.brain .related-docs .related-doc") + expect(links).to_have_count(2, timeout=30_000) + expect(links.nth(0)).to_contain_text("gitlab-compose.yaml") + expect(links.nth(1)).to_contain_text("uptime_probe.py") + expect(page.locator(".msg.brain .related-docs .source-chip")).to_have_count(0) + # The related links keep the chip's /document.html href + identity. + expect(links.first).to_have_attribute( + "title", "docs/homelab/container_gitlab/gitlab-compose.yaml" + ) + + # Durable record: not deflected; the FULL retrieval (cited + related) + # is logged — query_log records retrieval, not citations (LOCKED A3). + with SessionLocal() as db: + row_log = db.scalars(select(QueryLog)).one() + assert row_log.question == SINGLE_SOURCE_QUESTION + assert row_log.deflected is False + for path in ( + "homelab/ssh/ssh_aliases.txt", + "homelab/container_gitlab/gitlab-compose.yaml", + "homelab/scripts/uptime_probe.py", + ): + assert path in row_log.sources + + +def test_deflected_question_shows_zero_citation_chips( + page: Page, app_url: str, mock_llm: int, db_ready: None +) -> None: + """The acceptance pin: a deflected turn (known-out-of-KB) → ZERO + ``.source-chip`` elements under the bubble; the weak hits live in the + related row only — its links are ``.related-doc``, never + ``.source-chip`` (L2c: "at minimum: never render them as answer + citations"). The "Maybe try" chips and the durable record are + unchanged.""" + _reset_db(mock_llm) + page.set_default_timeout(30_000) + login(page, app_url, next="/") # phase 79: chat is require_user-gated + + _ask(page, OFF_TOPIC) + bubble = page.locator(".msg.brain.is-deflected .bubble").first + bubble.wait_for(state="visible", timeout=30_000) + expect(page.locator(".msg.brain.is-deflected")).to_have_count(1) + + # Wait for the done frame to have fully rendered (the related row and + # the suggestion chips append in it), THEN pin the absence of chips — + # a mid-stream reading would be a false zero. + chips = page.locator(".msg.brain.is-deflected .maybe-try .suggestion-chip") + expect(chips.first).to_be_visible(timeout=30_000) + assert chips.count() >= 2, "deflection must offer 2-3 alternative chips" + + row = page.locator(".msg.brain .related-docs") + expect(row).to_have_count(1, timeout=30_000) + links = page.locator(".msg.brain .related-docs .related-doc") + expect(links).to_have_count(2) # the weak hits, rank order, capped at 2 + expect(links.nth(0)).to_contain_text("new-service.md") + expect(links.nth(1)).to_contain_text("ssh_aliases.txt") + + # ZERO citation chips under the bubble — the weak hits are scored + # docs, not citations (the phase-112/113 contract on the wire). + expect(page.locator(".msg.brain .source-chip")).to_have_count(0) + expect(page.locator(".msg.brain .related-docs .source-chip")).to_have_count(0) + + # Durable record: deflected, weak top score — the retrieval stays + # logged for threshold tuning (LOCKED A3: observability unchanged). + with SessionLocal() as db: + row_log = db.scalars(select(QueryLog)).one() + assert row_log.question == OFF_TOPIC + assert row_log.deflected is True + assert 0.0 < row_log.top_score < get_settings().relevance_threshold + assert row_log.fts_hits == 0 + assert row_log.sources # the weak-hit paths, for threshold tuning diff --git a/tests/integration/test_chat_api.py b/tests/integration/test_chat_api.py index d59067c..693004b 100644 --- a/tests/integration/test_chat_api.py +++ b/tests/integration/test_chat_api.py @@ -443,6 +443,116 @@ def test_off_topic_question_deflects_honestly(client, db, seeded_kb: FakeRagLLM) assert row.sources +def test_done_frame_carries_related_tier_on_grounded_turn( + client, db, seeded_kb: FakeRagLLM +) -> None: + """Phase 113 (LOCKED A4): a grounded turn's done frame carries the + related tier — the ranked docs beyond the cited ceiling, capped at + ``related_max_docs`` (2), disjoint from the cited list. The durable + record keeps the FULL retrieval (cited + related, LOCKED A3).""" + fastapi_app.dependency_overrides[chat_api.get_llm] = lambda: seeded_kb + try: + _, _, frames = _stream_chat(client, QUESTION) + finally: + fastapi_app.dependency_overrides.clear() + + done = frames[-1] + assert done["deflected"] is False + sources = {(s["source"], s["path"]) for s in done["sources"]} + related = done["related"] + assert related, "the 2nd-and-lower scored docs ride the related tier" + assert len(related) <= get_settings().related_max_docs + # The related tier never overlaps the cited list (the dedupe is by + # (source, path) — the same pattern as the cited docs). + related_keys = {(s["source"], s["path"]) for s in related} + assert sources.isdisjoint(related_keys) + # Rank order: the cited top-2 are the kubernetes doc and the template; + # the next ranked doc is the ssh aliases file. + assert related[0]["path"] == "homelab/ssh/ssh_aliases.txt" + # Every ref carries the chip identity fields (the UI row reuses them). + assert all(s["title"] for s in related) + + # Durable record: the full retrieval (cited + related) is logged. + row = db.scalars(select(QueryLog)).one() + assert "docs/homelab/ssh/ssh_aliases.txt" in row.sources + assert "docs/homelab/kubernetes.md" in row.sources + + +def test_deflected_done_frame_carries_weak_hits_in_related( + client, db, seeded_kb: FakeRagLLM, monkeypatch: pytest.MonkeyPatch +) -> None: + """Phase 113: on a deflected turn nothing clears the bar — the cited + tier is empty (done.sources stays [], the phase-112 contract) and the + weak hits fall to the related tier (their visibility home). The + durable record still carries the retrieval (LOCKED A3).""" + monkeypatch.setenv("BOR_SOURCE_USEFULNESS_FLOOR", "0.20") + get_settings.cache_clear() + fastapi_app.dependency_overrides[chat_api.get_llm] = lambda: seeded_kb + try: + _, _, frames = _stream_chat(client, OFF_TOPIC) + done = frames[-1] + assert done["deflected"] is True + assert done["sources"] == [] # a deflected answer cites nothing + # The weak hits (the sourdough question's best mock cosines are + # ~0.11/0.04 — both below the 0.20 bar) ride the related tier, in + # rank order, capped at related_max_docs. + related = done["related"] + assert len(related) <= get_settings().related_max_docs + assert [s["path"] for s in related][:2] == [ + "deployments/new-service.md", + "homelab/quadlet/lan.network", + ] + assert all(s["title"] for s in related) + assert done["suggestions"] # the "Maybe try" chips are unchanged + + # Durable record: the weak retrieval stays logged for tuning (A3). + row = db.scalars(select(QueryLog)).one() + assert row.deflected is True + assert row.sources # the weak-hit paths, for threshold tuning + finally: + # The cache clear is LAST — an assertion that calls get_settings() + # after the clear would re-populate the lru_cache with the + # monkeypatched value and leak it into the next test. + fastapi_app.dependency_overrides.clear() + get_settings.cache_clear() + + +def test_related_doc_read_by_agent_is_cited_not_related( + client, db, seeded_kb: FakeRagLLM +) -> None: + """Phase 113 × phase 37: an agent-read doc is a citation by definition + — when the agent ``read``s a doc that would otherwise ride the related + tier, it joins done.sources (deduped, last) and is EXCLUDED from + done.related (a "nearby doc" that was actually used must not read as + nearby).""" + scripted = FakeRagLLM( + tool_script=[ + [ + ToolCallPiece( + id="call_1", + name="read", + arguments={"path": "docs/homelab/ssh/ssh_aliases.txt"}, + ) + ] + ] + ) + fastapi_app.dependency_overrides[chat_api.get_llm] = lambda: scripted + try: + _, _, frames = _stream_chat(client, QUESTION) + finally: + fastapi_app.dependency_overrides.clear() + + done = frames[-1] + assert done["deflected"] is False + sources = [(s["source"], s["path"]) for s in done["sources"]] + assert sources[-1] == ("docs", "homelab/ssh/ssh_aliases.txt") # read ⇒ cited + related = [(s["source"], s["path"]) for s in done["related"]] + assert ("docs", "homelab/ssh/ssh_aliases.txt") not in related + assert set(sources).isdisjoint(set(related)) + # The OTHER related-tier doc (gitlab, rank 4) stays in the tier. + assert ("docs", "homelab/container_gitlab/gitlab.md") in related + + def test_keyword_question_grounded_by_lexical_hit_despite_weak_cosine( client, db, seeded_kb: FakeRagLLM, monkeypatch: pytest.MonkeyPatch, @@ -462,26 +572,28 @@ def test_keyword_question_grounded_by_lexical_hit_despite_weak_cosine( fastapi_app.dependency_overrides[chat_api.get_llm] = lambda: seeded_kb try: _, _, frames = _stream_chat(client, "How does kafkabridge work?") + done = frames[-1] + assert done["type"] == "done" + assert done["deflected"] is False # weak cosine, but a lexical hit + assert done["suggestions"] == [] + sources = done["sources"] + assert sources and sources[0]["path"] == "homelab/networking/static-dns.json" + + (system, _user) = seeded_kb.seen_messages[0][0], seeded_kb.seen_messages[0][1] + assert "HIGH" in system["content"] # grounded prompt + + row = db.scalars(select(QueryLog)).one() + assert row.deflected is False + assert row.top_score < get_settings().relevance_threshold # weak vector score + assert (row.fts_hits or 0) >= 1 # …and it is the FTS hit that grounds it + assert "docs/homelab/networking/static-dns.json" in row.sources finally: + # The cache clear is LAST — an assertion that calls get_settings() + # after the clear would re-populate the lru_cache with the + # monkeypatched value and leak it into the next test. fastapi_app.dependency_overrides.clear() get_settings.cache_clear() - done = frames[-1] - assert done["type"] == "done" - assert done["deflected"] is False # weak cosine, but a lexical hit - assert done["suggestions"] == [] - sources = done["sources"] - assert sources and sources[0]["path"] == "homelab/networking/static-dns.json" - - (system, _user) = seeded_kb.seen_messages[0][0], seeded_kb.seen_messages[0][1] - assert "HIGH" in system["content"] # grounded prompt - - row = db.scalars(select(QueryLog)).one() - assert row.deflected is False - assert row.top_score < get_settings().relevance_threshold # weak vector score - assert (row.fts_hits or 0) >= 1 # …and it is the FTS hit that grounds it - assert "docs/homelab/networking/static-dns.json" in row.sources - def test_chat_empty_kb_streams_empty_sources(client, db) -> None: db.execute(text("TRUNCATE chunks, documents, query_log")) @@ -1875,6 +1987,9 @@ def test_done_event_serializes_column_maximum_source_refs() -> None: "type": "done", "deflected": False, "sources": [{"source": "s" * 120, "path": "p" * 1000, "title": "t" * 500}], + # Phase 113: the additive related tier defaults to [] (the + # key is always present on new frames; old clients ignore it). + "related": [], "suggestions": [], } # The caps sit exactly ON the column maxima: one over any of them @@ -1886,3 +2001,29 @@ def test_done_event_serializes_column_maximum_source_refs() -> None: SourceRef(source="s" * 120, path="p" * 1001, title="t" * 500) with pytest.raises(ValidationError): SourceRef(source="s" * 120, path="p" * 1000, title="t" * 501) + + +def test_done_event_related_defaults_empty_and_old_payload_parses() -> None: + """Phase 113 back-compat pin: ``related`` defaults to ``[]`` — a + pre-phase-113 done frame (no ``related`` key) still parses, and a + frame with the field round-trips it (PLAN §4: old clients ignore + unknown fields, so the field is additive in both directions).""" + old_payload = { + "type": "done", + "deflected": True, + "sources": [], + "suggestions": ["Maybe try X?"], + } + event = ChatDoneEvent(**old_payload) + assert event.related == [] + assert event.model_dump() == {**old_payload, "related": []} + + new_payload = { + "deflected": False, + "sources": [SourceRef(source="docs", path="a.md", title="A")], + "related": [SourceRef(source="docs", path="b.md", title="B")], + "suggestions": [], + } + dumped = ChatDoneEvent(**new_payload).model_dump() + assert [r["path"] for r in dumped["related"]] == ["b.md"] + assert [r["path"] for r in dumped["sources"]] == ["a.md"] diff --git a/tests/unit/test_chat_gate.py b/tests/unit/test_chat_gate.py index 6b898f3..0eb47e6 100644 --- a/tests/unit/test_chat_gate.py +++ b/tests/unit/test_chat_gate.py @@ -179,8 +179,11 @@ def test_gate_lexical_only_chunk_does_not_inflate_cosine() -> None: plan = chat_api.plan_turn(chunks, _settings(threshold=0.30)) assert plan.top_score == pytest.approx(0.55) assert plan.deflected is False # 0.55 >= 0.30 anyway - # ranking follows the fused score: Beta's doc is the top source - assert plan.docs[0].title == "Beta" + # Phase 113 (the usefulness bar): Beta's doc ranks first by fused + # score, but a lexical-only doc (cosine 0.0 by construction) cannot + # clear the bar — it lands in the RELATED tier, never the cited one. + assert plan.docs[0].title == "Alpha" + assert plan.related_docs[0].title == "Beta" # ---------- lexical support floor (A8 revised 2026-09-14) ---------- @@ -370,6 +373,136 @@ def test_gate_zero_chunks_deflects_with_fallback_chips() -> None: assert 2 <= len(plan.suggestions) <= MAX_SUGGESTIONS +# ---------- usefulness bar tiering (phase 113, LOCKED A2/A4) ---------- + + +def _bar_settings( + threshold: float = 0.62, + lex_floor: float = 0.35, + source_floor: float = 0.35, + related_cap: int = 2, + top_n: int = 2, +) -> Settings: + """Explicit code defaults (production calibration) — the env's mock- + calibrated floor (tests/conftest.py) is overridden per test.""" + return Settings( + _env_file=None, # pyright: ignore[reportCallIssue] + relevance_threshold=threshold, + lexical_support_floor=lex_floor, + source_usefulness_floor=source_floor, + related_max_docs=related_cap, + top_n_docs=top_n, + ) + + +def test_plan_turn_high_tiers_strong_plus_weak() -> None: + """Grounded turn: the bar-clearing doc is cited (and in the prompt), + the weak 2nd doc loses its citation slot and lands in related_docs — + the recurring incident's fix at the plan level.""" + strong = _doc("Kubernetes Homelab Cluster", "STRONG_DOC_CONTENT") + weak = _doc("Backup Strategy", "WEAK_DOC_CONTENT") + chunks = [_chunk(strong, 0.90, cosine=0.80), _chunk(weak, 0.80, cosine=0.20)] + plan = chat_api.plan_turn(chunks, _bar_settings()) + assert plan.deflected is False + assert [d.title for d in plan.docs] == ["Kubernetes Homelab Cluster"] + assert [d.title for d in plan.related_docs] == ["Backup Strategy"] + # The HIGH prompt carries the cited doc's content only. + assert "STRONG_DOC_CONTENT" in plan.system_prompt + assert "WEAK_DOC_CONTENT" not in plan.system_prompt + + +def test_plan_turn_high_single_strong_doc_yields_one_cited() -> None: + """top_n_docs is a CEILING, not a quota: one strong doc ⇒ one cited doc, + an empty related tier (LOCKED A2).""" + strong = _doc("Kubernetes Homelab Cluster", "STRONG_DOC_CONTENT") + plan = chat_api.plan_turn([_chunk(strong, 0.90, cosine=0.80)], _bar_settings()) + assert plan.deflected is False + assert [d.title for d in plan.docs] == ["Kubernetes Homelab Cluster"] + assert plan.related_docs == [] + + +def test_plan_turn_low_weak_hits_fall_to_related() -> None: + """Deflected turn: nothing clears the bar ⇒ the cited tier is empty + and the weak hits fall to related_docs (the done frame's home for + their visibility). The LOW prompt is unchanged (titles only).""" + a = _doc("Alpha", "ALPHA_DOC_NEVER_SENT") + b = _doc("Beta", "BETA_DOC_NEVER_SENT") + chunks = [_chunk(a, 0.30, cosine=0.20), _chunk(b, 0.20, cosine=0.15)] + plan = chat_api.plan_turn(chunks, _bar_settings()) + assert plan.deflected is True + assert plan.docs == [] # no citation slot below the bar + assert [d.title for d in plan.related_docs] == ["Alpha", "Beta"] # rank order + assert "ALPHA_DOC_NEVER_SENT" not in plan.system_prompt + assert "Beta" in plan.system_prompt # weak-hit titles still carried + assert plan.suggestions # chips unchanged + + +def test_plan_turn_related_cap_zero_kills_the_related_tier() -> None: + """related_max_docs=0 is the kill switch: weak docs are scored but + neither cited nor related (the pre-phase-113 visibility, minus the + false citation — a deflected turn cites nothing).""" + a = _doc("Alpha", "AAA") + b = _doc("Beta", "BBB") + chunks = [_chunk(a, 0.30, cosine=0.20), _chunk(b, 0.20, cosine=0.15)] + plan = chat_api.plan_turn(chunks, _bar_settings(related_cap=0)) + assert plan.deflected is True + assert plan.docs == [] + assert plan.related_docs == [] + + +def test_plan_turn_floor_zero_keeps_legacy_cited_docs() -> None: + """source_usefulness_floor=0 disables the bar: plan.docs is the legacy + rank-ordered top-N (any cosine, incl. 0.0 lexical-only) and the + related tier is empty.""" + a = _doc("Alpha", "AAA") + b = _doc("Beta", "BBB") + chunks = [ + _chunk(a, 0.90, cosine=0.0, fts_hit=True), # lexical-only, rank 1 + _chunk(b, 0.80, cosine=0.10), + ] + plan = chat_api.plan_turn( + chunks, _bar_settings(source_floor=0.0, lex_floor=0.05) + ) + assert plan.deflected is False # 0.10 + the fts hit clears the 0.05 lex floor + assert [d.title for d in plan.docs] == ["Alpha", "Beta"] # legacy order + assert plan.related_docs == [] + + +def test_plan_turn_lexically_grounded_below_source_floor_has_no_cited_docs() -> None: + """The degenerate operator config (citation bar STRICTER than the + grounding bar): a turn grounded by a corroborated-lexical hit whose + cosine sits between the two floors has an EMPTY cited tier — the HIGH + prompt carries no document content (the tools remain the escape + hatch). The bar is a citation filter, not a gate input.""" + doc = _doc("Static DNS", "DNS_DOC_CONTENT") + plan = chat_api.plan_turn( + [_chunk(doc, 0.50, cosine=0.35, fts_hit=True)], + _bar_settings(threshold=0.62, lex_floor=0.30, source_floor=0.50), + ) + assert plan.deflected is False # 0.35 >= lex floor 0.30, fts fired + assert plan.docs == [] # 0.35 < source floor 0.50 — no citation slot + assert "DNS_DOC_CONTENT" not in plan.system_prompt + # The doc still SCORED — it rides the related tier (the "nearby docs" + # row), it is not invisible. + assert [d.title for d in plan.related_docs] == ["Static DNS"] + + +def test_plan_turn_related_tier_capped_in_rank_order() -> None: + """Grounded turn, four bar-clearing docs, ceiling 2: cited = the top-2 + in rank order; related = the next two (the ceiling overflow, any + cosine), capped at related_max_docs.""" + docs_in = [ + _doc(f"Doc {i}", f"DOC_CONTENT_{i}") for i in range(4) + ] + chunks = [ + _chunk(d, 0.9 - 0.1 * i, cosine=0.8 - 0.05 * i) for i, d in enumerate(docs_in) + ] + plan = chat_api.plan_turn(chunks, _bar_settings(top_n=2, related_cap=2)) + assert plan.deflected is False + assert [d.title for d in plan.docs] == ["Doc 0", "Doc 1"] + assert [d.title for d in plan.related_docs] == ["Doc 2", "Doc 3"] + + # ---------- summary hits (phase 30: summary → full source document) ---------- diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 75aceeb..7f05ff1 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -213,6 +213,75 @@ def test_agent_max_rounds_rejects_negative(monkeypatch: pytest.MonkeyPatch) -> N _settings() +def test_source_usefulness_floor_default_and_env_override( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Phase 113 (LOCKED A2): the citation-slot bar — default 0.35 (the + same bar as the A8 lexical support floor), env-tunable, ``0`` = the + no-bar kill switch.""" + monkeypatch.delenv("BOR_SOURCE_USEFULNESS_FLOOR", raising=False) + # The test process pins the mock-calibrated threshold (0.30, see + # tests/conftest.py) — clear it so the PRODUCTION default pair + # (0.62 / 0.35) is what the validator sees. + monkeypatch.delenv("BOR_RELEVANCE_THRESHOLD", raising=False) + assert _settings().source_usefulness_floor == 0.35 + monkeypatch.setenv("BOR_SOURCE_USEFULNESS_FLOOR", "0.5") + assert _settings().source_usefulness_floor == 0.5 + monkeypatch.setenv("BOR_SOURCE_USEFULNESS_FLOOR", "0") + assert _settings().source_usefulness_floor == 0.0 + + +def test_source_usefulness_floor_at_threshold_is_legal( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The validator bound is inclusive (>=): a bar exactly at the + relevance threshold is legal — the gate and the bar agree on every + grounded document.""" + monkeypatch.delenv("BOR_SOURCE_USEFULNESS_FLOOR", raising=False) + monkeypatch.delenv("BOR_RELEVANCE_THRESHOLD", raising=False) + s = _settings(relevance_threshold=0.62, source_usefulness_floor=0.62) + assert s.source_usefulness_floor == 0.62 + + +def test_source_usefulness_floor_rejects_above_threshold( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A bar above the relevance threshold would demote to the related + tier documents the gate itself calls grounded — a typo, so the + validator fails loudly at startup (the lexical_support_floor guard).""" + monkeypatch.delenv("BOR_SOURCE_USEFULNESS_FLOOR", raising=False) + with pytest.raises(ValidationError, match="source_usefulness_floor"): + _settings(relevance_threshold=0.62, source_usefulness_floor=0.70) + + +def test_source_usefulness_floor_rejects_negative( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A negative bar is a typo (the agent_max_rounds pattern).""" + with pytest.raises(ValidationError, match="source_usefulness_floor"): + _settings(source_usefulness_floor=-0.1) + + +def test_related_max_docs_default_and_env_override( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Phase 113 (LOCKED A4): the related-doc tier cap — default 2, + ``0`` = the no-related-docs kill switch.""" + monkeypatch.delenv("BOR_RELATED_MAX_DOCS", raising=False) + assert _settings().related_max_docs == 2 + monkeypatch.setenv("BOR_RELATED_MAX_DOCS", "0") + assert _settings().related_max_docs == 0 + + +def test_related_max_docs_rejects_negative( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A negative cap is a typo (the agent_max_rounds pattern).""" + monkeypatch.setenv("BOR_RELATED_MAX_DOCS", "-1") + with pytest.raises(ValidationError, match="related_max_docs"): + _settings() + + def test_read_max_chars_default_and_env_override( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/unit/test_retriever.py b/tests/unit/test_retriever.py index 7d08e71..b929693 100644 --- a/tests/unit/test_retriever.py +++ b/tests/unit/test_retriever.py @@ -12,7 +12,12 @@ from types import SimpleNamespace import pytest from app.models import Document -from app.rag.retriever import TRUNCATION_MARKER, RetrievedChunk, select_documents +from app.rag.retriever import ( + TRUNCATION_MARKER, + RetrievedChunk, + select_documents, + select_documents_tiered, +) def _doc(path: str, content: str, source: str = "Homelab", title: str | None = None) -> Document: @@ -90,6 +95,169 @@ def test_empty_hits_yield_no_documents() -> None: assert select_documents([], n=2) == [] +# --------------------------------------------------------------------------- +# Phase 113 — the usefulness bar: cited vs related tiers +# --------------------------------------------------------------------------- + + +def _cos_chunk(doc: Document, score: float, cosine: float, position: int = 0) -> RetrievedChunk: + """A candidate with *score* (fused rank key) and *cosine* (gate input) decoupled.""" + return RetrievedChunk( + chunk_id=uuid.uuid4(), + position=position, + content=doc.content[:40], + score=score, + document=doc, + cosine=cosine, + ) + + +def test_tiered_both_clear_floor_both_cited() -> None: + """Both docs clear the bar → both cited, nothing related.""" + a = _doc("a.md", "A" * 50) + b = _doc("b.md", "B" * 50) + chunks = [_cos_chunk(a, 0.9, 0.50), _cos_chunk(b, 0.8, 0.40)] + cited, related = select_documents_tiered(chunks, n=2, floor=0.35, related_cap=2) + assert [d.path for d in cited] == ["a.md", "b.md"] + assert related == [] + + +def test_tiered_strong_plus_weak_one_cited_one_related() -> None: + """The recurring incident shape: a strong 1st doc and a weak 2nd — the + weak doc loses its citation slot and lands in the related tier.""" + a = _doc("a.md", "A" * 50) + b = _doc("b.md", "B" * 50) + chunks = [_cos_chunk(a, 0.9, 0.50), _cos_chunk(b, 0.8, 0.10)] + cited, related = select_documents_tiered(chunks, n=2, floor=0.35, related_cap=2) + assert [d.path for d in cited] == ["a.md"] + assert [d.path for d in related] == ["b.md"] + + +def test_tiered_both_weak_zero_cited_all_related() -> None: + """Neither doc clears the bar → no citation slot at all (the bar + filters, it never backfills), the weak hits become the related tier.""" + a = _doc("a.md", "A" * 50) + b = _doc("b.md", "B" * 50) + chunks = [_cos_chunk(a, 0.9, 0.20), _cos_chunk(b, 0.8, 0.15)] + cited, related = select_documents_tiered(chunks, n=2, floor=0.35, related_cap=2) + assert cited == [] + assert [d.path for d in related] == ["a.md", "b.md"] # rank order kept + + +def test_tiered_related_cap_respected() -> None: + """The related tier is capped (related_max_docs) in rank order.""" + docs_in = [_doc(f"d{i}.md", "X" * 20) for i in range(3)] + chunks = [_cos_chunk(d, 0.9 - 0.1 * i, 0.10 - 0.02 * i) for i, d in enumerate(docs_in)] + cited, related = select_documents_tiered(chunks, n=2, floor=0.35, related_cap=2) + assert cited == [] + assert [d.path for d in related] == ["d0.md", "d1.md"] + + +def test_tiered_n_is_ceiling_not_quota() -> None: + """A single strong doc yields ONE cited doc — the bar never pads the + cited tier up to ``n`` (LOCKED A2). And docs that clear the bar but + exceed the ceiling fall through to the related tier (the next docs in + rank order, never overlapping cited).""" + only = _doc("only.md", "O" * 50) + cited, related = select_documents_tiered( + [_cos_chunk(only, 0.9, 0.80)], n=2, floor=0.35, related_cap=2 + ) + assert [d.path for d in cited] == ["only.md"] + assert related == [] + + docs_in = [_doc(f"d{i}.md", "X" * 20) for i in range(4)] + chunks = [_cos_chunk(d, 0.9 - 0.1 * i, 0.8 - 0.05 * i) for i, d in enumerate(docs_in)] + cited, related = select_documents_tiered(chunks, n=2, floor=0.35, related_cap=2) + assert [d.path for d in cited] == ["d0.md", "d1.md"] # the ceiling + assert [d.path for d in related] == ["d2.md", "d3.md"] # the next in rank order + + +def test_tiered_bar_skips_weak_rank_one() -> None: + """A weak rank-1 doc does not consume a citation slot: the next-ranked + bar-clearing docs take it (the bar filters, it does not backfill).""" + w = _doc("w.md", "W" * 50) + s1 = _doc("s1.md", "1" * 50) + s2 = _doc("s2.md", "2" * 50) + chunks = [ + _cos_chunk(w, 0.9, 0.20), # rank 1 — below the bar + _cos_chunk(s1, 0.8, 0.90), + _cos_chunk(s2, 0.7, 0.80), + ] + cited, related = select_documents_tiered(chunks, n=2, floor=0.5, related_cap=2) + assert [d.path for d in cited] == ["s1.md", "s2.md"] + assert [d.path for d in related] == ["w.md"] + + +def test_tiered_tracks_best_chunk_cosine_across_a_docs_chunks() -> None: + """The bar is on the doc's BEST hit-chunk cosine — a weak first chunk + (rank 1) does not sink a doc whose later chunk is vector-strong.""" + a = _doc("a.md", "A" * 50) + b = _doc("b.md", "B" * 50) + chunks = [ + _cos_chunk(a, 0.9, 0.10, position=0), # a's weak chunk ranks first + _cos_chunk(a, 0.5, 0.90, position=2), # a's strong chunk + _cos_chunk(b, 0.4, 0.0), # lexical-only b + ] + cited, related = select_documents_tiered(chunks, n=2, floor=0.35, related_cap=2) + assert [d.path for d in cited] == ["a.md"] + assert [d.path for d in related] == ["b.md"] + + +def test_tiered_lexical_only_hit_goes_to_related_above_zero_floor() -> None: + """A lexical-only doc (cosine 0.0 by construction) is vector-unsupported + by definition: above a zero floor it never earns a cited slot (LOCKED A2).""" + a = _doc("a.md", "A" * 50) + b = _doc("b.md", "B" * 50) + lexical_only = RetrievedChunk( + chunk_id=uuid.uuid4(), + position=0, + content="X" * 10, + score=0.9, # top fused rank (the FTS hit) + document=a, + cosine=0.0, # no vector rank — lexical-only + fts_hit=True, + ) + chunks = [lexical_only, _cos_chunk(b, 0.8, 0.5)] + cited, related = select_documents_tiered(chunks, n=2, floor=0.35, related_cap=2) + assert [d.path for d in cited] == ["b.md"] + assert [d.path for d in related] == ["a.md"] + + +def test_tiered_floor_zero_is_no_bar() -> None: + """A zero floor admits every scored document — lexical-only (cosine + 0.0) and weak alike — so the bar can be disabled per deployment.""" + a = _doc("a.md", "A" * 50) + b = _doc("b.md", "B" * 50) + chunks = [_cos_chunk(a, 0.9, 0.0), _cos_chunk(b, 0.8, 0.1)] + cited, related = select_documents_tiered(chunks, n=2, floor=0.0, related_cap=2) + assert [d.path for d in cited] == ["a.md", "b.md"] + assert related == [] + + +def test_tiered_empty_chunks_yield_empty_tiers() -> None: + assert select_documents_tiered([], n=2, floor=0.35, related_cap=2) == ([], []) + + +def test_select_documents_wrapper_is_legacy_tiering() -> None: + """The wrapper (floor 0.0, cap 0) is the legacy "any score, top-N" + selection — byte-identical for the shapes the existing callers see: + rank order, dedupe, multi-chunk docs, ties, lexical-only hits.""" + docs_in = [_doc(f"d{i}.md", "X" * 20) for i in range(5)] + chunks = [ + _chunk(docs_in[0], 0.5), # cosine 0.0 (lexical-only) + _cos_chunk(docs_in[0], 0.9, 0.4, position=1), + _chunk(docs_in[1], 0.8), + _cos_chunk(docs_in[1], 0.8, 0.2, position=1), # fused tie across docs + _cos_chunk(docs_in[2], 0.7, 0.0), + _cos_chunk(docs_in[3], 0.6, 0.1), + _chunk(docs_in[4], 0.1), + ] + for n in (1, 2, 3, 10): + assert select_documents(chunks, n=n) == select_documents_tiered( + chunks, n, 0.0, 0 + )[0] + + # --------------------------------------------------------------------------- # Hybrid retrieval (A7): RRF fusion + lexical tsquery # --------------------------------------------------------------------------- diff --git a/tests/unit/test_source_chip_quality.py b/tests/unit/test_source_chip_quality.py new file mode 100644 index 0000000..8941832 --- /dev/null +++ b/tests/unit/test_source_chip_quality.py @@ -0,0 +1,825 @@ +"""Unit: the phase-113 source-chip-quality contract (TODO L5 + L2c — +"the 2nd chip is often noise the answer never used"). + +Phase 113 demotes sub-floor hits out of the citation surface: the +done frame carries the cited tier in ``sources`` (rendered by +``appendSources`` as ``.source-chip`` pills, UNCHANGED) and the +related tier in ``related`` (rendered by the NEW ``appendRelated`` as +the de-emphasized labeled row — ``.related-doc`` links, never +``.source-chip``). A deflected turn carries ``sources: []`` → zero +chips; its weak hits live in the related row only. + +This module pins the STATIC SOURCES the UI contract stands on, in the +house source-pin pattern (the test_chip_sizing_question_cap.py +``_rule`` style): + +* task 02 — the frontend: ``appendRelated`` exists and never builds a + ``source-chip``; the row renders only when ``related`` is non-empty; + the label copy is present; the done-frame handler and the restore path + both call it (the related tier persists with the turn, so a reload + re-renders the row exactly as it looked live); the CSS row is clearly + secondary (dashed border, muted ink, flat hover); +* task 03 — the acceptance pin (TODO L5): the FOUR OBSERVED LIVE SHAPES + (L110–123), each modeled as a ``plan_turn`` fixture with controlled + cosine/``fts_hit``/fused ``score`` under the production calibration + (the code defaults — the shapes were observed live): + + 1. **both docs weak** ("What is the capital of Mongolia?" → + ``Trooper_Nagraz.pl`` + ``Trooper_Begzei.pl``, both unrelated) — + cited tier empty, the weak hits ride the related tier, capped at + ``related_max_docs``; the FTS hit without vector corroboration + stays LOW (the A8-revised "Mongolia" case); + 2. **one strong + one weak** (the phase-gate question answered from + ``brain-of-reese/.agents/validate.sh``; the 2nd chip + ``ServMon/README.md`` unused) — exactly ONE cited ref, the weak + doc in ``related``; + 3. **the Nagraz case** (``Trooper_Nagraz.pl`` strong, + ``Trooper_Byzin.pl`` weak — same shape, different fixtures); + 4. **the meta/history question** (no doc clears the bar, the agent + reads nothing — chips ``app/api/suggestions.py`` + + ``108_history_wire_check/00_phase.md``, neither used) — pinned on + the DONE FRAME (endpoint-level, fake retriever/LLM/session): the + frame is row-only — ``sources: []`` (the UI's chip list — zero + chips) + the weak hits in ``related``; + 5. **the agent-read exemption** (LOCKED A2): a below-floor doc the + agent ``read`` via the tool joins ``sources`` (cited, last) and is + excluded from ``related``. + +The browser behavior (chip counts on a single-source question, zero +chips on a deflected turn) is E2E-gated by +tests/e2e/test_source_chip_quality.py (task 03). +""" + +from __future__ import annotations + +import json +import re +import uuid +from collections.abc import Iterator +from datetime import UTC, datetime +from pathlib import Path +from typing import TYPE_CHECKING, Any + +import pytest +from fastapi.testclient import TestClient + +from app.api import chat as chat_api +from app.config import Settings +from app.main import app as fastapi_app +from app.models import Document, QueryLog +from app.rag import agent +from app.rag.llm import StreamPiece, ToolCallPiece +from app.rag.retriever import RetrievedChunk +from tests.conftest import ADMIN_PASSWORD + +if TYPE_CHECKING: + from app.rag.scaffolding import ScaffoldingFilter + +FRONTEND = Path(__file__).resolve().parents[2] / "frontend" +STYLES_CSS = FRONTEND / "assets" / "styles.css" +APP_JS = FRONTEND / "assets" / "app.js" + + +def _css() -> str: + assert STYLES_CSS.is_file(), f"missing {STYLES_CSS}" + return STYLES_CSS.read_text(encoding="utf-8") + + +def _app_js() -> str: + assert APP_JS.is_file(), f"missing {APP_JS}" + return APP_JS.read_text(encoding="utf-8") + + +def _function_body(js: str, header: str) -> str: + """The full text of the function whose header is ``header`` — from + the header to its brace-matched closing ``}``. The naive brace + count is safe for the pinned functions: their template literals + carry balanced ``${…}`` pairs and no string literal holds a stray + brace.""" + start = js.index(header) + body_open = js.index("{", start) + depth = 0 + for j in range(body_open, len(js)): + if js[j] == "{": + depth += 1 + elif js[j] == "}": + depth -= 1 + if depth == 0: + return js[start : j + 1] + raise AssertionError(f"unbalanced braces in {header!r}") + + +def _rule(css: str, selector: str) -> str: + """The body of the rule whose selector line is exactly ``selector`` + (multi-line block).""" + block = re.search(rf"^{re.escape(selector)} \{{\n([\s\S]*?)\n\}}", css, re.MULTILINE) + assert block, f"styles.css must carry a `{selector} {{ … }}` rule" + return block.group(1) + + +def _done_branch(js: str) -> str: + """The SSE ``done`` branch of the stream handler — from the + ``ev.type === "done"`` test to the next ``else if`` (the + ``tool_result`` branch). The branch is a flat block (no nested + else-if chain), so a slice between the two branch markers is + exact.""" + start = js.index('ev.type === "done"') + end = js.index('ev.type === "tool_result"', start) + branch = js[start:end] + assert "appendSources(wrap, ev.sources);" in branch, ( + "the done branch must keep appending the cited tier (phase 113 " + "demotes to a row — it never removed the citation surface)" + ) + return branch + + +# ---------- task 02: appendRelated — the secondary row, never a chip ---------- + + +def test_append_related_exists_and_never_uses_source_chip() -> None: + """``appendRelated`` exists and builds ONLY ``.related-doc`` links — + the string ``source-chip`` must NOT appear anywhere in its body + (the acceptance criterion: a weak doc renders only as + ``.related-doc``, never as ``.source-chip``). It reuses the chip + behavior for navigation: the same ``documentUrl(s.source, s.path, + "/")`` href (the /document.html no-JS escape hatch) and the same + left-click → ``openDocumentModal`` (phase 26, the same-page modal). + Each link carries the full path in ``title`` AND ``aria-label`` + (the accessible name never depends on the visible text fitting).""" + body = _function_body(_app_js(), "function appendRelated(wrap, related) {") + assert "link.className = \"related-doc\";" in body, ( + "the related links must carry the .related-doc class" + ) + assert "source-chip" not in body, ( + "appendRelated must NEVER build a citation chip — the related " + "row is not a citation surface (phase 113 LOCKED A1/A2)" + ) + assert "link.href = documentUrl(s.source, s.path, \"/\");" in body, ( + "each related link keeps the chip's /document.html href — the " + "no-JS / context-menu escape hatch (back → the chat page)" + ) + assert "openDocumentModal(s.source, s.path, link);" in body, ( + "left-click opens the same-page document modal exactly like the " + "chips (phase 26 contract)" + ) + assert "e.preventDefault();" in body, ( + "the click must prevent default navigation — the modal takes " + "over, no new tab (the chip pattern)" + ) + assert "link.title = docLabel;" in body, ( + "the native tooltip carries the FULL path (the chip pattern)" + ) + assert 'link.setAttribute("aria-label", docLabel);' in body, ( + "the accessible name is the full path, always" + ) + + +def test_append_related_renders_only_when_related_is_non_empty() -> None: + """The row renders ONLY when ``related`` is non-empty: an + ``undefined``/``null``/``[]`` input (every pre-phase turn, every + turn with nothing under the bar) early-returns with NO DOM — the + bubble reads exactly as it did before phase 113. The pin is the + house guard, verbatim, as the FIRST statement of the body.""" + body = _function_body(_app_js(), "function appendRelated(wrap, related) {") + first_stmt = body.split("{", 1)[1].lstrip() + assert first_stmt.startswith("if (!related || !related.length) return;"), ( + "appendRelated must early-return on !related || !related.length — " + "an empty related tier adds zero DOM (the pre-phase look stays " + "byte-identical for those turns)" + ) + + +def test_append_related_row_contract() -> None: + """The row itself: a ``.msg-meta.related-docs`` div (it joins the + bubble's meta family — the .msg-body flex gap stacks it below the + citation row with the existing gap) that is an accessible list + (``role="list"`` + ``aria-label="Nearby docs, in case"``) headed + by the visible small-caps label ``Nearby docs, in case:`` (the + TODO's suggested wording, trimmed) with the + ``.related-docs-label`` class, and one ``role="listitem"`` link + per doc.""" + body = _function_body(_app_js(), "function appendRelated(wrap, related) {") + assert "row.className = \"msg-meta related-docs\";" in body, ( + "the row is a .msg-meta row (the family the bubble's meta rows " + "already form) with the .related-docs marker" + ) + assert 'row.setAttribute("role", "list");' in body + assert 'row.setAttribute("aria-label", "Nearby docs, in case");' in body, ( + "the row is an accessible list named 'Nearby docs, in case'" + ) + assert "label.className = \"related-docs-label\";" in body + assert "label.textContent = \"Nearby docs, in case:\";" in body, ( + "the visible label carries the TODO's suggested wording (trimmed)" + ) + assert 'link.setAttribute("role", "listitem");' in body, ( + "each link is a listitem of the row's list (ARIA stays valid)" + ) + assert "for (const s of related) {" in body, ( + "one link per related doc" + ) + + +# ---------- task 02: wiring — the done frame and the restore path ---------- + + +def test_done_handler_calls_append_related_last() -> None: + """The done-frame handler calls ``appendRelated(wrap, ev.related)`` + — and appends the row LAST among the bubble's meta rows: AFTER + ``markLastRetryable()`` (the appendTuneButton / appendSaveAsDoc / + appendRetryButton claimers all take the FIRST ``.msg-meta`` row, + so a deflected turn with related docs — no citation row — gets its + OWN meta row for the buttons instead of actions joining the + related row). The call sits after the Retry claim in the branch.""" + branch = _done_branch(_app_js()) + assert "appendRelated(wrap, ev.related);" in branch, ( + "the done handler must render the related tier (it arrives in " + "ev.related — on a deflected turn ev.sources is empty and the " + "weak hits live here, row only, zero chips)" + ) + assert branch.index("appendRelated(wrap, ev.related);") > branch.index( + "markLastRetryable();" + ), ( + "appendRelated must run AFTER the meta-action claimers (last " + "meta row) — a deflected turn's tune/retry buttons must land in " + "their own row, never in the related row" + ) + + +def test_done_handler_persists_related_with_the_turn() -> None: + """The related tier PERSISTS with the turn (the done handler's + ``rememberBrainTurn`` meta), using the house optional-meta pattern + (``undefined`` drops the key from the JSON — no ``related: []`` + noise on turns with nothing related). Without this the restore + path could never re-render the row after a reload.""" + branch = _done_branch(_app_js()) + pattern = ( + r"rememberBrainTurn\(finalText \|\| acc, \{([\s\S]*?)\}" + r"\s*,\s*leavePartialIndex\);" + ) + persist = re.search(pattern, branch) + assert persist, "the done handler must persist the turn through rememberBrainTurn" + meta = persist.group(1) + assert "related: ev.related?.length ? ev.related : undefined," in meta, ( + "the done frame's related tier must persist (undefined drops " + "the key — the house optional-meta pattern, cf. `tools`)" + ) + + +def test_restore_path_calls_append_related() -> None: + """The phase-14 restore path (``renderStoredMessage`` — used by + BOTH the localStorage restore and the /?chat= saved-chat boot + load) re-renders the related row from the stored payload when it + carries ``related``; a pre-phase record without the field + restores exactly as today (appendRelated no-ops — no row). The + call sits after the meta-row claimers, exactly like the live + done path, so restored buttons never join the related row.""" + body = _function_body(_app_js(), "function renderStoredMessage(m) {") + assert "appendRelated(wrap, m.related);" in body, ( + "the restore path must render the related tier from the stored " + "payload (pre-phase records carry no `related` → no row, " + "graceful)" + ) + assert body.index("appendRelated(wrap, m.related);") > body.index( + "appendStoppedNote(wrap);" + ), ( + "the related row appends LAST (after the claimers) on the " + "restore path too — identical order to the live done path" + ) + + +def test_citation_surface_untouched() -> None: + """The citation chip component is NOT touched (phase 113 "NOT + touched" list): ``appendSources`` still builds ``.source-chip`` + pills and still early-returns on an empty list — which is exactly + what a deflected turn (``ev.sources === []``) hits: zero + ``.source-chip`` elements under the bubble. The suggestion chips + (``appendMaybeTry`` / ``.suggestion-chip``) are a different + surface and stay as they were.""" + js = _app_js() + body = _function_body(js, "function appendSources(wrap, sources) {") + assert "chip.className = \"source-chip\";" in body + assert body.lstrip().startswith("function appendSources"), "sanity" + assert "if (!sources || !sources.length) return;" in body, ( + "appendSources still no-ops on an empty list — a deflected " + "turn's empty ev.sources renders ZERO citation chips (the " + "weak hits arrive in ev.related → the row only)" + ) + assert js.count("function appendRelated(wrap, related) {") == 1, ( + "exactly ONE appendRelated — no second related renderer" + ) + + +# ---------- task 02: the CSS — clearly secondary, AA, theme-neutral ---------- + + +def test_related_doc_rule_is_the_secondary_look() -> None: + """The ``.related-doc`` link: dashed border (the citation chip's + solid 1px ``--line`` pill is the citation look — the dash is the + visual split), transparent fill (no ``--brand-soft``), muted + ``--ink-soft`` text (8.6:1 on ``--bg`` — verified ≥4.5:1, WCAG + 2.1 AA; the ratio is recorded in the rule's provenance comment), + smaller mono than the chip (0.7rem < 0.72rem), and the + single-line ellipsis set (same overflow contract as the chip). + Palette variables only — zero new literals (phase-92 invariant), + so the monochrome theme grays the row automatically.""" + body = _rule(_css(), ".related-doc") + assert "border: 1px dashed var(--line);" in body, ( + "the related link is DASHED — the chip's solid border is the " + "citation look, the dash is the 'not a citation' signal" + ) + assert "background: transparent;" in body, ( + "no brand-soft fill — that surface is the citation pill's" + ) + assert "color: var(--ink-soft);" in body, ( + "the link text is the muted ink — AA on the page bg (8.6:1)" + ) + assert "font-family: var(--mono);" in body, ( + "source/path reads mono like the chips (same data, secondary " + "weight)" + ) + assert "font-size: 0.7rem;" in body, ( + "smaller than the chip's 0.72rem — visually secondary" + ) + for prop in ( + "white-space: nowrap;", + "overflow: hidden;", + "text-overflow: ellipsis;", + "max-width: 100%;", + "min-width: 0;", + ): + assert prop in body, f"the related link keeps the chip's single-line ellipsis set ({prop})" + comment = re.search(r"(/\*[^*]*?\*/)\s*\.related-docs-label \{", _css()) + assert comment, "the related-docs rules must carry their provenance comment" + note = comment.group(1) + assert "8.6:1" in note and "4.5:1" in note, ( + "the --ink-soft on --bg ratio must be recorded (verified " + "8.6:1 ≥ 4.5:1, WCAG AA — house style)" + ) + assert "#" not in note.replace("--", ""), ( + "the comment is theme-variable language — no literal colors " + "(phase-92 zero-literal invariant)" + ) + + +def test_related_doc_hover_is_flat() -> None: + """The ``.related-doc:hover`` rule is deliberately FLAT: no + background swap (the chip's ``background: var(--brand-soft)`` + hover is the citation affordance — 'no hover elevation of the + citation chips'), only the ink-soft → ink step-up (16.7:1 on + ``--bg``) plus the underline. The global 3px ``:focus-visible`` + outline rule covers the focus ring (no per-rule ring needed, the + chip precedent).""" + body = _rule(_css(), ".related-doc:hover") + assert "background" not in body, ( + "the hover must NOT change the background — that surface swap " + "is the citation chip's affordance" + ) + assert "box-shadow" not in body, "no hover elevation (no shadow gain)" + assert "color: var(--ink);" in body, ( + "the hover is the flat ink step-up (16.7:1 on --bg, AA)" + ) + assert "text-decoration: underline;" in body + + +def test_related_docs_label_is_small_caps_muted() -> None: + """The ``.related-docs-label``: small muted uppercase text (the + 'small caps or muted small text' from the task) — smaller than the + row's 0.75rem ``.msg-meta`` base, the same AA-safe ``--ink-soft`` + (8.6:1 on ``--bg``) as the links, letterspaced like every other + overline in the app.""" + body = _rule(_css(), ".related-docs-label") + assert "text-transform: uppercase;" in body + assert "color: var(--ink-soft);" in body, ( + "the label uses the AA-safe muted ink (8.6:1 on --bg)" + ) + size = re.search(r"font-size: ([0-9.]+)rem;", body) + assert size, "the label must set its own (smaller) font size" + assert float(size.group(1)) < 0.75, ( + "the label is smaller than the row's 0.75rem base — it is a " + "whisper, not a heading" + ) + + +# ---------- task 03: the four observed shapes (TODO L110–123) ---------- + +#: The canned answer the endpoint-level fakes stream (the gate-suite +#: convention — byte-stable, assertable against the wire). +ANSWER = "I haven't done anything like that — try one of these instead!" + + +def _shape_settings() -> Settings: + """The PRODUCTION calibration (the code defaults, explicit) — the + four shapes were observed LIVE under this threshold/floor pair. + ``_env_file=None`` keeps the mock-calibrated values from + ``tests/conftest.py`` (and any local ``.env``) out of the pin.""" + return Settings( + _env_file=None, # pyright: ignore[reportCallIssue] + relevance_threshold=0.62, + lexical_support_floor=0.35, + source_usefulness_floor=0.35, # LOCKED A2 default + related_max_docs=2, # LOCKED A4 default + top_n_docs=2, # the ceiling — never a quota (LOCKED A2) + ) + + +def _doc(source: str, path: str, title: str, content: str) -> Document: + return Document( + id=uuid.uuid4(), + source=source, + path=path, + full_path=f"/{source}/{path}", + title=title, + content=content, + content_hash="0" * 64, + # Phase 106, D5: the HIGH block / the ``read`` result's date + # line format the row's created_at — a fixed value keeps the + # fixtures deterministic. + created_at=datetime(2024, 6, 15, 12, 0, 0, tzinfo=UTC), + ) + + +def _chunk( + doc: Document, + score: float, + cosine: float, + fts_hit: bool = False, +) -> RetrievedChunk: + """One fake retrieval candidate: *score* is the RRF fused rank key, + *cosine* the vector similarity (the bar's input — independent of + *score* on purpose: the bar is on the cosine, LOCKED A2).""" + return RetrievedChunk( + chunk_id=uuid.uuid4(), + position=0, + content=doc.content[:32], + score=score, + document=doc, + cosine=cosine, + fts_hit=fts_hit, + ) + + +def test_shape_1_mongolia_both_docs_weak_cite_nothing() -> None: + """Observed shape 1 (TODO L110–113): "What is the capital of + Mongolia?" → chips ``Trooper_Nagraz.pl`` + ``Trooper_Begzei.pl``, + BOTH unrelated. Both below the bar: the cited tier is EMPTY (zero + citation chips) and the weak hits ride the related tier — in rank + order, capped at ``related_max_docs`` (the 3rd weak doc drops out). + The FTS hit without vector corroboration (0.20 < the 0.35 lexical + floor) stays LOW — the A8-revised "Mongolia" case; the weak content + never reaches the LLM.""" + nagraz = _doc("scripts", "scripts/Trooper_Nagraz.pl", "Trooper_Nagraz.pl", + "NAGRAZ_PL_CONTENT") + begzei = _doc("scripts", "scripts/Trooper_Begzei.pl", "Trooper_Begzei.pl", + "BEGZEI_PL_CONTENT") + third = _doc("scripts", "scripts/Trooper_Third.pl", "Trooper_Third.pl", + "THIRD_PL_CONTENT") + chunks = [ + _chunk(nagraz, 0.033, cosine=0.20, fts_hit=True), # rank 1, lexical hit + _chunk(begzei, 0.031, cosine=0.12), + _chunk(third, 0.030, cosine=0.10), # below the cap — related drops it + ] + plan = chat_api.plan_turn(chunks, _shape_settings()) + assert plan.deflected is True # 0.20 < 0.62 AND 0.20 < the 0.35 lex floor + assert plan.docs == [] # NO citation slot below the bar + assert [d.title for d in plan.related_docs] == [ + "Trooper_Nagraz.pl", + "Trooper_Begzei.pl", + ] # rank order, capped at related_max_docs (2) + assert len(plan.related_docs) <= 2 + # The LOW prompt is titles only — none of the weak content is sent. + assert "NAGRAZ_PL_CONTENT" not in plan.system_prompt + assert "Trooper_Nagraz.pl" in plan.system_prompt # weak-hit titles carried + assert plan.suggestions # the "Maybe try" chips are unchanged + + +def test_shape_2_validate_sh_strong_plus_unused_second_chip() -> None: + """Observed shape 2 (TODO L114–116): the phase-gate question is + answered from ``brain-of-reese/.agents/validate.sh`` — the 2nd chip + ``ServMon/README.md`` was NEVER used. The strong doc clears the bar + and takes the only cited slot (top_n_docs is a ceiling, not a + quota); the weak 2nd doc demotes to related — never a citation. + The HIGH prompt carries the cited content only.""" + validate = _doc("brain-of-reese", ".agents/validate.sh", "validate.sh", + "VALIDATE_SH_CONTENT") + servmon = _doc("ServMon", "README.md", "ServMon README", + "SERVMON_README_CONTENT") + chunks = [ + _chunk(validate, 0.90, cosine=0.70), # clears threshold AND bar + _chunk(servmon, 0.80, cosine=0.20), # high fused rank, weak cosine + ] + plan = chat_api.plan_turn(chunks, _shape_settings()) + assert plan.deflected is False # 0.70 >= 0.62 + assert [d.title for d in plan.docs] == ["validate.sh"] # exactly ONE cited + assert [d.title for d in plan.related_docs] == ["ServMon README"] + assert "VALIDATE_SH_CONTENT" in plan.system_prompt + assert "SERVMON_README_CONTENT" not in plan.system_prompt + + +def test_shape_3_nagraz_answered_by_own_doc_byzin_uncited() -> None: + """Observed shape 3 (TODO L117–119): the Trooper_Nagraz question is + answered from ``Trooper_Nagraz.pl`` — the 2nd chip + ``Trooper_Byzin.pl`` uncited. The SAME shape as shape 2 with + different fixtures — the bar filters the 2nd chip; it is not a + coincidence of the validate.sh pair.""" + nagraz = _doc("scripts", "scripts/Trooper_Nagraz.pl", "Trooper_Nagraz.pl", + "NAGRAZ_PL_CONTENT") + byzin = _doc("scripts", "scripts/Trooper_Byzin.pl", "Trooper_Byzin.pl", + "BYZIN_PL_CONTENT") + chunks = [ + _chunk(nagraz, 0.85, cosine=0.70), + _chunk(byzin, 0.75, cosine=0.15), # below the bar + ] + plan = chat_api.plan_turn(chunks, _shape_settings()) + assert plan.deflected is False + assert [d.title for d in plan.docs] == ["Trooper_Nagraz.pl"] # 1 cited + assert [d.title for d in plan.related_docs] == ["Trooper_Byzin.pl"] # 1 related + assert "BYZIN_PL_CONTENT" not in plan.system_prompt + + +# ---------- task 03: done-frame wire (endpoint-level fakes, no stack) ---------- + + +class _CannedLLM: + """Records the requests; streams the canned *answer*. + + Without *read_path* it never emits tool calls (the single-request + shape). With *read_path*, the first tools-offering request that + carries no tool result yet emits ONE ``read`` call on the combined + path; the follow-up request (carrying the tool result) streams the + answer — the phase-37 single-read shape, stateless (the e2e mock's + convention).""" + + def __init__(self, answer: str = ANSWER, read_path: str | None = None) -> None: + self.settings = Settings(_env_file=None) # pyright: ignore[reportCallIssue] + self.answer = answer + self.read_path = read_path + self.seen: list[list[dict[str, Any]]] = [] + self.seen_tools: list[list[dict[str, Any]] | None] = [] + + async def embed_one(self, _text: str) -> list[float]: + return [0.0] * 768 + + async def chat_stream( + self, + messages: list[dict[str, Any]], + tools: list[dict[str, Any]] | None = None, + scaffolding: ScaffoldingFilter | None = None, + ): + self.seen.append(messages) + self.seen_tools.append(tools) + if ( + self.read_path is not None + and tools is not None + and not any(m.get("role") == "tool" for m in messages) + ): + yield ToolCallPiece( + id="call_1", name="read", arguments={"path": self.read_path} + ) + return + for i in range(0, len(self.answer), 12): + yield StreamPiece("content", self.answer[i : i + 12]) + + +class _FakeSteeringResult: + """Empty steering-note result (no stored notes in these tests).""" + + def all(self) -> list[Any]: + return [] + + +class _FakeSession: + """Stands in for the DB session (the gate-suite pattern): + records the ``QueryLog`` row, yields no steering notes, no KB + overview (``get`` → ``None``). Tool execution's ``find_document`` + is monkeypatched separately (the agent module's, not the + session's).""" + + def __init__(self) -> None: + self.added: list[Any] = [] + self.commits = 0 + + def __enter__(self) -> _FakeSession: + return self + + def __exit__(self, *args: Any) -> None: + pass + + def add(self, obj: Any) -> None: + self.added.append(obj) + + def commit(self) -> None: + self.commits += 1 + + def scalars(self, _stmt: Any) -> _FakeSteeringResult: + return _FakeSteeringResult() + + def get(self, _model: Any, _pk: Any) -> Any: + return None + + +@pytest.fixture(autouse=True) +def _admin_signed_in(client: TestClient) -> None: + """Phase 79: ``POST /api/chat`` is user-gated — the endpoint-level + tests run as the signed-in ADMIN (the gate-suite pattern).""" + r = client.post("/api/login", json={"password": ADMIN_PASSWORD}) + assert r.status_code == 204, f"admin login failed: {r.status_code} {r.text}" + + +@pytest.fixture() +def chip_env( + monkeypatch: pytest.MonkeyPatch, +) -> Iterator[tuple[_FakeSession, _CannedLLM]]: + """``POST /api/chat`` with retriever, session, and LLM all faked — + the production calibration (``_shape_settings``) in force.""" + monkeypatch.setattr(chat_api, "db_available", lambda: True) + session = _FakeSession() + llm = _CannedLLM() + monkeypatch.setattr(chat_api, "SessionLocal", lambda: session) + monkeypatch.setitem(fastapi_app.dependency_overrides, chat_api.get_llm, lambda: llm) + monkeypatch.setattr(chat_api, "get_settings", _shape_settings) + yield session, llm + fastapi_app.dependency_overrides.clear() + + +def _ask(client: TestClient, message: str) -> list[dict[str, Any]]: + with client.stream("POST", "/api/chat", json={"message": message}) as r: + assert r.status_code == 200 + frames: list[dict[str, Any]] = [] + buf = "" + for part in r.iter_text(): + buf += part + while "\n\n" in buf: + frame, buf = buf.split("\n\n", 1) + frame = frame.strip() + if frame.startswith("data:"): + frames.append(json.loads(frame.removeprefix("data:").strip())) + assert buf.strip() == "" + return frames + + +def _fake_retriever(chunks: list[RetrievedChunk]) -> Any: + def retrieve(_db: Any, _question: str, _vec: list[float]) -> list[RetrievedChunk]: + return chunks + + return retrieve + + +def test_shape_4_meta_question_deflected_frame_is_row_only( + client: TestClient, + chip_env: tuple[_FakeSession, _CannedLLM], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Observed shape 4 (TODO L120–123), pinned on the DONE FRAME: a + meta question about the conversation's own history → chips + ``app/api/suggestions.py`` + ``108_history_wire_check/00_phase.md``, + neither used. No doc clears the bar and the agent reads nothing — + the frame is ROW-ONLY: ``sources: []`` (the UI chips every source + entry — zero chips) with the weak hits in ``related`` (rank order, + ≤ ``related_max_docs``) — the de-emphasized row's links (the row's + rendering itself is pinned by task 02's source tests + the E2E). + The weak retrieval stays durably recorded (LOCKED A3); the weak + content never reaches the LLM (LOW prompt, titles only).""" + session, llm = chip_env + suggestions = _doc("brain-of-reese", "app/api/suggestions.py", + "suggestions.py", "SUGGESTIONS_PY_CONTENT") + phase_md = _doc("brain-of-reese", + ".agents/108_history_wire_check/00_phase.md", "00_phase.md", + "PHASE_MD_CONTENT") + monkeypatch.setattr( + chat_api, + "retrieve", + _fake_retriever( + [ + _chunk(suggestions, 0.033, cosine=0.25), + _chunk(phase_md, 0.031, cosine=0.10), + ] + ), + ) + + frames = _ask(client, "What have we covered in this conversation so far?") + done = frames[-1] + assert done["type"] == "done" + assert done["deflected"] is True + assert done["sources"] == [] # zero citation chips on the wire + related = done["related"] + assert [(s["source"], s["path"]) for s in related] == [ + ("brain-of-reese", "app/api/suggestions.py"), + ("brain-of-reese", ".agents/108_history_wire_check/00_phase.md"), + ] # rank order + assert len(related) <= 2 # related_max_docs + assert all(s["title"] for s in related) # the row's links carry the identity + assert done["suggestions"] # the "Maybe try" chips are unchanged + + (system, _user) = llm.seen[0][0], llm.seen[0][1] + assert "SUGGESTIONS_PY_CONTENT" not in system["content"] + assert "PHASE_MD_CONTENT" not in system["content"] + + (row,) = session.added + assert isinstance(row, QueryLog) + assert row.deflected is True + # LOCKED A3: the weak retrieval stays recorded (observability). + assert "app/api/suggestions.py" in row.sources + assert "108_history_wire_check/00_phase.md" in row.sources + + +def test_done_frame_single_cited_ref_strong_plus_weak( + client: TestClient, + chip_env: tuple[_FakeSession, _CannedLLM], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Shape 2 on the wire — the single-document question's input to + "exactly one citation chip" (the E2E asserts the rendered chip): + the bar-clearing doc is the ONLY ``sources`` ref; the weak 2nd doc + rides ``related``; the tiers are disjoint (the done frame's dedupe). + The durable record keeps the FULL retrieval (LOCKED A3).""" + session, _llm = chip_env + validate = _doc("brain-of-reese", ".agents/validate.sh", "validate.sh", + "VALIDATE_SH_CONTENT") + servmon = _doc("ServMon", "README.md", "ServMon README", + "SERVMON_README_CONTENT") + monkeypatch.setattr( + chat_api, + "retrieve", + _fake_retriever( + [ + _chunk(validate, 0.90, cosine=0.70), + _chunk(servmon, 0.80, cosine=0.20), + ] + ), + ) + + frames = _ask(client, "How does the phase gate decide to validate?") + done = frames[-1] + assert done["deflected"] is False + assert [(s["source"], s["path"]) for s in done["sources"]] == [ + ("brain-of-reese", ".agents/validate.sh"), + ] # EXACTLY one citation chip on the wire + assert [(s["source"], s["path"]) for s in done["related"]] == [ + ("ServMon", "README.md"), + ] + cited = {(s["source"], s["path"]) for s in done["sources"]} + related = {(s["source"], s["path"]) for s in done["related"]} + assert cited.isdisjoint(related) + + (row,) = session.added + assert isinstance(row, QueryLog) + assert row.deflected is False + # The durable record keeps BOTH docs (retrieval, not citations — A3). + assert ".agents/validate.sh" in row.sources + assert "ServMon/README.md" in row.sources + + +def test_agent_read_below_floor_doc_joins_sources( + client: TestClient, + chip_env: tuple[_FakeSession, _CannedLLM], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The agent-read exemption (LOCKED A2): a doc UNDER the bar that + the agent ``read`` via the tool is cited by definition — the model + read it, so it was used. It joins ``sources`` (after the retrieved + cited docs, deduped) and is EXCLUDED from ``related`` (a used doc + must never read as "nearby"); the other below-floor doc stays in + the tier. The read content reached the model (the tool result in + the follow-up request).""" + session, _default_llm = chip_env + strong = _doc("docs", "strong.md", "Strong", "STRONG_DOC_CONTENT") + weak_b = _doc("docs", "weak-b.md", "Weak B", "WEAK_B_READ_BY_AGENT") + weak_c = _doc("docs", "weak-c.md", "Weak C", "WEAK_C_CONTENT") + monkeypatch.setattr( + chat_api, + "retrieve", + _fake_retriever( + [ + _chunk(strong, 0.90, cosine=0.70), # clears the bar + _chunk(weak_b, 0.80, cosine=0.20), # below the bar — read by the agent + _chunk(weak_c, 0.70, cosine=0.10), # below the bar — nobody reads it + ] + ), + ) + read_llm = _CannedLLM(read_path="docs/weak-b.md") + fastapi_app.dependency_overrides[chat_api.get_llm] = lambda: read_llm + + def _find_document(_db: Any, source: str, path: str) -> Document | None: + return weak_b if (source, path) == ("docs", "weak-b.md") else None + + monkeypatch.setattr(agent, "find_document", _find_document) + + frames = _ask(client, "What does the weak B document say?") + done = frames[-1] + assert done["deflected"] is False + # The tool ran (one ``tool`` frame) and the read doc reached the + # model's follow-up request. + assert any(f["type"] == "tool" for f in frames) + tool_msgs = [m for m in read_llm.seen[1] if m.get("role") == "tool"] + assert len(tool_msgs) == 1 + assert "WEAK_B_READ_BY_AGENT" in tool_msgs[0]["content"] + + sources = [(s["source"], s["path"]) for s in done["sources"]] + assert sources == [("docs", "strong.md"), ("docs", "weak-b.md")] # read ⇒ cited, last + related = [(s["source"], s["path"]) for s in done["related"]] + assert related == [("docs", "weak-c.md")] # the read doc is not "nearby" + assert set(sources).isdisjoint(set(related)) + + (row,) = session.added + assert isinstance(row, QueryLog) + assert "weak-b.md" in row.sources # the full retrieval is recorded (A3)