From aa65d2d69167fa11be5ee05001a221a7d9f40b19 Mon Sep 17 00:00:00 2001 From: ducoterra Date: Thu, 1 Oct 2026 04:25:03 -0400 Subject: [PATCH] phase: 142_kb_overview_freshness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All verification complete. Final report: **Phase 142 final verification pass — all green** - **Verified implemented** (working tree matches design; app diff is exactly 2 lines of logic): `app/rag/overview.py` — `regenerate_overview` now orders the lite-model document list `created_at` desc, `(source, path)` tie-break (newest docs survive the cap); `app/rag/prompts.py` — `_KB_INTRO` rewritten to the honest intro (recency bias, "map not inventory", verify-with-`ls`/`grep`; "(generated at import time)" gone). - **Completion criteria:** 1. Forced-cap outline leads with newest docs — ✅ E2E `test_kb_overview_freshness.py` test 1 (cap 300, digest from `zz_new/z.md`, no "old" tokens) + integration repro (cap 150 → `mydocs zz new 03 md zz new 03`) 2. New honest intro served in `` — ✅ unit pin (verbatim) + E2E test 2 (`build_kb_section` surface, tag/cap byte-stable) 3. Gates — ✅ `uv run pytest --cov=app --cov-report=term-missing`: **3359 passed**, `app/` TOTAL **99%** (>90%); `uv run pytest tests/e2e/test_kb_overview_freshness.py -v --no-cov`: **3 passed** in isolation; `uv run pytest tests/e2e/test_kb_overview.py -v --no-cov`: **4 passed** in isolation; `uv run ruff check . && uv run pyright`: **clean, 0 errors** 4. No behavior change in completed phases — ✅ cap env knob, change-gated sync trigger, `build_kb_section` cap/marker, deflect path all byte-identical (pinned pins pass in the 3359) - **Documentation:** README.md reviewed in full (every `.env.example` var documented with matching defaults, all endpoints/routes verified against `app/api/`, Quick Start/deploy/troubleshooting claims verified) and AGENTS.md re-reviewed (rules + quick reference accurate) — **verified consistent, no changes needed**. - **Decisions:** none required — no defects found; no live-infra changes (only the already-running local `db` container was used by tests). - **Next pending phase:** `.agents/phases/todo/143_mobile_keyboard_after_answer`. --- .../142_kb_overview_freshness/00_phase.md | 0 .../01_cap_ordering_repro.md | 0 .../02_recency_ordering.md | 0 .../03_honest_kb_intro.md | 0 .../04_e2e_outline_freshness.md | 0 ...142_kb_overview_freshness__00_phase.a1.err | 0 .../142_kb_overview_freshness__00_phase.a1.md | 13 + ...b_overview_freshness__00_phase.a1.validate | 118 +++++ ...ew_freshness__01_cap_ordering_repro.a1.err | 0 ...iew_freshness__01_cap_ordering_repro.a1.md | 19 + ...eshness__01_cap_ordering_repro.a1.validate | 240 +++++++++++ ...ew_freshness__01_cap_ordering_repro.a2.err | 0 ...iew_freshness__01_cap_ordering_repro.a2.md | 16 + ...eshness__01_cap_ordering_repro.a2.validate | 118 +++++ ...view_freshness__02_recency_ordering.a1.err | 0 ...rview_freshness__02_recency_ordering.a1.md | 18 + ...freshness__02_recency_ordering.a1.validate | 118 +++++ ...rview_freshness__03_honest_kb_intro.a1.err | 0 ...erview_freshness__03_honest_kb_intro.a1.md | 10 + ..._freshness__03_honest_kb_intro.a1.validate | 118 +++++ ...freshness__04_e2e_outline_freshness.a1.err | 0 ..._freshness__04_e2e_outline_freshness.a1.md | 10 + ...ness__04_e2e_outline_freshness.a1.validate | 118 +++++ app/rag/overview.py | 39 +- app/rag/prompts.py | 20 +- tests/e2e/test_kb_overview.py | 56 ++- tests/e2e/test_kb_overview_freshness.py | 402 ++++++++++++++++++ .../integration/test_import_docs_overview.py | 117 ++++- tests/unit/test_overview.py | 143 ++++++- tests/unit/test_prompts.py | 83 +++- 30 files changed, 1738 insertions(+), 38 deletions(-) rename .agents/phases/{todo => complete}/142_kb_overview_freshness/00_phase.md (100%) rename .agents/phases/{todo => complete}/142_kb_overview_freshness/01_cap_ordering_repro.md (100%) rename .agents/phases/{todo => complete}/142_kb_overview_freshness/02_recency_ordering.md (100%) rename .agents/phases/{todo => complete}/142_kb_overview_freshness/03_honest_kb_intro.md (100%) rename .agents/phases/{todo => complete}/142_kb_overview_freshness/04_e2e_outline_freshness.md (100%) create mode 100644 .agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__00_phase.a1.err create mode 100644 .agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__00_phase.a1.md create mode 100644 .agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__00_phase.a1.validate create mode 100644 .agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__01_cap_ordering_repro.a1.err create mode 100644 .agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__01_cap_ordering_repro.a1.md create mode 100644 .agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__01_cap_ordering_repro.a1.validate create mode 100644 .agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__01_cap_ordering_repro.a2.err create mode 100644 .agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__01_cap_ordering_repro.a2.md create mode 100644 .agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__01_cap_ordering_repro.a2.validate create mode 100644 .agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__02_recency_ordering.a1.err create mode 100644 .agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__02_recency_ordering.a1.md create mode 100644 .agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__02_recency_ordering.a1.validate create mode 100644 .agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__03_honest_kb_intro.a1.err create mode 100644 .agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__03_honest_kb_intro.a1.md create mode 100644 .agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__03_honest_kb_intro.a1.validate create mode 100644 .agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__04_e2e_outline_freshness.a1.err create mode 100644 .agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__04_e2e_outline_freshness.a1.md create mode 100644 .agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__04_e2e_outline_freshness.a1.validate create mode 100644 tests/e2e/test_kb_overview_freshness.py diff --git a/.agents/phases/todo/142_kb_overview_freshness/00_phase.md b/.agents/phases/complete/142_kb_overview_freshness/00_phase.md similarity index 100% rename from .agents/phases/todo/142_kb_overview_freshness/00_phase.md rename to .agents/phases/complete/142_kb_overview_freshness/00_phase.md diff --git a/.agents/phases/todo/142_kb_overview_freshness/01_cap_ordering_repro.md b/.agents/phases/complete/142_kb_overview_freshness/01_cap_ordering_repro.md similarity index 100% rename from .agents/phases/todo/142_kb_overview_freshness/01_cap_ordering_repro.md rename to .agents/phases/complete/142_kb_overview_freshness/01_cap_ordering_repro.md diff --git a/.agents/phases/todo/142_kb_overview_freshness/02_recency_ordering.md b/.agents/phases/complete/142_kb_overview_freshness/02_recency_ordering.md similarity index 100% rename from .agents/phases/todo/142_kb_overview_freshness/02_recency_ordering.md rename to .agents/phases/complete/142_kb_overview_freshness/02_recency_ordering.md diff --git a/.agents/phases/todo/142_kb_overview_freshness/03_honest_kb_intro.md b/.agents/phases/complete/142_kb_overview_freshness/03_honest_kb_intro.md similarity index 100% rename from .agents/phases/todo/142_kb_overview_freshness/03_honest_kb_intro.md rename to .agents/phases/complete/142_kb_overview_freshness/03_honest_kb_intro.md diff --git a/.agents/phases/todo/142_kb_overview_freshness/04_e2e_outline_freshness.md b/.agents/phases/complete/142_kb_overview_freshness/04_e2e_outline_freshness.md similarity index 100% rename from .agents/phases/todo/142_kb_overview_freshness/04_e2e_outline_freshness.md rename to .agents/phases/complete/142_kb_overview_freshness/04_e2e_outline_freshness.md diff --git a/.agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__00_phase.a1.err b/.agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__00_phase.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__00_phase.a1.md b/.agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__00_phase.a1.md new file mode 100644 index 0000000..a2705ea --- /dev/null +++ b/.agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__00_phase.a1.md @@ -0,0 +1,13 @@ +All verification complete. Final report: + +**Phase 142 final verification pass — all green** + +- **Verified implemented** (working tree matches design; app diff is exactly 2 lines of logic): `app/rag/overview.py` — `regenerate_overview` now orders the lite-model document list `created_at` desc, `(source, path)` tie-break (newest docs survive the cap); `app/rag/prompts.py` — `_KB_INTRO` rewritten to the honest intro (recency bias, "map not inventory", verify-with-`ls`/`grep`; "(generated at import time)" gone). +- **Completion criteria:** + 1. Forced-cap outline leads with newest docs — ✅ E2E `test_kb_overview_freshness.py` test 1 (cap 300, digest from `zz_new/z.md`, no "old" tokens) + integration repro (cap 150 → `mydocs zz new 03 md zz new 03`) + 2. New honest intro served in `` — ✅ unit pin (verbatim) + E2E test 2 (`build_kb_section` surface, tag/cap byte-stable) + 3. Gates — ✅ `uv run pytest --cov=app --cov-report=term-missing`: **3359 passed**, `app/` TOTAL **99%** (>90%); `uv run pytest tests/e2e/test_kb_overview_freshness.py -v --no-cov`: **3 passed** in isolation; `uv run pytest tests/e2e/test_kb_overview.py -v --no-cov`: **4 passed** in isolation; `uv run ruff check . && uv run pyright`: **clean, 0 errors** + 4. No behavior change in completed phases — ✅ cap env knob, change-gated sync trigger, `build_kb_section` cap/marker, deflect path all byte-identical (pinned pins pass in the 3359) +- **Documentation:** README.md reviewed in full (every `.env.example` var documented with matching defaults, all endpoints/routes verified against `app/api/`, Quick Start/deploy/troubleshooting claims verified) and AGENTS.md re-reviewed (rules + quick reference accurate) — **verified consistent, no changes needed**. +- **Decisions:** none required — no defects found; no live-infra changes (only the already-running local `db` container was used by tests). +- **Next pending phase:** `.agents/phases/todo/143_mobile_keyboard_after_answer`. diff --git a/.agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__00_phase.a1.validate b/.agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__00_phase.a1.validate new file mode 100644 index 0000000..4b0e5c1 --- /dev/null +++ b/.agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__00_phase.a1.validate @@ -0,0 +1,118 @@ +........................................................................ [ 2%] +........................................................................ [ 4%] +........................................................................ [ 6%] +........................................................................ [ 8%] +........................................................................ [ 10%] +........................................................................ [ 12%] +........................................................................ [ 15%] +........................................................................ [ 17%] +........................................................................ [ 19%] +........................................................................ [ 21%] +........................................................................ [ 23%] +........................................................................ [ 25%] +........................................................................ [ 27%] +........................................................................ [ 30%] +........................................................................ [ 32%] +........................................................................ [ 34%] +........................................................................ [ 36%] +........................................................................ [ 38%] +........................................................................ [ 40%] +........................................................................ [ 42%] +........................................................................ [ 45%] +........................................................................ [ 47%] +........................................................................ [ 49%] +........................................................................ [ 51%] +........................................................................ [ 53%] +........................................................................ [ 55%] +........................................................................ [ 57%] +........................................................................ [ 60%] +........................................................................ [ 62%] +........................................................................ [ 64%] +........................................................................ [ 66%] +........................................................................ [ 68%] +........................................................................ [ 70%] +........................................................................ [ 72%] +........................................................................ [ 75%] +........................................................................ [ 77%] +........................................................................ [ 79%] +........................................................................ [ 81%] +........................................................................ [ 83%] +........................................................................ [ 85%] +........................................................................ [ 87%] +........................................................................ [ 90%] +........................................................................ [ 92%] +........................................................................ [ 94%] +........................................................................ [ 96%] +........................................................................ [ 98%] +............................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 268 1 99% +app/api/chat_images.py 50 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 179 1 99% +app/api/git_sources.py 259 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 183 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 264 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 20 0 100% +app/core/theming.py 38 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 68 0 100% +app/models.py 133 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 422 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 134 0 100% +app/rag/git_sources.py 56 0 100% +app/rag/importer.py 705 9 99% +app/rag/llm.py 246 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 71 0 100% +app/rag/pdf.py 124 0 100% +app/rag/prompts.py 154 0 100% +app/rag/retriever.py 248 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 58 0 100% +app/schemas.py 424 0 100% +-------------------------------------------------- +TOTAL 5498 22 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__01_cap_ordering_repro.a1.err b/.agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__01_cap_ordering_repro.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__01_cap_ordering_repro.a1.md b/.agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__01_cap_ordering_repro.a1.md new file mode 100644 index 0000000..d35349e --- /dev/null +++ b/.agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__01_cap_ordering_repro.a1.md @@ -0,0 +1,19 @@ +All done. Final verification summary — everything is in the designed state. + +## Report — task `01_cap_ordering_repro` (phase 142) + +**Implemented (tests-only, no app code changed):** +- `tests/unit/test_overview.py`: `_phase_dir_rows()` (20 owner-shape rows, `01_old_a.md`…`20_new_z.md`, 100 chars/line); `test_cap_keeps_the_given_prefix_and_cuts_the_tail` — GREEN pin of the current cap behavior (lexicographic prefix survives, tail cut, marker on its own line, cut exactly at cap); `test_document_list_reaching_the_model_leads_with_newest_docs` — RED repro: `regenerate_overview` leads the model's list with the OLDEST doc (`01_old_a`), asserts the post-fix intent (newest first, oldest last). +- `tests/integration/test_import_docs_overview.py`: `OverviewDigestEmbedder` (reproduces the mock's `KB_OVERVIEW_MODE` 8-token digest, imports `TOKEN_RE` from `tests/e2e/mock_llm.py` so it's byte-identical); `owner_kb` fixture (one source, `old/` batch mtimed 2026-01 vs `zz_new/` batch 2026-09 via `os.utime` — recency opposes lexicographic order); `test_overview_outline_names_the_newest_docs_when_the_cap_bites` — RED repro with `BOR_OVERVIEW_INPUT_MAX_CHARS=150` forced: pre-fix the stored outline is `mydocs old 01 md old 01 summary of` (the exact "summaries only up to phase 113" shape); asserts the post-fix digest `mydocs zz new 03 md zz new 03` + no `old` token. + +**Test / lint / coverage (exact outcomes):** +- `uv run pytest` → `2 failed, 3356 passed` — the 2 failures are exactly the new repros, failing for the documented root cause (failure output names it); everything else 100% green. +- `uv run ruff check .` → clean; `uv run pyright` → `0 errors, 0 warnings`. +- `uv run pytest --cov=app` → `app/` coverage **99%** (gate >90%). +- `uv run pytest tests/e2e/test_kb_overview.py -v --no-cov` → 4 passed (no regression). + +**README/AGENTS.md:** no doc impact (tests-only task). + +**Notable:** red-is-by-design per the task's completion criteria (task 02 flips both green); `git diff` touches only the two test files. + +**Next pending task:** `.agents/phases/todo/142_kb_overview_freshness/02_recency_ordering.md`. diff --git a/.agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__01_cap_ordering_repro.a1.validate b/.agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__01_cap_ordering_repro.a1.validate new file mode 100644 index 0000000..978019d --- /dev/null +++ b/.agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__01_cap_ordering_repro.a1.validate @@ -0,0 +1,240 @@ +........................................................................ [ 2%] +........................................................................ [ 4%] +........................................................................ [ 6%] +........................................................................ [ 8%] +........................................................................ [ 10%] +........................................................................ [ 12%] +........................................................................ [ 15%] +........................................................................ [ 17%] +........................................................................ [ 19%] +........................................................................ [ 21%] +........................................................................ [ 23%] +........................................................................ [ 25%] +........................................................................ [ 27%] +........................................................................ [ 30%] +........................................................................ [ 32%] +........................................................................ [ 34%] +........................................................................ [ 36%] +........................................................................ [ 38%] +........................................................................ [ 40%] +........................................................................ [ 42%] +........................................................................ [ 45%] +........................................................................ [ 47%] +........................................................................ [ 49%] +...............................................................F........ [ 51%] +........................................................................ [ 53%] +........................................................................ [ 55%] +........................................................................ [ 57%] +........................................................................ [ 60%] +........................................................................ [ 62%] +........................................................................ [ 64%] +........................................................................ [ 66%] +........................................................................ [ 68%] +........................................................................ [ 70%] +........................................................................ [ 72%] +........................................................................ [ 75%] +........................................................................ [ 77%] +........................................................................ [ 79%] +........................................................................ [ 81%] +........................................................................ [ 83%] +........................................................................ [ 85%] +........................................................................ [ 87%] +........................................................................ [ 90%] +........................................................................ [ 92%] +.......F................................................................ [ 94%] +........................................................................ [ 96%] +........................................................................ [ 98%] +.............................................. [100%] +=================================== FAILURES =================================== +_________ test_document_list_reaching_the_model_leads_with_newest_docs _________ + +db = +clean_overview = None + + def test_document_list_reaching_the_model_leads_with_newest_docs( + db: Session, clean_overview + ) -> None: + """Phase 142, task 01 — RED pre-fix (the intent the fix must meet). + + The document list reaching the lite model must LEAD with the NEWEST + rows (``created_at`` desc — the document's date, phase 106) so that, + when the cap bites, the newest documents — not the lexicographically + first — are the ones the outline covers. Pre-fix the query is plain + ``order_by(source, path)``: with the phase-dir rows below (recency + strictly OPPOSITE to the path order) that puts ``01_old_a.md`` — the + OLDEST — first and ``20_new_z.md`` — the NEWEST — last, the traced + "summaries only up to phase 113" failure at unit scale. The + assertion is the EXPECTED POST-FIX order (task 02 flips it green). + """ + _truncate_documents(db) + try: + base = datetime(2026, 1, 1, tzinfo=UTC) + for i, (source, path, title, summary) in enumerate(_phase_dir_rows()): + _add_doc(db, source, path, title, summary) + # Row i (0-based) is i days after the base: index 0 + # (``01_old_a.md``) is the OLDEST, index 19 (``20_new_z.md``) + # the NEWEST — recency strictly opposite to the path order. + db.execute( + text("UPDATE documents SET created_at = :d WHERE path = :p"), + {"d": base + timedelta(days=i), "p": path}, + ) + db.commit() + llm = _FakeLLM() + assert asyncio.run(regenerate_overview(llm, db)) is True + finally: + _truncate_documents(db) + + user = next(m["content"] for m in llm.messages if m["role"] == "user") + lines = user.splitlines() + assert len(lines) == 20 # no cap at this size — the ORDER is what is asserted +> assert lines[0].startswith("s — 20_new_z.md"), ( + "the list must lead with the NEWEST doc (20_new_z) — pre-fix it " + "leads with the lexicographic prefix (01_old_a, the oldest)" + ) +E AssertionError: the list must lead with the NEWEST doc (20_new_z) — pre-fix it leads with the lexicographic prefix (01_old_a, the oldest) +E assert False +E + where False = ('s — 20_new_z.md') +E + where = 's — 01_old_a.md — Phase 01 — xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'.startswith + +tests/unit/test_overview.py:379: AssertionError +----------------------------- Captured stdout call ----------------------------- +2026-10-01 01:46:55 INFO app.rag.overview :: overview: regenerated docs=20 chars=61 +------------------------------ Captured log call ------------------------------- +INFO app.rag.overview:overview.py:199 overview: regenerated docs=20 chars=61 +________ test_overview_outline_names_the_newest_docs_when_the_cap_bites ________ + +db = +owner_kb = PosixPath('/tmp/pytest-of-ducoterra/pytest-49/test_overview_outline_names_th0/MyDocs') +monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x7f622c6472a0> +capsys = <_pytest.capture.CaptureFixture object at 0x7f5fb305e000> + + def test_overview_outline_names_the_newest_docs_when_the_cap_bites( + db: Session, + owner_kb: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + ) -> None: + """Phase 142, task 01 — RED pre-fix (the KB-shape repro of the + "summaries only up to phase 113" failure). + + The owner's KB shape: one source whose OLDER documents sort + lexicographically FIRST (``old/*``) and whose NEWER documents sort + LAST (``zz_new/*``). With ``BOR_OVERVIEW_INPUT_MAX_CHARS`` forced to + 150, the 6-doc list (275 chars) exceeds the cap and the cut keeps a + PREFIX. Pre-fix that prefix is the lexicographic one — the three + OLDER ``old/*`` docs — so the stored outline's 8-token digest names + the old docs (``mydocs old 01 md old 01 summary of``) and the newer + ``zz_new/*`` docs never reach the model. The assertion below is the + EXPECTED POST-FIX outcome: the ``created_at``-desc ordering (task 02) + puts the newest docs first, so they survive the cap and are the ones + the outline names (``mydocs zz new 03 md zz new 03``). + """ + llm = OverviewDigestEmbedder() + get_settings.cache_clear() + monkeypatch.setenv("BOR_OVERVIEW_INPUT_MAX_CHARS", "150") + try: + rc, out = _run_main(monkeypatch, llm, ["--source", str(owner_kb)], capsys) + finally: + get_settings.cache_clear() + assert rc == 0 + assert "added=6" in out + # 6 doc summaries (phase 118) + the overview + 3 folder summaries + # (source root, old/, zz_new/) — the overview gate fired (added > 0). + assert "overview=updated" in out + assert len(llm.chat_calls) == 10 + overview_call = next( + m for m in llm.chat_calls + if "KB_OVERVIEW_MODE" in m[0]["content"] + ) + user = next(m["content"] for m in overview_call if m["role"] == "user") + assert "[…truncated…]" in user # the 150-char cap bit (the repro's precondition) + + row = _row(db) + assert row is not None + # POST-fix digest: the capped list leads with the NEWEST docs — + # "MyDocs — zz_new/03.md — Zz New 03 — Summary of #" → first 8 + # tokens "mydocs zz new 03 md zz new 03". Pre-fix the stored row is + # "mydocs old 01 md old 01 summary of" (the oldest batch) — that + # mismatch is the red the fix must flip. +> assert row.content == "Knowledge base outline:\n- mydocs zz new 03 md zz new 03" +E AssertionError: assert 'Knowledge ba...01 summary of' == 'Knowledge ba... md zz new 03' +E +E Knowledge base outline: +E - - mydocs zz new 03 md zz new 03 +E + - mydocs old 01 md old 01 summary of + +tests/integration/test_import_docs_overview.py:457: AssertionError +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 268 1 99% +app/api/chat_images.py 50 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 179 1 99% +app/api/git_sources.py 259 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 183 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 264 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 20 0 100% +app/core/theming.py 38 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 68 0 100% +app/models.py 133 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 422 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 134 0 100% +app/rag/git_sources.py 56 0 100% +app/rag/importer.py 705 9 99% +app/rag/llm.py 246 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 71 0 100% +app/rag/pdf.py 124 0 100% +app/rag/prompts.py 154 0 100% +app/rag/retriever.py 248 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 58 0 100% +app/schemas.py 424 0 100% +-------------------------------------------------- +TOTAL 5498 22 99% +=========================== short test summary info ============================ +FAILED tests/unit/test_overview.py::test_document_list_reaching_the_model_leads_with_newest_docs +FAILED tests/integration/test_import_docs_overview.py::test_overview_outline_names_the_newest_docs_when_the_cap_bites +tests FAILED: unit + integration suite did not pass +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation FAILED (see output above) diff --git a/.agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__01_cap_ordering_repro.a2.err b/.agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__01_cap_ordering_repro.a2.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__01_cap_ordering_repro.a2.md b/.agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__01_cap_ordering_repro.a2.md new file mode 100644 index 0000000..cab04d6 --- /dev/null +++ b/.agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__01_cap_ordering_repro.a2.md @@ -0,0 +1,16 @@ +## Report — task `01_cap_ordering_repro` (phase 142), retry + +**Fixed:** the two repro tests now carry `@pytest.mark.xfail(strict=True)` with the root cause in the reason string — the test bodies still fail on pre-fix code (verified: pytest reports XFAIL, not XPASS), so the red repro is preserved, but the suite is green. `strict=True` turns a forgotten marker in task 02 into an XPASS failure, forcing the flip-to-green. The pre-fix cap-behavior pin (`test_cap_keeps_the_given_prefix_and_cuts_the_tail`) stays a plain green test; docstrings updated to match. + +**Repro content (unchanged from last attempt):** +- Unit: 20 owner-phase-dir rows (`01_old_a.md`…`20_new_z.md`, 100 chars/line); xfail body asserts the list reaching the model leads with `20_new_z` (newest) — pre-fix it leads with `01_old_a` (oldest). +- Integration: `owner_kb` fixture (`old/` mtimed 2026-01 vs `zz_new/` 2026-09 via `os.utime`), `OverviewDigestEmbedder` (mock-identical 8-token digest via the mock's `TOKEN_RE`), forced `BOR_OVERVIEW_INPUT_MAX_CHARS=150`; xfail body asserts the stored outline is `mydocs zz new 03 md zz new 03` — pre-fix it is `mydocs old 01 md old 01 summary of` (the "summaries only up to phase 113" shape). + +**Results:** +- `uv run pytest --cov=app --cov-report=term` → **3356 passed, 2 xfailed, 0 failed**; `app/` coverage **99%** (>90% gate). +- `uv run ruff check .` → clean; `uv run pyright` → 0 errors, 0 warnings. +- `uv run pytest tests/e2e/test_kb_overview.py -v --no-cov` → 4 passed (checked previously, app code untouched since). + +**Docs:** no doc impact (tests-only). **No application code changed** (git diff: two test files only). + +**Next pending task:** `.agents/phases/todo/142_kb_overview_freshness/02_recency_ordering.md` (landing the fix will XPASS both xfail tests → remove the markers there). diff --git a/.agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__01_cap_ordering_repro.a2.validate b/.agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__01_cap_ordering_repro.a2.validate new file mode 100644 index 0000000..4dd7877 --- /dev/null +++ b/.agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__01_cap_ordering_repro.a2.validate @@ -0,0 +1,118 @@ +........................................................................ [ 2%] +........................................................................ [ 4%] +........................................................................ [ 6%] +........................................................................ [ 8%] +........................................................................ [ 10%] +........................................................................ [ 12%] +........................................................................ [ 15%] +........................................................................ [ 17%] +........................................................................ [ 19%] +........................................................................ [ 21%] +........................................................................ [ 23%] +........................................................................ [ 25%] +........................................................................ [ 27%] +........................................................................ [ 30%] +........................................................................ [ 32%] +........................................................................ [ 34%] +........................................................................ [ 36%] +........................................................................ [ 38%] +........................................................................ [ 40%] +........................................................................ [ 42%] +........................................................................ [ 45%] +........................................................................ [ 47%] +........................................................................ [ 49%] +...............................................................x........ [ 51%] +........................................................................ [ 53%] +........................................................................ [ 55%] +........................................................................ [ 57%] +........................................................................ [ 60%] +........................................................................ [ 62%] +........................................................................ [ 64%] +........................................................................ [ 66%] +........................................................................ [ 68%] +........................................................................ [ 70%] +........................................................................ [ 72%] +........................................................................ [ 75%] +........................................................................ [ 77%] +........................................................................ [ 79%] +........................................................................ [ 81%] +........................................................................ [ 83%] +........................................................................ [ 85%] +........................................................................ [ 87%] +........................................................................ [ 90%] +........................................................................ [ 92%] +.......x................................................................ [ 94%] +........................................................................ [ 96%] +........................................................................ [ 98%] +.............................................. [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 268 1 99% +app/api/chat_images.py 50 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 179 1 99% +app/api/git_sources.py 259 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 183 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 264 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 20 0 100% +app/core/theming.py 38 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 68 0 100% +app/models.py 133 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 422 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 134 0 100% +app/rag/git_sources.py 56 0 100% +app/rag/importer.py 705 9 99% +app/rag/llm.py 246 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 71 0 100% +app/rag/pdf.py 124 0 100% +app/rag/prompts.py 154 0 100% +app/rag/retriever.py 248 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 58 0 100% +app/schemas.py 424 0 100% +-------------------------------------------------- +TOTAL 5498 22 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__02_recency_ordering.a1.err b/.agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__02_recency_ordering.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__02_recency_ordering.a1.md b/.agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__02_recency_ordering.a1.md new file mode 100644 index 0000000..fa4cde6 --- /dev/null +++ b/.agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__02_recency_ordering.a1.md @@ -0,0 +1,18 @@ +All changes verified and all gates green. Final report: + +**Implemented (phase 142, task 02 — recency ordering)** +- `app/rag/overview.py`: `regenerate_overview` now SELECTs `order_by(Document.created_at.desc(), Document.source, Document.path)` — newest docs lead the lite model's list so the cap drops the OLDEST; module + prompt docstrings updated (cap, cut+marker, fail-soft, upsert all unchanged) +- `tests/unit/test_overview.py`: strict-xfail repro flipped green (marker removed); new 6-doc/2-batch ordering pin (newest batch first, `(source,path)` tie-break); `build_overview_prompt` byte-identical pins untouched +- `tests/integration/test_import_docs_overview.py`: strict-xfail KB-shape repro flipped green (digest now names `zz_new/03`, the newest); fixture mtimes made run-relative (de-timebombs `normalize_doc_date`'s future-skew rule) +- `tests/e2e/test_kb_overview.py`: re-pinned test 4 — dates pinned explicitly (checkout mtimes aren't git-stable), stored digest re-pinned to the newest doc's line (`docs homelab tables md service port table this`), first-line recency assertion added +- Defect fixed from task 01: repro asserted path `20_new_z.md` but the row generator produces suffixes a–t (`20_new_t.md`) — assertions/docstrings corrected + +**Gates (all post-edit)** +- `uv run pytest --no-cov -v`: 3358 passed, 0 failed +- `uv run pytest --cov=app --cov-report=term-missing`: 3358 passed, TOTAL 99% (overview.py 100%) +- `uv run pytest tests/e2e/test_kb_overview.py -v --no-cov`: 4 passed (isolated); `test_sync_button.py` spot-check: 3 passed +- `uv run ruff check . && uv run pyright`: clean + +**Docs:** no doc impact (no env/CLI/API changes; README's overview trigger + knob docs stay accurate) +**Deviations:** none +**Next pending task:** `.agents/phases/todo/142_kb_overview_freshness/03_honest_kb_intro.md` diff --git a/.agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__02_recency_ordering.a1.validate b/.agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__02_recency_ordering.a1.validate new file mode 100644 index 0000000..798994e --- /dev/null +++ b/.agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__02_recency_ordering.a1.validate @@ -0,0 +1,118 @@ +........................................................................ [ 2%] +........................................................................ [ 4%] +........................................................................ [ 6%] +........................................................................ [ 8%] +........................................................................ [ 10%] +........................................................................ [ 12%] +........................................................................ [ 15%] +........................................................................ [ 17%] +........................................................................ [ 19%] +........................................................................ [ 21%] +........................................................................ [ 23%] +........................................................................ [ 25%] +........................................................................ [ 27%] +........................................................................ [ 30%] +........................................................................ [ 32%] +........................................................................ [ 34%] +........................................................................ [ 36%] +........................................................................ [ 38%] +........................................................................ [ 40%] +........................................................................ [ 42%] +........................................................................ [ 45%] +........................................................................ [ 47%] +........................................................................ [ 49%] +........................................................................ [ 51%] +........................................................................ [ 53%] +........................................................................ [ 55%] +........................................................................ [ 57%] +........................................................................ [ 60%] +........................................................................ [ 62%] +........................................................................ [ 64%] +........................................................................ [ 66%] +........................................................................ [ 68%] +........................................................................ [ 70%] +........................................................................ [ 72%] +........................................................................ [ 75%] +........................................................................ [ 77%] +........................................................................ [ 79%] +........................................................................ [ 81%] +........................................................................ [ 83%] +........................................................................ [ 85%] +........................................................................ [ 87%] +........................................................................ [ 90%] +........................................................................ [ 92%] +........................................................................ [ 94%] +........................................................................ [ 96%] +........................................................................ [ 98%] +.............................................. [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 268 1 99% +app/api/chat_images.py 50 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 179 1 99% +app/api/git_sources.py 259 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 183 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 264 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 20 0 100% +app/core/theming.py 38 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 68 0 100% +app/models.py 133 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 422 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 134 0 100% +app/rag/git_sources.py 56 0 100% +app/rag/importer.py 705 9 99% +app/rag/llm.py 246 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 71 0 100% +app/rag/pdf.py 124 0 100% +app/rag/prompts.py 154 0 100% +app/rag/retriever.py 248 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 58 0 100% +app/schemas.py 424 0 100% +-------------------------------------------------- +TOTAL 5498 22 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__03_honest_kb_intro.a1.err b/.agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__03_honest_kb_intro.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__03_honest_kb_intro.a1.md b/.agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__03_honest_kb_intro.a1.md new file mode 100644 index 0000000..8735150 --- /dev/null +++ b/.agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__03_honest_kb_intro.a1.md @@ -0,0 +1,10 @@ +All green. The diff is clean and complete. + +**Report — phase 142, task 03 (honest `` intro)** + +- Rewrote `_KB_INTRO` in `app/rag/prompts.py`: drops "(generated at import time)"; now states the outline is a compact, possibly partial picture favoring recent documents (a map, not a complete inventory) and to verify "latest / complete set" questions with `ls` (drill-down) / `grep` before answering from it. `build_kb_section` code untouched (tag/cap/marker/pathological behavior byte-stable); constant doc comment updated with the phase-142 rationale. +- `tests/unit/test_prompts.py`: new `test_kb_intro_phase142_honest_copy` pins the intro verbatim (incl. absence of the old phrase + the three contracts, in both HIGH and LOW builds); re-pinned the two budget-edge caps 200→600 to the new intro length (frame 422 chars; behavior unchanged, as the task permits); module docstring noted. No-overview byte-identity, section-order, and deflect-prompt pins pass unchanged. +- Test/lint results: `uv run pytest --cov=app --cov-report=term-missing` → **3359 passed, `app/` coverage 99%** (>90%); `uv run ruff check .` → all checks passed; `uv run pyright` → 0 errors, 0 warnings; `uv run pytest tests/e2e/test_kb_overview.py -v --no-cov` → **4 passed** in isolation (mock echo unaffected — intro is not a `-` bullet). +- Docs: no README/AGENTS.md impact (no commands/config/env/API changes; intro text not documented there). +- Notable: pre-existing stale comment "frame alone is 121 chars" (actual 135 pre-change) corrected to the accurate 422. +- Next pending task: `todo/142_kb_overview_freshness/04_e2e_outline_freshness.md`. diff --git a/.agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__03_honest_kb_intro.a1.validate b/.agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__03_honest_kb_intro.a1.validate new file mode 100644 index 0000000..4b0e5c1 --- /dev/null +++ b/.agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__03_honest_kb_intro.a1.validate @@ -0,0 +1,118 @@ +........................................................................ [ 2%] +........................................................................ [ 4%] +........................................................................ [ 6%] +........................................................................ [ 8%] +........................................................................ [ 10%] +........................................................................ [ 12%] +........................................................................ [ 15%] +........................................................................ [ 17%] +........................................................................ [ 19%] +........................................................................ [ 21%] +........................................................................ [ 23%] +........................................................................ [ 25%] +........................................................................ [ 27%] +........................................................................ [ 30%] +........................................................................ [ 32%] +........................................................................ [ 34%] +........................................................................ [ 36%] +........................................................................ [ 38%] +........................................................................ [ 40%] +........................................................................ [ 42%] +........................................................................ [ 45%] +........................................................................ [ 47%] +........................................................................ [ 49%] +........................................................................ [ 51%] +........................................................................ [ 53%] +........................................................................ [ 55%] +........................................................................ [ 57%] +........................................................................ [ 60%] +........................................................................ [ 62%] +........................................................................ [ 64%] +........................................................................ [ 66%] +........................................................................ [ 68%] +........................................................................ [ 70%] +........................................................................ [ 72%] +........................................................................ [ 75%] +........................................................................ [ 77%] +........................................................................ [ 79%] +........................................................................ [ 81%] +........................................................................ [ 83%] +........................................................................ [ 85%] +........................................................................ [ 87%] +........................................................................ [ 90%] +........................................................................ [ 92%] +........................................................................ [ 94%] +........................................................................ [ 96%] +........................................................................ [ 98%] +............................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 268 1 99% +app/api/chat_images.py 50 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 179 1 99% +app/api/git_sources.py 259 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 183 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 264 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 20 0 100% +app/core/theming.py 38 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 68 0 100% +app/models.py 133 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 422 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 134 0 100% +app/rag/git_sources.py 56 0 100% +app/rag/importer.py 705 9 99% +app/rag/llm.py 246 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 71 0 100% +app/rag/pdf.py 124 0 100% +app/rag/prompts.py 154 0 100% +app/rag/retriever.py 248 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 58 0 100% +app/schemas.py 424 0 100% +-------------------------------------------------- +TOTAL 5498 22 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__04_e2e_outline_freshness.a1.err b/.agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__04_e2e_outline_freshness.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__04_e2e_outline_freshness.a1.md b/.agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__04_e2e_outline_freshness.a1.md new file mode 100644 index 0000000..6b4995d --- /dev/null +++ b/.agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__04_e2e_outline_freshness.a1.md @@ -0,0 +1,10 @@ +**Task 04 complete — all gates green.** + +- Added `tests/e2e/test_kb_overview_freshness.py` (dedicated Playwright file, mock-only, run in isolation): + - `test_regenerated_outline_leads_with_newest_doc` — owner-shaped fixture (`old/a..c.md` past mtimes, sort first; `zz_new/y..z.md` fresh mtimes, sort last), `BOR_OVERVIEW_INPUT_MAX_CHARS=300` forced via env + `get_settings` cache clear so the cap bites; real `regenerate_overview` against the mock's `KB_OVERVIEW_MODE` → stored row's 8 digest tokens computed from the fixture (`reese zz new z md zz new z`) and proven to come from the NEWEST doc's uncut line; "old" absent. + - `test_kb_section_carries_honest_intro` — `build_kb_section` surface: lead line is the phase-142 honest intro pinned verbatim, "(generated at import time)" gone, recency/map/verify-with-`ls`/`grep` contracts present, tag + exact-fit/tight-budget cap behavior intact. + - `test_kb_section_reaches_a_rendered_turn` — seeded row's first-bullet `(kb: …)` echo reaches a rendered turn (phase-31 injection path, standalone). +- Tests/lint: `uv run pytest tests/e2e/test_kb_overview_freshness.py -v --no-cov` → 3 passed in isolation (ran twice); `uv run pytest tests/e2e/test_kb_overview.py -v --no-cov` → 4 passed in isolation; `uv run pytest --cov=app` → 100% green, `app/` coverage 99% (>90%); `uv run ruff check .` clean; `uv run pyright` 0 errors. +- Docs: no doc impact (E2E-only task; README/AGENTS.md overview claims still accurate — cap knob, change-gated trigger, env table all unchanged). +- Decisions: digest tokens derived in-test from the fixture's source/path/title (exactly 8 tokens, summary-independent); test 3 doesn't pin the deflection flag (echo is the injection contract in both modes). No app code touched; no phase files/commits. +- Next pending task: `143_mobile_keyboard_after_answer/01_coarse_pointer_focus_gate.md` (phase 142 is now fully complete — 01–04 done). diff --git a/.agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__04_e2e_outline_freshness.a1.validate b/.agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__04_e2e_outline_freshness.a1.validate new file mode 100644 index 0000000..4b0e5c1 --- /dev/null +++ b/.agents/reports/142_kb_overview_freshness/142_kb_overview_freshness__04_e2e_outline_freshness.a1.validate @@ -0,0 +1,118 @@ +........................................................................ [ 2%] +........................................................................ [ 4%] +........................................................................ [ 6%] +........................................................................ [ 8%] +........................................................................ [ 10%] +........................................................................ [ 12%] +........................................................................ [ 15%] +........................................................................ [ 17%] +........................................................................ [ 19%] +........................................................................ [ 21%] +........................................................................ [ 23%] +........................................................................ [ 25%] +........................................................................ [ 27%] +........................................................................ [ 30%] +........................................................................ [ 32%] +........................................................................ [ 34%] +........................................................................ [ 36%] +........................................................................ [ 38%] +........................................................................ [ 40%] +........................................................................ [ 42%] +........................................................................ [ 45%] +........................................................................ [ 47%] +........................................................................ [ 49%] +........................................................................ [ 51%] +........................................................................ [ 53%] +........................................................................ [ 55%] +........................................................................ [ 57%] +........................................................................ [ 60%] +........................................................................ [ 62%] +........................................................................ [ 64%] +........................................................................ [ 66%] +........................................................................ [ 68%] +........................................................................ [ 70%] +........................................................................ [ 72%] +........................................................................ [ 75%] +........................................................................ [ 77%] +........................................................................ [ 79%] +........................................................................ [ 81%] +........................................................................ [ 83%] +........................................................................ [ 85%] +........................................................................ [ 87%] +........................................................................ [ 90%] +........................................................................ [ 92%] +........................................................................ [ 94%] +........................................................................ [ 96%] +........................................................................ [ 98%] +............................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 268 1 99% +app/api/chat_images.py 50 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 179 1 99% +app/api/git_sources.py 259 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 183 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 264 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 20 0 100% +app/core/theming.py 38 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 68 0 100% +app/models.py 133 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 422 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 134 0 100% +app/rag/git_sources.py 56 0 100% +app/rag/importer.py 705 9 99% +app/rag/llm.py 246 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 71 0 100% +app/rag/pdf.py 124 0 100% +app/rag/prompts.py 154 0 100% +app/rag/retriever.py 248 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 58 0 100% +app/schemas.py 424 0 100% +-------------------------------------------------- +TOTAL 5498 22 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/app/rag/overview.py b/app/rag/overview.py index cb552e4..e59f189 100644 --- a/app/rag/overview.py +++ b/app/rag/overview.py @@ -1,10 +1,23 @@ """KB overview generator (phase 31, task 02). Builds the ``KB_OVERVIEW_MODE`` prompt from the document catalogue -(source, path, title, first summary line per document), calls the aipi -``lite`` model through the one-shot ``LLMClient.chat`` (phase 30, same -model as document summaries — no new model management), and stores the -plain-text outline in the single ``kb_overview`` row (migration 0005). +(one line per document: source, path, title, first summary line — +ordered RECENT-FIRST by the document's date, phase 142 task 02), +calls the aipi ``lite`` model through the one-shot ``LLMClient.chat`` +(phase 30, same model as document summaries — no new model management), +and stores the plain-text outline in the single ``kb_overview`` row +(migration 0005). + +The recency-first order is what makes the input cap honest: when +``BOR_OVERVIEW_INPUT_MAX_CHARS`` cuts the list, the documents that fall +off are the OLDEST, so the outline always represents the recent state +of the KB. The sort reads ``Document.created_at`` — phase 106's +git-history/mtime-derived, owner-correctable document date (the same +date the ``read`` tool's ``date:`` line shows); ``created_at_manual`` +rows are included exactly as stored, the ordering never re-derives +dates. Same-date documents tie-break by ``(source, path)``, keeping +the pre-phase-142 lexicographic order within a date (deterministic for +the mock's digest; byte-identical for single-date KBs). Chat turns never generate an overview — they read the stored row (:func:`load_kb_overview`, one indexed PK lookup) and inject it into the @@ -99,10 +112,12 @@ def build_overview_prompt( + the locked instruction. * ``user`` — one line per document, ``source — path — title — {first line of summary}``, joined with - newlines. The summary field is omitted when the document has no - summary (pre-phase-30 rows, the fail-soft path, and pre-backfill - NULL rows — no dangling dash). The list is capped at *max_chars* - (default + newlines, in the EXACT order the caller passes *rows* (order- + preserving; the caller — :func:`regenerate_overview` — passes the + catalogue recent-first, phase 142 task 02). The summary field is + omitted when the document has no summary (pre-phase-30 rows, the + fail-soft path, and pre-backfill NULL rows — no dangling dash). + The list is capped at *max_chars* (default ``BOR_OVERVIEW_INPUT_MAX_CHARS``): overflow is cut exactly at the cap and the shared ``[…truncated…]`` marker is appended on its own line, so the model never sees more than the cap and the cut is @@ -163,9 +178,15 @@ async def regenerate_overview( if session is None: session = SessionLocal() try: + # RECENT-FIRST (phase 142, task 02): the document's date + # (``created_at``, phase 106) descending, so when the input cap + # bites the OLDEST documents fall off — the outline always + # represents the recent KB. The (source, path) tie-break keeps + # same-date documents in the pre-phase-142 lexicographic order + # (deterministic digest; byte-identical for single-date KBs). result = session.execute( select(Document.source, Document.path, Document.title, Document.summary) - .order_by(Document.source, Document.path) + .order_by(Document.created_at.desc(), Document.source, Document.path) ).all() rows: list[tuple[str, str, str, str | None]] = [ (source, path, title, summary) diff --git a/app/rag/prompts.py b/app/rag/prompts.py index 842da03..ad37581 100644 --- a/app/rag/prompts.py +++ b/app/rag/prompts.py @@ -125,11 +125,23 @@ _STEERING_INTRO = ( "Where these instructions conflict with the defaults above, follow the owner:\n" ) -#: One-line intro of the ```` section (phase 31): the -#: lite-generated outline is the agent's a-priori picture of the KB. +#: One-line intro of the ```` section (phase 31; +#: phase 142, task 03 — the honest-intro rewrite, owner-confirmed +#: 2026-09-30): the outline is a COMPACT, possibly partial picture +#: that favors RECENT documents (task 02 orders the generation input +#: newest-first, so recent docs are the ones it covers when the cap +#: bites) — a map of what exists, not a complete inventory — and +#: "latest / complete set" questions verify with ``ls``/``grep`` +#: before answering from it; "(generated at import time)" is GONE +#: (false — the sync regenerates the row on every KB change — and it +#: taught the model to treat a partial outline as the whole frozen KB). _KB_INTRO = ( - "The basic categories of everything in this knowledge base " - "(generated at import time):\n" + 'A compact picture of the knowledge base\'s categories and topics — ' + 'it favors recent documents and may omit older or lesser ones: a map ' + 'of what exists, not a complete inventory. For questions about the ' + 'latest state or the complete set ("the last N …", "everything ' + 'about …", "all the …"), verify with `ls` (drill into the folders) ' + 'or `grep` before answering from the outline:\n' ) #: The ```` section's start-here intro (phase 118, task 03 — diff --git a/tests/e2e/test_kb_overview.py b/tests/e2e/test_kb_overview.py index 71b773e..460de9a 100644 --- a/tests/e2e/test_kb_overview.py +++ b/tests/e2e/test_kb_overview.py @@ -21,7 +21,9 @@ multi-line so the echo must pick the FIRST bullet — not the intro line, not a later line. The last test additionally runs the real ``regenerate_overview`` against the mock to cover the mock's ``KB_OVERVIEW_MODE`` generation branch end-to-end (stored outline is the -byte-stable 8-token digest of the generator's document list). +byte-stable 8-token digest of the generator's document list — re-pinned +in phase 142, task 02: the generator's list is now RECENT-FIRST, so the +digest is of the NEWEST document's line, not the lexicographic-first). Test → story mapping (Playwright Mapping Rule): 1. ``test_on_topic_answer_echoes_kb_overview`` @@ -33,6 +35,7 @@ from __future__ import annotations import asyncio import re +from datetime import UTC, datetime from pathlib import Path from threading import Thread from typing import Any @@ -47,7 +50,6 @@ from app.rag.importer import ImportSummary, import_sources from app.rag.llm import LLMClient from app.rag.overview import build_overview_prompt, regenerate_overview from e2e.auth_helpers import login -from tests.e2e.mock_llm import TOKEN_RE REPO = Path(__file__).resolve().parents[2] FIXTURES = REPO / "tests" / "fixtures" / "docs" @@ -153,10 +155,13 @@ def _overview_row() -> KbOverview | None: def _doc_rows() -> list[tuple[str, str, str, str | None]]: + """The catalogue the way the generator sees it (phase 142, task 02): + RECENT-FIRST by the document's date, ``(source, path)`` tie-break — + the same order ``regenerate_overview`` SELECTs.""" with SessionLocal() as db: rows = db.execute( select(Document.source, Document.path, Document.title, Document.summary) - .order_by(Document.source, Document.path) + .order_by(Document.created_at.desc(), Document.source, Document.path) ).all() return [(source, path, title, summary) for source, path, title, summary in rows] @@ -249,11 +254,38 @@ def test_mock_generated_outline_is_stored_and_echoed( """The mock's ``KB_OVERVIEW_MODE`` branch is exercised end-to-end: the real ``regenerate_overview`` (``LLMClient`` pointed at the mock) stores the byte-stable 8-token digest of the generator's document - list, and a chat turn echoes its first bullet.""" + list, and a chat turn echoes its first bullet. + + Phase 142, task 02 (recency-first re-pin): the generator's list is + now ``created_at`` desc, ``(source, path)`` tie-break. The fixture + files' on-disk mtimes are checkout-dependent (git preserves none), + so the documents' dates are pinned explicitly BEFORE the + regeneration: every doc shares one date EXCEPT + ``homelab/tables.md`` — the lexicographic LAST path, the owner's + phase-dir shape where the newest sorts last — which gets the newer + one. The stored digest is therefore of THAT doc's line, not of the + pre-fix lexicographic-first one (``deployments/new-service.md``). + """ summary = _reset_db(mock_llm, seed=True, overview=None) assert summary is not None and summary.added == 13 # phase 47 added quadlet+j2 assert _overview_row() is None # direct import never regenerates + # Pin the document dates (raw SQL — ``normalize_doc_date`` is an + # importer concern, the ordering reads the stored column): one + # shared date for all but the newest doc, a strictly newer date for + # it. Without this the order — and the digest — would depend on the + # fixture files' checkout mtimes. + with SessionLocal() as db: + db.execute( + text("UPDATE documents SET created_at = :d WHERE path != :newest"), + {"d": datetime(2026, 1, 1, tzinfo=UTC), "newest": "homelab/tables.md"}, + ) + db.execute( + text("UPDATE documents SET created_at = :d WHERE path = :newest"), + {"d": datetime(2026, 9, 1, tzinfo=UTC), "newest": "homelab/tables.md"}, + ) + db.commit() + kwargs: dict[str, Any] = { "_env_file": None, "llm_base_url": f"http://127.0.0.1:{mock_llm}/v1", @@ -264,11 +296,19 @@ def test_mock_generated_outline_is_stored_and_echoed( row = _overview_row() assert row is not None and row.id == 1 - # Byte-stable expectation: the mock digests the generator's user - # message (the same document list, the same cap) to its first 8 tokens. + # Byte-stable expectation, re-pinned for recency-first: the mock + # digests the generator's user message (the same document list, the + # same cap) to its first 8 tokens — and the list now LEADS with the + # newest doc, so the digest is of ITS line: + # "docs — homelab/tables.md — Service Port Table — This document + # covers …" → "docs homelab tables md service port table this" + # (pre-fix the lexicographic-first doc was deployments/new-service.md). _, user = build_overview_prompt(_doc_rows()) - expected = "Knowledge base outline:\n- " + " ".join( - TOKEN_RE.findall(user.lower())[:8] + assert user.splitlines()[0].startswith( + "docs — homelab/tables.md — Service Port Table" + ), "the generator's list must lead with the NEWEST doc (recency-first)" + expected = ( + "Knowledge base outline:\n- docs homelab tables md service port table this" ) assert row.content == expected first_bullet = row.content.splitlines()[1].removeprefix("- ").strip() diff --git a/tests/e2e/test_kb_overview_freshness.py b/tests/e2e/test_kb_overview_freshness.py new file mode 100644 index 0000000..ea40792 --- /dev/null +++ b/tests/e2e/test_kb_overview_freshness.py @@ -0,0 +1,402 @@ +"""Phase 142 E2E (Playwright): the forced-cap KB outline leads with the +NEWEST document, and the honest ```` intro is served. + +Source: ``TODO.md`` L4 (the "Summarize the last 10 phases" failure — the +stored outline described a KB that "ends at phase 113" because the input +cap cut the lexicographic prefix, so the newest documents never reached +the ``lite`` model). +Run in isolation (DB must be up: ``podman compose up -d db``): + + uv run pytest tests/e2e/test_kb_overview_freshness.py -v --no-cov + +The whole phase, end to end through the real prompt path: + +* the fixture KB is shaped like the owner's — one source, an OLDER batch + whose paths sort lexicographically FIRST (``old/a..c.md``, past file + mtimes) and a NEWER batch whose paths sort LAST (``zz_new/y..z.md``, + fresh mtimes) — phase 106 sources ``created_at`` from the local file's + mtime, so the pre-phase-142 lexicographic order and the recency order + DISAGREE about which document is first; +* ``BOR_OVERVIEW_INPUT_MAX_CHARS`` is forced small (the E2E settings + override: process env + ``get_settings`` cache clear around the + regeneration — the generator's ``build_overview_prompt`` reads the cap + from ``get_settings``) so the document list is CUT — the cap must bite + or the ordering is unobservable; +* test 1: the real ``regenerate_overview`` against the mock's + ``KB_OVERVIEW_MODE`` digest — the stored outline's 8 tokens come from + the NEWEST document's line (recency-first, end to end); +* test 2: the ```` section builder, called here directly + against the app (the phase-31 file's own convention) — the section's + lead line IS the new honest intro verbatim (no "generated at import + time"), tag + cap behavior intact; +* test 3: the seeded row's section reaches a RENDERED turn — the + phase-31 first-bullet echo, re-asserted here against the seeded row so + this file stands alone on the injection surface too. + +The pre-existing phase-31 file (``tests/e2e/test_kb_overview.py``) is +NOT modified here (its re-pins landed in task 02). + +Test → phase mapping: +1. ``test_regenerated_outline_leads_with_newest_doc`` (the objective) +2. ``test_kb_section_carries_honest_intro`` +3. ``test_kb_section_reaches_a_rendered_turn`` +""" +from __future__ import annotations + +import asyncio +import os +import re +from datetime import UTC, datetime, timedelta +from pathlib import Path +from threading import Thread +from typing import Any + +import pytest +from playwright.sync_api import Page, expect +from sqlalchemy import select, text + +from app.config import Settings, get_settings +from app.db import SessionLocal +from app.models import Document, KbOverview, QueryLog +from app.rag.importer import ImportSummary, import_sources +from app.rag.llm import LLMClient +from app.rag.overview import build_overview_prompt, regenerate_overview +from app.rag.prompts import _KB_INTRO, build_kb_section +from app.rag.retriever import TRUNCATION_MARKER +from e2e.auth_helpers import login +from tests.e2e.mock_llm import TOKEN_RE + +# --- the owner-shaped fixture -------------------------------------------- + +#: The fixture dir's basename — the ``source`` the importer stores (the +#: phase-31 fixture is "docs" for ``tests/fixtures/docs``). +SOURCE_NAME = "reese" + +#: The OLDER batch: paths that sort lexicographically FIRST, past +#: mtimes (phase 106's mtime fallback makes the documents' dates old). +OLD_PATHS = ("old/a.md", "old/b.md", "old/c.md") +#: The NEWER batch: paths that sort lexicographically LAST, fresh +#: mtimes. ``zz_new/z.md`` is the NEWEST document of the whole KB. +NEW_PATHS = ("zz_new/y.md", "zz_new/z.md") +NEWEST_PATH = "zz_new/z.md" +NEWEST_TITLE = "Zz New Z" + +#: The generator line the NEWEST doc must occupy FIRST — +#: ``source — path — title`` (the summary tail is the mock's). +NEWEST_LINE_PREFIX = f"{SOURCE_NAME} — {NEWEST_PATH} — {NEWEST_TITLE}" + +#: The forced input cap: the 5-doc list (~650 chars) exceeds it, so the +#: cap BITES (``[…truncated…]`` lands in the model's input), while the +#: ~130-char newest line survives intact (the digest is of ITS tokens). +OVERVIEW_INPUT_CAP = 300 + +#: The stored outline the mock's ``KB_OVERVIEW_MODE`` branch writes when +#: the newest doc leads: header + ONE bullet of the user message's first +#: 8 tokens — all 8 of them from ``NEWEST_LINE_PREFIX`` (source + path + +#: title supply exactly 8), computed from the fixture, not hardcoded. +EXPECTED_DIGEST = ( + "Knowledge base outline:\n- " + + " ".join(TOKEN_RE.findall(NEWEST_LINE_PREFIX.lower())[:8]) +) + +# --- test 2/3 constants ---------------------------------------------------- + +#: The phase-142 honest intro, pinned verbatim here (the same text the +#: unit suite pins — WITHOUT the trailing newline; the app's +#: ``_KB_INTRO`` carries it). It says what the outline actually is (a +#: compact picture that favors recent documents, a map not an inventory) +#: and teaches the verify-first behavior for "the last N …" questions. +E2E_KB_INTRO = ( + 'A compact picture of the knowledge base\'s categories and topics — ' + 'it favors recent documents and may omit older or lesser ones: a map ' + 'of what exists, not a complete inventory. For questions about the ' + 'latest state or the complete set ("the last N …", "everything ' + 'about …", "all the …"), verify with `ls` (drill into the folders) ' + 'or `grep` before answering from the outline:' +) + +#: The seeded stored outline (test 3): multi-line ``-`` bullets so the +#: echo must pick the FIRST bullet — the phase-31 story's parse contract. +SEED_OVERVIEW = ( + "- Newest zeta gateway routing and edge proxy notes\n" + "- Older ypsilon wireguard tunnel peers and keepalive tuning\n" + "- Old kubernetes cluster and node maintenance" +) +FIRST_BULLET = "Newest zeta gateway routing and edge proxy notes" +KB_ECHO = f"(kb: {FIRST_BULLET})" + +QUESTION = "What do my newest notes cover?" + + +# --- the owner-shaped fixture ---------------------------------------------- + + +def _write_doc(path: Path, title: str, body: str, age_days: float) -> None: + """Write one markdown doc with a pinned mtime *age_days* in the past. + + Phase 106 sources ``created_at`` from the local file's mtime + (``file_mtime_datetime`` → ``normalize_doc_date``), so the mtime IS + the document's date. The ages are RELATIVE to the run (the + integration fixture's pattern): they stay in the past forever, while + the staggered order — and thus the digest — is a pure function of + the fixture. + """ + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(f"# {title}\n\n{body}\n", encoding="utf-8") + ts = (datetime.now(UTC) - timedelta(days=age_days)).timestamp() + os.utime(path, (ts, ts)) + + +@pytest.fixture() +def owner_kb(tmp_path: Path) -> Path: + """The owner's KB shape at fixture scale: one source whose OLDER + documents sort lexicographically FIRST (``old/*``, ≈91–95 days old) + and whose NEWER documents sort LAST (``zz_new/*``, 1–3 days old). + Recency strictly OPPOSES the lexicographic path order — pre-fix, the + capped list led with ``old/a.md``; post-fix it leads with + ``zz_new/z.md`` (the newest).""" + root = tmp_path / SOURCE_NAME + _write_doc( + root / OLD_PATHS[0], "Old A", + "Old alpha notes: kubernetes cluster nodes and etcd quorum settings.", + 95, + ) + _write_doc( + root / OLD_PATHS[1], "Old B", + "Old beta notes: borg backup schedules and restore runbook steps.", + 93, + ) + _write_doc( + root / OLD_PATHS[2], "Old C", + "Old gamma notes: kafkabridge routing and static DNS records.", + 91, + ) + _write_doc( + root / NEW_PATHS[0], "Zz New Y", + "Newer ypsilon notes: wireguard tunnel peers and keepalive tuning.", + 3, + ) + _write_doc( + root / NEW_PATHS[1], NEWEST_TITLE, + "Newest zeta notes: edge proxy routing and gateway header policy.", + 1, + ) + return root + + +# --- Importer + thread helpers (test_kb_overview.py pattern) ---------------- + + +def _settings_for(mock_port: int) -> Settings: + kwargs: dict[str, Any] = { + "_env_file": None, + "llm_base_url": f"http://127.0.0.1:{mock_port}/v1", + } + return Settings(**kwargs) # pyright: ignore[reportCallIssue] + + +def _run_in_thread(coro: Any) -> Any: + """Run a coroutine on a worker thread. + + Playwright's sync API keeps an asyncio loop running on the test thread, + so ``asyncio.run`` cannot be called directly from a test body. + """ + box: dict[str, Any] = {} + + def runner() -> None: + try: + box["value"] = asyncio.run(coro) + except BaseException as e: # noqa: BLE001 — re-raised on the test thread + box["error"] = e + + t = Thread(target=runner) + t.start() + t.join() + if "error" in box: + raise box["error"] + return box["value"] + + +def _reset_db( + mock_port: int, root: Path, overview: str | None = None +) -> ImportSummary: + """Truncate the KB (+ query log, steering, kb_overview), re-import the + owner-shaped fixture, and optionally seed the single ``kb_overview`` + row (directly, so the injection path is what a test exercises).""" + with SessionLocal() as db: + db.execute( + text("TRUNCATE chunks, documents, query_log, steering_notes, kb_overview") + ) + db.commit() + if overview is not None: + db.add(KbOverview(id=1, content=overview)) + db.commit() + return _run_in_thread(import_sources([root], LLMClient(_settings_for(mock_port)))) + + +def _ask(page: Page, app_url: str, question: str, done_text: str) -> Any: + """Submit *question* and wait until the streamed answer has fully + landed (the phase-31 file's helper): ``done_text`` is the + deterministic last thing the answer contains — the ``(kb: …)`` echo, + which the mock appends after EVERYTHING else whenever the system + prompt carries ```` (HIGH and LOW alike).""" + page.set_default_timeout(30_000) + login(page, app_url, next="/") + page.fill("#message-input", question) + page.click("#send-btn") + bubble = page.locator(".msg.brain .bubble") + bubble.first.wait_for(state="visible", timeout=30_000) + expect(bubble.first).to_contain_text(done_text, timeout=30_000) + expect(page.locator("#send-btn")).to_be_enabled() + expect(page.locator("#send-label")).to_have_text("Send") + return bubble.first + + +def _last_query_log() -> QueryLog: + with SessionLocal() as db: + rows = db.scalars(select(QueryLog)).all() + assert len(rows) == 1, f"expected exactly one query_log row, got {len(rows)}" + return rows[0] + + +def _overview_row() -> KbOverview | None: + with SessionLocal() as db: + return db.get(KbOverview, 1) + + +def _doc_rows() -> list[tuple[str, str, str, str | None]]: + """The catalogue the way the generator sees it (phase 142, task 02): + RECENT-FIRST by the document's date, ``(source, path)`` tie-break — + the same order ``regenerate_overview`` SELECTs.""" + with SessionLocal() as db: + rows = db.execute( + select(Document.source, Document.path, Document.title, Document.summary) + .order_by(Document.created_at.desc(), Document.source, Document.path) + ).all() + return [(source, path, title, summary) for source, path, title, summary in rows] + + +# --- 1. The objective: with the cap biting, the outline leads with the newest --- + + +def test_regenerated_outline_leads_with_newest_doc( + page: Page, + app_url: str, + mock_llm: int, + db_ready: None, + owner_kb: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The owner's "ends at phase 113" failure, end to end: with + ``BOR_OVERVIEW_INPUT_MAX_CHARS`` forced to bite, the real + ``regenerate_overview`` (``LLMClient`` pointed at the mock's + ``KB_OVERVIEW_MODE`` digest) stores an outline whose 8 tokens come + from the NEWEST document's line — ``zz_new/z.md`` sorts + lexicographically LAST, so only the task-02 recency ordering can put + it first. Pre-fix the digest would have been of ``old/a.md``'s line + (the lexicographic prefix the cap kept) — the shape this pin + forbids.""" + summary = _reset_db(mock_llm, owner_kb) + assert summary.added == 5 # the whole owner-shaped fixture + assert _overview_row() is None # direct import never regenerates + + # The E2E settings override: the generator's ``build_overview_prompt`` + # reads the cap from ``get_settings`` (the process cache), so the + # forced cap rides in via the env + a cache clear around the call. + monkeypatch.setenv("BOR_OVERVIEW_INPUT_MAX_CHARS", str(OVERVIEW_INPUT_CAP)) + get_settings.cache_clear() + try: + ok = _run_in_thread(regenerate_overview(LLMClient(_settings_for(mock_llm)))) + finally: + get_settings.cache_clear() # restore the default-cap settings after + assert ok is True + + # The generator's list leads with the NEWEST doc (the ordering that + # makes the cap honest) — and its line renders uncut: + rows = _doc_rows() + assert rows[0][1] == NEWEST_PATH + _, first_line = build_overview_prompt([rows[0]]) + assert first_line.startswith(f"{NEWEST_LINE_PREFIX} — This document covers") + + # The cap BIT on the full list (the repro's precondition) while the + # leading line survived intact — so the mock digested THIS line: + _, user = build_overview_prompt(rows) + assert TRUNCATION_MARKER in user + assert len(user) <= OVERVIEW_INPUT_CAP + 1 + len(TRUNCATION_MARKER) + assert user.splitlines()[0] == first_line + + row = _overview_row() + assert row is not None and row.id == 1 + # The stored outline is the mock's 8-token digest of the capped list + # — all 8 tokens from the NEWEST doc's source/path/title (computed + # from the fixture above, not hardcoded wording): + assert row.content == EXPECTED_DIGEST + assert "old" not in row.content # no oldest-batch token survives the cut + + +# --- 2. The section builder serves the new honest intro ---------------------- + + +def test_kb_section_carries_honest_intro(db_ready: None) -> None: + """The intro is a prompt constant — the deterministic surface is the + section builder itself, called here directly against the app (the + phase-31 file's own convention for ``build_overview_prompt`` / + ``regenerate_overview``). The section's lead line IS the phase-142 + honest intro verbatim — "(generated at import time)" is GONE — and + the tag + cap behavior around it is intact (byte-stable contract).""" + # The app's constant, pinned verbatim (the trailing newline included): + assert _KB_INTRO == E2E_KB_INTRO + "\n" + + section = build_kb_section(SEED_OVERVIEW) + + # The lead line IS the new intro — verbatim, colon-terminated, + # between the open tag and the outline: + assert section.startswith(f"\n{E2E_KB_INTRO}\n") + assert section.splitlines()[1] == E2E_KB_INTRO + assert section.index(E2E_KB_INTRO) < section.index(SEED_OVERVIEW.splitlines()[0]) + # The false claim is gone from the served section (and thus the + # prompt), and the three contracts it must carry are present: + assert "(generated at import time)" not in section + assert "generated at import time" not in section + assert "favors recent documents" in section + assert "not a complete inventory" in section + assert "verify with `ls` (drill into the folders) or `grep`" in section + # Tag + body intact: the exact section is open tag, the new intro, + # the seed's bullets in order, close tag (the byte-stable shape): + assert section == f"\n{E2E_KB_INTRO}\n{SEED_OVERVIEW}\n" + # Cap behavior byte-stable: exact fit keeps the marker out … + exact = build_kb_section(SEED_OVERVIEW, max_chars=len(section)) + assert exact == section and TRUNCATION_MARKER not in exact + # … a tighter budget cuts with the shared marker, within the cap, + # the new intro still leading: + tight = build_kb_section(SEED_OVERVIEW, max_chars=len(section) - 10) + assert len(tight) <= len(section) - 10 + assert TRUNCATION_MARKER in tight + assert tight.startswith(f"\n{E2E_KB_INTRO}\n-") + # Empty overview still omits the whole section (byte-identity): + assert build_kb_section("") == "" + + +# --- 3. The seeded section reaches a rendered turn --------------------------- + + +def test_kb_section_reaches_a_rendered_turn( + page: Page, app_url: str, mock_llm: int, db_ready: None, owner_kb: Path +) -> None: + """The phase-31 injection path, re-asserted here so this file stands + alone on it: the SEEDED row's outline reaches a rendered turn — the + mock echoes the ```` section's first bullet into its + answer (`` (kb: )``), which only happens when the + section (lead line and all) was in the LLM prompt.""" + summary = _reset_db(mock_llm, owner_kb, overview=SEED_OVERVIEW) + assert summary.added == 5 + assert _overview_row() is not None # the row the turn must inject + + # The echo is the deterministic end of BOTH the grounded and the + # deflected mock answer — the completion signal and the proof: + bubble = _ask(page, app_url, QUESTION, KB_ECHO) + expect(bubble).to_have_text( + re.compile(rf"{re.escape(KB_ECHO)}\s*$"), timeout=30_000 + ) + row = _last_query_log() + assert row.question == QUESTION diff --git a/tests/integration/test_import_docs_overview.py b/tests/integration/test_import_docs_overview.py index e70361d..bb078de 100644 --- a/tests/integration/test_import_docs_overview.py +++ b/tests/integration/test_import_docs_overview.py @@ -38,19 +38,22 @@ calls on a fresh import). from __future__ import annotations import logging +import os from collections.abc import Iterator +from datetime import UTC, datetime, timedelta from pathlib import Path import pytest from sqlalchemy import text from sqlalchemy.orm import Session -from app.config import Settings +from app.config import Settings, get_settings from app.db import SessionLocal, db_available from app.models import KbOverview from app.rag.llm import LLMError from app.rag.sources_meta import current_sources_version from scripts import import_docs +from tests.e2e.mock_llm import TOKEN_RE from tests.fakes import FakeEmbedder @@ -62,6 +65,26 @@ class FailingChatEmbedder(FakeEmbedder): raise LLMError("simulated lite-model failure (test sentinel)") +class OverviewDigestEmbedder(FakeEmbedder): + """A ``lite`` stand-in reproducing the E2E mock's ``KB_OVERVIEW_MODE`` + branch (``tests/e2e/mock_llm.py``): the overview call returns the + document list's first 8 tokens (lowercased, the mock's + ``TOKEN_RE``) as the outline. The digest is ORDER-DEPENDENT, so the + stored row names exactly the documents that reached the model — the + phase 142 repro's witness. Every other call falls through to + :class:`FakeEmbedder`'s deterministic ``Summary of …`` digest.""" + + async def chat(self, messages: list[dict[str, str]], model: str | None = None) -> str: + system = next((m["content"] for m in messages if m.get("role") == "system"), "") + if "KB_OVERVIEW_MODE" in system: + self.chat_calls.append(list(messages)) # the base appends only on its path + user = next((m["content"] for m in messages if m.get("role") == "user"), "") + return "Knowledge base outline:\n- " + " ".join( + TOKEN_RE.findall(user.lower())[:8] + ) + return await super().chat(messages, model) + + def _row(db: Session) -> KbOverview | None: """The stored ``kb_overview`` row (freshly reloaded).""" db.expire_all() @@ -108,6 +131,41 @@ def src(tmp_path: Path) -> Path: return root +@pytest.fixture() +def owner_kb(tmp_path: Path) -> Path: + """Phase 142, task 01 — a KB shaped like the owner's: one source, an + OLDER batch whose paths sort FIRST (``old/01..03.md``) and a NEWER + batch whose paths sort LAST (``zz_new/01..03.md``) — the phase-dir + shape where ``131_…`` sorts after ``113_…``. Both batches get + explicit, staggered file mtimes (``os.utime``): phase 106 sources + ``created_at`` from the local file's mtime, so recency strictly + OPPOSES the lexicographic path order (old/01 is the oldest and sorts + first; zz_new/03 is the newest and sorts last). The dates are + RELATIVE to the run (old batch ≈ 90 days back, new batch ≈ 1–3 days + back, each file strictly later than the previous): they stay in the + past forever (a fixed calendar date would drift beyond + ``normalize_doc_date``'s future-skew tolerance and collapse to + "today") while the staggered order — and thus the digest — is a + pure function of the fixture, stable across runs.""" + root = tmp_path / "MyDocs" + old = root / "old" + new = root / "zz_new" + old.mkdir(parents=True) + new.mkdir() + now = datetime.now(UTC) + for i in (1, 2, 3): + f = old / f"{i:02d}.md" + f.write_text(f"# Old {i:02d}\n\nOld body {i}.\n", encoding="utf-8") + ts = (now - timedelta(days=92 - i)).timestamp() + os.utime(f, (ts, ts)) + for i in (1, 2, 3): + f = new / f"{i:02d}.md" + f.write_text(f"# Zz New {i:02d}\n\nNew body {i}.\n", encoding="utf-8") + ts = (now - timedelta(days=4 - i)).timestamp() + os.utime(f, (ts, ts)) + return root + + @pytest.fixture(autouse=True) def _clean_kb(db: Session) -> Iterator[None]: db.execute(text("TRUNCATE chunks, documents, kb_overview, folder_summaries")) @@ -349,3 +407,60 @@ def test_prune_only_run_bumps_sources_version( ) assert _version(db) == 2 # the prune-only change bumped exactly once assert _row(db) is not None # the outline row is untouched + + +# ---------- phase 142, task 01: the KB-shape repro (red pre-fix) ---------- + + +def test_overview_outline_names_the_newest_docs_when_the_cap_bites( + db: Session, + owner_kb: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + """Phase 142, task 01 repro, flipped green by task 02 (the strict + xfail marker is removed — a pass is now a pass). + + The owner's KB shape: one source whose OLDER documents sort + lexicographically FIRST (``old/*``) and whose NEWER documents sort + LAST (``zz_new/*``). With ``BOR_OVERVIEW_INPUT_MAX_CHARS`` forced to + 150, the 6-doc list (275 chars) exceeds the cap and the cut keeps a + PREFIX. Pre-fix that prefix was the lexicographic one — the three + OLDER ``old/*`` docs — so the stored outline's 8-token digest named + the old docs (``mydocs old 01 md old 01 summary of``) and the newer + ``zz_new/*`` docs never reached the model (the "summaries only up to + phase 113" failure at fixture scale). POST-fix the ``created_at``- + desc ordering (task 02) puts the newest docs first, so they survive + the cap and are the ones the outline names + (``mydocs zz new 03 md zz new 03``). + """ + llm = OverviewDigestEmbedder() + get_settings.cache_clear() + monkeypatch.setenv("BOR_OVERVIEW_INPUT_MAX_CHARS", "150") + try: + rc, out = _run_main(monkeypatch, llm, ["--source", str(owner_kb)], capsys) + finally: + get_settings.cache_clear() + assert rc == 0 + assert "added=6" in out + # 6 doc summaries (phase 118) + the overview + 3 folder summaries + # (source root, old/, zz_new/) — the overview gate fired (added > 0). + assert "overview=updated" in out + assert len(llm.chat_calls) == 10 + overview_call = next( + m for m in llm.chat_calls + if "KB_OVERVIEW_MODE" in m[0]["content"] + ) + user = next(m["content"] for m in overview_call if m["role"] == "user") + assert "[…truncated…]" in user # the 150-char cap bit (the repro's precondition) + + row = _row(db) + assert row is not None + # POST-fix digest: the capped list leads with the NEWEST docs — + # "MyDocs — zz_new/03.md — Zz New 03 — Summary of #" → first 8 + # tokens "mydocs zz new 03 md zz new 03". Pre-fix the stored row + # would have been "mydocs old 01 md old 01 summary of" (the oldest + # batch) — the "summaries only up to phase 113" shape this pin + # forbids. + assert row.content == "Knowledge base outline:\n- mydocs zz new 03 md zz new 03" + assert "old" not in row.content # no oldest-batch token may survive the cut diff --git a/tests/unit/test_overview.py b/tests/unit/test_overview.py index 419f1f3..9015faf 100644 --- a/tests/unit/test_overview.py +++ b/tests/unit/test_overview.py @@ -12,7 +12,7 @@ from __future__ import annotations import asyncio import logging import uuid -from datetime import UTC, datetime +from datetime import UTC, datetime, timedelta import pytest from sqlalchemy import text @@ -169,6 +169,55 @@ def test_user_prompt_truncated_at_default_cap() -> None: assert "p2999.md" not in body # the overflow never reaches the model +# ---------- phase 142, task 01: cap-ordering repro ----------------------- + + +def _phase_dir_rows() -> list[tuple[str, str, str, str]]: + """20 rows in the owner's KB phase-directory shape (task 01). + + ``01_old_a.md`` … ``10_old_j.md`` are the OLDER documents and + ``11_new_k.md`` … ``20_new_t.md`` the NEWER ones (the suffix is the + i-th letter, a–t), but the zero-padded + numbering makes the newest sort lexicographically LAST — the owner's + ``113_…`` vs ``128_…`` vs ``131_…`` shape (``131_…`` sorts after + ``13_…``, so the newest phases trail the lexicographic list). Each + rendered line (``source — path — title — summary``) is exactly 100 + chars, so a 500-char cap fits five lines plus four newlines (504 — + one line over budget): the cut is guaranteed to land mid-list, and + the lexicographic first (the oldest) vs last (the newest) contrast + is what the repro asserts on. + """ + rows: list[tuple[str, str, str, str]] = [] + for i in range(1, 21): + kind = "old" if i <= 10 else "new" + path = f"{i:02d}_{kind}_{chr(ord('a') + i - 1)}.md" + title = f"Phase {i:02d}" + prefix = f"s — {path} — {title} — " + rows.append(("s", path, title, "x" * (100 - len(prefix)))) + return rows + + +def test_cap_keeps_the_given_prefix_and_cuts_the_tail() -> None: + """Phase 142, task 01 — the cap behavior pinned AS-IS (green before + AND after the fix — the cap itself is unchanged, only the ordering + of the list feeding it is). Given rows in lexicographic order + (``01_old_a.md`` first, ``20_new_t.md`` last), a 500-char cap keeps + exactly the lexicographic FIRST lines and drops the tail: ``01_old_a`` + survives, ``20_new_t`` is cut, and the shared marker makes the cut + visible on its own line (the current behavior, byte-for-byte). + """ + rows = _phase_dir_rows() + _, full = build_overview_prompt(rows, max_chars=10_000) + cap = 500 + _, user = build_overview_prompt(rows, max_chars=cap) + assert "01_old_a.md" in user # the lexicographic prefix survives the cut + assert "20_new_t.md" not in user # the lexicographic tail never reaches the model + assert user.endswith("\n" + TRUNCATION_MARKER) # the cut is visible + body = user.removesuffix("\n" + TRUNCATION_MARKER) + assert len(body) == cap # cut exactly at the cap + assert body == full[:cap] # a plain prefix cut of the joined list + + # ---------- load_kb_overview ---------- @@ -196,7 +245,12 @@ def test_load_kb_overview_whitespace_only_row_empty(db: Session) -> None: def _add_doc( - db: Session, source: str, path: str, title: str, summary: str | None = None + db: Session, + source: str, + path: str, + title: str, + summary: str | None = None, + created_at: datetime | None = None, ) -> Document: doc = Document( id=uuid.uuid4(), @@ -208,6 +262,8 @@ def _add_doc( content_hash="0" * 64, summary=summary, ) + if created_at is not None: + doc.created_at = created_at db.add(doc) db.commit() return doc @@ -274,21 +330,92 @@ def test_regenerate_updates_existing_row_in_place(db: Session, clean_overview) - _truncate_documents(db) -def test_regenerate_orders_documents_by_source_path(db: Session, clean_overview) -> None: +def test_regenerate_orders_documents_newest_first_then_source_path( + db: Session, clean_overview +) -> None: + """Phase 142, task 02 — the generator's document list is RECENT-FIRST: + ``created_at`` descending, with the ``(source, path)`` tie-break + keeping same-date documents in the pre-phase-142 lexicographic + order. Six documents in two ``created_at`` batches (inserted out of + order, sources interleaved across the batches): the SELECT must + return the NEWER batch first, ``(source, path)`` within each batch — + and the date order must dominate the source/path order (pre-fix the + plain ``order_by(source, path)`` put ``S1 — a.md — Old S1a`` first).""" _truncate_documents(db) try: - # Inserted out of order — the model must see a deterministic order. - _add_doc(db, "Deployments", "z.md", "Z") - _add_doc(db, "Homelab", "b.md", "B") - _add_doc(db, "Homelab", "a.md", "A") + old = datetime(2026, 1, 1, tzinfo=UTC) + new = datetime(2026, 6, 1, tzinfo=UTC) + # The NEWER batch (2026-06-01) — inserted in a jumbled order. + _add_doc(db, "S2", "z.md", "New S2z", created_at=new) + _add_doc(db, "S1", "y.md", "New S1y", created_at=new) + _add_doc(db, "S2", "a.md", "New S2a", created_at=new) + # The OLDER batch (2026-01-01) — also jumbled. + _add_doc(db, "S2", "b.md", "Old S2b", created_at=old) + _add_doc(db, "S1", "c.md", "Old S1c", created_at=old) + _add_doc(db, "S1", "a.md", "Old S1a", created_at=old) llm = _FakeLLM() assert asyncio.run(regenerate_overview(llm, db)) is True finally: _truncate_documents(db) user = next(m["content"] for m in llm.messages if m["role"] == "user") + # Newest batch first (all three 2026-06-01 docs before any 2026-01-01 + # doc), (source, path) within a batch — the pre-fix lexicographic + # order (S1/a, S1/c, S1/y, S2/a, S2/b, S2/z) would fail this pin. assert user == ( - "Deployments — z.md — Z\nHomelab — a.md — A\nHomelab — b.md — B" + "S1 — y.md — New S1y\n" + "S2 — a.md — New S2a\n" + "S2 — z.md — New S2z\n" + "S1 — a.md — Old S1a\n" + "S1 — c.md — Old S1c\n" + "S2 — b.md — Old S2b" + ) + + +def test_document_list_reaching_the_model_leads_with_newest_docs( + db: Session, clean_overview +) -> None: + """Phase 142, task 01 repro, flipped green by task 02 (the strict + xfail marker is removed — a pass is now a pass). + + The document list reaching the lite model must LEAD with the NEWEST + rows (``created_at`` desc — the document's date, phase 106) so that, + when the cap bites, the newest documents — not the lexicographically + first — are the ones the outline covers. Pre-fix the query was plain + ``order_by(source, path)``: with the phase-dir rows below (recency + strictly OPPOSITE to the path order) that put ``01_old_a.md`` — the + OLDEST — first and ``20_new_t.md`` — the NEWEST — last, the traced + "summaries only up to phase 113" failure at unit scale. The + assertion is the POST-FIX order task 02's ``created_at``-desc + ordering delivers. + """ + _truncate_documents(db) + try: + base = datetime(2026, 1, 1, tzinfo=UTC) + for i, (source, path, title, summary) in enumerate(_phase_dir_rows()): + _add_doc(db, source, path, title, summary) + # Row i (0-based) is i days after the base: index 0 + # (``01_old_a.md``) is the OLDEST, index 19 (``20_new_t.md``) + # the NEWEST — recency strictly opposite to the path order. + db.execute( + text("UPDATE documents SET created_at = :d WHERE path = :p"), + {"d": base + timedelta(days=i), "p": path}, + ) + db.commit() + llm = _FakeLLM() + assert asyncio.run(regenerate_overview(llm, db)) is True + finally: + _truncate_documents(db) + + user = next(m["content"] for m in llm.messages if m["role"] == "user") + lines = user.splitlines() + assert len(lines) == 20 # no cap at this size — the ORDER is what is asserted + assert lines[0].startswith("s — 20_new_t.md"), ( + "the list must lead with the NEWEST doc (20_new_t) — pre-fix it " + "leads with the lexicographic prefix (01_old_a, the oldest)" + ) + assert lines[-1].startswith("s — 01_old_a.md"), ( + "the oldest doc must sort LAST — pre-fix it sorts FIRST" ) diff --git a/tests/unit/test_prompts.py b/tests/unit/test_prompts.py index f55252e..b372b54 100644 --- a/tests/unit/test_prompts.py +++ b/tests/unit/test_prompts.py @@ -5,6 +5,11 @@ lite-generated KB outline): its own builder contract (empty → ``""``, char budget + ``[…truncated…]`` marker, pathological budgets), its placement between ```` and ```` in both modes, and the byte-identical-when-absent convention (phase 15 precedent). +(Phase 142, task 03: the section's intro is the honest-copy rewrite — +a compact, possibly partial map that favors recent documents, with the +verify-with-``ls``/``grep`` teaching for "latest / complete set" +questions — pinned verbatim; the false "(generated at import time)" +phrase is gone.) And the phase-71 deflection plain-text line (owner-permitted 2026-09-03): the LOW prompt = pre-phase text + exactly the one new @@ -60,8 +65,19 @@ from app.rag.retriever import TRUNCATION_MARKER #: A small multi-line outline standing in for the lite-generated one. OVERVIEW = "- Homelab\n - Kubernetes (k3s)\n- Deployments\n - Borg backups" -#: The locked one-line intro of the ```` section. -KB_INTRO = "The basic categories of everything in this knowledge base (generated at import time):" +#: The locked one-line intro of the ```` section +#: (phase 142, task 03 — the honest-intro rewrite, pinned verbatim in +#: :func:`test_kb_intro_phase142_honest_copy`) — the intro line WITHOUT +#: its trailing newline (the section renderers / the f-strings below +#: supply the line break after it, as the pre-phase-142 constant did). +KB_INTRO = ( + 'A compact picture of the knowledge base\'s categories and topics — ' + 'it favors recent documents and may omit older or lesser ones: a map ' + 'of what exists, not a complete inventory. For questions about the ' + 'latest state or the complete set ("the last N …", "everything ' + 'about …", "all the …"), verify with `ls` (drill into the folders) ' + 'or `grep` before answering from the outline:' +) #: The fixture documents' fixed creation date (phase 106, D5) — the @@ -784,6 +800,55 @@ def test_kb_section_format_intro_and_content() -> None: ) +def test_kb_intro_phase142_honest_copy() -> None: + """Phase 142 (task 03): the intro is pinned verbatim — it says what + the outline actually is (a compact picture of the KB's categories + and topics that favors recent documents and may omit older or lesser + ones — a map of what exists, not a complete inventory) and gives the + one behavior that fixes the "the last N phases" question class: for + latest-state / complete-set questions, verify with ``ls`` (drill + into the folders) or ``grep`` BEFORE answering from the outline. The + "(generated at import time)" phrase is gone — it was false (the sync + regenerates the row on every KB change) and it taught the model to + treat a partial outline as the whole frozen KB.""" + from app.rag.prompts import _KB_INTRO + + # The app's constant, pinned verbatim (the trailing newline included): + assert _KB_INTRO == ( + 'A compact picture of the knowledge base\'s categories and topics — ' + 'it favors recent documents and may omit older or lesser ones: a map ' + 'of what exists, not a complete inventory. For questions about the ' + 'latest state or the complete set ("the last N …", "everything ' + 'about …", "all the …"), verify with `ls` (drill into the folders) ' + 'or `grep` before answering from the outline:\n' + ) + # The local pin is the same text minus the trailing newline. + assert KB_INTRO + "\n" == _KB_INTRO + # The false claim is gone from the constant (and thus the prompt). + assert "(generated at import time)" not in KB_INTRO + assert "generated at import time" not in KB_INTRO + # The three contracts it must carry: the recency bias (task 02's + # newest-first input is what makes it true), the map-not-inventory + # framing, and the verify-first teaching with both tool names. + assert "favors recent documents" in KB_INTRO + assert "not a complete inventory" in KB_INTRO + assert "verify with `ls` (drill into the folders) or `grep`" in KB_INTRO + assert "before answering from the outline" in KB_INTRO + # It rides the rendered section exactly where the old intro did: + # between the open tag and the outline, colon-terminated. + section = build_kb_section(OVERVIEW) + assert section.startswith(f"\n{KB_INTRO}") + assert section.index(KB_INTRO) < section.index(OVERVIEW) + # And it reaches BOTH modes' prompts (the section is mode-agnostic). + doc = _doc("kubernetes.md", "CONTENT", "T", summary="S.") + for prompt in ( + build_high_prompt([doc], kb_overview=OVERVIEW), + build_deflect_prompt(["T1"], kb_overview=OVERVIEW), + ): + assert KB_INTRO in prompt + assert "(generated at import time)" not in prompt + + def test_kb_section_trims_overview_edges() -> None: assert build_kb_section(f" {OVERVIEW} \n") == build_kb_section(OVERVIEW) @@ -797,9 +862,11 @@ def test_kb_section_fits_budget_exactly_no_marker() -> None: def test_kb_section_over_budget_capped_with_marker() -> None: text = "- " + "x" * 500 - # The section frame alone is 121 chars, so the cap must clear it for - # any outline prefix to fit (pathological budgets are tested below). - cap = 200 + # The section frame alone is 422 chars (the tag + the phase-142 + # honest intro + the marker + the close tag), so the cap must clear + # it for any outline prefix to fit (pathological budgets are tested + # below). + cap = 600 section = build_kb_section(text, max_chars=cap) assert len(section) <= cap # the budget is never exceeded assert TRUNCATION_MARKER in section @@ -961,8 +1028,8 @@ def test_prompt_kb_section_over_settings_budget_capped_with_marker( monkeypatch.setattr( prompts_mod, "get_settings", - # 200 > the 121-char section frame, so a prefix + marker can fit. - lambda: Settings(_env_file=None, kb_overview_max_chars=200), # pyright: ignore[reportCallIssue] + # 600 > the 422-char section frame, so a prefix + marker can fit. + lambda: Settings(_env_file=None, kb_overview_max_chars=600), # pyright: ignore[reportCallIssue] ) text = "- " + "z" * 500 doc = _doc("kubernetes.md", "TALOS_DOC_CONTENT", "Kubernetes Homelab Cluster") @@ -976,7 +1043,7 @@ def test_prompt_kb_section_over_settings_budget_capped_with_marker( section = prompt[prompt.index("") :] close = section.index("") section = section[: close + len("")] - assert len(section) <= 200 + assert len(section) <= 600 # ---------- phase 71: the deflection plain-text line (prevention) ----------