phase: 142_kb_overview_freshness
All verification complete. Final report: **Phase 142 final verification pass — all green** - **Verified implemented** (working tree matches design; app diff is exactly 2 lines of logic): `app/rag/overview.py` — `regenerate_overview` now orders the lite-model document list `created_at` desc, `(source, path)` tie-break (newest docs survive the cap); `app/rag/prompts.py` — `_KB_INTRO` rewritten to the honest intro (recency bias, "map not inventory", verify-with-`ls`/`grep`; "(generated at import time)" gone). - **Completion criteria:** 1. Forced-cap outline leads with newest docs — ✅ E2E `test_kb_overview_freshness.py` test 1 (cap 300, digest from `zz_new/z.md`, no "old" tokens) + integration repro (cap 150 → `mydocs zz new 03 md zz new 03`) 2. New honest intro served in `<knowledge_base>` — ✅ unit pin (verbatim) + E2E test 2 (`build_kb_section` surface, tag/cap byte-stable) 3. Gates — ✅ `uv run pytest --cov=app --cov-report=term-missing`: **3359 passed**, `app/` TOTAL **99%** (>90%); `uv run pytest tests/e2e/test_kb_overview_freshness.py -v --no-cov`: **3 passed** in isolation; `uv run pytest tests/e2e/test_kb_overview.py -v --no-cov`: **4 passed** in isolation; `uv run ruff check . && uv run pyright`: **clean, 0 errors** 4. No behavior change in completed phases — ✅ cap env knob, change-gated sync trigger, `build_kb_section` cap/marker, deflect path all byte-identical (pinned pins pass in the 3359) - **Documentation:** README.md reviewed in full (every `.env.example` var documented with matching defaults, all endpoints/routes verified against `app/api/`, Quick Start/deploy/troubleshooting claims verified) and AGENTS.md re-reviewed (rules + quick reference accurate) — **verified consistent, no changes needed**. - **Decisions:** none required — no defects found; no live-infra changes (only the already-running local `db` container was used by tests). - **Next pending phase:** `.agents/phases/todo/143_mobile_keyboard_after_answer`.
This commit is contained in:
+135
-8
@@ -12,7 +12,7 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
import logging
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import text
|
||||
@@ -169,6 +169,55 @@ def test_user_prompt_truncated_at_default_cap() -> None:
|
||||
assert "p2999.md" not in body # the overflow never reaches the model
|
||||
|
||||
|
||||
# ---------- phase 142, task 01: cap-ordering repro -----------------------
|
||||
|
||||
|
||||
def _phase_dir_rows() -> list[tuple[str, str, str, str]]:
|
||||
"""20 rows in the owner's KB phase-directory shape (task 01).
|
||||
|
||||
``01_old_a.md`` … ``10_old_j.md`` are the OLDER documents and
|
||||
``11_new_k.md`` … ``20_new_t.md`` the NEWER ones (the suffix is the
|
||||
i-th letter, a–t), but the zero-padded
|
||||
numbering makes the newest sort lexicographically LAST — the owner's
|
||||
``113_…`` vs ``128_…`` vs ``131_…`` shape (``131_…`` sorts after
|
||||
``13_…``, so the newest phases trail the lexicographic list). Each
|
||||
rendered line (``source — path — title — summary``) is exactly 100
|
||||
chars, so a 500-char cap fits five lines plus four newlines (504 —
|
||||
one line over budget): the cut is guaranteed to land mid-list, and
|
||||
the lexicographic first (the oldest) vs last (the newest) contrast
|
||||
is what the repro asserts on.
|
||||
"""
|
||||
rows: list[tuple[str, str, str, str]] = []
|
||||
for i in range(1, 21):
|
||||
kind = "old" if i <= 10 else "new"
|
||||
path = f"{i:02d}_{kind}_{chr(ord('a') + i - 1)}.md"
|
||||
title = f"Phase {i:02d}"
|
||||
prefix = f"s — {path} — {title} — "
|
||||
rows.append(("s", path, title, "x" * (100 - len(prefix))))
|
||||
return rows
|
||||
|
||||
|
||||
def test_cap_keeps_the_given_prefix_and_cuts_the_tail() -> None:
|
||||
"""Phase 142, task 01 — the cap behavior pinned AS-IS (green before
|
||||
AND after the fix — the cap itself is unchanged, only the ordering
|
||||
of the list feeding it is). Given rows in lexicographic order
|
||||
(``01_old_a.md`` first, ``20_new_t.md`` last), a 500-char cap keeps
|
||||
exactly the lexicographic FIRST lines and drops the tail: ``01_old_a``
|
||||
survives, ``20_new_t`` is cut, and the shared marker makes the cut
|
||||
visible on its own line (the current behavior, byte-for-byte).
|
||||
"""
|
||||
rows = _phase_dir_rows()
|
||||
_, full = build_overview_prompt(rows, max_chars=10_000)
|
||||
cap = 500
|
||||
_, user = build_overview_prompt(rows, max_chars=cap)
|
||||
assert "01_old_a.md" in user # the lexicographic prefix survives the cut
|
||||
assert "20_new_t.md" not in user # the lexicographic tail never reaches the model
|
||||
assert user.endswith("\n" + TRUNCATION_MARKER) # the cut is visible
|
||||
body = user.removesuffix("\n" + TRUNCATION_MARKER)
|
||||
assert len(body) == cap # cut exactly at the cap
|
||||
assert body == full[:cap] # a plain prefix cut of the joined list
|
||||
|
||||
|
||||
# ---------- load_kb_overview ----------
|
||||
|
||||
|
||||
@@ -196,7 +245,12 @@ def test_load_kb_overview_whitespace_only_row_empty(db: Session) -> None:
|
||||
|
||||
|
||||
def _add_doc(
|
||||
db: Session, source: str, path: str, title: str, summary: str | None = None
|
||||
db: Session,
|
||||
source: str,
|
||||
path: str,
|
||||
title: str,
|
||||
summary: str | None = None,
|
||||
created_at: datetime | None = None,
|
||||
) -> Document:
|
||||
doc = Document(
|
||||
id=uuid.uuid4(),
|
||||
@@ -208,6 +262,8 @@ def _add_doc(
|
||||
content_hash="0" * 64,
|
||||
summary=summary,
|
||||
)
|
||||
if created_at is not None:
|
||||
doc.created_at = created_at
|
||||
db.add(doc)
|
||||
db.commit()
|
||||
return doc
|
||||
@@ -274,21 +330,92 @@ def test_regenerate_updates_existing_row_in_place(db: Session, clean_overview) -
|
||||
_truncate_documents(db)
|
||||
|
||||
|
||||
def test_regenerate_orders_documents_by_source_path(db: Session, clean_overview) -> None:
|
||||
def test_regenerate_orders_documents_newest_first_then_source_path(
|
||||
db: Session, clean_overview
|
||||
) -> None:
|
||||
"""Phase 142, task 02 — the generator's document list is RECENT-FIRST:
|
||||
``created_at`` descending, with the ``(source, path)`` tie-break
|
||||
keeping same-date documents in the pre-phase-142 lexicographic
|
||||
order. Six documents in two ``created_at`` batches (inserted out of
|
||||
order, sources interleaved across the batches): the SELECT must
|
||||
return the NEWER batch first, ``(source, path)`` within each batch —
|
||||
and the date order must dominate the source/path order (pre-fix the
|
||||
plain ``order_by(source, path)`` put ``S1 — a.md — Old S1a`` first)."""
|
||||
_truncate_documents(db)
|
||||
try:
|
||||
# Inserted out of order — the model must see a deterministic order.
|
||||
_add_doc(db, "Deployments", "z.md", "Z")
|
||||
_add_doc(db, "Homelab", "b.md", "B")
|
||||
_add_doc(db, "Homelab", "a.md", "A")
|
||||
old = datetime(2026, 1, 1, tzinfo=UTC)
|
||||
new = datetime(2026, 6, 1, tzinfo=UTC)
|
||||
# The NEWER batch (2026-06-01) — inserted in a jumbled order.
|
||||
_add_doc(db, "S2", "z.md", "New S2z", created_at=new)
|
||||
_add_doc(db, "S1", "y.md", "New S1y", created_at=new)
|
||||
_add_doc(db, "S2", "a.md", "New S2a", created_at=new)
|
||||
# The OLDER batch (2026-01-01) — also jumbled.
|
||||
_add_doc(db, "S2", "b.md", "Old S2b", created_at=old)
|
||||
_add_doc(db, "S1", "c.md", "Old S1c", created_at=old)
|
||||
_add_doc(db, "S1", "a.md", "Old S1a", created_at=old)
|
||||
llm = _FakeLLM()
|
||||
assert asyncio.run(regenerate_overview(llm, db)) is True
|
||||
finally:
|
||||
_truncate_documents(db)
|
||||
|
||||
user = next(m["content"] for m in llm.messages if m["role"] == "user")
|
||||
# Newest batch first (all three 2026-06-01 docs before any 2026-01-01
|
||||
# doc), (source, path) within a batch — the pre-fix lexicographic
|
||||
# order (S1/a, S1/c, S1/y, S2/a, S2/b, S2/z) would fail this pin.
|
||||
assert user == (
|
||||
"Deployments — z.md — Z\nHomelab — a.md — A\nHomelab — b.md — B"
|
||||
"S1 — y.md — New S1y\n"
|
||||
"S2 — a.md — New S2a\n"
|
||||
"S2 — z.md — New S2z\n"
|
||||
"S1 — a.md — Old S1a\n"
|
||||
"S1 — c.md — Old S1c\n"
|
||||
"S2 — b.md — Old S2b"
|
||||
)
|
||||
|
||||
|
||||
def test_document_list_reaching_the_model_leads_with_newest_docs(
|
||||
db: Session, clean_overview
|
||||
) -> None:
|
||||
"""Phase 142, task 01 repro, flipped green by task 02 (the strict
|
||||
xfail marker is removed — a pass is now a pass).
|
||||
|
||||
The document list reaching the lite model must LEAD with the NEWEST
|
||||
rows (``created_at`` desc — the document's date, phase 106) so that,
|
||||
when the cap bites, the newest documents — not the lexicographically
|
||||
first — are the ones the outline covers. Pre-fix the query was plain
|
||||
``order_by(source, path)``: with the phase-dir rows below (recency
|
||||
strictly OPPOSITE to the path order) that put ``01_old_a.md`` — the
|
||||
OLDEST — first and ``20_new_t.md`` — the NEWEST — last, the traced
|
||||
"summaries only up to phase 113" failure at unit scale. The
|
||||
assertion is the POST-FIX order task 02's ``created_at``-desc
|
||||
ordering delivers.
|
||||
"""
|
||||
_truncate_documents(db)
|
||||
try:
|
||||
base = datetime(2026, 1, 1, tzinfo=UTC)
|
||||
for i, (source, path, title, summary) in enumerate(_phase_dir_rows()):
|
||||
_add_doc(db, source, path, title, summary)
|
||||
# Row i (0-based) is i days after the base: index 0
|
||||
# (``01_old_a.md``) is the OLDEST, index 19 (``20_new_t.md``)
|
||||
# the NEWEST — recency strictly opposite to the path order.
|
||||
db.execute(
|
||||
text("UPDATE documents SET created_at = :d WHERE path = :p"),
|
||||
{"d": base + timedelta(days=i), "p": path},
|
||||
)
|
||||
db.commit()
|
||||
llm = _FakeLLM()
|
||||
assert asyncio.run(regenerate_overview(llm, db)) is True
|
||||
finally:
|
||||
_truncate_documents(db)
|
||||
|
||||
user = next(m["content"] for m in llm.messages if m["role"] == "user")
|
||||
lines = user.splitlines()
|
||||
assert len(lines) == 20 # no cap at this size — the ORDER is what is asserted
|
||||
assert lines[0].startswith("s — 20_new_t.md"), (
|
||||
"the list must lead with the NEWEST doc (20_new_t) — pre-fix it "
|
||||
"leads with the lexicographic prefix (01_old_a, the oldest)"
|
||||
)
|
||||
assert lines[-1].startswith("s — 01_old_a.md"), (
|
||||
"the oldest doc must sort LAST — pre-fix it sorts FIRST"
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user