phase: 119_name_signal_read_chips
All verification complete. Final report: **Phase 119 final verification pass — all criteria verified, one stale pin fixed.** - Verified implementation of all 6 tasks: D1 component name-hit rule (`name_hit` flag, titles never matched, retired length tie-break), D2 `BOR_NAME_HIT_BONUS` (0.005 default, 0 = byte-identical kill switch, negative fails startup, selection-layer only, `eval_retrieval` `suggested:` line), D3 suggested-folder lines (after `SUGGEST_INTRO`, before first block), D4 cite-discipline `SUGGEST_INTRO` sentence (PERSONA/LOW/`TOOLS_SECTION` byte-pins intact), D5 `done.sources` = read docs only (frontend no-op on empty confirmed), D6 mock `repeat your folder map` echo + new suite + telemetry. - Battery (replica restored per skill, fingerprint docs=1000/chunks=8866 verified, `eval_retrieval --from-file tests/fixtures/retrieval_battery.txt` re-run): **GATE PASS** — gitea README #4 in suggested top-5, forgejo 5/5 (README #1), gateway README in top-5 (#4), qwen3.8-27b quadlets top-5, Mongolia HIGH/fts=5 unchanged. - New E2E in isolation: `4 passed` ×2 (deterministic). All 27 modified E2E suites in isolation: 26 green; **1 stale pin fixed** — `test_source_chip_quality.py` durable-record order pin pre-dated the D1 re-rank (`aliases` stem sub-component name-hits `ssh_aliases.txt`, deterministically lifting `backups.md` over `kubernetes.md`; probe-verified 0.016277 vs 0.016036, 4/4 stable) — re-pinned with the phase-119 rationale; suite green ×2. - Gates: `uv run pytest --cov=app --cov-report=term-missing` → **2547 passed, app coverage 99%** (>90%); `uv run ruff check .` → All checks passed; `uv run pyright` → 0 errors. - Completion criteria: 1 ✅ (battery, recorded), 2 ✅ (folder lines; block/LOW byte-identical pins green), 3 ✅ (read-only chips, zero-read chips nothing, related row + durable record untouched — unit+E2E agree), 4 ✅ (all green), 5 → commit/phase-move left to the harness per pass rules (nothing committed). - Deviations: battery output + real-model telemetry recorded in `.agents/reports/119_name_signal_read_chips/task06_battery_and_e2e.md` and `TOOL_CALLING_TESTING.md` §11 (task files in `complete/` are immutable to this pass); gateway canonical doc at #4 vs overview's #3 was already documented at task 06 (containment gate met). - Next pending phase: **none** — `todo/` holds only phase 119.
This commit is contained in:
@@ -6,10 +6,12 @@ mapping with fake rows; this suite covers the SQL side on real
|
||||
Postgres: the document-projection scan, the LATERAL representative-
|
||||
chunk fetch (the ``is_summary`` chunk wins, chunk 0 otherwise, and a
|
||||
chunk-less name match is EXCLUDED — the ``c.id IS NOT NULL`` guard),
|
||||
the (count, length, catalog) ranking, the name-hits-lead-the-lexical-
|
||||
list union with the FTS rows (chunk-id dedup), and the full
|
||||
``retrieve()`` → ``select_documents()`` path putting the versioned-
|
||||
name document into the seeded top-N.
|
||||
the (matched-token count, catalog) ranking (phase 119, LOCKED A2 —
|
||||
the two-class component rule: digit-bearing prefix, digitless exact,
|
||||
titles never matched), the name-hits-lead-the-lexical-list union with
|
||||
the FTS rows (chunk-id dedup), and the full ``retrieve()`` →
|
||||
``select_documents()`` path putting the versioned-name document into
|
||||
the seeded top-N.
|
||||
|
||||
Requires: ``podman compose up -d db``.
|
||||
"""
|
||||
@@ -137,19 +139,29 @@ def kb(db) -> Iterator[None]:
|
||||
|
||||
|
||||
def test_name_hit_chunks_real_sql(kb, db) -> None:
|
||||
"""Real Postgres: the projection scan finds exactly the qwen3.8
|
||||
quadlet (the qwen3.6 sibling and the chunk-less name match are
|
||||
excluded), and the LATERAL fetch hands back the SUMMARY chunk as
|
||||
the representative (position −1, is_summary)."""
|
||||
"""Real Postgres: the projection scan finds the qwen3.8 quadlet
|
||||
(the ``qwen38`` stem prefix — the incident's original case) AND
|
||||
the ``notes/llama.cpp.md`` doc (the question names "llama.cpp" —
|
||||
the dotted token ``llamacpp`` exact-matches the file stem, the
|
||||
phase-119 two-class rule). The qwen3.6 sibling (``qwen38`` is not
|
||||
a prefix of ``qwen36…``) and the chunk-less name match are
|
||||
excluded; the LATERAL fetch hands back the SUMMARY chunk as the
|
||||
representative for the quadlet (position −1, is_summary) and chunk
|
||||
0 for the single-chunk notes doc."""
|
||||
out = _name_hit_chunks(db, INCIDENT_QUESTION)
|
||||
assert [rc.document.path for rc in out] == [
|
||||
"reeseapps/ai/deployments/juggernaut/quadlets/qwen3.8-27b-juggernaut-vulkan.container"
|
||||
"reeseapps/ai/deployments/juggernaut/quadlets/qwen3.8-27b-juggernaut-vulkan.container",
|
||||
"notes/llama.cpp.md", # the question names it — stem exact (A2)
|
||||
]
|
||||
rc = out[0]
|
||||
assert rc.position == -1 # the summary chunk wins the LATERAL order
|
||||
assert rc.is_summary is True
|
||||
assert rc.fts_hit is True # the lexical signal — the A8 gate answers
|
||||
assert rc.cosine == 0.0 # no vector rank on the name-hit row
|
||||
rc_notes = out[1]
|
||||
assert rc_notes.position == 0 # chunk 0 (no summary chunk)
|
||||
assert rc_notes.is_summary is False
|
||||
assert all(rc.name_hit is True for rc in out) # phase 119 — D2 bonus input
|
||||
assert "qwen3.8-empty.container" not in [r.document.path for r in out] # chunk-less guard
|
||||
|
||||
|
||||
@@ -163,13 +175,19 @@ def test_lexical_candidates_name_hit_leads_real_sql(kb, db) -> None:
|
||||
assert paths[0] == (
|
||||
"reeseapps/ai/deployments/juggernaut/quadlets/qwen3.8-27b-juggernaut-vulkan.container"
|
||||
)
|
||||
assert paths[1] == "notes/llama.cpp.md" # the second name hit (stem exact)
|
||||
# The FTS pollution is still present (the incident's shape) — but
|
||||
# behind the name hit, no longer ahead of it.
|
||||
# behind the name hits, no longer ahead of them.
|
||||
assert (
|
||||
"reeseapps/ai/deployments/juggernaut/quadlets/qwen3.6-27b-juggernaut-vulkan.container"
|
||||
in paths
|
||||
)
|
||||
assert all(rc.fts_hit is True for rc in out)
|
||||
# Phase 119: the two name-hit representative rows are flagged, the
|
||||
# plain FTS rows are not.
|
||||
assert out[0].name_hit is True
|
||||
assert out[1].name_hit is True
|
||||
assert all(not rc.name_hit for rc in out[2:])
|
||||
|
||||
|
||||
def test_retrieve_selects_name_hit_doc_into_top_n(kb, db) -> None:
|
||||
@@ -177,7 +195,12 @@ def test_retrieve_selects_name_hit_doc_into_top_n(kb, db) -> None:
|
||||
fused) → ``select_documents`` puts the qwen3.8 quadlet in the
|
||||
seeded top-N — the incident's seed miss (the two overview docs
|
||||
only) is fixed. The question vector is parallel to the q38 chunk
|
||||
embeddings (cosine 1.0), orthogonal to q36 (0.0)."""
|
||||
embeddings (cosine 1.0), orthogonal to q36 (0.0).
|
||||
|
||||
Phase 119: the ``name_hit`` flag survives the fusion — the two
|
||||
name-hit representative chunks are double hits (vector ∪ lexical),
|
||||
and the double-hit merge ORs the flag into the surviving row;
|
||||
the plain q36 vector+FTS row stays False."""
|
||||
question_vec = _vec(1)
|
||||
chunks = retrieve(db, INCIDENT_QUESTION, question_vec)
|
||||
docs = select_documents(chunks, n=2)
|
||||
@@ -185,16 +208,31 @@ def test_retrieve_selects_name_hit_doc_into_top_n(kb, db) -> None:
|
||||
"reeseapps/ai/deployments/juggernaut/quadlets/qwen3.8-27b-juggernaut-vulkan.container",
|
||||
"notes/llama.cpp.md",
|
||||
]
|
||||
by_path: dict[str, list[bool]] = {}
|
||||
for rc in chunks:
|
||||
by_path.setdefault(rc.document.path, []).append(rc.name_hit)
|
||||
# The q38 doc's summary chunk is the double-hit name hit (flagged);
|
||||
# its plain chunk 0 (vector + FTS) is not — the doc has a flagged row.
|
||||
assert any(by_path[
|
||||
"reeseapps/ai/deployments/juggernaut/quadlets/qwen3.8-27b-juggernaut-vulkan.container"
|
||||
])
|
||||
assert any(by_path["notes/llama.cpp.md"]) # its only chunk is the name hit
|
||||
assert not any(by_path[
|
||||
"reeseapps/ai/deployments/juggernaut/quadlets/qwen3.6-27b-juggernaut-vulkan.container"
|
||||
]) # plain vector+FTS — never a name hit
|
||||
|
||||
|
||||
def test_name_hit_limit_real_sql(db) -> None:
|
||||
"""Twelve identical (1, 6) name hits — the LATERAL fetch (and the
|
||||
output) carries exactly ``NAME_HIT_LIMIT`` winners, catalog order."""
|
||||
"""Twelve tied name hits (one matched token each — the ``qwen38``
|
||||
stem PREFIX; the token must lead the stem, the old mid-stem
|
||||
containment no longer matches) — the LATERAL fetch (and the
|
||||
output) carries exactly ``NAME_HIT_LIMIT`` winners, catalog
|
||||
order."""
|
||||
db.execute(text("TRUNCATE chunks, documents"))
|
||||
db.commit()
|
||||
for i in range(12):
|
||||
doc = _doc(
|
||||
db, "S", f"quadlets/m{i:02d}-qwen38.container", f"m{i:02d}-qwen38", "llama cpp qwen38\n"
|
||||
db, "S", f"quadlets/qwen38-m{i:02d}.container", f"qwen38-m{i:02d}", "llama cpp qwen38\n"
|
||||
)
|
||||
db.flush()
|
||||
c = _chunk(db, doc, 0, f"llama cpp qwen38 doc {i}")
|
||||
@@ -203,5 +241,6 @@ def test_name_hit_limit_real_sql(db) -> None:
|
||||
out = _name_hit_chunks(db, "what are the llama.cpp arguments for qwen 3.8")
|
||||
assert len(out) == NAME_HIT_LIMIT
|
||||
assert [rc.document.path for rc in out] == [
|
||||
f"quadlets/m{i:02d}-qwen38.container" for i in range(NAME_HIT_LIMIT)
|
||||
f"quadlets/qwen38-m{i:02d}.container" for i in range(NAME_HIT_LIMIT)
|
||||
]
|
||||
assert all(rc.name_hit is True for rc in out)
|
||||
|
||||
Reference in New Issue
Block a user