finally getting accurate answers
Build and Push Containers / build-and-push-app (push) Successful in 1m46s
Build and Push Containers / build-and-push-db (push) Successful in 12s

This commit is contained in:
2026-09-05 10:26:39 -04:00
parent bb2803bebd
commit 766702c750
9 changed files with 1628 additions and 41 deletions
+196 -8
View File
@@ -209,17 +209,26 @@ def test_agent_tools_names_and_parameters() -> None:
# ("pass ONLY `pattern`") + the source-name-is-not-a-document
# clause (the model kept scoping grep with an ls-style source name
# — the 2026-09-03 incident loop shape, but on grep) plus the
# one-call-at-a-time discipline clause.
# one-call-at-a-time discipline clause. The 2026-09-05 incident
# (the "Qwen 3.8" sample question — the harness prior is that grep
# takes a REGEX; this grep is a fixed substring, owner-locked A5):
# the plain-substring-never-a-regex clause states the contract up
# front, so the regex-shaped first grep that does fire gets the
# teaching no-match line instead of a trusted miss.
assert grep["description"] == (
"Search the indexed documents for an exact string "
"(case-insensitive) and return up to 20 matching lines "
"as `source/path:line: text` — a locator, not a "
"context-adder: read the winner with `read`. For a "
"normal search pass ONLY `pattern` — it searches every "
"document and that is how you search the knowledge "
"base; never pass a source name as `path` (a source "
"name is not a document). Call one tool at a time — "
"wait for this result before your next call."
"context-adder: read the winner with `read`. The "
"pattern is a plain substring, NEVER a regex — if a "
"pattern with regex syntax (like '.*' or '\\.') comes "
"back with no matches, retry with the plain text you "
"expect to see. For a normal search pass ONLY `pattern` "
"— it searches every document and that is how you "
"search the knowledge base; never pass a source name "
"as `path` (a source name is not a document). Call one "
"tool at a time — wait for this result before your "
"next call."
)
grep_params = grep["parameters"]
assert grep_params["type"] == "object"
@@ -227,7 +236,8 @@ def test_agent_tools_names_and_parameters() -> None:
assert set(grep_params["properties"]) == {"pattern", "path"}
assert all(p["type"] == "string" for p in grep_params["properties"].values())
assert grep_params["properties"]["pattern"]["description"] == (
"The exact text to search for (a plain substring, not a regex)"
"The exact text to search for (a plain substring, "
"not a regex — no '.*', no '\\.', no character classes)"
)
# Phase 72 (task 02): the bare-path contract is stated up front;
# task 05 (live gate iterations 1-8): the one-known-document clause
@@ -1164,6 +1174,184 @@ def test_grep_truncates_match_lines_at_200_chars(monkeypatch: pytest.MonkeyPatch
assert holder.tool_calls == 1
# ---------- grep no-match teaching: the regex-shaped pattern
# (the 2026-09-05 "Qwen 3.8" incident — the harness prior is that
# grep takes a REGEX; this grep is a fixed substring, owner-locked
# A5, and the contract does not change) ----------
def test_plain_form_reduces_regex_to_literal_text() -> None:
"""The plain-form hint: the pattern reduced to literal text — the
incident's exact recovery (``qwen.*3\\.8`` → ``qwen3.8``) plus the
edge cases (raw ``.*`` runs dropped before unescape, so an escaped
dot survives; first alternative only; classes/quantifiers/parens/
anchors gone; whitespace preserved; pure metacharacters → ``""``).
"""
assert agent.plain_form(r"qwen.*3\.8") == "qwen3.8" # the incident
assert agent.plain_form(r"qwen 3\.8") == "qwen 3.8"
assert agent.plain_form(r"Qwen 3\.8") == "Qwen 3.8" # case kept
assert agent.plain_form(r"qwen3\.8") == "qwen3.8"
assert agent.plain_form(r"llama\.cpp") == "llama.cpp" # escaped dot kept
assert agent.plain_form(r"qwen[0-9]+") == "qwen" # class + quantifier
assert agent.plain_form("a|b") == "a" # first alternative only
assert agent.plain_form(r"\d+") == "" # no literal text — no hint
assert agent.plain_form(r".*") == "" # pure wildcard — no hint
assert agent.plain_form(r"(qwen)3\.8") == "qwen3.8" # group contents kept
assert agent.plain_form("a{2,3}b") == "ab"
assert agent.plain_form(r"^qwen$") == "qwen" # anchors dropped
assert agent.plain_form(r"a\.b") == "a.b" # escaped dot is a literal
assert agent.plain_form("plain") == "plain" # identity for plain text
def test_looks_like_regex_detection() -> None:
"""One metacharacter anywhere marks the pattern regex-shaped; a
plain substring (even with a space) does not."""
for p in (
r"qwen.*3\.8", r"qwen 3\.8", "qwen+", "a?b", "x|y", "(a)", "[a-z]", "a^b", "b$c", "a{2}"
):
assert agent.looks_like_regex(p) is True, p
for p in ("qwen 3.8", "qwen3.8", "plain substring", ""):
assert agent.looks_like_regex(p) is False, p
def test_grep_no_match_regex_pattern_gets_teaching_line(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""The incident's shape: a regex-shaped pattern that (necessarily)
misses gets the TEACHING no-match line — the plain-substring
contract stated, the plain-form retry hint handed over. Still a
counted result; the context is untouched (locked A5)."""
d1 = _doc("Alpha", "a/one.md", "One", "qwen3.8-27b juggernaut\nno regex text")
monkeypatch.setattr(agent, "all_documents", lambda db: [d1])
holder = AgentHolder()
llm = ScriptedLLM(
[
ToolCallPiece(
id="call_1", name="grep", arguments={"pattern": r"qwen.*3\.8"}
)
],
[StreamPiece("content", "ans")],
)
asyncio.run(_run(llm, holder, _settings()))
assert llm.requests[1][0][3]["content"] == agent.NO_MATCHES_REGEX.format(
pattern=r"qwen.*3\.8", plain="qwen3.8"
)
assert llm.requests[1][0][3]["content"] == (
"No matches for 'qwen.*3\\.8'. grep matches a plain substring "
"(case-insensitive), not a regex — '.*', '\\.' and the like are "
"literal text here, so that pattern can never match. Retry with "
"the plain text you expect to see (e.g. 'qwen3.8')."
)
assert holder.tool_calls == 1 # a no-match with teaching is still a result
assert holder.read_docs == [] # locked A5: a grep adds no context
def test_grep_no_match_regex_scoped_gets_teaching_line(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""The scoped teaching variant: the resolved identity is echoed, the
hint handed over."""
d1 = _doc("Alpha", "a/one.md", "One", "nothing regex-shaped here")
def _find(db: Any, source: str, path: str) -> Document | None:
return d1 if (source, path) == ("Alpha", "a/one.md") else None
monkeypatch.setattr(agent, "find_document", _find)
holder = AgentHolder()
llm = ScriptedLLM(
[
ToolCallPiece(
id="call_1",
name="grep",
arguments={"pattern": r"qwen 3\.8", "path": "Alpha/a/one.md"},
)
],
[StreamPiece("content", "ans")],
)
asyncio.run(_run(llm, holder, _settings()))
assert llm.requests[1][0][3]["content"] == (
"No matches for 'qwen 3\\.8' in Alpha/a/one.md. grep matches a "
"plain substring (case-insensitive), not a regex — retry with "
"the plain text you expect to see (e.g. 'qwen 3.8')."
)
assert holder.tool_calls == 1
assert holder.read_docs == []
def test_grep_no_match_plain_pattern_keeps_ordinary_line(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""A no-match for a PLAIN pattern (no metacharacters — "qwen 3.8" with
the space included) keeps the ordinary line byte-identical: the
teaching never fires for a well-formed pattern (the retrieval side —
the name-hit lexical signal — is what covers that case)."""
d1 = _doc("Alpha", "a/one.md", "One", "qwen3.8-27b juggernaut")
monkeypatch.setattr(agent, "all_documents", lambda db: [d1])
holder = AgentHolder()
llm = ScriptedLLM(
[ToolCallPiece(id="call_1", name="grep", arguments={"pattern": "qwen 3.8"})],
[StreamPiece("content", "ans")],
)
asyncio.run(_run(llm, holder, _settings()))
assert llm.requests[1][0][3]["content"] == (
"No matches for 'qwen 3.8' in the knowledge base."
)
assert holder.tool_calls == 1
def test_grep_matched_regex_pattern_returns_matches_not_teaching(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""A pattern with metacharacters that MATCHES literally gets the
ordinary match output — the teaching can never suppress a real hit
(the detection keys on a NO-MATCH only)."""
d1 = _doc("S", "a.md", "A", "the C++ compiler is here")
monkeypatch.setattr(agent, "all_documents", lambda db: [d1])
holder = AgentHolder()
llm = ScriptedLLM(
[ToolCallPiece(id="call_1", name="grep", arguments={"pattern": "C++"})],
[StreamPiece("content", "ans")],
)
asyncio.run(_run(llm, holder, _settings()))
assert llm.requests[1][0][3]["content"] == "S/a.md:1: the C++ compiler is here"
assert holder.tool_calls == 1
def test_grep_no_match_regex_reducing_to_empty_falls_back(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""A regex-shaped pattern with no literal text left after the
reduction (``.*``) gets the ORDINARY line — no empty hint."""
d1 = _doc("Alpha", "a/one.md", "One", "any text at all")
monkeypatch.setattr(agent, "all_documents", lambda db: [d1])
holder = AgentHolder()
llm = ScriptedLLM(
[ToolCallPiece(id="call_1", name="grep", arguments={"pattern": r".*"})],
[StreamPiece("content", "ans")],
)
asyncio.run(_run(llm, holder, _settings()))
assert llm.requests[1][0][3]["content"] == (
"No matches for '.*' in the knowledge base."
)
assert holder.tool_calls == 1
def test_no_matches_regex_templates_pin() -> None:
"""The teaching templates are verbatim pins (the model-facing copy —
the mock E2E keys off the plain-substring clause)."""
assert agent.NO_MATCHES_REGEX == (
"No matches for '{pattern}'. grep matches a plain substring "
"(case-insensitive), not a regex — '.*', '\\.' and the like are "
"literal text here, so that pattern can never match. Retry with "
"the plain text you expect to see (e.g. '{plain}')."
)
assert agent.NO_MATCHES_REGEX_SCOPED == (
"No matches for '{pattern}' in {source}/{path}. grep matches a "
"plain substring (case-insensitive), not a regex — retry with "
"the plain text you expect to see (e.g. '{plain}')."
)
def test_grep_scoped_to_one_document(monkeypatch: pytest.MonkeyPatch) -> None:
"""Scoped grep: only the named document is loaded (find_document on
the first-slash split), ``all_documents`` never runs, and the match
+227 -7
View File
@@ -133,6 +133,20 @@ def test_lexical_tsquery_stopwords_left_to_postgres() -> None:
assert lexical_tsquery("how do i") == "how | do | i"
def test_lexical_tsquery_dotted_tokens_kept_whole() -> None:
"""The 2026-09-05 incident: the default parser lexes dotted words
as ONE lexeme ("llama.cpp" → 'llama.cpp', "Qwen 3.8" → '3.8'), so
the query carries them whole — split tokens (llama | cpp) can never
match the document side."""
assert lexical_tsquery(
"What are the correct llama.cpp arguments for Qwen 3.8?"
) == "what | are | the | correct | llama.cpp | arguments | for | qwen | 3.8"
# The dash still splits (only dots group): ai | internal.network.
assert lexical_tsquery("how did I set up ai-internal.network?") == (
"how | did | i | set | up | ai | internal.network"
)
def test_fuse_combines_both_lists_for_double_hits() -> None:
v1 = _rc("a.md", cosine=0.9)
v2 = _rc("b.md", cosine=0.5)
@@ -196,7 +210,14 @@ def test_fuse_empty_lists() -> None:
# ---------------------------------------------------------------------------
from app.models import Chunk # noqa: E402
from app.rag.retriever import _lexical_candidates, _vector_candidates # noqa: E402
from app.rag.retriever import ( # noqa: E402
NAME_HIT_LIMIT,
_lexical_candidates,
_name_hit_chunks,
_normalize_name,
_vector_candidates,
name_hit_tokens,
)
class _FakeResult:
@@ -210,15 +231,29 @@ class _FakeResult:
class _FakeSession:
"""Returns canned rows from ``execute`` without touching Postgres."""
"""Returns canned rows from ``execute`` without touching Postgres.
def __init__(self, rows: list) -> None:
self._rows = rows
One list of rows (legacy form) is returned for EVERY call; several
lists (one per successive ``execute``) model a query sequence — the
name-hit lexical path (2026-09-05) issues the document-projection
query and, when hits exist, the LATERAL chunk query, BEFORE the FTS
query.
"""
def __init__(self, *rowsets: list) -> None:
if len(rowsets) == 1 and not (
rowsets[0] and isinstance(rowsets[0][0], list)
):
rowsets = (rowsets[0],) # the single-rowset legacy form
self._rowsets = rowsets
self._call = 0
self.statements: list = []
def execute(self, stmt, params: dict | None = None) -> _FakeResult:
self.statements.append((stmt, params))
return _FakeResult(self._rows)
rows = self._rowsets[min(self._call, len(self._rowsets) - 1)]
self._call += 1
return _FakeResult(rows)
def _chunk_row(is_summary: bool) -> Chunk:
@@ -286,9 +321,17 @@ def _lexical_row(is_summary: bool, doc_path: str) -> object:
def test_lexical_candidates_carry_is_summary_flag() -> None:
"""The lexical list reads ``c.is_summary`` from the raw row."""
"""The lexical list reads ``c.is_summary`` from the raw row.
The question carries no digit-bearing name token (no bare, no
numeric-join), so the name-hit path issues NO queries at all — the
single FTS rowset answers the only (FTS) call, and the list is the
plain FTS rows: the pre-name-hit behavior, unchanged.
"""
rows = [_lexical_row(True, "summary-src.yaml"), _lexical_row(False, "other.md")]
out = _lexical_candidates(_FakeSession(rows), "how do i configure the thing", limit=10) # pyright: ignore[reportArgumentType]
out = _lexical_candidates(
_FakeSession(rows), "how do i configure the thing", limit=10 # pyright: ignore[reportArgumentType]
)
assert len(out) == 2
by_path = {rc.document.path: rc for rc in out}
assert by_path["summary-src.yaml"].is_summary is True
@@ -323,3 +366,180 @@ def test_fuse_default_is_summary_stays_false_for_legacy_chunks() -> None:
out = fuse([_rc("a.md", cosine=0.8)], [_rc("b.md")], k=60)
assert len(out) == 2
assert all(rc.is_summary is False for rc in out)
# ---------------------------------------------------------------------------
# Name-hit lexical signal (the 2026-09-05 incident — the versioned-name
# case the default parser lexes incompatibly: "Qwen 3.8" → qwen/3/8 can
# never match a document's qwen3/8/27b tokens)
# ---------------------------------------------------------------------------
INCIDENT_QUESTION = "What are the correct llama.cpp arguments for Qwen 3.8?"
def test_normalize_name() -> None:
assert _normalize_name("Qwen 3.8") == "qwen38"
assert _normalize_name("qwen3.8-27b-juggernaut-vulkan") == "qwen3827bjuggernautvulkan"
assert _normalize_name("Mixed CASE-99") == "mixedcase99"
assert _normalize_name("!!!") == ""
def test_name_hit_tokens_incident_question() -> None:
"""The incident question yields EXACTLY the versioned join
``qwen38`` — the token the document names actually carry. Plain
prose words (``what``, ``llamacpp``, ``arguments``, ``server`` —
no digit) never name-match (the precision guard); the single
digits ("3", "8") and the bare "38" are < 4 chars; the
digit-leading ``38show`` boundary artifact is dropped."""
tokens = name_hit_tokens(INCIDENT_QUESTION)
assert tokens == ["qwen38"]
for absent in ("what", "qwen", "llamacpp", "arguments", "3", "8", "38", "38show", "server"):
assert absent not in tokens
def test_name_hit_tokens_no_digit_question_returns_empty() -> None:
"""A question with no digit-bearing token (bare or joined) yields
no name candidates — prose joins like ``correctllama`` never count."""
assert name_hit_tokens("what is the correct caddy config") == []
assert name_hit_tokens("a e i o u 3 8") == []
def test_name_hit_tokens_bare_digit_bearing_token() -> None:
"""A single written token that carries a digit (``1panel``) is a
name candidate on its own — no join needed."""
tokens = name_hit_tokens("what is my 1panel dashboard setup")
assert tokens == ["1panel"]
def _name_row(doc: Document) -> tuple:
"""One row of the name-hit document projection (catalog order)."""
return (doc.id, doc.source, doc.path, doc.title)
def _name_hit_lateral_row(doc: Document, is_summary: bool = False) -> SimpleNamespace:
"""One row of the name-hit LATERAL chunk query."""
return SimpleNamespace(
doc_id=doc.id,
source=doc.source,
path=doc.path,
full_path=doc.full_path,
title=doc.title,
doc_content=doc.content,
content_hash=doc.content_hash,
indexed_at=None,
chunk_id=uuid.uuid4(),
position=-1 if is_summary else 0,
content="summary chunk" if is_summary else "content chunk",
is_summary=is_summary,
)
def test_name_hit_chunks_no_tokens_skips_all_queries() -> None:
"""A question with no name tokens issues no queries at all."""
session = _FakeSession([]) # any call would surface a statement
assert _name_hit_chunks(session, "a e i o u 3 8") == [] # pyright: ignore[reportArgumentType]
assert session.statements == []
def test_name_hit_chunks_no_matching_doc_returns_empty() -> None:
"""Name tokens exist but no document name carries one: the
projection runs, the LATERAL fetch does not."""
doc = _doc("quadlets/other.container", "body")
name_rows = [_name_row(doc)]
session = _FakeSession(name_rows, [])
assert _name_hit_chunks(session, INCIDENT_QUESTION) == [] # pyright: ignore[reportArgumentType]
assert len(session.statements) == 1 # projection only — no LATERAL fetch
def test_name_hit_chunks_ranked_by_count_length_catalog() -> None:
"""A two-candidate question (``qwen38`` + ``1panel``): the document
whose name carries BOTH (2 matches, 12 total chars) leads; the two
single-match documents tie on (1, 6) and fall to catalog order
(``dashboards/1panel-notes.md`` before ``quadlets/qwen3.8…``).
Hits carry ``fts_hit=True`` (the A8 gate answers), ``cosine=0.0``,
and the summary flag of their representative chunk."""
both = _doc("dashboards/1panel-qwen3.8.md", "body", title="1Panel Qwen 3.8")
panel = _doc("dashboards/1panel-notes.md", "body")
q38 = _doc("quadlets/qwen3.8-27b-juggernaut-vulkan.container", "body")
name_rows = [_name_row(d) for d in (panel, both, q38)] # catalog order
question = "what are the correct llama.cpp arguments for qwen 3.8 and the 1panel dashboard?"
lateral_rows = [
_name_hit_lateral_row(q38, is_summary=True), # LATERAL may return any order
_name_hit_lateral_row(both),
_name_hit_lateral_row(panel),
]
session = _FakeSession(name_rows, lateral_rows)
out = _name_hit_chunks(session, question) # pyright: ignore[reportArgumentType]
assert [rc.document.path for rc in out] == [
"dashboards/1panel-qwen3.8.md", # 2 matched tokens — leads
"dashboards/1panel-notes.md", # (1, 6) — catalog order
"quadlets/qwen3.8-27b-juggernaut-vulkan.container", # (1, 6) — after
]
assert all(rc.fts_hit is True for rc in out) # the lexical signal
assert all(rc.cosine == 0.0 for rc in out) # no vector rank
assert all(rc.score == 0.0 for rc in out) # fuse fills the score
by_path = {rc.document.path: rc for rc in out}
assert by_path["quadlets/qwen3.8-27b-juggernaut-vulkan.container"].is_summary is True
assert by_path["quadlets/qwen3.8-27b-juggernaut-vulkan.container"].position == -1
assert by_path["dashboards/1panel-notes.md"].is_summary is False
def test_name_hit_chunks_capped_at_limit() -> None:
"""Twelve tied name hits (one matched token each) yield exactly
``NAME_HIT_LIMIT`` of them — catalog order (the deterministic
tie-break)."""
docs = [_doc(f"quadlets/m{i:02d}.container", "body") for i in range(12)]
for d in docs: # give every document a name that carries the token
d.title = "qwen38 model i"
name_rows = [_name_row(d) for d in docs]
# Only the ten winners (catalog order — the deterministic tie-break
# of the twelve identical scores) reach the LATERAL fetch; the fake
# answers with exactly those rows.
lateral_rows = [_name_hit_lateral_row(d) for d in docs[:NAME_HIT_LIMIT]]
session = _FakeSession(name_rows, lateral_rows)
out = _name_hit_chunks(
session, "tell me about the qwen 3.8 models" # pyright: ignore[reportArgumentType]
)
assert len(out) == NAME_HIT_LIMIT
assert [rc.document.path for rc in out] == [f"quadlets/m{i:02d}.container" for i in range(10)]
def test_lexical_candidates_name_hits_lead_and_dedupe_with_fts() -> None:
"""The full lexical list: name hits LEAD (their representative
chunks), the FTS rows follow, and an FTS row sharing the name hit's
chunk id appears exactly once (deduped)."""
q38 = _doc("quadlets/qwen3.8-27b-juggernaut-vulkan.container", "body")
other = _doc("quadlets/qwen38-other.container", "body") # 1 matched token
name_rows = [_name_row(q38), _name_row(other)]
q38_chunk = uuid.uuid4()
def _lateral(doc: Document) -> SimpleNamespace:
row = _name_hit_lateral_row(doc)
if doc is q38:
row.chunk_id = q38_chunk
return row
lateral_rows = [_lateral(q38), _lateral(other)]
fts_rows = [
# an FTS hit on the SAME chunk as the q38 name hit (deduped away)
SimpleNamespace(
chunk_id=q38_chunk, position=1, content="c", doc_id=q38.id,
source=q38.source, path=q38.path, full_path=q38.full_path,
title=q38.title, doc_content=q38.content, content_hash=q38.content_hash,
indexed_at=None, is_summary=False, rank=0.1,
),
# an FTS hit on a different chunk of the OTHER doc (kept)
_lexical_row(False, "quadlets/qwen38-other.container"),
]
session = _FakeSession(name_rows, lateral_rows, fts_rows)
out = _lexical_candidates(session, INCIDENT_QUESTION, limit=10) # pyright: ignore[reportArgumentType]
assert len(out) == 3 # q38 (once), other (name hit), other (FTS chunk)
# Both name hits tie on (1, 6) — catalog order: "qwen3." (ASCII 46)
# sorts before "qwen38" (ASCII 56).
assert out[0].document.path == "quadlets/qwen3.8-27b-juggernaut-vulkan.container"
assert out[1].document.path == "quadlets/qwen38-other.container"
assert out[0].chunk_id == q38_chunk # the name-hit representative row
assert {
rc.chunk_id for rc in out
} == {q38_chunk, fts_rows[1].chunk_id, lateral_rows[1].chunk_id}
assert all(rc.fts_hit is True for rc in out)