From 766702c750b36f69557992362c78cbc8f63be64f Mon Sep 17 00:00:00 2001 From: ducoterra Date: Sat, 5 Sep 2026 10:26:39 -0400 Subject: [PATCH] finally getting accurate answers --- .gitignore | 3 + app/rag/agent.py | 121 ++++- app/rag/prompts.py | 7 +- app/rag/retriever.py | 247 ++++++++++- tests/e2e/mock_llm.py | 161 +++++++ tests/e2e/test_grep_regex_teaching.py | 485 +++++++++++++++++++++ tests/integration/test_name_hit_lexical.py | 207 +++++++++ tests/unit/test_agent.py | 204 ++++++++- tests/unit/test_retriever.py | 234 +++++++++- 9 files changed, 1628 insertions(+), 41 deletions(-) create mode 100644 tests/e2e/test_grep_regex_teaching.py create mode 100644 tests/integration/test_name_hit_lexical.py diff --git a/.gitignore b/.gitignore index 96f7dee..d587ba0 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,6 @@ frontend/dist/ *.log .DS_Store Thumbs.db + +--- Local data dumps (KB replicas, pg_dumps) — never committed --- +data/ diff --git a/app/rag/agent.py b/app/rag/agent.py index af6ebf5..4666de0 100644 --- a/app/rag/agent.py +++ b/app/rag/agent.py @@ -66,6 +66,16 @@ task 04): each line truncated to 200 chars. A grep is a **locator**, not a context-adder: it never appends to the answer context (only ``read`` does — ``holder.read_docs`` is untouched by a grep). + The 2026-09-05 incident teaching (the harness prior is that grep + takes a REGEX; this grep is a fixed substring and the contract does + not change): when a grep RAN but found nothing and the pattern is + regex-shaped (:func:`looks_like_regex`) with a non-empty + :func:`plain_form`, the no-match result is the TEACHING line — + :data:`NO_MATCHES_REGEX` / :data:`NO_MATCHES_REGEX_SCOPED` — which + states the plain-substring contract and hands over the plain-form + retry hint; a grep that matched, or a no-match for a plain pattern + (or one that reduces to nothing), is byte-identical to the ordinary + line. Still a (counted) result, never a refusal. 3. Rejected calls get a one-line refusal and count in nothing (``holder.tool_calls`` tracks executed calls only): unknown tool name → ``"Unknown tool."``; a ``read`` without a usable ``path`` (missing, @@ -163,6 +173,7 @@ from __future__ import annotations import json import logging +import re from collections.abc import AsyncIterator, Sequence from dataclasses import dataclass, field from typing import Any, cast @@ -268,12 +279,16 @@ AGENT_TOOLS: list[dict[str, Any]] = [ "Search the indexed documents for an exact string " "(case-insensitive) and return up to 20 matching lines " "as `source/path:line: text` — a locator, not a " - "context-adder: read the winner with `read`. For a " - "normal search pass ONLY `pattern` — it searches every " - "document and that is how you search the knowledge " - "base; never pass a source name as `path` (a source " - "name is not a document). Call one tool at a time — " - "wait for this result before your next call." + "context-adder: read the winner with `read`. The " + "pattern is a plain substring, NEVER a regex — if a " + "pattern with regex syntax (like '.*' or '\\.') comes " + "back with no matches, retry with the plain text you " + "expect to see. For a normal search pass ONLY `pattern` " + "— it searches every document and that is how you " + "search the knowledge base; never pass a source name " + "as `path` (a source name is not a document). Call one " + "tool at a time — wait for this result before your " + "next call." ), "parameters": { "type": "object", @@ -282,7 +297,8 @@ AGENT_TOOLS: list[dict[str, Any]] = [ "type": "string", "description": ( "The exact text to search for (a plain " - "substring, not a regex)" + "substring, not a regex — no '.*', no " + "'\\.', no character classes)" ), }, "path": { @@ -417,6 +433,79 @@ SEARCH_LINE_LIMIT = 200 NO_MATCHES = "No matches for '{pattern}' in the knowledge base." NO_MATCHES_SCOPED = "No matches for '{pattern}' in {source}/{path}." +#: Teaching no-match lines (the 2026-09-05 incident — owner chat: +#: "Qwen 3.8" question failed repeatedly). The harness-trained prior is +#: that ``grep(pattern)`` takes a REGEX (pi.dev's grep, ripgrep, grep +#: itself); this app's grep is a case-insensitive FIXED SUBSTRING +#: (owner-locked A5 — the contract does not change). A regex-shaped +#: pattern (``qwen.*3\.8``, ``qwen 3\.8``) can therefore never match, and +#: the bare no-match line above made the turbo model trust the miss and +#: end the turn with a wrong "I searched the entire knowledge base" +#: refusal. These lines are the deterministic teaching: the same result +#: (a no-match is still a *result* — counted, no context added) with the +#: correct contract stated and a plain-text retry hint (the +#: :func:`plain_form` of the pattern, when non-empty). Deterministic only +#: (owner permission 2026-09-03: "deterministic guardrails only right +#: now"): no model participates in detection or repair. +NO_MATCHES_REGEX = ( + "No matches for '{pattern}'. grep matches a plain substring " + "(case-insensitive), not a regex — '.*', '\\.' and the like are " + "literal text here, so that pattern can never match. Retry with the " + "plain text you expect to see (e.g. '{plain}')." +) +NO_MATCHES_REGEX_SCOPED = ( + "No matches for '{pattern}' in {source}/{path}. grep matches a plain " + "substring (case-insensitive), not a regex — retry with the plain " + "text you expect to see (e.g. '{plain}')." +) + +#: One of these metacharacters anywhere in the pattern marks it +#: regex-shaped (the detection for the teaching no-match lines above). +#: Plain substrings that happen to contain one ("C++", "a|b") are +#: affected only on a NO-MATCH — a pattern that matched literally still +#: gets its ordinary result, so the teaching can never suppress a real +#: hit. +_REGEX_META_RE = re.compile(r"[*+?(){}\[\]|\\^$]") + + +# Backslash + a non-alphanumeric char is an escaped literal (``\\.`` → +# ``.``); backslash + an alphanumeric is a class shorthand (``\\d``, +# ``\\w``, ``\\s``) with no plain-text equivalent (→ dropped). +_ESCAPE_RE = re.compile(r"\\(.)") + + +def looks_like_regex(pattern: str) -> bool: + """True when *pattern* carries regex metacharacters (see + :data:`_REGEX_META_RE`). Pure detection — the grep itself stays a + fixed substring (owner-locked A5).""" + return _REGEX_META_RE.search(pattern) is not None + + +def plain_form(pattern: str) -> str: + """A deterministic plain-text retry hint for a regex-shaped pattern. + + The hint is the pattern reduced to literal text, in this order: + drop ``.*`` runs on the RAW pattern first (the wildcard — an escaped + ``\\.`` + ``\\*`` pair carries no raw ``.*`` run, so a literal dot + survives), unescape (``\\.`` → ``.``; class shorthands like ``\\d`` + dropped), keep only the FIRST ``|`` alternative, drop character + classes (``[0-9]``), drop quantifier runs (``*``, ``+``, ``?``, + ``{2,3}``), drop group parens and anchors (contents kept). Whitespace + is preserved. ``qwen.*3\\.8`` → ``qwen3.8`` (the incident's exact + recovery), ``llama\\.cpp`` → ``llama.cpp``, ``qwen[0-9]+`` → + ``qwen``. A pattern made of pure metacharacters reduces to ``""`` — + callers then fall back to the ordinary no-match line (no hint). + """ + p = re.sub(r"\.\*", "", pattern) # .* wildcard runs (raw form) + p = _ESCAPE_RE.sub(lambda m: m.group(1) if not m.group(1).isalnum() else "", p) + p = p.split("|", 1)[0] # first alternative only — a hint, not an answer + p = re.sub(r"\[[^\]]*\]", "", p) # character classes carry no literal text + p = re.sub(r"\{[^{}]*\}", "", p) # {n} / {n,} / {n,m} quantifiers + p = re.sub(r"[*+?]+", "", p) # stray * + ? quantifiers + p = p.replace("(", "").replace(")", "") + p = p.replace("^", "").replace("$", "") + return p.strip() + def list_catalog(db: Session) -> list[tuple[str, str, str]]: """Every indexed document as ``(source, path, title)``. @@ -694,6 +783,24 @@ def _execute_tool( # Locked A5: a grep never adds context — read_docs untouched. if not matches: shown = pattern[:100] # keep a long pattern short in the line + # The 2026-09-05 incident teaching: a regex-shaped pattern + # (the harness prior) can NEVER match a fixed-substring + # grep, so a no-match for one is not "the KB lacks this" — + # it is "the pattern was in the wrong form". Teach the + # contract and hand over the plain-form retry hint (when the + # reduction is non-empty); a plain pattern (or a pattern + # that reduces to nothing) keeps the ordinary line + # byte-identical. + plain = plain_form(pattern) if looks_like_regex(pattern) else "" + if plain: + if scoped_to is not None: + return NO_MATCHES_REGEX_SCOPED.format( + pattern=shown, + source=scoped_to[0], + path=scoped_to[1], + plain=plain[:100], + ) + return NO_MATCHES_REGEX.format(pattern=shown, plain=plain[:100]) if scoped_to is not None: # The scoped no-match line is keyed on the resolved # source/path (== the argument, stripped). diff --git a/app/rag/prompts.py b/app/rag/prompts.py index 0cb4412..be182a4 100644 --- a/app/rag/prompts.py +++ b/app/rag/prompts.py @@ -149,8 +149,11 @@ TOOLS_SECTION: str = ( "path (without the source name) will not resolve. `grep` locates an " "exact string (case-insensitive) in the indexed documents and " "returns up to 20 matching `source/path:line: text` lines — a " - "locator, not a context-adder: read the winner with `read`; for a " - "normal search pass only `pattern` — its optional `path` argument " + "locator, not a context-adder: read the winner with `read`. A grep " + "pattern is a plain substring, NEVER a regex — '.*' and '\\.' are " + "literal text there; if such a pattern returns no matches, retry " + "with the plain text you expect to see. For a normal search pass " + "only `pattern` — its optional `path` argument " "limits the search to one document you already know, by the same " "combined `source/path` string; never a source name — a bare " "document path (without the source name) will not resolve there " diff --git a/app/rag/retriever.py b/app/rag/retriever.py index 56c25fc..ec2187e 100644 --- a/app/rag/retriever.py +++ b/app/rag/retriever.py @@ -3,8 +3,27 @@ * **Vector list** — top-N chunks by cosine distance (``embedding <=> $1``), each carrying its cosine ``1 − distance`` (the honesty-gate input). * **Lexical list** — top-N chunks matching an OR-``tsquery`` over the - question's tokens, ordered by ``ts_rank``. This is what finds - name-your-tool questions ("gitlab") that vector similarity buries. + question's tokens, ordered by ``ts_rank``, UNION the name-hit list: + documents whose normalized name (title + path stem, alnum-only, + lowercased) contains a DIGIT-BEARING normalized question token of + length >= 4 — bare tokens (``1panel``) and the join of adjacent + question tokens starting with a letter ("Qwen 3.8" → ``qwen38``). + The digit requirement is the precision guard: plain prose words + ("server", "arguments" — 4+ chars but no digit) never name-match, + while versioned product names (the incident's whole point) always + carry one. The name-hit documents LEAD the lexical list (ranked by + match count, then total matched length, then catalog order; capped + at :data:`NAME_HIT_LIMIT`), + the FTS rows follow. This is what finds name-your-tool questions + ("gitlab") that vector similarity buries — and, the 2026-09-05 + incident, the versioned-name case the default parser lexes + incompatibly (question "Qwen 3.8" → ``qwen``/``3``/``8`` can never + match a document's ``qwen3``/``8``/``27b`` tokens, while every + unrelated llama.cpp quadlet out-ranks the target on the shared + ``llama``/``cpp`` tokens). The name-hit rows carry ``fts_hit=True`` + (they ARE the lexical signal — the A8 honesty gate then answers + instead of deflecting) and ``cosine=0.0``; the RRF fusion is + unchanged (same lists, same ``1/(k+rank)`` terms). * **Fusion** — Reciprocal Rank Fusion (``score = Σ 1/(k + rank)`` over the lists a chunk appears in; chunks hit by both lists get both terms). The fused score ranks; :meth:`select_documents` and :func:`weak_hit_titles` @@ -26,6 +45,7 @@ import re import uuid from collections.abc import Sequence from dataclasses import dataclass, replace +from pathlib import Path from sqlalchemy import select, text from sqlalchemy.orm import Session @@ -38,8 +58,78 @@ from app.models import Chunk, Document #: path never truncates (A7 revised, owner permission 2026-08-24). TRUNCATION_MARKER = "[…truncated…]" -#: Alphanumeric tokens of a question (``to_tsquery`` input, OR-joined). -_TOKEN_RE = re.compile(r"[a-z0-9]+") +#: Alphanumeric tokens of a question (``to_tsquery`` input, OR-joined), +#: with DOTTED tokens kept whole (``llama.cpp`` → ``llama.cpp``). The +#: default parser lexes a dotted word as ONE lexeme (``to_tsvector`` of +#: "llama.cpp" → ``'llama.cpp'``; of "Qwen 3.8" → ``'3.8'``), so a split +#: token (``llama`` | ``cpp``) can never match the document side — the +#: 2026-09-05 incident's question phrase "llama.cpp" lexed +#: incompatibly on both sides of the query. Dotted tokens are passed +#: through as single ``to_tsquery`` lexemes (verified: Postgres accepts +#: ``'llama.cpp'`` and ``'3.8'`` as lexemes). +_TOKEN_RE = re.compile(r"[a-z0-9]+(?:\.[a-z0-9]+)*") + +#: Name-hit policy (the 2026-09-05 incident): a question token shorter +#: than this (normalized) is too weak a name signal ("3", "8", "the") — +#: it would match half the KB by accident. +NAME_TOKEN_MIN_LEN = 4 + +#: The name-hit list may contribute at most this many documents to the +#: lexical side (before the FTS rows); the rest are dropped. +NAME_HIT_LIMIT = 10 + +#: Alphanumeric runs of a lowercased string (name normalization). +_ALNUM_RE = re.compile(r"[a-z0-9]+") + + +def _normalize_name(s: str) -> str: + """Lowercased, alnum-only form of *s* (``Qwen 3.8`` → ``qwen38``).""" + return "".join(_ALNUM_RE.findall(s.lower())) + + +def name_hit_tokens(question: str) -> list[str]: + """The normalized name-match candidates of one question. + + Only DIGIT-BEARING candidates count (the precision guard — a plain + prose word like "server" or "arguments" must never name-match a + document; a versioned product name always carries a digit): + + * the :func:`_normalize_name` form of every whitespace token, kept + when at least :data:`NAME_TOKEN_MIN_LEN` chars (``1panel``, + ``qwen38`` from a single written token); + * the versioned-name JOIN — the normalized concatenation of every + ADJACENT token pair whose SECOND token is purely numeric (a + version number: ``"… for Qwen 3.8"`` → ``Qwen`` + ``3.8`` joins + to ``qwen38``; prose joins like ``correct`` + ``llama`` and the + word-after-version boundaries like ``3.8`` + ``show`` are + dropped). A purely-numeric second token means the join always + starts with the first token's text (a letter in practice), so no + digit-leading artifact (``38show``) can survive. + + Order of first appearance, de-duplicated. + """ + tokens = question.split() + out: list[str] = [] + seen: set[str] = set() + + def add(tok: str) -> None: + if ( + len(tok) >= NAME_TOKEN_MIN_LEN + and any(ch.isdigit() for ch in tok) + and tok not in seen + ): + seen.add(tok) + out.append(tok) + + for i, raw in enumerate(tokens): + norm = _normalize_name(raw) + add(norm) + if i + 1 < len(tokens): + next_norm = _normalize_name(tokens[i + 1]) + if next_norm and next_norm.isdigit(): + add(norm + next_norm) # the versioned-name join + return out + #: One row of the lexical candidate query (all fields needed to build a #: detached :class:`Document` plus the chunk fields and ``ts_rank``). @@ -96,11 +186,14 @@ class RetrievedChunk: def lexical_tsquery(question: str) -> str | None: """OR-joined token string for ``to_tsquery('english', …)``, or ``None``. - Tokens are lowercased ``[a-z0-9]+`` runs, de-duplicated in order of - first appearance. Postgres does the lexing/stemming; a question whose - tokens are all stopwords lexes to an *empty* tsquery (which matches - nothing), so no special-casing is needed there. Pure-symbol questions - ("???", "🔧") yield no tokens → ``None`` → no lexical query at all. + Tokens are lowercased ``[a-z0-9]+`` runs with DOTTED words kept whole + (see :data:`_TOKEN_RE` — the default parser lexes ``llama.cpp`` as + one lexeme, so the query must carry it whole too), de-duplicated in + order of first appearance. Postgres does the lexing/stemming; a + question whose tokens are all stopwords lexes to an *empty* tsquery + (which matches nothing), so no special-casing is needed there. + Pure-symbol questions ("???", "🔧") yield no tokens → ``None`` → no + lexical query at all. """ seen: set[str] = set() tokens: list[str] = [] @@ -181,21 +274,141 @@ def _vector_candidates( ] -def _lexical_candidates(db: Session, question: str, limit: int) -> list[RetrievedChunk]: - """Top-*limit* chunks matching the question's OR-tsquery (A7). - - Ordered by ``ts_rank`` (with ``d.path, c.position`` as the - deterministic tie-break); an empty tsquery (stopword-only question) - simply matches nothing. +#: One name-hit row: the document fields (detached :class:`Document`) +#: plus the representative chunk — the ``is_summary`` chunk when the +#: document has one (its natural-language summary is the best chunk of +#: a machine file, and the chunk the vector list is likeliest to have +#: ranked too, so the RRF merge dedupes cleanly), else chunk 0. +_NAME_HIT_SQL = text( """ + SELECT d.id AS doc_id, + d.source AS source, + d.path AS path, + d.full_path AS full_path, + d.title AS title, + d.content AS doc_content, + d.content_hash AS content_hash, + d.indexed_at AS indexed_at, + c.id AS chunk_id, + c.position AS position, + c.content AS content, + c.is_summary AS is_summary + FROM documents d + LEFT JOIN LATERAL ( + SELECT id, position, content, is_summary + FROM chunks + WHERE document_id = d.id + ORDER BY is_summary DESC, position ASC + LIMIT 1 + ) c ON true + WHERE d.id = ANY(:ids) AND c.id IS NOT NULL + """ +) + + +def _name_hit_chunks(db: Session, question: str) -> list[RetrievedChunk]: + """The documents whose NAME matches the question (the 2026-09-05 + incident, the versioned-name signal). + + A document is a name hit when its normalized name — + :func:`_normalize_name` of its ``title`` followed by the normalized + path stem (``qwen3.8-27b-juggernaut-vulkan.container`` → + ``qwen3827bjuggernautvulkan``) — contains at least one + :func:`name_hit_tokens` candidate (``qwen38``). Ranked by + (distinct matched tokens, total matched length, source, path) — + catalog order is the final deterministic tie-break — capped at + :data:`NAME_HIT_LIMIT`. Each hit becomes one lexical + :class:`RetrievedChunk` (its representative chunk, ``fts_hit=True``, + ``cosine=0.0``). Two lightweight queries: one projection over + (id, source, path, title) in catalog order, one LATERAL chunk + fetch for the ≤ :data:`NAME_HIT_LIMIT` winners (no full-content + load; the content joins in via the row fetch below). + """ + tokens = name_hit_tokens(question) + if not tokens: + return [] + rows = db.execute( + select(Document.id, Document.source, Document.path, Document.title).order_by( + Document.source, Document.path + ) + ).all() + scored: list[tuple[int, int, uuid.UUID]] = [] + by_id: dict[uuid.UUID, tuple[str, str]] = {} # id -> (source, path) + for doc_id, source, path, title in rows: + by_id[doc_id] = (source, path) + name = _normalize_name(title) + _normalize_name(Path(path).stem) + matched = [t for t in tokens if t in name] + if matched: + scored.append((len(matched), sum(len(t) for t in matched), doc_id)) + if not scored: + return [] + # Ranked by (distinct matched tokens, total matched length), the + # deterministic catalog tie-break (source, path) last. + scored.sort( + key=lambda s: (-s[0], -s[1], by_id[s[2]][0], by_id[s[2]][1]) + ) + ids = [s[2] for s in scored[:NAME_HIT_LIMIT]] + hit_rows = list(db.execute(_NAME_HIT_SQL, {"ids": ids}).all()) + # The LATERAL query returns winners in id order; re-order by the + # ranked order computed above so the lexical list is deterministic. + order = {doc_id: rank for rank, doc_id in enumerate(ids)} + hit_rows.sort(key=lambda row: order[row.doc_id]) + out: list[RetrievedChunk] = [] + for row in hit_rows: + doc = Document( + id=row.doc_id, + source=row.source, + path=row.path, + full_path=row.full_path, + title=row.title, + content=row.doc_content, + content_hash=row.content_hash, + indexed_at=row.indexed_at, + ) + out.append( + RetrievedChunk( + chunk_id=row.chunk_id, + position=row.position, + content=row.content, + score=0.0, # filled in by :func:`fuse` + document=doc, + cosine=0.0, # no vector rank — name-only hit + fts_hit=True, # the lexical signal — the A8 gate answers + is_summary=bool(row.is_summary), + ) + ) + return out + + +def _lexical_candidates(db: Session, question: str, limit: int) -> list[RetrievedChunk]: + """The lexical candidate list: name hits, then FTS rows (A7). + + The name-hit documents (:func:`_name_hit_chunks`, the versioned-name + signal) LEAD the list — they are the strongest lexical evidence for + a name-your-thing question — followed by the top-*limit* chunks + matching the question's OR-tsquery, ordered by ``ts_rank`` (with + ``d.path, c.position`` as the deterministic tie-break). Deduped by + chunk id (a name-hit representative chunk the FTS list also ranked + appears once). An empty tsquery (stopword-only question) simply + contributes no FTS rows; a question with no name tokens contributes + no name hits — both halves are independent. + """ + out: list[RetrievedChunk] = [] + seen: set[uuid.UUID] = set() + for rc in _name_hit_chunks(db, question): + if rc.chunk_id not in seen: + seen.add(rc.chunk_id) + out.append(rc) tsquery = lexical_tsquery(question) if tsquery is None: - return [] + return out rows = db.execute( _LEXICAL_SQL, {"tsquery": tsquery, "limit": limit} ).all() - out: list[RetrievedChunk] = [] for row in rows: + if row.chunk_id in seen: + continue + seen.add(row.chunk_id) doc = Document( id=row.doc_id, source=row.source, diff --git a/tests/e2e/mock_llm.py b/tests/e2e/mock_llm.py index 346512c..e660f28 100644 --- a/tests/e2e/mock_llm.py +++ b/tests/e2e/mock_llm.py @@ -183,6 +183,40 @@ Implements just enough of the aipi surface: phrases are disjoint substrings — the phase-71 ordering convention); no existing E2E question or fixture file contains the phrase, so every other suite is unaffected. + - user message containing ``what are the correct llama.cpp + arguments`` (``GREP_TEACH_TRIGGER``, the 2026-09-05 incident — + the harness prior is that grep takes a REGEX; this app's grep is a + case-insensitive fixed substring, owner-locked A5) **and** the + system prompt carries the ```` section -> the deterministic + GREP-REGEX-TEACHING flow, discriminated statelessly from the + messages (streaming only): + * request 1 (``tools`` offered, no ``tool``-role result yet): + stream ONLY ``tool_calls`` deltas — ``grep`` with + ``{"pattern": GREP_TEACH_PATTERN}`` (``qwen.*3\\.8``, id + ``call_0``) — the incident's regex-shaped first grep, which a + fixed-substring grep can NEVER match; + * request 2 (the last tool result is the server's TEACHING + no-match line — it carries ``GREP_TEACH_MARKER``): + ``grep`` with the plain form ``GREP_TEACH_PLAIN`` + (``qwen3.8``, id ``call_1``) — the one-round correction; + * request 3 (the last tool result carries + ``source/path:line: text`` match lines): ``read`` the FIRST + match line's document by its combined ``source/path`` (id + ``call_2``); + * request 4 (the last tool result is a read result, the + ``"Document :\n"`` shape): the + deterministic echo answer ``Read . ``, ``finish_reason: "stop"`` — the loop ended in ONE + correction, not at the round cap. + * A PLAIN no-match as the last result (no match line, no + teaching marker — e.g. the plain pattern genuinely absent) is + the deterministic terminal answer ``No matches — the knowledge + base has no such text.`` (the flow cannot loop on a + well-formed pattern). + Checked BEFORE the SEARCH / TOOLS_TRIGGER flows (disjoint trigger + phrases — the phase-72 ordering convention); no existing E2E + question or fixture file contains the phrase, so every other suite + is unaffected. - user message containing ``show me a table`` (phase 44, markdown tables, TODO.md L6) -> the fixed table answer (``TABLE_ANSWER``): a 3-column service table, an ```` XSS probe line, and @@ -483,6 +517,34 @@ assert _CORRECTION_MARKER in CORRECTION_INSTRUCTION, ( #: file contains the phrase, so every other suite is unaffected. LS_TEACH_TRIGGER = "list the files in this directory" +#: The deterministic GREP-REGEX-TEACH flow (the 2026-09-05 "Qwen 3.8" +#: incident — the harness prior is that grep takes a REGEX; this app's +#: grep is a case-insensitive fixed substring, owner-locked A5, so a +#: regex-shaped pattern can NEVER match, and the bare no-match line +#: made the turbo model trust the miss and end the turn with a wrong +#: "I searched the entire knowledge base" refusal). The flow pins the +#: self-correction on the SSE wire: the regex-shaped first grep → the +#: server's TEACHING no-match line (``GREP_TEACH_MARKER``) → the +#: plain-form retry grep → the match → the read → the deterministic +#: echo answer. Checked BEFORE the SEARCH / TOOLS_TRIGGER flows +#: (disjoint trigger phrases — the phase-71/72 ordering convention); +#: verified: no existing E2E question or fixture file contains the +#: phrase, so every other suite is unaffected. +GREP_TEACH_TRIGGER = "what are the correct llama.cpp arguments" + +#: The incident's regex-shaped first grep (it can never match a +#: fixed-substring grep — that is the point of the flow). +GREP_TEACH_PATTERN = "qwen.*3\\.8" + +#: The plain-form retry — the server's teaching line hands over exactly +#: this hint (``app.rag.agent.plain_form(GREP_TEACH_PATTERN)``). +GREP_TEACH_PLAIN = "qwen3.8" + +#: The marker of the agent's teaching no-match line (app.rag.agent +#: ``NO_MATCHES_REGEX`` / ``NO_MATCHES_REGEX_SCOPED``) — the mock's +#: plain step keys on it (a plain no-match line carries it not). +GREP_TEACH_MARKER = "grep matches a plain substring" + #: The agent's ``ls`` listing header (app.rag.agent ``_execute_tool``): #: ``"N documents:"`` — the first line of every catalog tool result. _CATALOG_HEADER_RE = re.compile(r"^\d+ documents:") @@ -836,6 +898,61 @@ def _ls_teach_flow(body: dict[str, Any]) -> tuple[str, ...] | None: return ("misuse",) +def _grep_teach_flow(body: dict[str, Any]) -> tuple[str, ...] | None: + """Classify a GREP-TEACH request (the 2026-09-05 incident — see the + ``GREP_TEACH_*`` constants). Stateless over the messages, like the + other marker flows: + + * ``("regex",)`` — ``tools`` are offered and no ``tool``-role result + is in the messages yet: the incident's regex-shaped first grep — + ``grep`` with ``{"pattern": GREP_TEACH_PATTERN}`` (id ``call_0``). + * ``("plain",)`` — the LAST tool result is the server's TEACHING + no-match line (it carries ``GREP_TEACH_MARKER``): the one-round + correction — ``grep`` with the plain form (id ``call_1``). + * ``("read", combined, "call_2")`` — the last tool result carries + ``source/path:line: text`` match lines: ``read`` the FIRST match + line's document by its combined ``source/path`` identity. + * ``("answer", combined, content)`` — the last tool result is a + read result (``"Document :\n"``): the + deterministic echo answer ``Read . ``. + * ``("nomatch",)`` — the last tool result is a PLAIN no-match (no + match line, no teaching marker): the deterministic terminal + ``No matches — the knowledge base has no such text.`` answer. + * ``None`` — not the flow: the trigger is absent, the ```` + section is missing (deflected turns never carry it), or ``tools`` + are not offered and no tool results are in the messages yet (e.g. + ``agent_max_rounds=0``). + """ + user = _user(body).lower() + if GREP_TEACH_TRIGGER not in user: + return None + if "" not in _system(body): + return None + results = [ + str(m.get("content") or "") + for m in _messages(body) + if m.get("role") == "tool" + ] + if not results: + if not body.get("tools"): + return None + return ("regex",) + last = results[-1] + if last.startswith(_READ_RESULT_PREFIX): + head, _, content = last.partition("\n") + # The read result is ``"Document :\n"`` — the + # head carries the server's appended ``:`` (removed here; a + # document path never legitimately ends with one). + return ("answer", head[len(_READ_RESULT_PREFIX):].removesuffix(":"), content) + if GREP_TEACH_MARKER in last: + return ("plain",) + for line in last.splitlines(): + m = _SEARCH_LINE_RE.match(line) + if m: + return ("read", m.group("sp"), "call_2") + return ("nomatch",) + + def long_answer() -> str: """~900-word deterministic walkthrough (phase 11): numbered steps plus a unique final line that must survive the stream untruncated.""" @@ -1307,6 +1424,50 @@ def chat_completions(body: dict[str, Any]) -> Any: media_type="text/event-stream", headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, ) + # 2026-09-05 (the "Qwen 3.8" incident — the grep regex prior): + # the deterministic GREP-TEACH self-correction flow — checked + # BEFORE the SEARCH / TOOLS_TRIGGER flows (disjoint trigger + # phrases — the phase-72 ordering convention; the trigger needs + # the ```` section, so deflected turns never hit it). + grep_teach = _grep_teach_flow(body) + if grep_teach is not None: + if grep_teach[0] == "regex": + # The incident's misuse, deterministic: the regex-shaped + # pattern (it can never match a fixed-substring grep). + stream = _tool_call_stream( + "grep", {"pattern": GREP_TEACH_PATTERN}, "call_0" + ) + elif grep_teach[0] == "plain": + # The one-round correction: the plain-form retry (the + # teaching line handed over exactly this hint). + stream = _tool_call_stream( + "grep", {"pattern": GREP_TEACH_PLAIN}, "call_1" + ) + elif grep_teach[0] == "read": + stream = _tool_call_stream( + "read", {"path": grep_teach[1]}, grep_teach[2] + ) + elif grep_teach[0] == "nomatch": + stream = _sse_stream( + _apply_max_tokens( + "No matches — the knowledge base has no such text.", + body.get("max_tokens"), + ), + 0.0, + ) + else: # "answer" — quote the read document (first 80 chars) + stream = _sse_stream( + _apply_max_tokens( + f"Read {grep_teach[1]}. {grep_teach[2][:80]}", + body.get("max_tokens"), + ), + 0.0, + ) + return StreamingResponse( + stream, + media_type="text/event-stream", + headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, + ) # Phase 68 (search tool): the deterministic search marker flow — # checked BEFORE the phase-37 tool flow (the more specific # trigger phrase wins, same convention as THINK_PARAS_TRIGGER). diff --git a/tests/e2e/test_grep_regex_teaching.py b/tests/e2e/test_grep_regex_teaching.py new file mode 100644 index 0000000..51b2607 --- /dev/null +++ b/tests/e2e/test_grep_regex_teaching.py @@ -0,0 +1,485 @@ +"""The 2026-09-05 incident E2E (Playwright, mock-only): the +grep-regex-teaching self-correction loop through the real UI. + +Context: the sample question "What are the correct llama.cpp arguments +for Qwen 3.8?" failed repeatedly against the live KB. The harness +prior is that ``grep(pattern)`` takes a REGEX (pi.dev's grep, ripgrep, +grep itself); this app's grep is a case-insensitive fixed substring +(owner-locked A5 — the contract does not change). A regex-shaped +first grep (``qwen.*3\\.8``) can therefore NEVER match, and the bare +"no matches" result made the turbo model trust the miss and end the +turn with a wrong "I searched the entire knowledge base" refusal. +The fix under test (``app.rag.agent``): a no-match for a regex-shaped +pattern is the TEACHING line (``NO_MATCHES_REGEX`` / +``NO_MATCHES_REGEX_SCOPED``) — the plain-substring contract stated, +the ``plain_form`` retry hint handed over — so the model self-corrects +in one round. + +Run in isolation (DB must be up: ``podman compose up -d db``): + + uv run pytest tests/e2e/test_grep_regex_teaching.py -v --no-cov + +MOCK-ONLY suite: ``E2E_REAL_LLM=1`` is not supported — the gate is the +deterministic GREP-TEACH flow in ``tests/e2e/mock_llm.py`` +(``GREP_TEACH_TRIGGER`` — "what are the correct llama.cpp arguments" +— + the HIGH prompt's ```` section): the incident's regex- +shaped first grep (``grep`` with ``{"pattern": "qwen.*3\\.8"}``, id +``call_0``) → the agent's TEACHING no-match line (the mock's plain +step keys on the marker ``grep matches a plain substring``) → the +plain-form retry (``grep`` with ``{"pattern": "qwen3.8"}``, id +``call_1``) → the match → the ``read`` of the first match line's +document (id ``call_2``) → the deterministic echo answer. + +KB fixture (TRUNCATE-then-seed, house pattern): ONE source with TWO +documents of known ``source``/``path``/``title`` (catalog order = +``(source, path)``, so the first match line is deterministic): + +* ``Homelab/llama-server-args.md`` — the CATALOG-FIRST document, + indexed WITHOUT chunks (catalog-only; never in the retrieval + context, so the flow's ``read`` of it is NOT deduped as + already-in-context). It carries the literal line the plain grep + finds (``qwen3.8-27b`` — NOT the regex-shaped ``qwen.*3\\.8``, + which can never match a fixed-substring grep) and the search-flow + sentinel line (the no-regression turn). Its FIRST line is longer + than 80 chars, so the mock's first-80-chars quote stays + newline-free. +* ``Homelab/ai-stack-notes.md`` — the retrievable document: one chunk + whose embedding is the mock's own bag-of-words vector (the trigger + question cosines well past the E2E 0.30 threshold → grounded, the + ```` section rides along). It carries NO ``qwen3.8`` line + (the plain grep's match is the catalog-first document alone) and + its name carries no name-hit token (it stays the vector seed only). + +Test → phase mapping (Playwright Mapping Rule): +1. ``test_regex_grep_self_corrects_to_plain_form`` — the grounded + GREP-TEACH turn: the turn settles (composer re-enables, ``done`` + observed), the answer bubble carries the read document's echo + (``Read Homelab/llama-server-args.md. ``), the UI + shows the three tool lines (two ``Searching for`` lines + the + ``Reading`` line), and no error banner. Wire level: the ``tool`` + frames arrive in order — ``grep`` ``qwen.*3\\.8`` → ``grep`` + ``qwen3.8`` → ``read`` the combined identity — and there is NO + fourth ``tool`` frame (the loop ended in one correction, not at + the round cap). +2. ``test_plain_search_flow_not_swallowed_by_new_trigger`` — in the + SAME session, the GREP-TEACH turn settles and a follow-up question + carrying ``SEARCH_TRIGGER`` still settles with the search flow's + ``Found …`` answer (the new flow did not swallow the existing + trigger). +""" +from __future__ import annotations + +import hashlib +import json +import re +import time +from datetime import UTC, datetime +from pathlib import Path + +from playwright.sync_api import Page, expect +from sqlalchemy import text +from sqlalchemy.orm import Session + +from app.db import SessionLocal +from app.models import Chunk, Document +from tests.e2e.mock_llm import ( + GREP_TEACH_MARKER, + GREP_TEACH_PATTERN, + GREP_TEACH_PLAIN, + GREP_TEACH_TRIGGER, + SEARCH_PATTERN, + SEARCH_TRIGGER, + embed_text, +) + +REPO = Path(__file__).resolve().parents[2] + +# -------------------------------------------------------------------------- +# The one-source, two-document fixture (see the module docstring) +# -------------------------------------------------------------------------- + +SEED_SOURCE = "Homelab" +DOC1_PATH = "llama-server-args.md" +DOC1_TITLE = "Llama Server Args" +DOC1_SP = f"{SEED_SOURCE}/{DOC1_PATH}" + +DOC2_PATH = "ai-stack-notes.md" +DOC2_TITLE = "AI Stack Notes" +DOC2_SP = f"{SEED_SOURCE}/{DOC2_PATH}" + +#: The catalog-first document (catalog order = (source, path) — DOC1 +#: sorts before DOC2): the plain grep's ONLY match, and the document +#: the flow reads (so it must NOT be the seed — a seed read dedupes to +#: "Already in your context.", which the mock flow does not model). +#: Indexed WITHOUT chunks: catalog-only, never in the retrieval +#: context. Line 1 (the read quote) is >80 chars and newline-free; it +#: carries the literal ``qwen3.8-27b`` text (NEVER the regex-shaped +#: ``qwen.*3\.8`` — that is the whole point of the incident). +DOC1_CONTENT = ( + "The llama.cpp server launch line for the qwen3.8-27b juggernaut " + "deployment pins the sampling and speculative-decoding flags.\n" + "Server command (verbatim): --port 8000 -ctk q8_0 -ctv q8_0 " + "--kv-unified -fa on --n-gpu-layers all --jinja.\n" + "Model file: /models/qwen3.8-27b/Qwen3.8-27B-UD-Q6_K.gguf with " + "the mmproj-BF16.gguf projector and the custom jinja template.\n" + f"Regression sentinel line: {SEARCH_PATTERN} must stay findable " + "by the plain search flow.\n" +) +assert "\n" not in DOC1_CONTENT[:80] # the read quote stays one line +assert GREP_TEACH_PLAIN in DOC1_CONTENT # the plain grep matches DOC1 +assert GREP_TEACH_PATTERN not in DOC1_CONTENT # the regex never matches +assert GREP_TEACH_MARKER not in DOC1_CONTENT # the marker stays tool-side + +#: The retrievable document (the grounded seed context): repeated lines +#: carry the trigger question's key tokens (llama, cpp, arguments, +#: qwen) — well past the E2E 0.30 cosine threshold — but NO literal +#: ``qwen3.8`` line (the plain grep's match stays DOC1 alone) and the +#: name carries no name-hit token (the seed stays the vector side +#: only). FIRST line >80 chars, newline-free (the seed context stays +#: one clean line). +DOC2_CONTENT = ( + "Notes on the self-hosted ai stack: the llama cpp server arguments " + "for every qwen model are kept next to the quadlet files.\n" + + ( + "The llama cpp server arguments — sampling, context, kv cache " + "quantization — are documented per model in the quadlet notes.\n" + ) + * 10 + + "\n## Server notes\n\n" + "Every qwen deployment shares the same llama cpp sampling " + "defaults; the per-model file overrides the speculative flags.\n" +) +assert "\n" not in DOC2_CONTENT[:80] +assert GREP_TEACH_PLAIN not in DOC2_CONTENT # the match stays DOC1 alone + +#: Carries ``GREP_TEACH_TRIGGER`` (the incident's sample question, +#: near-verbatim) and nothing else — no other mock marker. +GREP_TEACH_QUESTION = ( + "What are the correct llama.cpp arguments for Qwen 3.8? Show me " + "the exact server launch line from my notes." +) +assert GREP_TEACH_TRIGGER in GREP_TEACH_QUESTION.lower() +for _other in ( + "use your tools", + "read two documents", + "search your documents", + "list the files in this directory", + "emit raw tool markup", + "always emit raw tool markup", + "show me a table", + "think in paragraphs", + "think out loud then hesitate", + "think out loud", + "show the end of your notes", + "write a long answer", + "fail then answer", + "always fail", + "embed fail once", + "pretend to think slowly", +): + assert _other not in GREP_TEACH_QUESTION.lower(), _other + +#: Carries ``SEARCH_TRIGGER`` (the phase-68 search flow) and nothing +#: else — the no-regression follow-up question in the same session. +SEARCH_QUESTION = ( + "Search your documents for the reese-sentinel-42 marker and tell " + "me the line that carries it." +) +assert SEARCH_TRIGGER in SEARCH_QUESTION.lower() +for _other in ( + GREP_TEACH_TRIGGER, + "use your tools", + "read two documents", + "list the files in this directory", + "emit raw tool markup", + "always emit raw tool markup", + "show me a table", + "think in paragraphs", + "think out loud then hesitate", + "think out loud", + "show the end of your notes", + "write a long answer", + "fail then answer", + "always fail", + "embed fail once", + "pretend to think slowly", +): + assert _other not in SEARCH_QUESTION.lower(), _other + +#: The mock's deterministic read echo (the read document reached the +#: model and landed in the answer) — the flow reads DOC1 (the plain +#: grep's first — only — match line). +READ_ANSWER_PREFIX = f"Read {DOC1_SP}." +READ_ANSWER_QUOTE = DOC1_CONTENT[:80] + + +def _seed_fixture(db: Session) -> None: + """The one-source, two-document fixture (see the module docstring). + + DOC1 (catalog-first, the grep match, the read target) is indexed + WITHOUT chunks; DOC2 carries the single chunk (the mock's own + embedding → the trigger question cosines well past the E2E 0.30 + threshold → grounded, the ```` section rides along). + """ + db.add( + Document( + source=SEED_SOURCE, + path=DOC1_PATH, + full_path=f"/tmp/{DOC1_PATH}", + title=DOC1_TITLE, + content=DOC1_CONTENT, + content_hash=hashlib.sha256(DOC1_CONTENT.encode()).hexdigest(), + indexed_at=datetime.now(UTC), + ) + ) + doc2 = Document( + source=SEED_SOURCE, + path=DOC2_PATH, + full_path=f"/tmp/{DOC2_PATH}", + title=DOC2_TITLE, + content=DOC2_CONTENT, + content_hash=hashlib.sha256(DOC2_CONTENT.encode()).hexdigest(), + indexed_at=datetime.now(UTC), + ) + db.add(doc2) + db.flush() + db.add( + Chunk( + document_id=doc2.id, + position=0, + content=DOC2_CONTENT, + embedding=embed_text(DOC2_CONTENT), + ) + ) + + +def _reset_db_fixture() -> None: + """Truncate the KB (plus the prompt-shaping tables), then seed the + one-source, two-document fixture. ``steering_notes`` / + ``kb_overview`` are truncated too, so the HIGH prompt is exactly + ```` + ```` + ```` — byte-stable + prompts, byte-stable answers.""" + with SessionLocal() as db: + db.execute( + text("TRUNCATE chunks, documents, query_log, steering_notes, kb_overview") + ) + db.commit() + _seed_fixture(db) + db.commit() + + +# -------------------------------------------------------------------------- +# Page helpers (the house pattern — cf. test_tool_path_teaching.py) +# -------------------------------------------------------------------------- + +#: Captures the raw SSE ``data:`` payloads of the /api/chat stream +#: (a response clone read in the background) — wire-level assertions +#: for the ``tool`` frames, independent of the UI rendering. +SSE_HOOK = """ +() => { + if (window.__sseInstalled) return; + window.__sseInstalled = true; + window.__sseFrames = []; + const origFetch = window.fetch; + window.fetch = async function (...args) { + const res = await origFetch.apply(this, args); + try { + const url = typeof args[0] === 'string' ? args[0] : args[0].url; + if (url.includes('/api/chat')) { + res.clone().text().then((bodyText) => { + for (const block of bodyText.split('\\n\\n')) { + const line = block.trim(); + if (line.startsWith('data: ')) { + window.__sseFrames.push(line.slice(6)); + } + } + }); + } + } catch (e) { /* non-clonable responses: ignored */ } + return res; + }; +} +""" + + +def _install_sse_hook(page: Page) -> None: + page.evaluate(SSE_HOOK) + + +def _drain_frames(page: Page) -> list[dict]: + """One turn's SSE frames: wait for that turn's ``done`` frame, then + return EVERY frame captured since the last drain (the hook's + background read appends the whole stream at once after it closes, so + clearing-and-reading is race-free per turn).""" + deadline = time.monotonic() + 10.0 + while True: + raw = page.evaluate( + "() => { const f = window.__sseFrames || []; " + "window.__sseFrames = []; return f; }" + ) + parsed = [json.loads(line) for line in raw if line] + if any(f.get("type") == "done" for f in parsed): + return parsed + if time.monotonic() > deadline: + raise AssertionError( + f"SSE hook captured no `done` frame (frames so far: " + f"{len(parsed)}) — hook install failed?" + ) + time.sleep(0.05) + + +def _tool_frames(frames: list[dict]) -> list[dict]: + return [f for f in frames if f.get("type") == "tool"] + + +def _submit(page: Page, question: str) -> None: + page.fill("#message-input", question) + page.click("#send-btn") + expect(page.locator(".msg.user .bubble").last).to_contain_text(question) + + +def _wait_settled(page: Page) -> None: + """The turn is complete: answer text in the bubble, button recovered. + + Phase 48: the label assertion carries the settle wait with an + explicit timeout — the in-flight button is the enabled Stop control + (never disabled), so ``to_be_enabled`` no longer blocks until the + turn settles.""" + expect(page.locator(".msg.brain .bubble").last).not_to_have_text("", timeout=30_000) + expect(page.locator("#send-btn")).to_be_enabled(timeout=30_000) + expect(page.locator("#send-label")).to_have_text("Send", timeout=30_000) + + +def _assert_no_error_banner(page: Page) -> None: + """The turn settled through the normal done path — never the red + role=alert error banner (the KB-offline banner is a separate, + health-driven state the db_ready fixture keeps away).""" + banner = page.locator("#kb-banner") + expect(banner).to_be_hidden() + expect(banner).not_to_have_attribute("role", "alert") + expect(banner).not_to_have_class(re.compile(r"is-error")) + + +# -------------------------------------------------------------------------- +# 1. The grounded GREP-TEACH turn: the incident's regex-shaped first +# grep → the teaching no-match line → the plain-form retry → the +# match → the read → the echo answer — the loop settles in ONE +# correction (three tool rounds), pinned on the SSE wire +# -------------------------------------------------------------------------- + + +def test_regex_grep_self_corrects_to_plain_form( + page: Page, app_url: str, mock_llm: int, db_ready: None +) -> None: + page.set_default_timeout(30_000) + _reset_db_fixture() + page.goto(app_url) + _install_sse_hook(page) + + _submit(page, GREP_TEACH_QUESTION) + _wait_settled(page) + + # Self-correction: the answer quotes the READ document — the plain + # grep's match was located, the document reached the model, and the + # echo landed in the answer (the wrong-deflection end state — no + # tool success, an "I searched everything" refusal — is impossible + # on this wire: the done frame below is not deflected). + bubble = page.locator(".msg.brain .bubble").last + expect(bubble).to_contain_text(READ_ANSWER_PREFIX) + expect(bubble).to_contain_text(READ_ANSWER_QUOTE) + _assert_no_error_banner(page) + + # The UI shows the three tool lines in order: the regex-shaped + # first grep (Searching for qwen.*3\.8), the + # plain-form retry (Searching for qwen3.8), the read + # of the combined identity. + lines = page.locator(".msg.brain .tool-call") + expect(lines).to_have_count(3) + expect(lines.nth(0)).to_contain_text("Searching for") + expect(lines.nth(0).locator("code")).to_have_text(GREP_TEACH_PATTERN) + expect(lines.nth(1)).to_contain_text("Searching for") + expect(lines.nth(1).locator("code")).to_have_text(GREP_TEACH_PLAIN) + expect(lines.nth(2)).to_contain_text("Reading") + expect(lines.nth(2).locator("code")).to_have_text(DOC1_SP) + + # Three rounds on the wire: the tool frames arrive in order — + # grep qwen.*3\.8 (the incident's regex-shaped first call), grep + # qwen3.8 (the plain-form correction the teaching line triggered), + # read the combined identity — and there is NO fourth tool frame: + # the loop ended in one correction, not at the round cap. + frames = _drain_frames(page) + assert _tool_frames(frames) == [ + {"type": "tool", "name": "grep", "argument": GREP_TEACH_PATTERN}, + {"type": "tool", "name": "grep", "argument": GREP_TEACH_PLAIN}, + {"type": "tool", "name": "read", "argument": DOC1_SP}, + ] + first_delta = next(i for i, f in enumerate(frames) if f.get("type") == "delta") + assert all( + i < first_delta for i, f in enumerate(frames) if f.get("type") == "tool" + ) + done = next(f for f in frames if f.get("type") == "done") + assert done["deflected"] is False + assert not [f for f in frames if f.get("type") == "error"] + + +# -------------------------------------------------------------------------- +# 2. No regression to the plain search flow — the SAME session: after +# the GREP-TEACH turn, the SEARCH_TRIGGER follow-up (the phase-68 +# search flow) still settles with the "Found …" answer +# -------------------------------------------------------------------------- + + +def test_plain_search_flow_not_swallowed_by_new_trigger( + page: Page, app_url: str, mock_llm: int, db_ready: None +) -> None: + page.set_default_timeout(30_000) + _reset_db_fixture() + page.goto(app_url) + _install_sse_hook(page) + + # Turn 1 — the GREP-TEACH flow (the incident's regex-shaped first + # grep → the teaching line → the plain-form retry → the read → the + # echo answer). + _submit(page, GREP_TEACH_QUESTION) + _wait_settled(page) + teach_frames = _drain_frames(page) + assert _tool_frames(teach_frames) == [ + {"type": "tool", "name": "grep", "argument": GREP_TEACH_PATTERN}, + {"type": "tool", "name": "grep", "argument": GREP_TEACH_PLAIN}, + {"type": "tool", "name": "read", "argument": DOC1_SP}, + ] + expect( + page.locator(".msg.brain .bubble").last + ).to_contain_text(READ_ANSWER_PREFIX) + + # Turn 2 — the SAME session: the phase-68 search flow on + # SEARCH_TRIGGER. The new flow must not have swallowed the existing + # trigger: the follow-up settles with the search flow's answer + # (grep the sentinel → "Found "). + _submit(page, SEARCH_QUESTION) + _wait_settled(page) + + second_msg = page.locator(".msg.brain").last + lines = second_msg.locator(".tool-call") + expect(lines).to_have_count(1) + expect(lines.nth(0)).to_contain_text("Searching for") + expect(lines.nth(0).locator("code")).to_have_text(SEARCH_PATTERN) + + # The answer is the search flow's echo: "Found " — the sentinel line from DOC1. + sentinel_line = next( + line for line in DOC1_CONTENT.splitlines() if SEARCH_PATTERN in line + ) + bubble = second_msg.locator(".bubble").last + expect(bubble).to_contain_text(f"Found {sentinel_line[:80]}") + _assert_no_error_banner(page) + + # Wire level for the follow-up: one grep of the sentinel pattern — + # the search flow, unchanged. + frames = _drain_frames(page) + assert _tool_frames(frames) == [ + {"type": "tool", "name": "grep", "argument": SEARCH_PATTERN}, + ] + done = next(f for f in frames if f.get("type") == "done") + assert done["deflected"] is False + assert not [f for f in frames if f.get("type") == "error"] diff --git a/tests/integration/test_name_hit_lexical.py b/tests/integration/test_name_hit_lexical.py new file mode 100644 index 0000000..844cdaa --- /dev/null +++ b/tests/integration/test_name_hit_lexical.py @@ -0,0 +1,207 @@ +"""Integration: the name-hit lexical signal against real Postgres (the +2026-09-05 "Qwen 3.8" incident). + +The unit suite (``tests/unit/test_retriever.py``) covers the pure +mapping with fake rows; this suite covers the SQL side on real +Postgres: the document-projection scan, the LATERAL representative- +chunk fetch (the ``is_summary`` chunk wins, chunk 0 otherwise, and a +chunk-less name match is EXCLUDED — the ``c.id IS NOT NULL`` guard), +the (count, length, catalog) ranking, the name-hits-lead-the-lexical- +list union with the FTS rows (chunk-id dedup), and the full +``retrieve()`` → ``select_documents()`` path putting the versioned- +name document into the seeded top-N. + +Requires: ``podman compose up -d db``. +""" +from __future__ import annotations + +import uuid +from collections.abc import Iterator +from datetime import UTC, datetime + +import pytest +from sqlalchemy import text +from sqlalchemy.orm import Session + +from app.models import Chunk, Document +from app.rag.retriever import ( + NAME_HIT_LIMIT, + _lexical_candidates, + _name_hit_chunks, + retrieve, + select_documents, +) + +INCIDENT_QUESTION = "What are the correct llama.cpp arguments for Qwen 3.8?" + +#: 768-dim test vectors (the pgvector column's dimension) — axis unit +#: vectors so the cosines are exact (1.0 parallel, 0.0 orthogonal, +#: 0.7071 half-parallel). +D = 768 + + +def _vec(axis: int, second: bool = False) -> list[float]: + v = [0.0] * D + v[axis] = 1.0 + if second: + v[axis + 1] = 1.0 + return v + + +def _doc(db: Session, source: str, path: str, title: str, content: str) -> Document: + doc = Document( + id=uuid.uuid4(), + source=source, + path=path, + full_path=f"/tmp/{source}/{path}", + title=title, + content=content, + content_hash="0" * 64, + indexed_at=datetime.now(UTC), + ) + db.add(doc) + return doc + + +def _chunk( + db: Session, doc: Document, position: int, content: str, is_summary: bool = False +) -> Chunk: + chunk = Chunk( + id=uuid.uuid4(), + document_id=doc.id, + position=position, + content=content, + is_summary=is_summary, + ) + db.add(chunk) + return chunk + + +@pytest.fixture() +def kb(db) -> Iterator[None]: + """A fresh KB with the incident shape: the qwen3.8 quadlet (the + name hit, with a summary chunk + an ordinary chunk), a qwen3.6 + quadlet (same family, different version — NOT a hit), an + unrelated document (FTS-only candidate), and a chunk-less document + whose name DOES carry the token (the exclusion guard).""" + db.execute(text("TRUNCATE chunks, documents")) + db.commit() + + q38 = _doc( + db, + "deploy", + "reeseapps/ai/deployments/juggernaut/quadlets/qwen3.8-27b-juggernaut-vulkan.container", + "qwen3.8-27b-juggernaut-vulkan", + "# llama.cpp juggernaut\nExec=--port 8000 -ctk q8_0 -ctv q8_0 --jinja\n" + "-m /models/qwen3.8-27b/Qwen3.8-27B-UD-Q6_K.gguf\n", + ) + _chunk( + db, + q38, + -1, + "Podman quadlet: llama.cpp server for Qwen 3.8 27B (juggernaut).", + is_summary=True, + ) + _chunk(db, q38, 0, "# llama.cpp juggernaut\nExec=--port 8000 -ctk q8_0") + db.flush() + # A vector the question vector (below) cosines with — non-NULL so + # the chunk is eligible for the vector list too. + for c in q38.chunks: + c.embedding = _vec(1) + db.commit() + + q36 = _doc( + db, + "deploy", + "reeseapps/ai/deployments/juggernaut/quadlets/qwen3.6-27b-juggernaut-vulkan.container", + "qwen3.6-27b-juggernaut-vulkan", + "# llama.cpp juggernaut\n-m /models/qwen3.6-27b/model.gguf\n", + ) + c36 = _chunk(db, q36, 0, "# llama.cpp juggernaut\n-m /models/qwen3.6-27b/model.gguf") + c36.embedding = _vec(0) # orthogonal to the question vector + db.commit() + + other = _doc( + db, "homelab", "notes/llama.cpp.md", "llama.cpp notes", "llama cpp server arguments notes\n" + ) + c_other = _chunk(db, other, 0, "llama cpp server arguments notes") + c_other.embedding = _vec(1, second=True) # half-parallel to the question + db.commit() + + # Name carries the token, ZERO chunks — the exclusion guard. + _doc(db, "deploy", "qwen3.8-empty.container", "qwen3.8-empty", "(empty file)") + db.commit() + yield + db.execute(text("TRUNCATE chunks, documents")) + db.commit() + + +def test_name_hit_chunks_real_sql(kb, db) -> None: + """Real Postgres: the projection scan finds exactly the qwen3.8 + quadlet (the qwen3.6 sibling and the chunk-less name match are + excluded), and the LATERAL fetch hands back the SUMMARY chunk as + the representative (position −1, is_summary).""" + out = _name_hit_chunks(db, INCIDENT_QUESTION) + assert [rc.document.path for rc in out] == [ + "reeseapps/ai/deployments/juggernaut/quadlets/qwen3.8-27b-juggernaut-vulkan.container" + ] + rc = out[0] + assert rc.position == -1 # the summary chunk wins the LATERAL order + assert rc.is_summary is True + assert rc.fts_hit is True # the lexical signal — the A8 gate answers + assert rc.cosine == 0.0 # no vector rank on the name-hit row + assert "qwen3.8-empty.container" not in [r.document.path for r in out] # chunk-less guard + + +def test_lexical_candidates_name_hit_leads_real_sql(kb, db) -> None: + """The full lexical list on real Postgres: the name hit leads, the + FTS rows follow (the qwen3.6 and llama.cpp docs both match the + OR-tsquery on llama|cpp|arguments|… — the pre-incident pollution — + but the name hit still ranks them behind it).""" + out = _lexical_candidates(db, INCIDENT_QUESTION, limit=30) + paths = [rc.document.path for rc in out] + assert paths[0] == ( + "reeseapps/ai/deployments/juggernaut/quadlets/qwen3.8-27b-juggernaut-vulkan.container" + ) + # The FTS pollution is still present (the incident's shape) — but + # behind the name hit, no longer ahead of it. + assert ( + "reeseapps/ai/deployments/juggernaut/quadlets/qwen3.6-27b-juggernaut-vulkan.container" + in paths + ) + assert all(rc.fts_hit is True for rc in out) + + +def test_retrieve_selects_name_hit_doc_into_top_n(kb, db) -> None: + """The product path: hybrid ``retrieve()`` (vector ∪ lexical, RRF + fused) → ``select_documents`` puts the qwen3.8 quadlet in the + seeded top-N — the incident's seed miss (the two overview docs + only) is fixed. The question vector is parallel to the q38 chunk + embeddings (cosine 1.0), orthogonal to q36 (0.0).""" + question_vec = _vec(1) + chunks = retrieve(db, INCIDENT_QUESTION, question_vec) + docs = select_documents(chunks, n=2) + assert [d.path for d in docs] == [ + "reeseapps/ai/deployments/juggernaut/quadlets/qwen3.8-27b-juggernaut-vulkan.container", + "notes/llama.cpp.md", + ] + + +def test_name_hit_limit_real_sql(db) -> None: + """Twelve identical (1, 6) name hits — the LATERAL fetch (and the + output) carries exactly ``NAME_HIT_LIMIT`` winners, catalog order.""" + db.execute(text("TRUNCATE chunks, documents")) + db.commit() + for i in range(12): + doc = _doc( + db, "S", f"quadlets/m{i:02d}-qwen38.container", f"m{i:02d}-qwen38", "llama cpp qwen38\n" + ) + db.flush() + c = _chunk(db, doc, 0, f"llama cpp qwen38 doc {i}") + c.embedding = _vec(2) + db.commit() + out = _name_hit_chunks(db, "what are the llama.cpp arguments for qwen 3.8") + assert len(out) == NAME_HIT_LIMIT + assert [rc.document.path for rc in out] == [ + f"quadlets/m{i:02d}-qwen38.container" for i in range(NAME_HIT_LIMIT) + ] diff --git a/tests/unit/test_agent.py b/tests/unit/test_agent.py index dd9e82e..f662b4e 100644 --- a/tests/unit/test_agent.py +++ b/tests/unit/test_agent.py @@ -209,17 +209,26 @@ def test_agent_tools_names_and_parameters() -> None: # ("pass ONLY `pattern`") + the source-name-is-not-a-document # clause (the model kept scoping grep with an ls-style source name # — the 2026-09-03 incident loop shape, but on grep) plus the - # one-call-at-a-time discipline clause. + # one-call-at-a-time discipline clause. The 2026-09-05 incident + # (the "Qwen 3.8" sample question — the harness prior is that grep + # takes a REGEX; this grep is a fixed substring, owner-locked A5): + # the plain-substring-never-a-regex clause states the contract up + # front, so the regex-shaped first grep that does fire gets the + # teaching no-match line instead of a trusted miss. assert grep["description"] == ( "Search the indexed documents for an exact string " "(case-insensitive) and return up to 20 matching lines " "as `source/path:line: text` — a locator, not a " - "context-adder: read the winner with `read`. For a " - "normal search pass ONLY `pattern` — it searches every " - "document and that is how you search the knowledge " - "base; never pass a source name as `path` (a source " - "name is not a document). Call one tool at a time — " - "wait for this result before your next call." + "context-adder: read the winner with `read`. The " + "pattern is a plain substring, NEVER a regex — if a " + "pattern with regex syntax (like '.*' or '\\.') comes " + "back with no matches, retry with the plain text you " + "expect to see. For a normal search pass ONLY `pattern` " + "— it searches every document and that is how you " + "search the knowledge base; never pass a source name " + "as `path` (a source name is not a document). Call one " + "tool at a time — wait for this result before your " + "next call." ) grep_params = grep["parameters"] assert grep_params["type"] == "object" @@ -227,7 +236,8 @@ def test_agent_tools_names_and_parameters() -> None: assert set(grep_params["properties"]) == {"pattern", "path"} assert all(p["type"] == "string" for p in grep_params["properties"].values()) assert grep_params["properties"]["pattern"]["description"] == ( - "The exact text to search for (a plain substring, not a regex)" + "The exact text to search for (a plain substring, " + "not a regex — no '.*', no '\\.', no character classes)" ) # Phase 72 (task 02): the bare-path contract is stated up front; # task 05 (live gate iterations 1-8): the one-known-document clause @@ -1164,6 +1174,184 @@ def test_grep_truncates_match_lines_at_200_chars(monkeypatch: pytest.MonkeyPatch assert holder.tool_calls == 1 +# ---------- grep no-match teaching: the regex-shaped pattern +# (the 2026-09-05 "Qwen 3.8" incident — the harness prior is that +# grep takes a REGEX; this grep is a fixed substring, owner-locked +# A5, and the contract does not change) ---------- + + +def test_plain_form_reduces_regex_to_literal_text() -> None: + """The plain-form hint: the pattern reduced to literal text — the + incident's exact recovery (``qwen.*3\\.8`` → ``qwen3.8``) plus the + edge cases (raw ``.*`` runs dropped before unescape, so an escaped + dot survives; first alternative only; classes/quantifiers/parens/ + anchors gone; whitespace preserved; pure metacharacters → ``""``). + """ + assert agent.plain_form(r"qwen.*3\.8") == "qwen3.8" # the incident + assert agent.plain_form(r"qwen 3\.8") == "qwen 3.8" + assert agent.plain_form(r"Qwen 3\.8") == "Qwen 3.8" # case kept + assert agent.plain_form(r"qwen3\.8") == "qwen3.8" + assert agent.plain_form(r"llama\.cpp") == "llama.cpp" # escaped dot kept + assert agent.plain_form(r"qwen[0-9]+") == "qwen" # class + quantifier + assert agent.plain_form("a|b") == "a" # first alternative only + assert agent.plain_form(r"\d+") == "" # no literal text — no hint + assert agent.plain_form(r".*") == "" # pure wildcard — no hint + assert agent.plain_form(r"(qwen)3\.8") == "qwen3.8" # group contents kept + assert agent.plain_form("a{2,3}b") == "ab" + assert agent.plain_form(r"^qwen$") == "qwen" # anchors dropped + assert agent.plain_form(r"a\.b") == "a.b" # escaped dot is a literal + assert agent.plain_form("plain") == "plain" # identity for plain text + + +def test_looks_like_regex_detection() -> None: + """One metacharacter anywhere marks the pattern regex-shaped; a + plain substring (even with a space) does not.""" + for p in ( + r"qwen.*3\.8", r"qwen 3\.8", "qwen+", "a?b", "x|y", "(a)", "[a-z]", "a^b", "b$c", "a{2}" + ): + assert agent.looks_like_regex(p) is True, p + for p in ("qwen 3.8", "qwen3.8", "plain substring", ""): + assert agent.looks_like_regex(p) is False, p + + +def test_grep_no_match_regex_pattern_gets_teaching_line( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The incident's shape: a regex-shaped pattern that (necessarily) + misses gets the TEACHING no-match line — the plain-substring + contract stated, the plain-form retry hint handed over. Still a + counted result; the context is untouched (locked A5).""" + d1 = _doc("Alpha", "a/one.md", "One", "qwen3.8-27b juggernaut\nno regex text") + monkeypatch.setattr(agent, "all_documents", lambda db: [d1]) + holder = AgentHolder() + llm = ScriptedLLM( + [ + ToolCallPiece( + id="call_1", name="grep", arguments={"pattern": r"qwen.*3\.8"} + ) + ], + [StreamPiece("content", "ans")], + ) + asyncio.run(_run(llm, holder, _settings())) + assert llm.requests[1][0][3]["content"] == agent.NO_MATCHES_REGEX.format( + pattern=r"qwen.*3\.8", plain="qwen3.8" + ) + assert llm.requests[1][0][3]["content"] == ( + "No matches for 'qwen.*3\\.8'. grep matches a plain substring " + "(case-insensitive), not a regex — '.*', '\\.' and the like are " + "literal text here, so that pattern can never match. Retry with " + "the plain text you expect to see (e.g. 'qwen3.8')." + ) + assert holder.tool_calls == 1 # a no-match with teaching is still a result + assert holder.read_docs == [] # locked A5: a grep adds no context + + +def test_grep_no_match_regex_scoped_gets_teaching_line( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The scoped teaching variant: the resolved identity is echoed, the + hint handed over.""" + d1 = _doc("Alpha", "a/one.md", "One", "nothing regex-shaped here") + + def _find(db: Any, source: str, path: str) -> Document | None: + return d1 if (source, path) == ("Alpha", "a/one.md") else None + + monkeypatch.setattr(agent, "find_document", _find) + holder = AgentHolder() + llm = ScriptedLLM( + [ + ToolCallPiece( + id="call_1", + name="grep", + arguments={"pattern": r"qwen 3\.8", "path": "Alpha/a/one.md"}, + ) + ], + [StreamPiece("content", "ans")], + ) + asyncio.run(_run(llm, holder, _settings())) + assert llm.requests[1][0][3]["content"] == ( + "No matches for 'qwen 3\\.8' in Alpha/a/one.md. grep matches a " + "plain substring (case-insensitive), not a regex — retry with " + "the plain text you expect to see (e.g. 'qwen 3.8')." + ) + assert holder.tool_calls == 1 + assert holder.read_docs == [] + + +def test_grep_no_match_plain_pattern_keeps_ordinary_line( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A no-match for a PLAIN pattern (no metacharacters — "qwen 3.8" with + the space included) keeps the ordinary line byte-identical: the + teaching never fires for a well-formed pattern (the retrieval side — + the name-hit lexical signal — is what covers that case).""" + d1 = _doc("Alpha", "a/one.md", "One", "qwen3.8-27b juggernaut") + monkeypatch.setattr(agent, "all_documents", lambda db: [d1]) + holder = AgentHolder() + llm = ScriptedLLM( + [ToolCallPiece(id="call_1", name="grep", arguments={"pattern": "qwen 3.8"})], + [StreamPiece("content", "ans")], + ) + asyncio.run(_run(llm, holder, _settings())) + assert llm.requests[1][0][3]["content"] == ( + "No matches for 'qwen 3.8' in the knowledge base." + ) + assert holder.tool_calls == 1 + + +def test_grep_matched_regex_pattern_returns_matches_not_teaching( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A pattern with metacharacters that MATCHES literally gets the + ordinary match output — the teaching can never suppress a real hit + (the detection keys on a NO-MATCH only).""" + d1 = _doc("S", "a.md", "A", "the C++ compiler is here") + monkeypatch.setattr(agent, "all_documents", lambda db: [d1]) + holder = AgentHolder() + llm = ScriptedLLM( + [ToolCallPiece(id="call_1", name="grep", arguments={"pattern": "C++"})], + [StreamPiece("content", "ans")], + ) + asyncio.run(_run(llm, holder, _settings())) + assert llm.requests[1][0][3]["content"] == "S/a.md:1: the C++ compiler is here" + assert holder.tool_calls == 1 + + +def test_grep_no_match_regex_reducing_to_empty_falls_back( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A regex-shaped pattern with no literal text left after the + reduction (``.*``) gets the ORDINARY line — no empty hint.""" + d1 = _doc("Alpha", "a/one.md", "One", "any text at all") + monkeypatch.setattr(agent, "all_documents", lambda db: [d1]) + holder = AgentHolder() + llm = ScriptedLLM( + [ToolCallPiece(id="call_1", name="grep", arguments={"pattern": r".*"})], + [StreamPiece("content", "ans")], + ) + asyncio.run(_run(llm, holder, _settings())) + assert llm.requests[1][0][3]["content"] == ( + "No matches for '.*' in the knowledge base." + ) + assert holder.tool_calls == 1 + + +def test_no_matches_regex_templates_pin() -> None: + """The teaching templates are verbatim pins (the model-facing copy — + the mock E2E keys off the plain-substring clause).""" + assert agent.NO_MATCHES_REGEX == ( + "No matches for '{pattern}'. grep matches a plain substring " + "(case-insensitive), not a regex — '.*', '\\.' and the like are " + "literal text here, so that pattern can never match. Retry with " + "the plain text you expect to see (e.g. '{plain}')." + ) + assert agent.NO_MATCHES_REGEX_SCOPED == ( + "No matches for '{pattern}' in {source}/{path}. grep matches a " + "plain substring (case-insensitive), not a regex — retry with " + "the plain text you expect to see (e.g. '{plain}')." + ) + + def test_grep_scoped_to_one_document(monkeypatch: pytest.MonkeyPatch) -> None: """Scoped grep: only the named document is loaded (find_document on the first-slash split), ``all_documents`` never runs, and the match diff --git a/tests/unit/test_retriever.py b/tests/unit/test_retriever.py index a673bac..b2795c2 100644 --- a/tests/unit/test_retriever.py +++ b/tests/unit/test_retriever.py @@ -133,6 +133,20 @@ def test_lexical_tsquery_stopwords_left_to_postgres() -> None: assert lexical_tsquery("how do i") == "how | do | i" +def test_lexical_tsquery_dotted_tokens_kept_whole() -> None: + """The 2026-09-05 incident: the default parser lexes dotted words + as ONE lexeme ("llama.cpp" → 'llama.cpp', "Qwen 3.8" → '3.8'), so + the query carries them whole — split tokens (llama | cpp) can never + match the document side.""" + assert lexical_tsquery( + "What are the correct llama.cpp arguments for Qwen 3.8?" + ) == "what | are | the | correct | llama.cpp | arguments | for | qwen | 3.8" + # The dash still splits (only dots group): ai | internal.network. + assert lexical_tsquery("how did I set up ai-internal.network?") == ( + "how | did | i | set | up | ai | internal.network" + ) + + def test_fuse_combines_both_lists_for_double_hits() -> None: v1 = _rc("a.md", cosine=0.9) v2 = _rc("b.md", cosine=0.5) @@ -196,7 +210,14 @@ def test_fuse_empty_lists() -> None: # --------------------------------------------------------------------------- from app.models import Chunk # noqa: E402 -from app.rag.retriever import _lexical_candidates, _vector_candidates # noqa: E402 +from app.rag.retriever import ( # noqa: E402 + NAME_HIT_LIMIT, + _lexical_candidates, + _name_hit_chunks, + _normalize_name, + _vector_candidates, + name_hit_tokens, +) class _FakeResult: @@ -210,15 +231,29 @@ class _FakeResult: class _FakeSession: - """Returns canned rows from ``execute`` without touching Postgres.""" + """Returns canned rows from ``execute`` without touching Postgres. - def __init__(self, rows: list) -> None: - self._rows = rows + One list of rows (legacy form) is returned for EVERY call; several + lists (one per successive ``execute``) model a query sequence — the + name-hit lexical path (2026-09-05) issues the document-projection + query and, when hits exist, the LATERAL chunk query, BEFORE the FTS + query. + """ + + def __init__(self, *rowsets: list) -> None: + if len(rowsets) == 1 and not ( + rowsets[0] and isinstance(rowsets[0][0], list) + ): + rowsets = (rowsets[0],) # the single-rowset legacy form + self._rowsets = rowsets + self._call = 0 self.statements: list = [] def execute(self, stmt, params: dict | None = None) -> _FakeResult: self.statements.append((stmt, params)) - return _FakeResult(self._rows) + rows = self._rowsets[min(self._call, len(self._rowsets) - 1)] + self._call += 1 + return _FakeResult(rows) def _chunk_row(is_summary: bool) -> Chunk: @@ -286,9 +321,17 @@ def _lexical_row(is_summary: bool, doc_path: str) -> object: def test_lexical_candidates_carry_is_summary_flag() -> None: - """The lexical list reads ``c.is_summary`` from the raw row.""" + """The lexical list reads ``c.is_summary`` from the raw row. + + The question carries no digit-bearing name token (no bare, no + numeric-join), so the name-hit path issues NO queries at all — the + single FTS rowset answers the only (FTS) call, and the list is the + plain FTS rows: the pre-name-hit behavior, unchanged. + """ rows = [_lexical_row(True, "summary-src.yaml"), _lexical_row(False, "other.md")] - out = _lexical_candidates(_FakeSession(rows), "how do i configure the thing", limit=10) # pyright: ignore[reportArgumentType] + out = _lexical_candidates( + _FakeSession(rows), "how do i configure the thing", limit=10 # pyright: ignore[reportArgumentType] + ) assert len(out) == 2 by_path = {rc.document.path: rc for rc in out} assert by_path["summary-src.yaml"].is_summary is True @@ -323,3 +366,180 @@ def test_fuse_default_is_summary_stays_false_for_legacy_chunks() -> None: out = fuse([_rc("a.md", cosine=0.8)], [_rc("b.md")], k=60) assert len(out) == 2 assert all(rc.is_summary is False for rc in out) + + +# --------------------------------------------------------------------------- +# Name-hit lexical signal (the 2026-09-05 incident — the versioned-name +# case the default parser lexes incompatibly: "Qwen 3.8" → qwen/3/8 can +# never match a document's qwen3/8/27b tokens) +# --------------------------------------------------------------------------- + +INCIDENT_QUESTION = "What are the correct llama.cpp arguments for Qwen 3.8?" + + +def test_normalize_name() -> None: + assert _normalize_name("Qwen 3.8") == "qwen38" + assert _normalize_name("qwen3.8-27b-juggernaut-vulkan") == "qwen3827bjuggernautvulkan" + assert _normalize_name("Mixed CASE-99") == "mixedcase99" + assert _normalize_name("!!!") == "" + + +def test_name_hit_tokens_incident_question() -> None: + """The incident question yields EXACTLY the versioned join + ``qwen38`` — the token the document names actually carry. Plain + prose words (``what``, ``llamacpp``, ``arguments``, ``server`` — + no digit) never name-match (the precision guard); the single + digits ("3", "8") and the bare "38" are < 4 chars; the + digit-leading ``38show`` boundary artifact is dropped.""" + tokens = name_hit_tokens(INCIDENT_QUESTION) + assert tokens == ["qwen38"] + for absent in ("what", "qwen", "llamacpp", "arguments", "3", "8", "38", "38show", "server"): + assert absent not in tokens + + +def test_name_hit_tokens_no_digit_question_returns_empty() -> None: + """A question with no digit-bearing token (bare or joined) yields + no name candidates — prose joins like ``correctllama`` never count.""" + assert name_hit_tokens("what is the correct caddy config") == [] + assert name_hit_tokens("a e i o u 3 8") == [] + + +def test_name_hit_tokens_bare_digit_bearing_token() -> None: + """A single written token that carries a digit (``1panel``) is a + name candidate on its own — no join needed.""" + tokens = name_hit_tokens("what is my 1panel dashboard setup") + assert tokens == ["1panel"] + + +def _name_row(doc: Document) -> tuple: + """One row of the name-hit document projection (catalog order).""" + return (doc.id, doc.source, doc.path, doc.title) + + +def _name_hit_lateral_row(doc: Document, is_summary: bool = False) -> SimpleNamespace: + """One row of the name-hit LATERAL chunk query.""" + return SimpleNamespace( + doc_id=doc.id, + source=doc.source, + path=doc.path, + full_path=doc.full_path, + title=doc.title, + doc_content=doc.content, + content_hash=doc.content_hash, + indexed_at=None, + chunk_id=uuid.uuid4(), + position=-1 if is_summary else 0, + content="summary chunk" if is_summary else "content chunk", + is_summary=is_summary, + ) + + +def test_name_hit_chunks_no_tokens_skips_all_queries() -> None: + """A question with no name tokens issues no queries at all.""" + session = _FakeSession([]) # any call would surface a statement + assert _name_hit_chunks(session, "a e i o u 3 8") == [] # pyright: ignore[reportArgumentType] + assert session.statements == [] + + +def test_name_hit_chunks_no_matching_doc_returns_empty() -> None: + """Name tokens exist but no document name carries one: the + projection runs, the LATERAL fetch does not.""" + doc = _doc("quadlets/other.container", "body") + name_rows = [_name_row(doc)] + session = _FakeSession(name_rows, []) + assert _name_hit_chunks(session, INCIDENT_QUESTION) == [] # pyright: ignore[reportArgumentType] + assert len(session.statements) == 1 # projection only — no LATERAL fetch + + +def test_name_hit_chunks_ranked_by_count_length_catalog() -> None: + """A two-candidate question (``qwen38`` + ``1panel``): the document + whose name carries BOTH (2 matches, 12 total chars) leads; the two + single-match documents tie on (1, 6) and fall to catalog order + (``dashboards/1panel-notes.md`` before ``quadlets/qwen3.8…``). + Hits carry ``fts_hit=True`` (the A8 gate answers), ``cosine=0.0``, + and the summary flag of their representative chunk.""" + both = _doc("dashboards/1panel-qwen3.8.md", "body", title="1Panel Qwen 3.8") + panel = _doc("dashboards/1panel-notes.md", "body") + q38 = _doc("quadlets/qwen3.8-27b-juggernaut-vulkan.container", "body") + name_rows = [_name_row(d) for d in (panel, both, q38)] # catalog order + question = "what are the correct llama.cpp arguments for qwen 3.8 and the 1panel dashboard?" + lateral_rows = [ + _name_hit_lateral_row(q38, is_summary=True), # LATERAL may return any order + _name_hit_lateral_row(both), + _name_hit_lateral_row(panel), + ] + session = _FakeSession(name_rows, lateral_rows) + out = _name_hit_chunks(session, question) # pyright: ignore[reportArgumentType] + assert [rc.document.path for rc in out] == [ + "dashboards/1panel-qwen3.8.md", # 2 matched tokens — leads + "dashboards/1panel-notes.md", # (1, 6) — catalog order + "quadlets/qwen3.8-27b-juggernaut-vulkan.container", # (1, 6) — after + ] + assert all(rc.fts_hit is True for rc in out) # the lexical signal + assert all(rc.cosine == 0.0 for rc in out) # no vector rank + assert all(rc.score == 0.0 for rc in out) # fuse fills the score + by_path = {rc.document.path: rc for rc in out} + assert by_path["quadlets/qwen3.8-27b-juggernaut-vulkan.container"].is_summary is True + assert by_path["quadlets/qwen3.8-27b-juggernaut-vulkan.container"].position == -1 + assert by_path["dashboards/1panel-notes.md"].is_summary is False + + +def test_name_hit_chunks_capped_at_limit() -> None: + """Twelve tied name hits (one matched token each) yield exactly + ``NAME_HIT_LIMIT`` of them — catalog order (the deterministic + tie-break).""" + docs = [_doc(f"quadlets/m{i:02d}.container", "body") for i in range(12)] + for d in docs: # give every document a name that carries the token + d.title = "qwen38 model i" + name_rows = [_name_row(d) for d in docs] + # Only the ten winners (catalog order — the deterministic tie-break + # of the twelve identical scores) reach the LATERAL fetch; the fake + # answers with exactly those rows. + lateral_rows = [_name_hit_lateral_row(d) for d in docs[:NAME_HIT_LIMIT]] + session = _FakeSession(name_rows, lateral_rows) + out = _name_hit_chunks( + session, "tell me about the qwen 3.8 models" # pyright: ignore[reportArgumentType] + ) + assert len(out) == NAME_HIT_LIMIT + assert [rc.document.path for rc in out] == [f"quadlets/m{i:02d}.container" for i in range(10)] + + +def test_lexical_candidates_name_hits_lead_and_dedupe_with_fts() -> None: + """The full lexical list: name hits LEAD (their representative + chunks), the FTS rows follow, and an FTS row sharing the name hit's + chunk id appears exactly once (deduped).""" + q38 = _doc("quadlets/qwen3.8-27b-juggernaut-vulkan.container", "body") + other = _doc("quadlets/qwen38-other.container", "body") # 1 matched token + name_rows = [_name_row(q38), _name_row(other)] + q38_chunk = uuid.uuid4() + + def _lateral(doc: Document) -> SimpleNamespace: + row = _name_hit_lateral_row(doc) + if doc is q38: + row.chunk_id = q38_chunk + return row + + lateral_rows = [_lateral(q38), _lateral(other)] + fts_rows = [ + # an FTS hit on the SAME chunk as the q38 name hit (deduped away) + SimpleNamespace( + chunk_id=q38_chunk, position=1, content="c", doc_id=q38.id, + source=q38.source, path=q38.path, full_path=q38.full_path, + title=q38.title, doc_content=q38.content, content_hash=q38.content_hash, + indexed_at=None, is_summary=False, rank=0.1, + ), + # an FTS hit on a different chunk of the OTHER doc (kept) + _lexical_row(False, "quadlets/qwen38-other.container"), + ] + session = _FakeSession(name_rows, lateral_rows, fts_rows) + out = _lexical_candidates(session, INCIDENT_QUESTION, limit=10) # pyright: ignore[reportArgumentType] + assert len(out) == 3 # q38 (once), other (name hit), other (FTS chunk) + # Both name hits tie on (1, 6) — catalog order: "qwen3." (ASCII 46) + # sorts before "qwen38" (ASCII 56). + assert out[0].document.path == "quadlets/qwen3.8-27b-juggernaut-vulkan.container" + assert out[1].document.path == "quadlets/qwen38-other.container" + assert out[0].chunk_id == q38_chunk # the name-hit representative row + assert { + rc.chunk_id for rc in out + } == {q38_chunk, fts_rows[1].chunk_id, lateral_rows[1].chunk_id} + assert all(rc.fts_hit is True for rc in out)