finally getting accurate answers
Build and Push Containers / build-and-push-app (push) Successful in 1m46s
Build and Push Containers / build-and-push-db (push) Successful in 12s

This commit is contained in:
2026-09-05 10:26:39 -04:00
parent bb2803bebd
commit 766702c750
9 changed files with 1628 additions and 41 deletions
+3
View File
@@ -28,3 +28,6 @@ frontend/dist/
*.log *.log
.DS_Store .DS_Store
Thumbs.db Thumbs.db
--- Local data dumps (KB replicas, pg_dumps) — never committed ---
data/
+114 -7
View File
@@ -66,6 +66,16 @@ task 04):
each line truncated to 200 chars. A grep is a **locator**, not a each line truncated to 200 chars. A grep is a **locator**, not a
context-adder: it never appends to the answer context (only context-adder: it never appends to the answer context (only
``read`` does — ``holder.read_docs`` is untouched by a grep). ``read`` does — ``holder.read_docs`` is untouched by a grep).
The 2026-09-05 incident teaching (the harness prior is that grep
takes a REGEX; this grep is a fixed substring and the contract does
not change): when a grep RAN but found nothing and the pattern is
regex-shaped (:func:`looks_like_regex`) with a non-empty
:func:`plain_form`, the no-match result is the TEACHING line —
:data:`NO_MATCHES_REGEX` / :data:`NO_MATCHES_REGEX_SCOPED` — which
states the plain-substring contract and hands over the plain-form
retry hint; a grep that matched, or a no-match for a plain pattern
(or one that reduces to nothing), is byte-identical to the ordinary
line. Still a (counted) result, never a refusal.
3. Rejected calls get a one-line refusal and count in nothing 3. Rejected calls get a one-line refusal and count in nothing
(``holder.tool_calls`` tracks executed calls only): unknown tool name (``holder.tool_calls`` tracks executed calls only): unknown tool name
→ ``"Unknown tool."``; a ``read`` without a usable ``path`` (missing, → ``"Unknown tool."``; a ``read`` without a usable ``path`` (missing,
@@ -163,6 +173,7 @@ from __future__ import annotations
import json import json
import logging import logging
import re
from collections.abc import AsyncIterator, Sequence from collections.abc import AsyncIterator, Sequence
from dataclasses import dataclass, field from dataclasses import dataclass, field
from typing import Any, cast from typing import Any, cast
@@ -268,12 +279,16 @@ AGENT_TOOLS: list[dict[str, Any]] = [
"Search the indexed documents for an exact string " "Search the indexed documents for an exact string "
"(case-insensitive) and return up to 20 matching lines " "(case-insensitive) and return up to 20 matching lines "
"as `source/path:line: text` — a locator, not a " "as `source/path:line: text` — a locator, not a "
"context-adder: read the winner with `read`. For a " "context-adder: read the winner with `read`. The "
"normal search pass ONLY `pattern` — it searches every " "pattern is a plain substring, NEVER a regex — if a "
"document and that is how you search the knowledge " "pattern with regex syntax (like '.*' or '\\.') comes "
"base; never pass a source name as `path` (a source " "back with no matches, retry with the plain text you "
"name is not a document). Call one tool at a time — " "expect to see. For a normal search pass ONLY `pattern` "
"wait for this result before your next call." "— it searches every document and that is how you "
"search the knowledge base; never pass a source name "
"as `path` (a source name is not a document). Call one "
"tool at a time — wait for this result before your "
"next call."
), ),
"parameters": { "parameters": {
"type": "object", "type": "object",
@@ -282,7 +297,8 @@ AGENT_TOOLS: list[dict[str, Any]] = [
"type": "string", "type": "string",
"description": ( "description": (
"The exact text to search for (a plain " "The exact text to search for (a plain "
"substring, not a regex)" "substring, not a regex — no '.*', no "
"'\\.', no character classes)"
), ),
}, },
"path": { "path": {
@@ -417,6 +433,79 @@ SEARCH_LINE_LIMIT = 200
NO_MATCHES = "No matches for '{pattern}' in the knowledge base." NO_MATCHES = "No matches for '{pattern}' in the knowledge base."
NO_MATCHES_SCOPED = "No matches for '{pattern}' in {source}/{path}." NO_MATCHES_SCOPED = "No matches for '{pattern}' in {source}/{path}."
#: Teaching no-match lines (the 2026-09-05 incident — owner chat:
#: "Qwen 3.8" question failed repeatedly). The harness-trained prior is
#: that ``grep(pattern)`` takes a REGEX (pi.dev's grep, ripgrep, grep
#: itself); this app's grep is a case-insensitive FIXED SUBSTRING
#: (owner-locked A5 — the contract does not change). A regex-shaped
#: pattern (``qwen.*3\.8``, ``qwen 3\.8``) can therefore never match, and
#: the bare no-match line above made the turbo model trust the miss and
#: end the turn with a wrong "I searched the entire knowledge base"
#: refusal. These lines are the deterministic teaching: the same result
#: (a no-match is still a *result* — counted, no context added) with the
#: correct contract stated and a plain-text retry hint (the
#: :func:`plain_form` of the pattern, when non-empty). Deterministic only
#: (owner permission 2026-09-03: "deterministic guardrails only right
#: now"): no model participates in detection or repair.
NO_MATCHES_REGEX = (
"No matches for '{pattern}'. grep matches a plain substring "
"(case-insensitive), not a regex — '.*', '\\.' and the like are "
"literal text here, so that pattern can never match. Retry with the "
"plain text you expect to see (e.g. '{plain}')."
)
NO_MATCHES_REGEX_SCOPED = (
"No matches for '{pattern}' in {source}/{path}. grep matches a plain "
"substring (case-insensitive), not a regex — retry with the plain "
"text you expect to see (e.g. '{plain}')."
)
#: One of these metacharacters anywhere in the pattern marks it
#: regex-shaped (the detection for the teaching no-match lines above).
#: Plain substrings that happen to contain one ("C++", "a|b") are
#: affected only on a NO-MATCH — a pattern that matched literally still
#: gets its ordinary result, so the teaching can never suppress a real
#: hit.
_REGEX_META_RE = re.compile(r"[*+?(){}\[\]|\\^$]")
# Backslash + a non-alphanumeric char is an escaped literal (``\\.`` →
# ``.``); backslash + an alphanumeric is a class shorthand (``\\d``,
# ``\\w``, ``\\s``) with no plain-text equivalent (→ dropped).
_ESCAPE_RE = re.compile(r"\\(.)")
def looks_like_regex(pattern: str) -> bool:
"""True when *pattern* carries regex metacharacters (see
:data:`_REGEX_META_RE`). Pure detection — the grep itself stays a
fixed substring (owner-locked A5)."""
return _REGEX_META_RE.search(pattern) is not None
def plain_form(pattern: str) -> str:
"""A deterministic plain-text retry hint for a regex-shaped pattern.
The hint is the pattern reduced to literal text, in this order:
drop ``.*`` runs on the RAW pattern first (the wildcard — an escaped
``\\.`` + ``\\*`` pair carries no raw ``.*`` run, so a literal dot
survives), unescape (``\\.`` → ``.``; class shorthands like ``\\d``
dropped), keep only the FIRST ``|`` alternative, drop character
classes (``[0-9]``), drop quantifier runs (``*``, ``+``, ``?``,
``{2,3}``), drop group parens and anchors (contents kept). Whitespace
is preserved. ``qwen.*3\\.8`` → ``qwen3.8`` (the incident's exact
recovery), ``llama\\.cpp`` → ``llama.cpp``, ``qwen[0-9]+`` →
``qwen``. A pattern made of pure metacharacters reduces to ``""`` —
callers then fall back to the ordinary no-match line (no hint).
"""
p = re.sub(r"\.\*", "", pattern) # .* wildcard runs (raw form)
p = _ESCAPE_RE.sub(lambda m: m.group(1) if not m.group(1).isalnum() else "", p)
p = p.split("|", 1)[0] # first alternative only — a hint, not an answer
p = re.sub(r"\[[^\]]*\]", "", p) # character classes carry no literal text
p = re.sub(r"\{[^{}]*\}", "", p) # {n} / {n,} / {n,m} quantifiers
p = re.sub(r"[*+?]+", "", p) # stray * + ? quantifiers
p = p.replace("(", "").replace(")", "")
p = p.replace("^", "").replace("$", "")
return p.strip()
def list_catalog(db: Session) -> list[tuple[str, str, str]]: def list_catalog(db: Session) -> list[tuple[str, str, str]]:
"""Every indexed document as ``(source, path, title)``. """Every indexed document as ``(source, path, title)``.
@@ -694,6 +783,24 @@ def _execute_tool(
# Locked A5: a grep never adds context — read_docs untouched. # Locked A5: a grep never adds context — read_docs untouched.
if not matches: if not matches:
shown = pattern[:100] # keep a long pattern short in the line shown = pattern[:100] # keep a long pattern short in the line
# The 2026-09-05 incident teaching: a regex-shaped pattern
# (the harness prior) can NEVER match a fixed-substring
# grep, so a no-match for one is not "the KB lacks this" —
# it is "the pattern was in the wrong form". Teach the
# contract and hand over the plain-form retry hint (when the
# reduction is non-empty); a plain pattern (or a pattern
# that reduces to nothing) keeps the ordinary line
# byte-identical.
plain = plain_form(pattern) if looks_like_regex(pattern) else ""
if plain:
if scoped_to is not None:
return NO_MATCHES_REGEX_SCOPED.format(
pattern=shown,
source=scoped_to[0],
path=scoped_to[1],
plain=plain[:100],
)
return NO_MATCHES_REGEX.format(pattern=shown, plain=plain[:100])
if scoped_to is not None: if scoped_to is not None:
# The scoped no-match line is keyed on the resolved # The scoped no-match line is keyed on the resolved
# source/path (== the argument, stripped). # source/path (== the argument, stripped).
+5 -2
View File
@@ -149,8 +149,11 @@ TOOLS_SECTION: str = (
"path (without the source name) will not resolve. `grep` locates an " "path (without the source name) will not resolve. `grep` locates an "
"exact string (case-insensitive) in the indexed documents and " "exact string (case-insensitive) in the indexed documents and "
"returns up to 20 matching `source/path:line: text` lines — a " "returns up to 20 matching `source/path:line: text` lines — a "
"locator, not a context-adder: read the winner with `read`; for a " "locator, not a context-adder: read the winner with `read`. A grep "
"normal search pass only `pattern` — its optional `path` argument " "pattern is a plain substring, NEVER a regex — '.*' and '\\.' are "
"literal text there; if such a pattern returns no matches, retry "
"with the plain text you expect to see. For a normal search pass "
"only `pattern` — its optional `path` argument "
"limits the search to one document you already know, by the same " "limits the search to one document you already know, by the same "
"combined `source/path` string; never a source name — a bare " "combined `source/path` string; never a source name — a bare "
"document path (without the source name) will not resolve there " "document path (without the source name) will not resolve there "
+230 -17
View File
@@ -3,8 +3,27 @@
* **Vector list** — top-N chunks by cosine distance (``embedding <=> $1``), * **Vector list** — top-N chunks by cosine distance (``embedding <=> $1``),
each carrying its cosine ``1 − distance`` (the honesty-gate input). each carrying its cosine ``1 − distance`` (the honesty-gate input).
* **Lexical list** — top-N chunks matching an OR-``tsquery`` over the * **Lexical list** — top-N chunks matching an OR-``tsquery`` over the
question's tokens, ordered by ``ts_rank``. This is what finds question's tokens, ordered by ``ts_rank``, UNION the name-hit list:
name-your-tool questions ("gitlab") that vector similarity buries. documents whose normalized name (title + path stem, alnum-only,
lowercased) contains a DIGIT-BEARING normalized question token of
length >= 4 — bare tokens (``1panel``) and the join of adjacent
question tokens starting with a letter ("Qwen 3.8" → ``qwen38``).
The digit requirement is the precision guard: plain prose words
("server", "arguments" — 4+ chars but no digit) never name-match,
while versioned product names (the incident's whole point) always
carry one. The name-hit documents LEAD the lexical list (ranked by
match count, then total matched length, then catalog order; capped
at :data:`NAME_HIT_LIMIT`),
the FTS rows follow. This is what finds name-your-tool questions
("gitlab") that vector similarity buries — and, the 2026-09-05
incident, the versioned-name case the default parser lexes
incompatibly (question "Qwen 3.8" → ``qwen``/``3``/``8`` can never
match a document's ``qwen3``/``8``/``27b`` tokens, while every
unrelated llama.cpp quadlet out-ranks the target on the shared
``llama``/``cpp`` tokens). The name-hit rows carry ``fts_hit=True``
(they ARE the lexical signal — the A8 honesty gate then answers
instead of deflecting) and ``cosine=0.0``; the RRF fusion is
unchanged (same lists, same ``1/(k+rank)`` terms).
* **Fusion** — Reciprocal Rank Fusion (``score = Σ 1/(k + rank)`` over the * **Fusion** — Reciprocal Rank Fusion (``score = Σ 1/(k + rank)`` over the
lists a chunk appears in; chunks hit by both lists get both terms). The lists a chunk appears in; chunks hit by both lists get both terms). The
fused score ranks; :meth:`select_documents` and :func:`weak_hit_titles` fused score ranks; :meth:`select_documents` and :func:`weak_hit_titles`
@@ -26,6 +45,7 @@ import re
import uuid import uuid
from collections.abc import Sequence from collections.abc import Sequence
from dataclasses import dataclass, replace from dataclasses import dataclass, replace
from pathlib import Path
from sqlalchemy import select, text from sqlalchemy import select, text
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
@@ -38,8 +58,78 @@ from app.models import Chunk, Document
#: path never truncates (A7 revised, owner permission 2026-08-24). #: path never truncates (A7 revised, owner permission 2026-08-24).
TRUNCATION_MARKER = "[…truncated…]" TRUNCATION_MARKER = "[…truncated…]"
#: Alphanumeric tokens of a question (``to_tsquery`` input, OR-joined). #: Alphanumeric tokens of a question (``to_tsquery`` input, OR-joined),
_TOKEN_RE = re.compile(r"[a-z0-9]+") #: with DOTTED tokens kept whole (``llama.cpp`` → ``llama.cpp``). The
#: default parser lexes a dotted word as ONE lexeme (``to_tsvector`` of
#: "llama.cpp" → ``'llama.cpp'``; of "Qwen 3.8" → ``'3.8'``), so a split
#: token (``llama`` | ``cpp``) can never match the document side — the
#: 2026-09-05 incident's question phrase "llama.cpp" lexed
#: incompatibly on both sides of the query. Dotted tokens are passed
#: through as single ``to_tsquery`` lexemes (verified: Postgres accepts
#: ``'llama.cpp'`` and ``'3.8'`` as lexemes).
_TOKEN_RE = re.compile(r"[a-z0-9]+(?:\.[a-z0-9]+)*")
#: Name-hit policy (the 2026-09-05 incident): a question token shorter
#: than this (normalized) is too weak a name signal ("3", "8", "the") —
#: it would match half the KB by accident.
NAME_TOKEN_MIN_LEN = 4
#: The name-hit list may contribute at most this many documents to the
#: lexical side (before the FTS rows); the rest are dropped.
NAME_HIT_LIMIT = 10
#: Alphanumeric runs of a lowercased string (name normalization).
_ALNUM_RE = re.compile(r"[a-z0-9]+")
def _normalize_name(s: str) -> str:
"""Lowercased, alnum-only form of *s* (``Qwen 3.8`` → ``qwen38``)."""
return "".join(_ALNUM_RE.findall(s.lower()))
def name_hit_tokens(question: str) -> list[str]:
"""The normalized name-match candidates of one question.
Only DIGIT-BEARING candidates count (the precision guard — a plain
prose word like "server" or "arguments" must never name-match a
document; a versioned product name always carries a digit):
* the :func:`_normalize_name` form of every whitespace token, kept
when at least :data:`NAME_TOKEN_MIN_LEN` chars (``1panel``,
``qwen38`` from a single written token);
* the versioned-name JOIN — the normalized concatenation of every
ADJACENT token pair whose SECOND token is purely numeric (a
version number: ``"… for Qwen 3.8"`` → ``Qwen`` + ``3.8`` joins
to ``qwen38``; prose joins like ``correct`` + ``llama`` and the
word-after-version boundaries like ``3.8`` + ``show`` are
dropped). A purely-numeric second token means the join always
starts with the first token's text (a letter in practice), so no
digit-leading artifact (``38show``) can survive.
Order of first appearance, de-duplicated.
"""
tokens = question.split()
out: list[str] = []
seen: set[str] = set()
def add(tok: str) -> None:
if (
len(tok) >= NAME_TOKEN_MIN_LEN
and any(ch.isdigit() for ch in tok)
and tok not in seen
):
seen.add(tok)
out.append(tok)
for i, raw in enumerate(tokens):
norm = _normalize_name(raw)
add(norm)
if i + 1 < len(tokens):
next_norm = _normalize_name(tokens[i + 1])
if next_norm and next_norm.isdigit():
add(norm + next_norm) # the versioned-name join
return out
#: One row of the lexical candidate query (all fields needed to build a #: One row of the lexical candidate query (all fields needed to build a
#: detached :class:`Document` plus the chunk fields and ``ts_rank``). #: detached :class:`Document` plus the chunk fields and ``ts_rank``).
@@ -96,11 +186,14 @@ class RetrievedChunk:
def lexical_tsquery(question: str) -> str | None: def lexical_tsquery(question: str) -> str | None:
"""OR-joined token string for ``to_tsquery('english', …)``, or ``None``. """OR-joined token string for ``to_tsquery('english', …)``, or ``None``.
Tokens are lowercased ``[a-z0-9]+`` runs, de-duplicated in order of Tokens are lowercased ``[a-z0-9]+`` runs with DOTTED words kept whole
first appearance. Postgres does the lexing/stemming; a question whose (see :data:`_TOKEN_RE` — the default parser lexes ``llama.cpp`` as
tokens are all stopwords lexes to an *empty* tsquery (which matches one lexeme, so the query must carry it whole too), de-duplicated in
nothing), so no special-casing is needed there. Pure-symbol questions order of first appearance. Postgres does the lexing/stemming; a
("???", "🔧") yield no tokens → ``None`` → no lexical query at all. question whose tokens are all stopwords lexes to an *empty* tsquery
(which matches nothing), so no special-casing is needed there.
Pure-symbol questions ("???", "🔧") yield no tokens → ``None`` → no
lexical query at all.
""" """
seen: set[str] = set() seen: set[str] = set()
tokens: list[str] = [] tokens: list[str] = []
@@ -181,21 +274,141 @@ def _vector_candidates(
] ]
def _lexical_candidates(db: Session, question: str, limit: int) -> list[RetrievedChunk]: #: One name-hit row: the document fields (detached :class:`Document`)
"""Top-*limit* chunks matching the question's OR-tsquery (A7). #: plus the representative chunk — the ``is_summary`` chunk when the
#: document has one (its natural-language summary is the best chunk of
Ordered by ``ts_rank`` (with ``d.path, c.position`` as the #: a machine file, and the chunk the vector list is likeliest to have
deterministic tie-break); an empty tsquery (stopword-only question) #: ranked too, so the RRF merge dedupes cleanly), else chunk 0.
simply matches nothing. _NAME_HIT_SQL = text(
""" """
SELECT d.id AS doc_id,
d.source AS source,
d.path AS path,
d.full_path AS full_path,
d.title AS title,
d.content AS doc_content,
d.content_hash AS content_hash,
d.indexed_at AS indexed_at,
c.id AS chunk_id,
c.position AS position,
c.content AS content,
c.is_summary AS is_summary
FROM documents d
LEFT JOIN LATERAL (
SELECT id, position, content, is_summary
FROM chunks
WHERE document_id = d.id
ORDER BY is_summary DESC, position ASC
LIMIT 1
) c ON true
WHERE d.id = ANY(:ids) AND c.id IS NOT NULL
"""
)
def _name_hit_chunks(db: Session, question: str) -> list[RetrievedChunk]:
"""The documents whose NAME matches the question (the 2026-09-05
incident, the versioned-name signal).
A document is a name hit when its normalized name —
:func:`_normalize_name` of its ``title`` followed by the normalized
path stem (``qwen3.8-27b-juggernaut-vulkan.container`` →
``qwen3827bjuggernautvulkan``) — contains at least one
:func:`name_hit_tokens` candidate (``qwen38``). Ranked by
(distinct matched tokens, total matched length, source, path) —
catalog order is the final deterministic tie-break — capped at
:data:`NAME_HIT_LIMIT`. Each hit becomes one lexical
:class:`RetrievedChunk` (its representative chunk, ``fts_hit=True``,
``cosine=0.0``). Two lightweight queries: one projection over
(id, source, path, title) in catalog order, one LATERAL chunk
fetch for the ≤ :data:`NAME_HIT_LIMIT` winners (no full-content
load; the content joins in via the row fetch below).
"""
tokens = name_hit_tokens(question)
if not tokens:
return []
rows = db.execute(
select(Document.id, Document.source, Document.path, Document.title).order_by(
Document.source, Document.path
)
).all()
scored: list[tuple[int, int, uuid.UUID]] = []
by_id: dict[uuid.UUID, tuple[str, str]] = {} # id -> (source, path)
for doc_id, source, path, title in rows:
by_id[doc_id] = (source, path)
name = _normalize_name(title) + _normalize_name(Path(path).stem)
matched = [t for t in tokens if t in name]
if matched:
scored.append((len(matched), sum(len(t) for t in matched), doc_id))
if not scored:
return []
# Ranked by (distinct matched tokens, total matched length), the
# deterministic catalog tie-break (source, path) last.
scored.sort(
key=lambda s: (-s[0], -s[1], by_id[s[2]][0], by_id[s[2]][1])
)
ids = [s[2] for s in scored[:NAME_HIT_LIMIT]]
hit_rows = list(db.execute(_NAME_HIT_SQL, {"ids": ids}).all())
# The LATERAL query returns winners in id order; re-order by the
# ranked order computed above so the lexical list is deterministic.
order = {doc_id: rank for rank, doc_id in enumerate(ids)}
hit_rows.sort(key=lambda row: order[row.doc_id])
out: list[RetrievedChunk] = []
for row in hit_rows:
doc = Document(
id=row.doc_id,
source=row.source,
path=row.path,
full_path=row.full_path,
title=row.title,
content=row.doc_content,
content_hash=row.content_hash,
indexed_at=row.indexed_at,
)
out.append(
RetrievedChunk(
chunk_id=row.chunk_id,
position=row.position,
content=row.content,
score=0.0, # filled in by :func:`fuse`
document=doc,
cosine=0.0, # no vector rank — name-only hit
fts_hit=True, # the lexical signal — the A8 gate answers
is_summary=bool(row.is_summary),
)
)
return out
def _lexical_candidates(db: Session, question: str, limit: int) -> list[RetrievedChunk]:
"""The lexical candidate list: name hits, then FTS rows (A7).
The name-hit documents (:func:`_name_hit_chunks`, the versioned-name
signal) LEAD the list — they are the strongest lexical evidence for
a name-your-thing question — followed by the top-*limit* chunks
matching the question's OR-tsquery, ordered by ``ts_rank`` (with
``d.path, c.position`` as the deterministic tie-break). Deduped by
chunk id (a name-hit representative chunk the FTS list also ranked
appears once). An empty tsquery (stopword-only question) simply
contributes no FTS rows; a question with no name tokens contributes
no name hits — both halves are independent.
"""
out: list[RetrievedChunk] = []
seen: set[uuid.UUID] = set()
for rc in _name_hit_chunks(db, question):
if rc.chunk_id not in seen:
seen.add(rc.chunk_id)
out.append(rc)
tsquery = lexical_tsquery(question) tsquery = lexical_tsquery(question)
if tsquery is None: if tsquery is None:
return [] return out
rows = db.execute( rows = db.execute(
_LEXICAL_SQL, {"tsquery": tsquery, "limit": limit} _LEXICAL_SQL, {"tsquery": tsquery, "limit": limit}
).all() ).all()
out: list[RetrievedChunk] = []
for row in rows: for row in rows:
if row.chunk_id in seen:
continue
seen.add(row.chunk_id)
doc = Document( doc = Document(
id=row.doc_id, id=row.doc_id,
source=row.source, source=row.source,
+161
View File
@@ -183,6 +183,40 @@ Implements just enough of the aipi surface:
phrases are disjoint substrings — the phase-71 ordering phrases are disjoint substrings — the phase-71 ordering
convention); no existing E2E question or fixture file contains the convention); no existing E2E question or fixture file contains the
phrase, so every other suite is unaffected. phrase, so every other suite is unaffected.
- user message containing ``what are the correct llama.cpp
arguments`` (``GREP_TEACH_TRIGGER``, the 2026-09-05 incident —
the harness prior is that grep takes a REGEX; this app's grep is a
case-insensitive fixed substring, owner-locked A5) **and** the
system prompt carries the ``<tools>`` section -> the deterministic
GREP-REGEX-TEACHING flow, discriminated statelessly from the
messages (streaming only):
* request 1 (``tools`` offered, no ``tool``-role result yet):
stream ONLY ``tool_calls`` deltas — ``grep`` with
``{"pattern": GREP_TEACH_PATTERN}`` (``qwen.*3\\.8``, id
``call_0``) — the incident's regex-shaped first grep, which a
fixed-substring grep can NEVER match;
* request 2 (the last tool result is the server's TEACHING
no-match line — it carries ``GREP_TEACH_MARKER``):
``grep`` with the plain form ``GREP_TEACH_PLAIN``
(``qwen3.8``, id ``call_1``) — the one-round correction;
* request 3 (the last tool result carries
``source/path:line: text`` match lines): ``read`` the FIRST
match line's document by its combined ``source/path`` (id
``call_2``);
* request 4 (the last tool result is a read result, the
``"Document <combined>:\n<content>"`` shape): the
deterministic echo answer ``Read <combined>. <first 80
chars>``, ``finish_reason: "stop"`` — the loop ended in ONE
correction, not at the round cap.
* A PLAIN no-match as the last result (no match line, no
teaching marker — e.g. the plain pattern genuinely absent) is
the deterministic terminal answer ``No matches — the knowledge
base has no such text.`` (the flow cannot loop on a
well-formed pattern).
Checked BEFORE the SEARCH / TOOLS_TRIGGER flows (disjoint trigger
phrases — the phase-72 ordering convention); no existing E2E
question or fixture file contains the phrase, so every other suite
is unaffected.
- user message containing ``show me a table`` (phase 44, markdown - user message containing ``show me a table`` (phase 44, markdown
tables, TODO.md L6) -> the fixed table answer (``TABLE_ANSWER``): tables, TODO.md L6) -> the fixed table answer (``TABLE_ANSWER``):
a 3-column service table, an ``<img onerror>`` XSS probe line, and a 3-column service table, an ``<img onerror>`` XSS probe line, and
@@ -483,6 +517,34 @@ assert _CORRECTION_MARKER in CORRECTION_INSTRUCTION, (
#: file contains the phrase, so every other suite is unaffected. #: file contains the phrase, so every other suite is unaffected.
LS_TEACH_TRIGGER = "list the files in this directory" LS_TEACH_TRIGGER = "list the files in this directory"
#: The deterministic GREP-REGEX-TEACH flow (the 2026-09-05 "Qwen 3.8"
#: incident — the harness prior is that grep takes a REGEX; this app's
#: grep is a case-insensitive fixed substring, owner-locked A5, so a
#: regex-shaped pattern can NEVER match, and the bare no-match line
#: made the turbo model trust the miss and end the turn with a wrong
#: "I searched the entire knowledge base" refusal). The flow pins the
#: self-correction on the SSE wire: the regex-shaped first grep → the
#: server's TEACHING no-match line (``GREP_TEACH_MARKER``) → the
#: plain-form retry grep → the match → the read → the deterministic
#: echo answer. Checked BEFORE the SEARCH / TOOLS_TRIGGER flows
#: (disjoint trigger phrases — the phase-71/72 ordering convention);
#: verified: no existing E2E question or fixture file contains the
#: phrase, so every other suite is unaffected.
GREP_TEACH_TRIGGER = "what are the correct llama.cpp arguments"
#: The incident's regex-shaped first grep (it can never match a
#: fixed-substring grep — that is the point of the flow).
GREP_TEACH_PATTERN = "qwen.*3\\.8"
#: The plain-form retry — the server's teaching line hands over exactly
#: this hint (``app.rag.agent.plain_form(GREP_TEACH_PATTERN)``).
GREP_TEACH_PLAIN = "qwen3.8"
#: The marker of the agent's teaching no-match line (app.rag.agent
#: ``NO_MATCHES_REGEX`` / ``NO_MATCHES_REGEX_SCOPED``) — the mock's
#: plain step keys on it (a plain no-match line carries it not).
GREP_TEACH_MARKER = "grep matches a plain substring"
#: The agent's ``ls`` listing header (app.rag.agent ``_execute_tool``): #: The agent's ``ls`` listing header (app.rag.agent ``_execute_tool``):
#: ``"N documents:"`` — the first line of every catalog tool result. #: ``"N documents:"`` — the first line of every catalog tool result.
_CATALOG_HEADER_RE = re.compile(r"^\d+ documents:") _CATALOG_HEADER_RE = re.compile(r"^\d+ documents:")
@@ -836,6 +898,61 @@ def _ls_teach_flow(body: dict[str, Any]) -> tuple[str, ...] | None:
return ("misuse",) return ("misuse",)
def _grep_teach_flow(body: dict[str, Any]) -> tuple[str, ...] | None:
"""Classify a GREP-TEACH request (the 2026-09-05 incident — see the
``GREP_TEACH_*`` constants). Stateless over the messages, like the
other marker flows:
* ``("regex",)`` — ``tools`` are offered and no ``tool``-role result
is in the messages yet: the incident's regex-shaped first grep —
``grep`` with ``{"pattern": GREP_TEACH_PATTERN}`` (id ``call_0``).
* ``("plain",)`` — the LAST tool result is the server's TEACHING
no-match line (it carries ``GREP_TEACH_MARKER``): the one-round
correction — ``grep`` with the plain form (id ``call_1``).
* ``("read", combined, "call_2")`` — the last tool result carries
``source/path:line: text`` match lines: ``read`` the FIRST match
line's document by its combined ``source/path`` identity.
* ``("answer", combined, content)`` — the last tool result is a
read result (``"Document <combined>:\n<content>"``): the
deterministic echo answer ``Read <combined>. <first 80 chars>``.
* ``("nomatch",)`` — the last tool result is a PLAIN no-match (no
match line, no teaching marker): the deterministic terminal
``No matches — the knowledge base has no such text.`` answer.
* ``None`` — not the flow: the trigger is absent, the ``<tools>``
section is missing (deflected turns never carry it), or ``tools``
are not offered and no tool results are in the messages yet (e.g.
``agent_max_rounds=0``).
"""
user = _user(body).lower()
if GREP_TEACH_TRIGGER not in user:
return None
if "<tools>" not in _system(body):
return None
results = [
str(m.get("content") or "")
for m in _messages(body)
if m.get("role") == "tool"
]
if not results:
if not body.get("tools"):
return None
return ("regex",)
last = results[-1]
if last.startswith(_READ_RESULT_PREFIX):
head, _, content = last.partition("\n")
# The read result is ``"Document <combined>:\n<content>"`` — the
# head carries the server's appended ``:`` (removed here; a
# document path never legitimately ends with one).
return ("answer", head[len(_READ_RESULT_PREFIX):].removesuffix(":"), content)
if GREP_TEACH_MARKER in last:
return ("plain",)
for line in last.splitlines():
m = _SEARCH_LINE_RE.match(line)
if m:
return ("read", m.group("sp"), "call_2")
return ("nomatch",)
def long_answer() -> str: def long_answer() -> str:
"""~900-word deterministic walkthrough (phase 11): numbered steps plus """~900-word deterministic walkthrough (phase 11): numbered steps plus
a unique final line that must survive the stream untruncated.""" a unique final line that must survive the stream untruncated."""
@@ -1307,6 +1424,50 @@ def chat_completions(body: dict[str, Any]) -> Any:
media_type="text/event-stream", media_type="text/event-stream",
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
) )
# 2026-09-05 (the "Qwen 3.8" incident — the grep regex prior):
# the deterministic GREP-TEACH self-correction flow — checked
# BEFORE the SEARCH / TOOLS_TRIGGER flows (disjoint trigger
# phrases — the phase-72 ordering convention; the trigger needs
# the ``<tools>`` section, so deflected turns never hit it).
grep_teach = _grep_teach_flow(body)
if grep_teach is not None:
if grep_teach[0] == "regex":
# The incident's misuse, deterministic: the regex-shaped
# pattern (it can never match a fixed-substring grep).
stream = _tool_call_stream(
"grep", {"pattern": GREP_TEACH_PATTERN}, "call_0"
)
elif grep_teach[0] == "plain":
# The one-round correction: the plain-form retry (the
# teaching line handed over exactly this hint).
stream = _tool_call_stream(
"grep", {"pattern": GREP_TEACH_PLAIN}, "call_1"
)
elif grep_teach[0] == "read":
stream = _tool_call_stream(
"read", {"path": grep_teach[1]}, grep_teach[2]
)
elif grep_teach[0] == "nomatch":
stream = _sse_stream(
_apply_max_tokens(
"No matches — the knowledge base has no such text.",
body.get("max_tokens"),
),
0.0,
)
else: # "answer" — quote the read document (first 80 chars)
stream = _sse_stream(
_apply_max_tokens(
f"Read {grep_teach[1]}. {grep_teach[2][:80]}",
body.get("max_tokens"),
),
0.0,
)
return StreamingResponse(
stream,
media_type="text/event-stream",
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
)
# Phase 68 (search tool): the deterministic search marker flow — # Phase 68 (search tool): the deterministic search marker flow —
# checked BEFORE the phase-37 tool flow (the more specific # checked BEFORE the phase-37 tool flow (the more specific
# trigger phrase wins, same convention as THINK_PARAS_TRIGGER). # trigger phrase wins, same convention as THINK_PARAS_TRIGGER).
+485
View File
@@ -0,0 +1,485 @@
"""The 2026-09-05 incident E2E (Playwright, mock-only): the
grep-regex-teaching self-correction loop through the real UI.
Context: the sample question "What are the correct llama.cpp arguments
for Qwen 3.8?" failed repeatedly against the live KB. The harness
prior is that ``grep(pattern)`` takes a REGEX (pi.dev's grep, ripgrep,
grep itself); this app's grep is a case-insensitive fixed substring
(owner-locked A5 — the contract does not change). A regex-shaped
first grep (``qwen.*3\\.8``) can therefore NEVER match, and the bare
"no matches" result made the turbo model trust the miss and end the
turn with a wrong "I searched the entire knowledge base" refusal.
The fix under test (``app.rag.agent``): a no-match for a regex-shaped
pattern is the TEACHING line (``NO_MATCHES_REGEX`` /
``NO_MATCHES_REGEX_SCOPED``) — the plain-substring contract stated,
the ``plain_form`` retry hint handed over — so the model self-corrects
in one round.
Run in isolation (DB must be up: ``podman compose up -d db``):
uv run pytest tests/e2e/test_grep_regex_teaching.py -v --no-cov
MOCK-ONLY suite: ``E2E_REAL_LLM=1`` is not supported — the gate is the
deterministic GREP-TEACH flow in ``tests/e2e/mock_llm.py``
(``GREP_TEACH_TRIGGER`` — "what are the correct llama.cpp arguments"
— + the HIGH prompt's ``<tools>`` section): the incident's regex-
shaped first grep (``grep`` with ``{"pattern": "qwen.*3\\.8"}``, id
``call_0``) → the agent's TEACHING no-match line (the mock's plain
step keys on the marker ``grep matches a plain substring``) → the
plain-form retry (``grep`` with ``{"pattern": "qwen3.8"}``, id
``call_1``) → the match → the ``read`` of the first match line's
document (id ``call_2``) → the deterministic echo answer.
KB fixture (TRUNCATE-then-seed, house pattern): ONE source with TWO
documents of known ``source``/``path``/``title`` (catalog order =
``(source, path)``, so the first match line is deterministic):
* ``Homelab/llama-server-args.md`` — the CATALOG-FIRST document,
indexed WITHOUT chunks (catalog-only; never in the retrieval
context, so the flow's ``read`` of it is NOT deduped as
already-in-context). It carries the literal line the plain grep
finds (``qwen3.8-27b`` — NOT the regex-shaped ``qwen.*3\\.8``,
which can never match a fixed-substring grep) and the search-flow
sentinel line (the no-regression turn). Its FIRST line is longer
than 80 chars, so the mock's first-80-chars quote stays
newline-free.
* ``Homelab/ai-stack-notes.md`` — the retrievable document: one chunk
whose embedding is the mock's own bag-of-words vector (the trigger
question cosines well past the E2E 0.30 threshold → grounded, the
``<tools>`` section rides along). It carries NO ``qwen3.8`` line
(the plain grep's match is the catalog-first document alone) and
its name carries no name-hit token (it stays the vector seed only).
Test → phase mapping (Playwright Mapping Rule):
1. ``test_regex_grep_self_corrects_to_plain_form`` — the grounded
GREP-TEACH turn: the turn settles (composer re-enables, ``done``
observed), the answer bubble carries the read document's echo
(``Read Homelab/llama-server-args.md. <first 80 chars>``), the UI
shows the three tool lines (two ``Searching for`` lines + the
``Reading`` line), and no error banner. Wire level: the ``tool``
frames arrive in order — ``grep`` ``qwen.*3\\.8`` → ``grep``
``qwen3.8`` → ``read`` the combined identity — and there is NO
fourth ``tool`` frame (the loop ended in one correction, not at
the round cap).
2. ``test_plain_search_flow_not_swallowed_by_new_trigger`` — in the
SAME session, the GREP-TEACH turn settles and a follow-up question
carrying ``SEARCH_TRIGGER`` still settles with the search flow's
``Found …`` answer (the new flow did not swallow the existing
trigger).
"""
from __future__ import annotations
import hashlib
import json
import re
import time
from datetime import UTC, datetime
from pathlib import Path
from playwright.sync_api import Page, expect
from sqlalchemy import text
from sqlalchemy.orm import Session
from app.db import SessionLocal
from app.models import Chunk, Document
from tests.e2e.mock_llm import (
GREP_TEACH_MARKER,
GREP_TEACH_PATTERN,
GREP_TEACH_PLAIN,
GREP_TEACH_TRIGGER,
SEARCH_PATTERN,
SEARCH_TRIGGER,
embed_text,
)
REPO = Path(__file__).resolve().parents[2]
# --------------------------------------------------------------------------
# The one-source, two-document fixture (see the module docstring)
# --------------------------------------------------------------------------
SEED_SOURCE = "Homelab"
DOC1_PATH = "llama-server-args.md"
DOC1_TITLE = "Llama Server Args"
DOC1_SP = f"{SEED_SOURCE}/{DOC1_PATH}"
DOC2_PATH = "ai-stack-notes.md"
DOC2_TITLE = "AI Stack Notes"
DOC2_SP = f"{SEED_SOURCE}/{DOC2_PATH}"
#: The catalog-first document (catalog order = (source, path) — DOC1
#: sorts before DOC2): the plain grep's ONLY match, and the document
#: the flow reads (so it must NOT be the seed — a seed read dedupes to
#: "Already in your context.", which the mock flow does not model).
#: Indexed WITHOUT chunks: catalog-only, never in the retrieval
#: context. Line 1 (the read quote) is >80 chars and newline-free; it
#: carries the literal ``qwen3.8-27b`` text (NEVER the regex-shaped
#: ``qwen.*3\.8`` — that is the whole point of the incident).
DOC1_CONTENT = (
"The llama.cpp server launch line for the qwen3.8-27b juggernaut "
"deployment pins the sampling and speculative-decoding flags.\n"
"Server command (verbatim): --port 8000 -ctk q8_0 -ctv q8_0 "
"--kv-unified -fa on --n-gpu-layers all --jinja.\n"
"Model file: /models/qwen3.8-27b/Qwen3.8-27B-UD-Q6_K.gguf with "
"the mmproj-BF16.gguf projector and the custom jinja template.\n"
f"Regression sentinel line: {SEARCH_PATTERN} must stay findable "
"by the plain search flow.\n"
)
assert "\n" not in DOC1_CONTENT[:80] # the read quote stays one line
assert GREP_TEACH_PLAIN in DOC1_CONTENT # the plain grep matches DOC1
assert GREP_TEACH_PATTERN not in DOC1_CONTENT # the regex never matches
assert GREP_TEACH_MARKER not in DOC1_CONTENT # the marker stays tool-side
#: The retrievable document (the grounded seed context): repeated lines
#: carry the trigger question's key tokens (llama, cpp, arguments,
#: qwen) — well past the E2E 0.30 cosine threshold — but NO literal
#: ``qwen3.8`` line (the plain grep's match stays DOC1 alone) and the
#: name carries no name-hit token (the seed stays the vector side
#: only). FIRST line >80 chars, newline-free (the seed context stays
#: one clean line).
DOC2_CONTENT = (
"Notes on the self-hosted ai stack: the llama cpp server arguments "
"for every qwen model are kept next to the quadlet files.\n"
+ (
"The llama cpp server arguments — sampling, context, kv cache "
"quantization — are documented per model in the quadlet notes.\n"
)
* 10
+ "\n## Server notes\n\n"
"Every qwen deployment shares the same llama cpp sampling "
"defaults; the per-model file overrides the speculative flags.\n"
)
assert "\n" not in DOC2_CONTENT[:80]
assert GREP_TEACH_PLAIN not in DOC2_CONTENT # the match stays DOC1 alone
#: Carries ``GREP_TEACH_TRIGGER`` (the incident's sample question,
#: near-verbatim) and nothing else — no other mock marker.
GREP_TEACH_QUESTION = (
"What are the correct llama.cpp arguments for Qwen 3.8? Show me "
"the exact server launch line from my notes."
)
assert GREP_TEACH_TRIGGER in GREP_TEACH_QUESTION.lower()
for _other in (
"use your tools",
"read two documents",
"search your documents",
"list the files in this directory",
"emit raw tool markup",
"always emit raw tool markup",
"show me a table",
"think in paragraphs",
"think out loud then hesitate",
"think out loud",
"show the end of your notes",
"write a long answer",
"fail then answer",
"always fail",
"embed fail once",
"pretend to think slowly",
):
assert _other not in GREP_TEACH_QUESTION.lower(), _other
#: Carries ``SEARCH_TRIGGER`` (the phase-68 search flow) and nothing
#: else — the no-regression follow-up question in the same session.
SEARCH_QUESTION = (
"Search your documents for the reese-sentinel-42 marker and tell "
"me the line that carries it."
)
assert SEARCH_TRIGGER in SEARCH_QUESTION.lower()
for _other in (
GREP_TEACH_TRIGGER,
"use your tools",
"read two documents",
"list the files in this directory",
"emit raw tool markup",
"always emit raw tool markup",
"show me a table",
"think in paragraphs",
"think out loud then hesitate",
"think out loud",
"show the end of your notes",
"write a long answer",
"fail then answer",
"always fail",
"embed fail once",
"pretend to think slowly",
):
assert _other not in SEARCH_QUESTION.lower(), _other
#: The mock's deterministic read echo (the read document reached the
#: model and landed in the answer) — the flow reads DOC1 (the plain
#: grep's first — only — match line).
READ_ANSWER_PREFIX = f"Read {DOC1_SP}."
READ_ANSWER_QUOTE = DOC1_CONTENT[:80]
def _seed_fixture(db: Session) -> None:
"""The one-source, two-document fixture (see the module docstring).
DOC1 (catalog-first, the grep match, the read target) is indexed
WITHOUT chunks; DOC2 carries the single chunk (the mock's own
embedding → the trigger question cosines well past the E2E 0.30
threshold → grounded, the ``<tools>`` section rides along).
"""
db.add(
Document(
source=SEED_SOURCE,
path=DOC1_PATH,
full_path=f"/tmp/{DOC1_PATH}",
title=DOC1_TITLE,
content=DOC1_CONTENT,
content_hash=hashlib.sha256(DOC1_CONTENT.encode()).hexdigest(),
indexed_at=datetime.now(UTC),
)
)
doc2 = Document(
source=SEED_SOURCE,
path=DOC2_PATH,
full_path=f"/tmp/{DOC2_PATH}",
title=DOC2_TITLE,
content=DOC2_CONTENT,
content_hash=hashlib.sha256(DOC2_CONTENT.encode()).hexdigest(),
indexed_at=datetime.now(UTC),
)
db.add(doc2)
db.flush()
db.add(
Chunk(
document_id=doc2.id,
position=0,
content=DOC2_CONTENT,
embedding=embed_text(DOC2_CONTENT),
)
)
def _reset_db_fixture() -> None:
"""Truncate the KB (plus the prompt-shaping tables), then seed the
one-source, two-document fixture. ``steering_notes`` /
``kb_overview`` are truncated too, so the HIGH prompt is exactly
``<relevance>`` + ``<documents>`` + ``<tools>`` — byte-stable
prompts, byte-stable answers."""
with SessionLocal() as db:
db.execute(
text("TRUNCATE chunks, documents, query_log, steering_notes, kb_overview")
)
db.commit()
_seed_fixture(db)
db.commit()
# --------------------------------------------------------------------------
# Page helpers (the house pattern — cf. test_tool_path_teaching.py)
# --------------------------------------------------------------------------
#: Captures the raw SSE ``data:`` payloads of the /api/chat stream
#: (a response clone read in the background) — wire-level assertions
#: for the ``tool`` frames, independent of the UI rendering.
SSE_HOOK = """
() => {
if (window.__sseInstalled) return;
window.__sseInstalled = true;
window.__sseFrames = [];
const origFetch = window.fetch;
window.fetch = async function (...args) {
const res = await origFetch.apply(this, args);
try {
const url = typeof args[0] === 'string' ? args[0] : args[0].url;
if (url.includes('/api/chat')) {
res.clone().text().then((bodyText) => {
for (const block of bodyText.split('\\n\\n')) {
const line = block.trim();
if (line.startsWith('data: ')) {
window.__sseFrames.push(line.slice(6));
}
}
});
}
} catch (e) { /* non-clonable responses: ignored */ }
return res;
};
}
"""
def _install_sse_hook(page: Page) -> None:
page.evaluate(SSE_HOOK)
def _drain_frames(page: Page) -> list[dict]:
"""One turn's SSE frames: wait for that turn's ``done`` frame, then
return EVERY frame captured since the last drain (the hook's
background read appends the whole stream at once after it closes, so
clearing-and-reading is race-free per turn)."""
deadline = time.monotonic() + 10.0
while True:
raw = page.evaluate(
"() => { const f = window.__sseFrames || []; "
"window.__sseFrames = []; return f; }"
)
parsed = [json.loads(line) for line in raw if line]
if any(f.get("type") == "done" for f in parsed):
return parsed
if time.monotonic() > deadline:
raise AssertionError(
f"SSE hook captured no `done` frame (frames so far: "
f"{len(parsed)}) — hook install failed?"
)
time.sleep(0.05)
def _tool_frames(frames: list[dict]) -> list[dict]:
return [f for f in frames if f.get("type") == "tool"]
def _submit(page: Page, question: str) -> None:
page.fill("#message-input", question)
page.click("#send-btn")
expect(page.locator(".msg.user .bubble").last).to_contain_text(question)
def _wait_settled(page: Page) -> None:
"""The turn is complete: answer text in the bubble, button recovered.
Phase 48: the label assertion carries the settle wait with an
explicit timeout — the in-flight button is the enabled Stop control
(never disabled), so ``to_be_enabled`` no longer blocks until the
turn settles."""
expect(page.locator(".msg.brain .bubble").last).not_to_have_text("", timeout=30_000)
expect(page.locator("#send-btn")).to_be_enabled(timeout=30_000)
expect(page.locator("#send-label")).to_have_text("Send", timeout=30_000)
def _assert_no_error_banner(page: Page) -> None:
"""The turn settled through the normal done path — never the red
role=alert error banner (the KB-offline banner is a separate,
health-driven state the db_ready fixture keeps away)."""
banner = page.locator("#kb-banner")
expect(banner).to_be_hidden()
expect(banner).not_to_have_attribute("role", "alert")
expect(banner).not_to_have_class(re.compile(r"is-error"))
# --------------------------------------------------------------------------
# 1. The grounded GREP-TEACH turn: the incident's regex-shaped first
# grep → the teaching no-match line → the plain-form retry → the
# match → the read → the echo answer — the loop settles in ONE
# correction (three tool rounds), pinned on the SSE wire
# --------------------------------------------------------------------------
def test_regex_grep_self_corrects_to_plain_form(
page: Page, app_url: str, mock_llm: int, db_ready: None
) -> None:
page.set_default_timeout(30_000)
_reset_db_fixture()
page.goto(app_url)
_install_sse_hook(page)
_submit(page, GREP_TEACH_QUESTION)
_wait_settled(page)
# Self-correction: the answer quotes the READ document — the plain
# grep's match was located, the document reached the model, and the
# echo landed in the answer (the wrong-deflection end state — no
# tool success, an "I searched everything" refusal — is impossible
# on this wire: the done frame below is not deflected).
bubble = page.locator(".msg.brain .bubble").last
expect(bubble).to_contain_text(READ_ANSWER_PREFIX)
expect(bubble).to_contain_text(READ_ANSWER_QUOTE)
_assert_no_error_banner(page)
# The UI shows the three tool lines in order: the regex-shaped
# first grep (Searching for <code>qwen.*3\.8</code>), the
# plain-form retry (Searching for <code>qwen3.8</code>), the read
# of the combined identity.
lines = page.locator(".msg.brain .tool-call")
expect(lines).to_have_count(3)
expect(lines.nth(0)).to_contain_text("Searching for")
expect(lines.nth(0).locator("code")).to_have_text(GREP_TEACH_PATTERN)
expect(lines.nth(1)).to_contain_text("Searching for")
expect(lines.nth(1).locator("code")).to_have_text(GREP_TEACH_PLAIN)
expect(lines.nth(2)).to_contain_text("Reading")
expect(lines.nth(2).locator("code")).to_have_text(DOC1_SP)
# Three rounds on the wire: the tool frames arrive in order —
# grep qwen.*3\.8 (the incident's regex-shaped first call), grep
# qwen3.8 (the plain-form correction the teaching line triggered),
# read the combined identity — and there is NO fourth tool frame:
# the loop ended in one correction, not at the round cap.
frames = _drain_frames(page)
assert _tool_frames(frames) == [
{"type": "tool", "name": "grep", "argument": GREP_TEACH_PATTERN},
{"type": "tool", "name": "grep", "argument": GREP_TEACH_PLAIN},
{"type": "tool", "name": "read", "argument": DOC1_SP},
]
first_delta = next(i for i, f in enumerate(frames) if f.get("type") == "delta")
assert all(
i < first_delta for i, f in enumerate(frames) if f.get("type") == "tool"
)
done = next(f for f in frames if f.get("type") == "done")
assert done["deflected"] is False
assert not [f for f in frames if f.get("type") == "error"]
# --------------------------------------------------------------------------
# 2. No regression to the plain search flow — the SAME session: after
# the GREP-TEACH turn, the SEARCH_TRIGGER follow-up (the phase-68
# search flow) still settles with the "Found …" answer
# --------------------------------------------------------------------------
def test_plain_search_flow_not_swallowed_by_new_trigger(
page: Page, app_url: str, mock_llm: int, db_ready: None
) -> None:
page.set_default_timeout(30_000)
_reset_db_fixture()
page.goto(app_url)
_install_sse_hook(page)
# Turn 1 — the GREP-TEACH flow (the incident's regex-shaped first
# grep → the teaching line → the plain-form retry → the read → the
# echo answer).
_submit(page, GREP_TEACH_QUESTION)
_wait_settled(page)
teach_frames = _drain_frames(page)
assert _tool_frames(teach_frames) == [
{"type": "tool", "name": "grep", "argument": GREP_TEACH_PATTERN},
{"type": "tool", "name": "grep", "argument": GREP_TEACH_PLAIN},
{"type": "tool", "name": "read", "argument": DOC1_SP},
]
expect(
page.locator(".msg.brain .bubble").last
).to_contain_text(READ_ANSWER_PREFIX)
# Turn 2 — the SAME session: the phase-68 search flow on
# SEARCH_TRIGGER. The new flow must not have swallowed the existing
# trigger: the follow-up settles with the search flow's answer
# (grep the sentinel → "Found <first matched line>").
_submit(page, SEARCH_QUESTION)
_wait_settled(page)
second_msg = page.locator(".msg.brain").last
lines = second_msg.locator(".tool-call")
expect(lines).to_have_count(1)
expect(lines.nth(0)).to_contain_text("Searching for")
expect(lines.nth(0).locator("code")).to_have_text(SEARCH_PATTERN)
# The answer is the search flow's echo: "Found <first matched
# line's content up to 80 chars>" — the sentinel line from DOC1.
sentinel_line = next(
line for line in DOC1_CONTENT.splitlines() if SEARCH_PATTERN in line
)
bubble = second_msg.locator(".bubble").last
expect(bubble).to_contain_text(f"Found {sentinel_line[:80]}")
_assert_no_error_banner(page)
# Wire level for the follow-up: one grep of the sentinel pattern —
# the search flow, unchanged.
frames = _drain_frames(page)
assert _tool_frames(frames) == [
{"type": "tool", "name": "grep", "argument": SEARCH_PATTERN},
]
done = next(f for f in frames if f.get("type") == "done")
assert done["deflected"] is False
assert not [f for f in frames if f.get("type") == "error"]
+207
View File
@@ -0,0 +1,207 @@
"""Integration: the name-hit lexical signal against real Postgres (the
2026-09-05 "Qwen 3.8" incident).
The unit suite (``tests/unit/test_retriever.py``) covers the pure
mapping with fake rows; this suite covers the SQL side on real
Postgres: the document-projection scan, the LATERAL representative-
chunk fetch (the ``is_summary`` chunk wins, chunk 0 otherwise, and a
chunk-less name match is EXCLUDED — the ``c.id IS NOT NULL`` guard),
the (count, length, catalog) ranking, the name-hits-lead-the-lexical-
list union with the FTS rows (chunk-id dedup), and the full
``retrieve()`` → ``select_documents()`` path putting the versioned-
name document into the seeded top-N.
Requires: ``podman compose up -d db``.
"""
from __future__ import annotations
import uuid
from collections.abc import Iterator
from datetime import UTC, datetime
import pytest
from sqlalchemy import text
from sqlalchemy.orm import Session
from app.models import Chunk, Document
from app.rag.retriever import (
NAME_HIT_LIMIT,
_lexical_candidates,
_name_hit_chunks,
retrieve,
select_documents,
)
INCIDENT_QUESTION = "What are the correct llama.cpp arguments for Qwen 3.8?"
#: 768-dim test vectors (the pgvector column's dimension) — axis unit
#: vectors so the cosines are exact (1.0 parallel, 0.0 orthogonal,
#: 0.7071 half-parallel).
D = 768
def _vec(axis: int, second: bool = False) -> list[float]:
v = [0.0] * D
v[axis] = 1.0
if second:
v[axis + 1] = 1.0
return v
def _doc(db: Session, source: str, path: str, title: str, content: str) -> Document:
doc = Document(
id=uuid.uuid4(),
source=source,
path=path,
full_path=f"/tmp/{source}/{path}",
title=title,
content=content,
content_hash="0" * 64,
indexed_at=datetime.now(UTC),
)
db.add(doc)
return doc
def _chunk(
db: Session, doc: Document, position: int, content: str, is_summary: bool = False
) -> Chunk:
chunk = Chunk(
id=uuid.uuid4(),
document_id=doc.id,
position=position,
content=content,
is_summary=is_summary,
)
db.add(chunk)
return chunk
@pytest.fixture()
def kb(db) -> Iterator[None]:
"""A fresh KB with the incident shape: the qwen3.8 quadlet (the
name hit, with a summary chunk + an ordinary chunk), a qwen3.6
quadlet (same family, different version — NOT a hit), an
unrelated document (FTS-only candidate), and a chunk-less document
whose name DOES carry the token (the exclusion guard)."""
db.execute(text("TRUNCATE chunks, documents"))
db.commit()
q38 = _doc(
db,
"deploy",
"reeseapps/ai/deployments/juggernaut/quadlets/qwen3.8-27b-juggernaut-vulkan.container",
"qwen3.8-27b-juggernaut-vulkan",
"# llama.cpp juggernaut\nExec=--port 8000 -ctk q8_0 -ctv q8_0 --jinja\n"
"-m /models/qwen3.8-27b/Qwen3.8-27B-UD-Q6_K.gguf\n",
)
_chunk(
db,
q38,
-1,
"Podman quadlet: llama.cpp server for Qwen 3.8 27B (juggernaut).",
is_summary=True,
)
_chunk(db, q38, 0, "# llama.cpp juggernaut\nExec=--port 8000 -ctk q8_0")
db.flush()
# A vector the question vector (below) cosines with — non-NULL so
# the chunk is eligible for the vector list too.
for c in q38.chunks:
c.embedding = _vec(1)
db.commit()
q36 = _doc(
db,
"deploy",
"reeseapps/ai/deployments/juggernaut/quadlets/qwen3.6-27b-juggernaut-vulkan.container",
"qwen3.6-27b-juggernaut-vulkan",
"# llama.cpp juggernaut\n-m /models/qwen3.6-27b/model.gguf\n",
)
c36 = _chunk(db, q36, 0, "# llama.cpp juggernaut\n-m /models/qwen3.6-27b/model.gguf")
c36.embedding = _vec(0) # orthogonal to the question vector
db.commit()
other = _doc(
db, "homelab", "notes/llama.cpp.md", "llama.cpp notes", "llama cpp server arguments notes\n"
)
c_other = _chunk(db, other, 0, "llama cpp server arguments notes")
c_other.embedding = _vec(1, second=True) # half-parallel to the question
db.commit()
# Name carries the token, ZERO chunks — the exclusion guard.
_doc(db, "deploy", "qwen3.8-empty.container", "qwen3.8-empty", "(empty file)")
db.commit()
yield
db.execute(text("TRUNCATE chunks, documents"))
db.commit()
def test_name_hit_chunks_real_sql(kb, db) -> None:
"""Real Postgres: the projection scan finds exactly the qwen3.8
quadlet (the qwen3.6 sibling and the chunk-less name match are
excluded), and the LATERAL fetch hands back the SUMMARY chunk as
the representative (position −1, is_summary)."""
out = _name_hit_chunks(db, INCIDENT_QUESTION)
assert [rc.document.path for rc in out] == [
"reeseapps/ai/deployments/juggernaut/quadlets/qwen3.8-27b-juggernaut-vulkan.container"
]
rc = out[0]
assert rc.position == -1 # the summary chunk wins the LATERAL order
assert rc.is_summary is True
assert rc.fts_hit is True # the lexical signal — the A8 gate answers
assert rc.cosine == 0.0 # no vector rank on the name-hit row
assert "qwen3.8-empty.container" not in [r.document.path for r in out] # chunk-less guard
def test_lexical_candidates_name_hit_leads_real_sql(kb, db) -> None:
"""The full lexical list on real Postgres: the name hit leads, the
FTS rows follow (the qwen3.6 and llama.cpp docs both match the
OR-tsquery on llama|cpp|arguments|… — the pre-incident pollution —
but the name hit still ranks them behind it)."""
out = _lexical_candidates(db, INCIDENT_QUESTION, limit=30)
paths = [rc.document.path for rc in out]
assert paths[0] == (
"reeseapps/ai/deployments/juggernaut/quadlets/qwen3.8-27b-juggernaut-vulkan.container"
)
# The FTS pollution is still present (the incident's shape) — but
# behind the name hit, no longer ahead of it.
assert (
"reeseapps/ai/deployments/juggernaut/quadlets/qwen3.6-27b-juggernaut-vulkan.container"
in paths
)
assert all(rc.fts_hit is True for rc in out)
def test_retrieve_selects_name_hit_doc_into_top_n(kb, db) -> None:
"""The product path: hybrid ``retrieve()`` (vector ∪ lexical, RRF
fused) → ``select_documents`` puts the qwen3.8 quadlet in the
seeded top-N — the incident's seed miss (the two overview docs
only) is fixed. The question vector is parallel to the q38 chunk
embeddings (cosine 1.0), orthogonal to q36 (0.0)."""
question_vec = _vec(1)
chunks = retrieve(db, INCIDENT_QUESTION, question_vec)
docs = select_documents(chunks, n=2)
assert [d.path for d in docs] == [
"reeseapps/ai/deployments/juggernaut/quadlets/qwen3.8-27b-juggernaut-vulkan.container",
"notes/llama.cpp.md",
]
def test_name_hit_limit_real_sql(db) -> None:
"""Twelve identical (1, 6) name hits — the LATERAL fetch (and the
output) carries exactly ``NAME_HIT_LIMIT`` winners, catalog order."""
db.execute(text("TRUNCATE chunks, documents"))
db.commit()
for i in range(12):
doc = _doc(
db, "S", f"quadlets/m{i:02d}-qwen38.container", f"m{i:02d}-qwen38", "llama cpp qwen38\n"
)
db.flush()
c = _chunk(db, doc, 0, f"llama cpp qwen38 doc {i}")
c.embedding = _vec(2)
db.commit()
out = _name_hit_chunks(db, "what are the llama.cpp arguments for qwen 3.8")
assert len(out) == NAME_HIT_LIMIT
assert [rc.document.path for rc in out] == [
f"quadlets/m{i:02d}-qwen38.container" for i in range(NAME_HIT_LIMIT)
]
+196 -8
View File
@@ -209,17 +209,26 @@ def test_agent_tools_names_and_parameters() -> None:
# ("pass ONLY `pattern`") + the source-name-is-not-a-document # ("pass ONLY `pattern`") + the source-name-is-not-a-document
# clause (the model kept scoping grep with an ls-style source name # clause (the model kept scoping grep with an ls-style source name
# — the 2026-09-03 incident loop shape, but on grep) plus the # — the 2026-09-03 incident loop shape, but on grep) plus the
# one-call-at-a-time discipline clause. # one-call-at-a-time discipline clause. The 2026-09-05 incident
# (the "Qwen 3.8" sample question — the harness prior is that grep
# takes a REGEX; this grep is a fixed substring, owner-locked A5):
# the plain-substring-never-a-regex clause states the contract up
# front, so the regex-shaped first grep that does fire gets the
# teaching no-match line instead of a trusted miss.
assert grep["description"] == ( assert grep["description"] == (
"Search the indexed documents for an exact string " "Search the indexed documents for an exact string "
"(case-insensitive) and return up to 20 matching lines " "(case-insensitive) and return up to 20 matching lines "
"as `source/path:line: text` — a locator, not a " "as `source/path:line: text` — a locator, not a "
"context-adder: read the winner with `read`. For a " "context-adder: read the winner with `read`. The "
"normal search pass ONLY `pattern` — it searches every " "pattern is a plain substring, NEVER a regex — if a "
"document and that is how you search the knowledge " "pattern with regex syntax (like '.*' or '\\.') comes "
"base; never pass a source name as `path` (a source " "back with no matches, retry with the plain text you "
"name is not a document). Call one tool at a time — " "expect to see. For a normal search pass ONLY `pattern` "
"wait for this result before your next call." "— it searches every document and that is how you "
"search the knowledge base; never pass a source name "
"as `path` (a source name is not a document). Call one "
"tool at a time — wait for this result before your "
"next call."
) )
grep_params = grep["parameters"] grep_params = grep["parameters"]
assert grep_params["type"] == "object" assert grep_params["type"] == "object"
@@ -227,7 +236,8 @@ def test_agent_tools_names_and_parameters() -> None:
assert set(grep_params["properties"]) == {"pattern", "path"} assert set(grep_params["properties"]) == {"pattern", "path"}
assert all(p["type"] == "string" for p in grep_params["properties"].values()) assert all(p["type"] == "string" for p in grep_params["properties"].values())
assert grep_params["properties"]["pattern"]["description"] == ( assert grep_params["properties"]["pattern"]["description"] == (
"The exact text to search for (a plain substring, not a regex)" "The exact text to search for (a plain substring, "
"not a regex — no '.*', no '\\.', no character classes)"
) )
# Phase 72 (task 02): the bare-path contract is stated up front; # Phase 72 (task 02): the bare-path contract is stated up front;
# task 05 (live gate iterations 1-8): the one-known-document clause # task 05 (live gate iterations 1-8): the one-known-document clause
@@ -1164,6 +1174,184 @@ def test_grep_truncates_match_lines_at_200_chars(monkeypatch: pytest.MonkeyPatch
assert holder.tool_calls == 1 assert holder.tool_calls == 1
# ---------- grep no-match teaching: the regex-shaped pattern
# (the 2026-09-05 "Qwen 3.8" incident — the harness prior is that
# grep takes a REGEX; this grep is a fixed substring, owner-locked
# A5, and the contract does not change) ----------
def test_plain_form_reduces_regex_to_literal_text() -> None:
"""The plain-form hint: the pattern reduced to literal text — the
incident's exact recovery (``qwen.*3\\.8`` → ``qwen3.8``) plus the
edge cases (raw ``.*`` runs dropped before unescape, so an escaped
dot survives; first alternative only; classes/quantifiers/parens/
anchors gone; whitespace preserved; pure metacharacters → ``""``).
"""
assert agent.plain_form(r"qwen.*3\.8") == "qwen3.8" # the incident
assert agent.plain_form(r"qwen 3\.8") == "qwen 3.8"
assert agent.plain_form(r"Qwen 3\.8") == "Qwen 3.8" # case kept
assert agent.plain_form(r"qwen3\.8") == "qwen3.8"
assert agent.plain_form(r"llama\.cpp") == "llama.cpp" # escaped dot kept
assert agent.plain_form(r"qwen[0-9]+") == "qwen" # class + quantifier
assert agent.plain_form("a|b") == "a" # first alternative only
assert agent.plain_form(r"\d+") == "" # no literal text — no hint
assert agent.plain_form(r".*") == "" # pure wildcard — no hint
assert agent.plain_form(r"(qwen)3\.8") == "qwen3.8" # group contents kept
assert agent.plain_form("a{2,3}b") == "ab"
assert agent.plain_form(r"^qwen$") == "qwen" # anchors dropped
assert agent.plain_form(r"a\.b") == "a.b" # escaped dot is a literal
assert agent.plain_form("plain") == "plain" # identity for plain text
def test_looks_like_regex_detection() -> None:
"""One metacharacter anywhere marks the pattern regex-shaped; a
plain substring (even with a space) does not."""
for p in (
r"qwen.*3\.8", r"qwen 3\.8", "qwen+", "a?b", "x|y", "(a)", "[a-z]", "a^b", "b$c", "a{2}"
):
assert agent.looks_like_regex(p) is True, p
for p in ("qwen 3.8", "qwen3.8", "plain substring", ""):
assert agent.looks_like_regex(p) is False, p
def test_grep_no_match_regex_pattern_gets_teaching_line(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""The incident's shape: a regex-shaped pattern that (necessarily)
misses gets the TEACHING no-match line — the plain-substring
contract stated, the plain-form retry hint handed over. Still a
counted result; the context is untouched (locked A5)."""
d1 = _doc("Alpha", "a/one.md", "One", "qwen3.8-27b juggernaut\nno regex text")
monkeypatch.setattr(agent, "all_documents", lambda db: [d1])
holder = AgentHolder()
llm = ScriptedLLM(
[
ToolCallPiece(
id="call_1", name="grep", arguments={"pattern": r"qwen.*3\.8"}
)
],
[StreamPiece("content", "ans")],
)
asyncio.run(_run(llm, holder, _settings()))
assert llm.requests[1][0][3]["content"] == agent.NO_MATCHES_REGEX.format(
pattern=r"qwen.*3\.8", plain="qwen3.8"
)
assert llm.requests[1][0][3]["content"] == (
"No matches for 'qwen.*3\\.8'. grep matches a plain substring "
"(case-insensitive), not a regex — '.*', '\\.' and the like are "
"literal text here, so that pattern can never match. Retry with "
"the plain text you expect to see (e.g. 'qwen3.8')."
)
assert holder.tool_calls == 1 # a no-match with teaching is still a result
assert holder.read_docs == [] # locked A5: a grep adds no context
def test_grep_no_match_regex_scoped_gets_teaching_line(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""The scoped teaching variant: the resolved identity is echoed, the
hint handed over."""
d1 = _doc("Alpha", "a/one.md", "One", "nothing regex-shaped here")
def _find(db: Any, source: str, path: str) -> Document | None:
return d1 if (source, path) == ("Alpha", "a/one.md") else None
monkeypatch.setattr(agent, "find_document", _find)
holder = AgentHolder()
llm = ScriptedLLM(
[
ToolCallPiece(
id="call_1",
name="grep",
arguments={"pattern": r"qwen 3\.8", "path": "Alpha/a/one.md"},
)
],
[StreamPiece("content", "ans")],
)
asyncio.run(_run(llm, holder, _settings()))
assert llm.requests[1][0][3]["content"] == (
"No matches for 'qwen 3\\.8' in Alpha/a/one.md. grep matches a "
"plain substring (case-insensitive), not a regex — retry with "
"the plain text you expect to see (e.g. 'qwen 3.8')."
)
assert holder.tool_calls == 1
assert holder.read_docs == []
def test_grep_no_match_plain_pattern_keeps_ordinary_line(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""A no-match for a PLAIN pattern (no metacharacters — "qwen 3.8" with
the space included) keeps the ordinary line byte-identical: the
teaching never fires for a well-formed pattern (the retrieval side —
the name-hit lexical signal — is what covers that case)."""
d1 = _doc("Alpha", "a/one.md", "One", "qwen3.8-27b juggernaut")
monkeypatch.setattr(agent, "all_documents", lambda db: [d1])
holder = AgentHolder()
llm = ScriptedLLM(
[ToolCallPiece(id="call_1", name="grep", arguments={"pattern": "qwen 3.8"})],
[StreamPiece("content", "ans")],
)
asyncio.run(_run(llm, holder, _settings()))
assert llm.requests[1][0][3]["content"] == (
"No matches for 'qwen 3.8' in the knowledge base."
)
assert holder.tool_calls == 1
def test_grep_matched_regex_pattern_returns_matches_not_teaching(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""A pattern with metacharacters that MATCHES literally gets the
ordinary match output — the teaching can never suppress a real hit
(the detection keys on a NO-MATCH only)."""
d1 = _doc("S", "a.md", "A", "the C++ compiler is here")
monkeypatch.setattr(agent, "all_documents", lambda db: [d1])
holder = AgentHolder()
llm = ScriptedLLM(
[ToolCallPiece(id="call_1", name="grep", arguments={"pattern": "C++"})],
[StreamPiece("content", "ans")],
)
asyncio.run(_run(llm, holder, _settings()))
assert llm.requests[1][0][3]["content"] == "S/a.md:1: the C++ compiler is here"
assert holder.tool_calls == 1
def test_grep_no_match_regex_reducing_to_empty_falls_back(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""A regex-shaped pattern with no literal text left after the
reduction (``.*``) gets the ORDINARY line — no empty hint."""
d1 = _doc("Alpha", "a/one.md", "One", "any text at all")
monkeypatch.setattr(agent, "all_documents", lambda db: [d1])
holder = AgentHolder()
llm = ScriptedLLM(
[ToolCallPiece(id="call_1", name="grep", arguments={"pattern": r".*"})],
[StreamPiece("content", "ans")],
)
asyncio.run(_run(llm, holder, _settings()))
assert llm.requests[1][0][3]["content"] == (
"No matches for '.*' in the knowledge base."
)
assert holder.tool_calls == 1
def test_no_matches_regex_templates_pin() -> None:
"""The teaching templates are verbatim pins (the model-facing copy —
the mock E2E keys off the plain-substring clause)."""
assert agent.NO_MATCHES_REGEX == (
"No matches for '{pattern}'. grep matches a plain substring "
"(case-insensitive), not a regex — '.*', '\\.' and the like are "
"literal text here, so that pattern can never match. Retry with "
"the plain text you expect to see (e.g. '{plain}')."
)
assert agent.NO_MATCHES_REGEX_SCOPED == (
"No matches for '{pattern}' in {source}/{path}. grep matches a "
"plain substring (case-insensitive), not a regex — retry with "
"the plain text you expect to see (e.g. '{plain}')."
)
def test_grep_scoped_to_one_document(monkeypatch: pytest.MonkeyPatch) -> None: def test_grep_scoped_to_one_document(monkeypatch: pytest.MonkeyPatch) -> None:
"""Scoped grep: only the named document is loaded (find_document on """Scoped grep: only the named document is loaded (find_document on
the first-slash split), ``all_documents`` never runs, and the match the first-slash split), ``all_documents`` never runs, and the match
+227 -7
View File
@@ -133,6 +133,20 @@ def test_lexical_tsquery_stopwords_left_to_postgres() -> None:
assert lexical_tsquery("how do i") == "how | do | i" assert lexical_tsquery("how do i") == "how | do | i"
def test_lexical_tsquery_dotted_tokens_kept_whole() -> None:
"""The 2026-09-05 incident: the default parser lexes dotted words
as ONE lexeme ("llama.cpp" → 'llama.cpp', "Qwen 3.8" → '3.8'), so
the query carries them whole — split tokens (llama | cpp) can never
match the document side."""
assert lexical_tsquery(
"What are the correct llama.cpp arguments for Qwen 3.8?"
) == "what | are | the | correct | llama.cpp | arguments | for | qwen | 3.8"
# The dash still splits (only dots group): ai | internal.network.
assert lexical_tsquery("how did I set up ai-internal.network?") == (
"how | did | i | set | up | ai | internal.network"
)
def test_fuse_combines_both_lists_for_double_hits() -> None: def test_fuse_combines_both_lists_for_double_hits() -> None:
v1 = _rc("a.md", cosine=0.9) v1 = _rc("a.md", cosine=0.9)
v2 = _rc("b.md", cosine=0.5) v2 = _rc("b.md", cosine=0.5)
@@ -196,7 +210,14 @@ def test_fuse_empty_lists() -> None:
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
from app.models import Chunk # noqa: E402 from app.models import Chunk # noqa: E402
from app.rag.retriever import _lexical_candidates, _vector_candidates # noqa: E402 from app.rag.retriever import ( # noqa: E402
NAME_HIT_LIMIT,
_lexical_candidates,
_name_hit_chunks,
_normalize_name,
_vector_candidates,
name_hit_tokens,
)
class _FakeResult: class _FakeResult:
@@ -210,15 +231,29 @@ class _FakeResult:
class _FakeSession: class _FakeSession:
"""Returns canned rows from ``execute`` without touching Postgres.""" """Returns canned rows from ``execute`` without touching Postgres.
def __init__(self, rows: list) -> None: One list of rows (legacy form) is returned for EVERY call; several
self._rows = rows lists (one per successive ``execute``) model a query sequence — the
name-hit lexical path (2026-09-05) issues the document-projection
query and, when hits exist, the LATERAL chunk query, BEFORE the FTS
query.
"""
def __init__(self, *rowsets: list) -> None:
if len(rowsets) == 1 and not (
rowsets[0] and isinstance(rowsets[0][0], list)
):
rowsets = (rowsets[0],) # the single-rowset legacy form
self._rowsets = rowsets
self._call = 0
self.statements: list = [] self.statements: list = []
def execute(self, stmt, params: dict | None = None) -> _FakeResult: def execute(self, stmt, params: dict | None = None) -> _FakeResult:
self.statements.append((stmt, params)) self.statements.append((stmt, params))
return _FakeResult(self._rows) rows = self._rowsets[min(self._call, len(self._rowsets) - 1)]
self._call += 1
return _FakeResult(rows)
def _chunk_row(is_summary: bool) -> Chunk: def _chunk_row(is_summary: bool) -> Chunk:
@@ -286,9 +321,17 @@ def _lexical_row(is_summary: bool, doc_path: str) -> object:
def test_lexical_candidates_carry_is_summary_flag() -> None: def test_lexical_candidates_carry_is_summary_flag() -> None:
"""The lexical list reads ``c.is_summary`` from the raw row.""" """The lexical list reads ``c.is_summary`` from the raw row.
The question carries no digit-bearing name token (no bare, no
numeric-join), so the name-hit path issues NO queries at all — the
single FTS rowset answers the only (FTS) call, and the list is the
plain FTS rows: the pre-name-hit behavior, unchanged.
"""
rows = [_lexical_row(True, "summary-src.yaml"), _lexical_row(False, "other.md")] rows = [_lexical_row(True, "summary-src.yaml"), _lexical_row(False, "other.md")]
out = _lexical_candidates(_FakeSession(rows), "how do i configure the thing", limit=10) # pyright: ignore[reportArgumentType] out = _lexical_candidates(
_FakeSession(rows), "how do i configure the thing", limit=10 # pyright: ignore[reportArgumentType]
)
assert len(out) == 2 assert len(out) == 2
by_path = {rc.document.path: rc for rc in out} by_path = {rc.document.path: rc for rc in out}
assert by_path["summary-src.yaml"].is_summary is True assert by_path["summary-src.yaml"].is_summary is True
@@ -323,3 +366,180 @@ def test_fuse_default_is_summary_stays_false_for_legacy_chunks() -> None:
out = fuse([_rc("a.md", cosine=0.8)], [_rc("b.md")], k=60) out = fuse([_rc("a.md", cosine=0.8)], [_rc("b.md")], k=60)
assert len(out) == 2 assert len(out) == 2
assert all(rc.is_summary is False for rc in out) assert all(rc.is_summary is False for rc in out)
# ---------------------------------------------------------------------------
# Name-hit lexical signal (the 2026-09-05 incident — the versioned-name
# case the default parser lexes incompatibly: "Qwen 3.8" → qwen/3/8 can
# never match a document's qwen3/8/27b tokens)
# ---------------------------------------------------------------------------
INCIDENT_QUESTION = "What are the correct llama.cpp arguments for Qwen 3.8?"
def test_normalize_name() -> None:
assert _normalize_name("Qwen 3.8") == "qwen38"
assert _normalize_name("qwen3.8-27b-juggernaut-vulkan") == "qwen3827bjuggernautvulkan"
assert _normalize_name("Mixed CASE-99") == "mixedcase99"
assert _normalize_name("!!!") == ""
def test_name_hit_tokens_incident_question() -> None:
"""The incident question yields EXACTLY the versioned join
``qwen38`` — the token the document names actually carry. Plain
prose words (``what``, ``llamacpp``, ``arguments``, ``server`` —
no digit) never name-match (the precision guard); the single
digits ("3", "8") and the bare "38" are < 4 chars; the
digit-leading ``38show`` boundary artifact is dropped."""
tokens = name_hit_tokens(INCIDENT_QUESTION)
assert tokens == ["qwen38"]
for absent in ("what", "qwen", "llamacpp", "arguments", "3", "8", "38", "38show", "server"):
assert absent not in tokens
def test_name_hit_tokens_no_digit_question_returns_empty() -> None:
"""A question with no digit-bearing token (bare or joined) yields
no name candidates — prose joins like ``correctllama`` never count."""
assert name_hit_tokens("what is the correct caddy config") == []
assert name_hit_tokens("a e i o u 3 8") == []
def test_name_hit_tokens_bare_digit_bearing_token() -> None:
"""A single written token that carries a digit (``1panel``) is a
name candidate on its own — no join needed."""
tokens = name_hit_tokens("what is my 1panel dashboard setup")
assert tokens == ["1panel"]
def _name_row(doc: Document) -> tuple:
"""One row of the name-hit document projection (catalog order)."""
return (doc.id, doc.source, doc.path, doc.title)
def _name_hit_lateral_row(doc: Document, is_summary: bool = False) -> SimpleNamespace:
"""One row of the name-hit LATERAL chunk query."""
return SimpleNamespace(
doc_id=doc.id,
source=doc.source,
path=doc.path,
full_path=doc.full_path,
title=doc.title,
doc_content=doc.content,
content_hash=doc.content_hash,
indexed_at=None,
chunk_id=uuid.uuid4(),
position=-1 if is_summary else 0,
content="summary chunk" if is_summary else "content chunk",
is_summary=is_summary,
)
def test_name_hit_chunks_no_tokens_skips_all_queries() -> None:
"""A question with no name tokens issues no queries at all."""
session = _FakeSession([]) # any call would surface a statement
assert _name_hit_chunks(session, "a e i o u 3 8") == [] # pyright: ignore[reportArgumentType]
assert session.statements == []
def test_name_hit_chunks_no_matching_doc_returns_empty() -> None:
"""Name tokens exist but no document name carries one: the
projection runs, the LATERAL fetch does not."""
doc = _doc("quadlets/other.container", "body")
name_rows = [_name_row(doc)]
session = _FakeSession(name_rows, [])
assert _name_hit_chunks(session, INCIDENT_QUESTION) == [] # pyright: ignore[reportArgumentType]
assert len(session.statements) == 1 # projection only — no LATERAL fetch
def test_name_hit_chunks_ranked_by_count_length_catalog() -> None:
"""A two-candidate question (``qwen38`` + ``1panel``): the document
whose name carries BOTH (2 matches, 12 total chars) leads; the two
single-match documents tie on (1, 6) and fall to catalog order
(``dashboards/1panel-notes.md`` before ``quadlets/qwen3.8…``).
Hits carry ``fts_hit=True`` (the A8 gate answers), ``cosine=0.0``,
and the summary flag of their representative chunk."""
both = _doc("dashboards/1panel-qwen3.8.md", "body", title="1Panel Qwen 3.8")
panel = _doc("dashboards/1panel-notes.md", "body")
q38 = _doc("quadlets/qwen3.8-27b-juggernaut-vulkan.container", "body")
name_rows = [_name_row(d) for d in (panel, both, q38)] # catalog order
question = "what are the correct llama.cpp arguments for qwen 3.8 and the 1panel dashboard?"
lateral_rows = [
_name_hit_lateral_row(q38, is_summary=True), # LATERAL may return any order
_name_hit_lateral_row(both),
_name_hit_lateral_row(panel),
]
session = _FakeSession(name_rows, lateral_rows)
out = _name_hit_chunks(session, question) # pyright: ignore[reportArgumentType]
assert [rc.document.path for rc in out] == [
"dashboards/1panel-qwen3.8.md", # 2 matched tokens — leads
"dashboards/1panel-notes.md", # (1, 6) — catalog order
"quadlets/qwen3.8-27b-juggernaut-vulkan.container", # (1, 6) — after
]
assert all(rc.fts_hit is True for rc in out) # the lexical signal
assert all(rc.cosine == 0.0 for rc in out) # no vector rank
assert all(rc.score == 0.0 for rc in out) # fuse fills the score
by_path = {rc.document.path: rc for rc in out}
assert by_path["quadlets/qwen3.8-27b-juggernaut-vulkan.container"].is_summary is True
assert by_path["quadlets/qwen3.8-27b-juggernaut-vulkan.container"].position == -1
assert by_path["dashboards/1panel-notes.md"].is_summary is False
def test_name_hit_chunks_capped_at_limit() -> None:
"""Twelve tied name hits (one matched token each) yield exactly
``NAME_HIT_LIMIT`` of them — catalog order (the deterministic
tie-break)."""
docs = [_doc(f"quadlets/m{i:02d}.container", "body") for i in range(12)]
for d in docs: # give every document a name that carries the token
d.title = "qwen38 model i"
name_rows = [_name_row(d) for d in docs]
# Only the ten winners (catalog order — the deterministic tie-break
# of the twelve identical scores) reach the LATERAL fetch; the fake
# answers with exactly those rows.
lateral_rows = [_name_hit_lateral_row(d) for d in docs[:NAME_HIT_LIMIT]]
session = _FakeSession(name_rows, lateral_rows)
out = _name_hit_chunks(
session, "tell me about the qwen 3.8 models" # pyright: ignore[reportArgumentType]
)
assert len(out) == NAME_HIT_LIMIT
assert [rc.document.path for rc in out] == [f"quadlets/m{i:02d}.container" for i in range(10)]
def test_lexical_candidates_name_hits_lead_and_dedupe_with_fts() -> None:
"""The full lexical list: name hits LEAD (their representative
chunks), the FTS rows follow, and an FTS row sharing the name hit's
chunk id appears exactly once (deduped)."""
q38 = _doc("quadlets/qwen3.8-27b-juggernaut-vulkan.container", "body")
other = _doc("quadlets/qwen38-other.container", "body") # 1 matched token
name_rows = [_name_row(q38), _name_row(other)]
q38_chunk = uuid.uuid4()
def _lateral(doc: Document) -> SimpleNamespace:
row = _name_hit_lateral_row(doc)
if doc is q38:
row.chunk_id = q38_chunk
return row
lateral_rows = [_lateral(q38), _lateral(other)]
fts_rows = [
# an FTS hit on the SAME chunk as the q38 name hit (deduped away)
SimpleNamespace(
chunk_id=q38_chunk, position=1, content="c", doc_id=q38.id,
source=q38.source, path=q38.path, full_path=q38.full_path,
title=q38.title, doc_content=q38.content, content_hash=q38.content_hash,
indexed_at=None, is_summary=False, rank=0.1,
),
# an FTS hit on a different chunk of the OTHER doc (kept)
_lexical_row(False, "quadlets/qwen38-other.container"),
]
session = _FakeSession(name_rows, lateral_rows, fts_rows)
out = _lexical_candidates(session, INCIDENT_QUESTION, limit=10) # pyright: ignore[reportArgumentType]
assert len(out) == 3 # q38 (once), other (name hit), other (FTS chunk)
# Both name hits tie on (1, 6) — catalog order: "qwen3." (ASCII 46)
# sorts before "qwen38" (ASCII 56).
assert out[0].document.path == "quadlets/qwen3.8-27b-juggernaut-vulkan.container"
assert out[1].document.path == "quadlets/qwen38-other.container"
assert out[0].chunk_id == q38_chunk # the name-hit representative row
assert {
rc.chunk_id for rc in out
} == {q38_chunk, fts_rows[1].chunk_id, lateral_rows[1].chunk_id}
assert all(rc.fts_hit is True for rc in out)