finally getting accurate answers
This commit is contained in:
@@ -28,3 +28,6 @@ frontend/dist/
|
||||
*.log
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
--- Local data dumps (KB replicas, pg_dumps) — never committed ---
|
||||
data/
|
||||
|
||||
+114
-7
@@ -66,6 +66,16 @@ task 04):
|
||||
each line truncated to 200 chars. A grep is a **locator**, not a
|
||||
context-adder: it never appends to the answer context (only
|
||||
``read`` does — ``holder.read_docs`` is untouched by a grep).
|
||||
The 2026-09-05 incident teaching (the harness prior is that grep
|
||||
takes a REGEX; this grep is a fixed substring and the contract does
|
||||
not change): when a grep RAN but found nothing and the pattern is
|
||||
regex-shaped (:func:`looks_like_regex`) with a non-empty
|
||||
:func:`plain_form`, the no-match result is the TEACHING line —
|
||||
:data:`NO_MATCHES_REGEX` / :data:`NO_MATCHES_REGEX_SCOPED` — which
|
||||
states the plain-substring contract and hands over the plain-form
|
||||
retry hint; a grep that matched, or a no-match for a plain pattern
|
||||
(or one that reduces to nothing), is byte-identical to the ordinary
|
||||
line. Still a (counted) result, never a refusal.
|
||||
3. Rejected calls get a one-line refusal and count in nothing
|
||||
(``holder.tool_calls`` tracks executed calls only): unknown tool name
|
||||
→ ``"Unknown tool."``; a ``read`` without a usable ``path`` (missing,
|
||||
@@ -163,6 +173,7 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import AsyncIterator, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, cast
|
||||
@@ -268,12 +279,16 @@ AGENT_TOOLS: list[dict[str, Any]] = [
|
||||
"Search the indexed documents for an exact string "
|
||||
"(case-insensitive) and return up to 20 matching lines "
|
||||
"as `source/path:line: text` — a locator, not a "
|
||||
"context-adder: read the winner with `read`. For a "
|
||||
"normal search pass ONLY `pattern` — it searches every "
|
||||
"document and that is how you search the knowledge "
|
||||
"base; never pass a source name as `path` (a source "
|
||||
"name is not a document). Call one tool at a time — "
|
||||
"wait for this result before your next call."
|
||||
"context-adder: read the winner with `read`. The "
|
||||
"pattern is a plain substring, NEVER a regex — if a "
|
||||
"pattern with regex syntax (like '.*' or '\\.') comes "
|
||||
"back with no matches, retry with the plain text you "
|
||||
"expect to see. For a normal search pass ONLY `pattern` "
|
||||
"— it searches every document and that is how you "
|
||||
"search the knowledge base; never pass a source name "
|
||||
"as `path` (a source name is not a document). Call one "
|
||||
"tool at a time — wait for this result before your "
|
||||
"next call."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
@@ -282,7 +297,8 @@ AGENT_TOOLS: list[dict[str, Any]] = [
|
||||
"type": "string",
|
||||
"description": (
|
||||
"The exact text to search for (a plain "
|
||||
"substring, not a regex)"
|
||||
"substring, not a regex — no '.*', no "
|
||||
"'\\.', no character classes)"
|
||||
),
|
||||
},
|
||||
"path": {
|
||||
@@ -417,6 +433,79 @@ SEARCH_LINE_LIMIT = 200
|
||||
NO_MATCHES = "No matches for '{pattern}' in the knowledge base."
|
||||
NO_MATCHES_SCOPED = "No matches for '{pattern}' in {source}/{path}."
|
||||
|
||||
#: Teaching no-match lines (the 2026-09-05 incident — owner chat:
|
||||
#: "Qwen 3.8" question failed repeatedly). The harness-trained prior is
|
||||
#: that ``grep(pattern)`` takes a REGEX (pi.dev's grep, ripgrep, grep
|
||||
#: itself); this app's grep is a case-insensitive FIXED SUBSTRING
|
||||
#: (owner-locked A5 — the contract does not change). A regex-shaped
|
||||
#: pattern (``qwen.*3\.8``, ``qwen 3\.8``) can therefore never match, and
|
||||
#: the bare no-match line above made the turbo model trust the miss and
|
||||
#: end the turn with a wrong "I searched the entire knowledge base"
|
||||
#: refusal. These lines are the deterministic teaching: the same result
|
||||
#: (a no-match is still a *result* — counted, no context added) with the
|
||||
#: correct contract stated and a plain-text retry hint (the
|
||||
#: :func:`plain_form` of the pattern, when non-empty). Deterministic only
|
||||
#: (owner permission 2026-09-03: "deterministic guardrails only right
|
||||
#: now"): no model participates in detection or repair.
|
||||
NO_MATCHES_REGEX = (
|
||||
"No matches for '{pattern}'. grep matches a plain substring "
|
||||
"(case-insensitive), not a regex — '.*', '\\.' and the like are "
|
||||
"literal text here, so that pattern can never match. Retry with the "
|
||||
"plain text you expect to see (e.g. '{plain}')."
|
||||
)
|
||||
NO_MATCHES_REGEX_SCOPED = (
|
||||
"No matches for '{pattern}' in {source}/{path}. grep matches a plain "
|
||||
"substring (case-insensitive), not a regex — retry with the plain "
|
||||
"text you expect to see (e.g. '{plain}')."
|
||||
)
|
||||
|
||||
#: One of these metacharacters anywhere in the pattern marks it
|
||||
#: regex-shaped (the detection for the teaching no-match lines above).
|
||||
#: Plain substrings that happen to contain one ("C++", "a|b") are
|
||||
#: affected only on a NO-MATCH — a pattern that matched literally still
|
||||
#: gets its ordinary result, so the teaching can never suppress a real
|
||||
#: hit.
|
||||
_REGEX_META_RE = re.compile(r"[*+?(){}\[\]|\\^$]")
|
||||
|
||||
|
||||
# Backslash + a non-alphanumeric char is an escaped literal (``\\.`` →
|
||||
# ``.``); backslash + an alphanumeric is a class shorthand (``\\d``,
|
||||
# ``\\w``, ``\\s``) with no plain-text equivalent (→ dropped).
|
||||
_ESCAPE_RE = re.compile(r"\\(.)")
|
||||
|
||||
|
||||
def looks_like_regex(pattern: str) -> bool:
|
||||
"""True when *pattern* carries regex metacharacters (see
|
||||
:data:`_REGEX_META_RE`). Pure detection — the grep itself stays a
|
||||
fixed substring (owner-locked A5)."""
|
||||
return _REGEX_META_RE.search(pattern) is not None
|
||||
|
||||
|
||||
def plain_form(pattern: str) -> str:
|
||||
"""A deterministic plain-text retry hint for a regex-shaped pattern.
|
||||
|
||||
The hint is the pattern reduced to literal text, in this order:
|
||||
drop ``.*`` runs on the RAW pattern first (the wildcard — an escaped
|
||||
``\\.`` + ``\\*`` pair carries no raw ``.*`` run, so a literal dot
|
||||
survives), unescape (``\\.`` → ``.``; class shorthands like ``\\d``
|
||||
dropped), keep only the FIRST ``|`` alternative, drop character
|
||||
classes (``[0-9]``), drop quantifier runs (``*``, ``+``, ``?``,
|
||||
``{2,3}``), drop group parens and anchors (contents kept). Whitespace
|
||||
is preserved. ``qwen.*3\\.8`` → ``qwen3.8`` (the incident's exact
|
||||
recovery), ``llama\\.cpp`` → ``llama.cpp``, ``qwen[0-9]+`` →
|
||||
``qwen``. A pattern made of pure metacharacters reduces to ``""`` —
|
||||
callers then fall back to the ordinary no-match line (no hint).
|
||||
"""
|
||||
p = re.sub(r"\.\*", "", pattern) # .* wildcard runs (raw form)
|
||||
p = _ESCAPE_RE.sub(lambda m: m.group(1) if not m.group(1).isalnum() else "", p)
|
||||
p = p.split("|", 1)[0] # first alternative only — a hint, not an answer
|
||||
p = re.sub(r"\[[^\]]*\]", "", p) # character classes carry no literal text
|
||||
p = re.sub(r"\{[^{}]*\}", "", p) # {n} / {n,} / {n,m} quantifiers
|
||||
p = re.sub(r"[*+?]+", "", p) # stray * + ? quantifiers
|
||||
p = p.replace("(", "").replace(")", "")
|
||||
p = p.replace("^", "").replace("$", "")
|
||||
return p.strip()
|
||||
|
||||
|
||||
def list_catalog(db: Session) -> list[tuple[str, str, str]]:
|
||||
"""Every indexed document as ``(source, path, title)``.
|
||||
@@ -694,6 +783,24 @@ def _execute_tool(
|
||||
# Locked A5: a grep never adds context — read_docs untouched.
|
||||
if not matches:
|
||||
shown = pattern[:100] # keep a long pattern short in the line
|
||||
# The 2026-09-05 incident teaching: a regex-shaped pattern
|
||||
# (the harness prior) can NEVER match a fixed-substring
|
||||
# grep, so a no-match for one is not "the KB lacks this" —
|
||||
# it is "the pattern was in the wrong form". Teach the
|
||||
# contract and hand over the plain-form retry hint (when the
|
||||
# reduction is non-empty); a plain pattern (or a pattern
|
||||
# that reduces to nothing) keeps the ordinary line
|
||||
# byte-identical.
|
||||
plain = plain_form(pattern) if looks_like_regex(pattern) else ""
|
||||
if plain:
|
||||
if scoped_to is not None:
|
||||
return NO_MATCHES_REGEX_SCOPED.format(
|
||||
pattern=shown,
|
||||
source=scoped_to[0],
|
||||
path=scoped_to[1],
|
||||
plain=plain[:100],
|
||||
)
|
||||
return NO_MATCHES_REGEX.format(pattern=shown, plain=plain[:100])
|
||||
if scoped_to is not None:
|
||||
# The scoped no-match line is keyed on the resolved
|
||||
# source/path (== the argument, stripped).
|
||||
|
||||
+5
-2
@@ -149,8 +149,11 @@ TOOLS_SECTION: str = (
|
||||
"path (without the source name) will not resolve. `grep` locates an "
|
||||
"exact string (case-insensitive) in the indexed documents and "
|
||||
"returns up to 20 matching `source/path:line: text` lines — a "
|
||||
"locator, not a context-adder: read the winner with `read`; for a "
|
||||
"normal search pass only `pattern` — its optional `path` argument "
|
||||
"locator, not a context-adder: read the winner with `read`. A grep "
|
||||
"pattern is a plain substring, NEVER a regex — '.*' and '\\.' are "
|
||||
"literal text there; if such a pattern returns no matches, retry "
|
||||
"with the plain text you expect to see. For a normal search pass "
|
||||
"only `pattern` — its optional `path` argument "
|
||||
"limits the search to one document you already know, by the same "
|
||||
"combined `source/path` string; never a source name — a bare "
|
||||
"document path (without the source name) will not resolve there "
|
||||
|
||||
+230
-17
@@ -3,8 +3,27 @@
|
||||
* **Vector list** — top-N chunks by cosine distance (``embedding <=> $1``),
|
||||
each carrying its cosine ``1 − distance`` (the honesty-gate input).
|
||||
* **Lexical list** — top-N chunks matching an OR-``tsquery`` over the
|
||||
question's tokens, ordered by ``ts_rank``. This is what finds
|
||||
name-your-tool questions ("gitlab") that vector similarity buries.
|
||||
question's tokens, ordered by ``ts_rank``, UNION the name-hit list:
|
||||
documents whose normalized name (title + path stem, alnum-only,
|
||||
lowercased) contains a DIGIT-BEARING normalized question token of
|
||||
length >= 4 — bare tokens (``1panel``) and the join of adjacent
|
||||
question tokens starting with a letter ("Qwen 3.8" → ``qwen38``).
|
||||
The digit requirement is the precision guard: plain prose words
|
||||
("server", "arguments" — 4+ chars but no digit) never name-match,
|
||||
while versioned product names (the incident's whole point) always
|
||||
carry one. The name-hit documents LEAD the lexical list (ranked by
|
||||
match count, then total matched length, then catalog order; capped
|
||||
at :data:`NAME_HIT_LIMIT`),
|
||||
the FTS rows follow. This is what finds name-your-tool questions
|
||||
("gitlab") that vector similarity buries — and, the 2026-09-05
|
||||
incident, the versioned-name case the default parser lexes
|
||||
incompatibly (question "Qwen 3.8" → ``qwen``/``3``/``8`` can never
|
||||
match a document's ``qwen3``/``8``/``27b`` tokens, while every
|
||||
unrelated llama.cpp quadlet out-ranks the target on the shared
|
||||
``llama``/``cpp`` tokens). The name-hit rows carry ``fts_hit=True``
|
||||
(they ARE the lexical signal — the A8 honesty gate then answers
|
||||
instead of deflecting) and ``cosine=0.0``; the RRF fusion is
|
||||
unchanged (same lists, same ``1/(k+rank)`` terms).
|
||||
* **Fusion** — Reciprocal Rank Fusion (``score = Σ 1/(k + rank)`` over the
|
||||
lists a chunk appears in; chunks hit by both lists get both terms). The
|
||||
fused score ranks; :meth:`select_documents` and :func:`weak_hit_titles`
|
||||
@@ -26,6 +45,7 @@ import re
|
||||
import uuid
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass, replace
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import select, text
|
||||
from sqlalchemy.orm import Session
|
||||
@@ -38,8 +58,78 @@ from app.models import Chunk, Document
|
||||
#: path never truncates (A7 revised, owner permission 2026-08-24).
|
||||
TRUNCATION_MARKER = "[…truncated…]"
|
||||
|
||||
#: Alphanumeric tokens of a question (``to_tsquery`` input, OR-joined).
|
||||
_TOKEN_RE = re.compile(r"[a-z0-9]+")
|
||||
#: Alphanumeric tokens of a question (``to_tsquery`` input, OR-joined),
|
||||
#: with DOTTED tokens kept whole (``llama.cpp`` → ``llama.cpp``). The
|
||||
#: default parser lexes a dotted word as ONE lexeme (``to_tsvector`` of
|
||||
#: "llama.cpp" → ``'llama.cpp'``; of "Qwen 3.8" → ``'3.8'``), so a split
|
||||
#: token (``llama`` | ``cpp``) can never match the document side — the
|
||||
#: 2026-09-05 incident's question phrase "llama.cpp" lexed
|
||||
#: incompatibly on both sides of the query. Dotted tokens are passed
|
||||
#: through as single ``to_tsquery`` lexemes (verified: Postgres accepts
|
||||
#: ``'llama.cpp'`` and ``'3.8'`` as lexemes).
|
||||
_TOKEN_RE = re.compile(r"[a-z0-9]+(?:\.[a-z0-9]+)*")
|
||||
|
||||
#: Name-hit policy (the 2026-09-05 incident): a question token shorter
|
||||
#: than this (normalized) is too weak a name signal ("3", "8", "the") —
|
||||
#: it would match half the KB by accident.
|
||||
NAME_TOKEN_MIN_LEN = 4
|
||||
|
||||
#: The name-hit list may contribute at most this many documents to the
|
||||
#: lexical side (before the FTS rows); the rest are dropped.
|
||||
NAME_HIT_LIMIT = 10
|
||||
|
||||
#: Alphanumeric runs of a lowercased string (name normalization).
|
||||
_ALNUM_RE = re.compile(r"[a-z0-9]+")
|
||||
|
||||
|
||||
def _normalize_name(s: str) -> str:
|
||||
"""Lowercased, alnum-only form of *s* (``Qwen 3.8`` → ``qwen38``)."""
|
||||
return "".join(_ALNUM_RE.findall(s.lower()))
|
||||
|
||||
|
||||
def name_hit_tokens(question: str) -> list[str]:
|
||||
"""The normalized name-match candidates of one question.
|
||||
|
||||
Only DIGIT-BEARING candidates count (the precision guard — a plain
|
||||
prose word like "server" or "arguments" must never name-match a
|
||||
document; a versioned product name always carries a digit):
|
||||
|
||||
* the :func:`_normalize_name` form of every whitespace token, kept
|
||||
when at least :data:`NAME_TOKEN_MIN_LEN` chars (``1panel``,
|
||||
``qwen38`` from a single written token);
|
||||
* the versioned-name JOIN — the normalized concatenation of every
|
||||
ADJACENT token pair whose SECOND token is purely numeric (a
|
||||
version number: ``"… for Qwen 3.8"`` → ``Qwen`` + ``3.8`` joins
|
||||
to ``qwen38``; prose joins like ``correct`` + ``llama`` and the
|
||||
word-after-version boundaries like ``3.8`` + ``show`` are
|
||||
dropped). A purely-numeric second token means the join always
|
||||
starts with the first token's text (a letter in practice), so no
|
||||
digit-leading artifact (``38show``) can survive.
|
||||
|
||||
Order of first appearance, de-duplicated.
|
||||
"""
|
||||
tokens = question.split()
|
||||
out: list[str] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
def add(tok: str) -> None:
|
||||
if (
|
||||
len(tok) >= NAME_TOKEN_MIN_LEN
|
||||
and any(ch.isdigit() for ch in tok)
|
||||
and tok not in seen
|
||||
):
|
||||
seen.add(tok)
|
||||
out.append(tok)
|
||||
|
||||
for i, raw in enumerate(tokens):
|
||||
norm = _normalize_name(raw)
|
||||
add(norm)
|
||||
if i + 1 < len(tokens):
|
||||
next_norm = _normalize_name(tokens[i + 1])
|
||||
if next_norm and next_norm.isdigit():
|
||||
add(norm + next_norm) # the versioned-name join
|
||||
return out
|
||||
|
||||
|
||||
#: One row of the lexical candidate query (all fields needed to build a
|
||||
#: detached :class:`Document` plus the chunk fields and ``ts_rank``).
|
||||
@@ -96,11 +186,14 @@ class RetrievedChunk:
|
||||
def lexical_tsquery(question: str) -> str | None:
|
||||
"""OR-joined token string for ``to_tsquery('english', …)``, or ``None``.
|
||||
|
||||
Tokens are lowercased ``[a-z0-9]+`` runs, de-duplicated in order of
|
||||
first appearance. Postgres does the lexing/stemming; a question whose
|
||||
tokens are all stopwords lexes to an *empty* tsquery (which matches
|
||||
nothing), so no special-casing is needed there. Pure-symbol questions
|
||||
("???", "🔧") yield no tokens → ``None`` → no lexical query at all.
|
||||
Tokens are lowercased ``[a-z0-9]+`` runs with DOTTED words kept whole
|
||||
(see :data:`_TOKEN_RE` — the default parser lexes ``llama.cpp`` as
|
||||
one lexeme, so the query must carry it whole too), de-duplicated in
|
||||
order of first appearance. Postgres does the lexing/stemming; a
|
||||
question whose tokens are all stopwords lexes to an *empty* tsquery
|
||||
(which matches nothing), so no special-casing is needed there.
|
||||
Pure-symbol questions ("???", "🔧") yield no tokens → ``None`` → no
|
||||
lexical query at all.
|
||||
"""
|
||||
seen: set[str] = set()
|
||||
tokens: list[str] = []
|
||||
@@ -181,21 +274,141 @@ def _vector_candidates(
|
||||
]
|
||||
|
||||
|
||||
def _lexical_candidates(db: Session, question: str, limit: int) -> list[RetrievedChunk]:
|
||||
"""Top-*limit* chunks matching the question's OR-tsquery (A7).
|
||||
|
||||
Ordered by ``ts_rank`` (with ``d.path, c.position`` as the
|
||||
deterministic tie-break); an empty tsquery (stopword-only question)
|
||||
simply matches nothing.
|
||||
#: One name-hit row: the document fields (detached :class:`Document`)
|
||||
#: plus the representative chunk — the ``is_summary`` chunk when the
|
||||
#: document has one (its natural-language summary is the best chunk of
|
||||
#: a machine file, and the chunk the vector list is likeliest to have
|
||||
#: ranked too, so the RRF merge dedupes cleanly), else chunk 0.
|
||||
_NAME_HIT_SQL = text(
|
||||
"""
|
||||
SELECT d.id AS doc_id,
|
||||
d.source AS source,
|
||||
d.path AS path,
|
||||
d.full_path AS full_path,
|
||||
d.title AS title,
|
||||
d.content AS doc_content,
|
||||
d.content_hash AS content_hash,
|
||||
d.indexed_at AS indexed_at,
|
||||
c.id AS chunk_id,
|
||||
c.position AS position,
|
||||
c.content AS content,
|
||||
c.is_summary AS is_summary
|
||||
FROM documents d
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT id, position, content, is_summary
|
||||
FROM chunks
|
||||
WHERE document_id = d.id
|
||||
ORDER BY is_summary DESC, position ASC
|
||||
LIMIT 1
|
||||
) c ON true
|
||||
WHERE d.id = ANY(:ids) AND c.id IS NOT NULL
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def _name_hit_chunks(db: Session, question: str) -> list[RetrievedChunk]:
|
||||
"""The documents whose NAME matches the question (the 2026-09-05
|
||||
incident, the versioned-name signal).
|
||||
|
||||
A document is a name hit when its normalized name —
|
||||
:func:`_normalize_name` of its ``title`` followed by the normalized
|
||||
path stem (``qwen3.8-27b-juggernaut-vulkan.container`` →
|
||||
``qwen3827bjuggernautvulkan``) — contains at least one
|
||||
:func:`name_hit_tokens` candidate (``qwen38``). Ranked by
|
||||
(distinct matched tokens, total matched length, source, path) —
|
||||
catalog order is the final deterministic tie-break — capped at
|
||||
:data:`NAME_HIT_LIMIT`. Each hit becomes one lexical
|
||||
:class:`RetrievedChunk` (its representative chunk, ``fts_hit=True``,
|
||||
``cosine=0.0``). Two lightweight queries: one projection over
|
||||
(id, source, path, title) in catalog order, one LATERAL chunk
|
||||
fetch for the ≤ :data:`NAME_HIT_LIMIT` winners (no full-content
|
||||
load; the content joins in via the row fetch below).
|
||||
"""
|
||||
tokens = name_hit_tokens(question)
|
||||
if not tokens:
|
||||
return []
|
||||
rows = db.execute(
|
||||
select(Document.id, Document.source, Document.path, Document.title).order_by(
|
||||
Document.source, Document.path
|
||||
)
|
||||
).all()
|
||||
scored: list[tuple[int, int, uuid.UUID]] = []
|
||||
by_id: dict[uuid.UUID, tuple[str, str]] = {} # id -> (source, path)
|
||||
for doc_id, source, path, title in rows:
|
||||
by_id[doc_id] = (source, path)
|
||||
name = _normalize_name(title) + _normalize_name(Path(path).stem)
|
||||
matched = [t for t in tokens if t in name]
|
||||
if matched:
|
||||
scored.append((len(matched), sum(len(t) for t in matched), doc_id))
|
||||
if not scored:
|
||||
return []
|
||||
# Ranked by (distinct matched tokens, total matched length), the
|
||||
# deterministic catalog tie-break (source, path) last.
|
||||
scored.sort(
|
||||
key=lambda s: (-s[0], -s[1], by_id[s[2]][0], by_id[s[2]][1])
|
||||
)
|
||||
ids = [s[2] for s in scored[:NAME_HIT_LIMIT]]
|
||||
hit_rows = list(db.execute(_NAME_HIT_SQL, {"ids": ids}).all())
|
||||
# The LATERAL query returns winners in id order; re-order by the
|
||||
# ranked order computed above so the lexical list is deterministic.
|
||||
order = {doc_id: rank for rank, doc_id in enumerate(ids)}
|
||||
hit_rows.sort(key=lambda row: order[row.doc_id])
|
||||
out: list[RetrievedChunk] = []
|
||||
for row in hit_rows:
|
||||
doc = Document(
|
||||
id=row.doc_id,
|
||||
source=row.source,
|
||||
path=row.path,
|
||||
full_path=row.full_path,
|
||||
title=row.title,
|
||||
content=row.doc_content,
|
||||
content_hash=row.content_hash,
|
||||
indexed_at=row.indexed_at,
|
||||
)
|
||||
out.append(
|
||||
RetrievedChunk(
|
||||
chunk_id=row.chunk_id,
|
||||
position=row.position,
|
||||
content=row.content,
|
||||
score=0.0, # filled in by :func:`fuse`
|
||||
document=doc,
|
||||
cosine=0.0, # no vector rank — name-only hit
|
||||
fts_hit=True, # the lexical signal — the A8 gate answers
|
||||
is_summary=bool(row.is_summary),
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def _lexical_candidates(db: Session, question: str, limit: int) -> list[RetrievedChunk]:
|
||||
"""The lexical candidate list: name hits, then FTS rows (A7).
|
||||
|
||||
The name-hit documents (:func:`_name_hit_chunks`, the versioned-name
|
||||
signal) LEAD the list — they are the strongest lexical evidence for
|
||||
a name-your-thing question — followed by the top-*limit* chunks
|
||||
matching the question's OR-tsquery, ordered by ``ts_rank`` (with
|
||||
``d.path, c.position`` as the deterministic tie-break). Deduped by
|
||||
chunk id (a name-hit representative chunk the FTS list also ranked
|
||||
appears once). An empty tsquery (stopword-only question) simply
|
||||
contributes no FTS rows; a question with no name tokens contributes
|
||||
no name hits — both halves are independent.
|
||||
"""
|
||||
out: list[RetrievedChunk] = []
|
||||
seen: set[uuid.UUID] = set()
|
||||
for rc in _name_hit_chunks(db, question):
|
||||
if rc.chunk_id not in seen:
|
||||
seen.add(rc.chunk_id)
|
||||
out.append(rc)
|
||||
tsquery = lexical_tsquery(question)
|
||||
if tsquery is None:
|
||||
return []
|
||||
return out
|
||||
rows = db.execute(
|
||||
_LEXICAL_SQL, {"tsquery": tsquery, "limit": limit}
|
||||
).all()
|
||||
out: list[RetrievedChunk] = []
|
||||
for row in rows:
|
||||
if row.chunk_id in seen:
|
||||
continue
|
||||
seen.add(row.chunk_id)
|
||||
doc = Document(
|
||||
id=row.doc_id,
|
||||
source=row.source,
|
||||
|
||||
@@ -183,6 +183,40 @@ Implements just enough of the aipi surface:
|
||||
phrases are disjoint substrings — the phase-71 ordering
|
||||
convention); no existing E2E question or fixture file contains the
|
||||
phrase, so every other suite is unaffected.
|
||||
- user message containing ``what are the correct llama.cpp
|
||||
arguments`` (``GREP_TEACH_TRIGGER``, the 2026-09-05 incident —
|
||||
the harness prior is that grep takes a REGEX; this app's grep is a
|
||||
case-insensitive fixed substring, owner-locked A5) **and** the
|
||||
system prompt carries the ``<tools>`` section -> the deterministic
|
||||
GREP-REGEX-TEACHING flow, discriminated statelessly from the
|
||||
messages (streaming only):
|
||||
* request 1 (``tools`` offered, no ``tool``-role result yet):
|
||||
stream ONLY ``tool_calls`` deltas — ``grep`` with
|
||||
``{"pattern": GREP_TEACH_PATTERN}`` (``qwen.*3\\.8``, id
|
||||
``call_0``) — the incident's regex-shaped first grep, which a
|
||||
fixed-substring grep can NEVER match;
|
||||
* request 2 (the last tool result is the server's TEACHING
|
||||
no-match line — it carries ``GREP_TEACH_MARKER``):
|
||||
``grep`` with the plain form ``GREP_TEACH_PLAIN``
|
||||
(``qwen3.8``, id ``call_1``) — the one-round correction;
|
||||
* request 3 (the last tool result carries
|
||||
``source/path:line: text`` match lines): ``read`` the FIRST
|
||||
match line's document by its combined ``source/path`` (id
|
||||
``call_2``);
|
||||
* request 4 (the last tool result is a read result, the
|
||||
``"Document <combined>:\n<content>"`` shape): the
|
||||
deterministic echo answer ``Read <combined>. <first 80
|
||||
chars>``, ``finish_reason: "stop"`` — the loop ended in ONE
|
||||
correction, not at the round cap.
|
||||
* A PLAIN no-match as the last result (no match line, no
|
||||
teaching marker — e.g. the plain pattern genuinely absent) is
|
||||
the deterministic terminal answer ``No matches — the knowledge
|
||||
base has no such text.`` (the flow cannot loop on a
|
||||
well-formed pattern).
|
||||
Checked BEFORE the SEARCH / TOOLS_TRIGGER flows (disjoint trigger
|
||||
phrases — the phase-72 ordering convention); no existing E2E
|
||||
question or fixture file contains the phrase, so every other suite
|
||||
is unaffected.
|
||||
- user message containing ``show me a table`` (phase 44, markdown
|
||||
tables, TODO.md L6) -> the fixed table answer (``TABLE_ANSWER``):
|
||||
a 3-column service table, an ``<img onerror>`` XSS probe line, and
|
||||
@@ -483,6 +517,34 @@ assert _CORRECTION_MARKER in CORRECTION_INSTRUCTION, (
|
||||
#: file contains the phrase, so every other suite is unaffected.
|
||||
LS_TEACH_TRIGGER = "list the files in this directory"
|
||||
|
||||
#: The deterministic GREP-REGEX-TEACH flow (the 2026-09-05 "Qwen 3.8"
|
||||
#: incident — the harness prior is that grep takes a REGEX; this app's
|
||||
#: grep is a case-insensitive fixed substring, owner-locked A5, so a
|
||||
#: regex-shaped pattern can NEVER match, and the bare no-match line
|
||||
#: made the turbo model trust the miss and end the turn with a wrong
|
||||
#: "I searched the entire knowledge base" refusal). The flow pins the
|
||||
#: self-correction on the SSE wire: the regex-shaped first grep → the
|
||||
#: server's TEACHING no-match line (``GREP_TEACH_MARKER``) → the
|
||||
#: plain-form retry grep → the match → the read → the deterministic
|
||||
#: echo answer. Checked BEFORE the SEARCH / TOOLS_TRIGGER flows
|
||||
#: (disjoint trigger phrases — the phase-71/72 ordering convention);
|
||||
#: verified: no existing E2E question or fixture file contains the
|
||||
#: phrase, so every other suite is unaffected.
|
||||
GREP_TEACH_TRIGGER = "what are the correct llama.cpp arguments"
|
||||
|
||||
#: The incident's regex-shaped first grep (it can never match a
|
||||
#: fixed-substring grep — that is the point of the flow).
|
||||
GREP_TEACH_PATTERN = "qwen.*3\\.8"
|
||||
|
||||
#: The plain-form retry — the server's teaching line hands over exactly
|
||||
#: this hint (``app.rag.agent.plain_form(GREP_TEACH_PATTERN)``).
|
||||
GREP_TEACH_PLAIN = "qwen3.8"
|
||||
|
||||
#: The marker of the agent's teaching no-match line (app.rag.agent
|
||||
#: ``NO_MATCHES_REGEX`` / ``NO_MATCHES_REGEX_SCOPED``) — the mock's
|
||||
#: plain step keys on it (a plain no-match line carries it not).
|
||||
GREP_TEACH_MARKER = "grep matches a plain substring"
|
||||
|
||||
#: The agent's ``ls`` listing header (app.rag.agent ``_execute_tool``):
|
||||
#: ``"N documents:"`` — the first line of every catalog tool result.
|
||||
_CATALOG_HEADER_RE = re.compile(r"^\d+ documents:")
|
||||
@@ -836,6 +898,61 @@ def _ls_teach_flow(body: dict[str, Any]) -> tuple[str, ...] | None:
|
||||
return ("misuse",)
|
||||
|
||||
|
||||
def _grep_teach_flow(body: dict[str, Any]) -> tuple[str, ...] | None:
|
||||
"""Classify a GREP-TEACH request (the 2026-09-05 incident — see the
|
||||
``GREP_TEACH_*`` constants). Stateless over the messages, like the
|
||||
other marker flows:
|
||||
|
||||
* ``("regex",)`` — ``tools`` are offered and no ``tool``-role result
|
||||
is in the messages yet: the incident's regex-shaped first grep —
|
||||
``grep`` with ``{"pattern": GREP_TEACH_PATTERN}`` (id ``call_0``).
|
||||
* ``("plain",)`` — the LAST tool result is the server's TEACHING
|
||||
no-match line (it carries ``GREP_TEACH_MARKER``): the one-round
|
||||
correction — ``grep`` with the plain form (id ``call_1``).
|
||||
* ``("read", combined, "call_2")`` — the last tool result carries
|
||||
``source/path:line: text`` match lines: ``read`` the FIRST match
|
||||
line's document by its combined ``source/path`` identity.
|
||||
* ``("answer", combined, content)`` — the last tool result is a
|
||||
read result (``"Document <combined>:\n<content>"``): the
|
||||
deterministic echo answer ``Read <combined>. <first 80 chars>``.
|
||||
* ``("nomatch",)`` — the last tool result is a PLAIN no-match (no
|
||||
match line, no teaching marker): the deterministic terminal
|
||||
``No matches — the knowledge base has no such text.`` answer.
|
||||
* ``None`` — not the flow: the trigger is absent, the ``<tools>``
|
||||
section is missing (deflected turns never carry it), or ``tools``
|
||||
are not offered and no tool results are in the messages yet (e.g.
|
||||
``agent_max_rounds=0``).
|
||||
"""
|
||||
user = _user(body).lower()
|
||||
if GREP_TEACH_TRIGGER not in user:
|
||||
return None
|
||||
if "<tools>" not in _system(body):
|
||||
return None
|
||||
results = [
|
||||
str(m.get("content") or "")
|
||||
for m in _messages(body)
|
||||
if m.get("role") == "tool"
|
||||
]
|
||||
if not results:
|
||||
if not body.get("tools"):
|
||||
return None
|
||||
return ("regex",)
|
||||
last = results[-1]
|
||||
if last.startswith(_READ_RESULT_PREFIX):
|
||||
head, _, content = last.partition("\n")
|
||||
# The read result is ``"Document <combined>:\n<content>"`` — the
|
||||
# head carries the server's appended ``:`` (removed here; a
|
||||
# document path never legitimately ends with one).
|
||||
return ("answer", head[len(_READ_RESULT_PREFIX):].removesuffix(":"), content)
|
||||
if GREP_TEACH_MARKER in last:
|
||||
return ("plain",)
|
||||
for line in last.splitlines():
|
||||
m = _SEARCH_LINE_RE.match(line)
|
||||
if m:
|
||||
return ("read", m.group("sp"), "call_2")
|
||||
return ("nomatch",)
|
||||
|
||||
|
||||
def long_answer() -> str:
|
||||
"""~900-word deterministic walkthrough (phase 11): numbered steps plus
|
||||
a unique final line that must survive the stream untruncated."""
|
||||
@@ -1307,6 +1424,50 @@ def chat_completions(body: dict[str, Any]) -> Any:
|
||||
media_type="text/event-stream",
|
||||
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
|
||||
)
|
||||
# 2026-09-05 (the "Qwen 3.8" incident — the grep regex prior):
|
||||
# the deterministic GREP-TEACH self-correction flow — checked
|
||||
# BEFORE the SEARCH / TOOLS_TRIGGER flows (disjoint trigger
|
||||
# phrases — the phase-72 ordering convention; the trigger needs
|
||||
# the ``<tools>`` section, so deflected turns never hit it).
|
||||
grep_teach = _grep_teach_flow(body)
|
||||
if grep_teach is not None:
|
||||
if grep_teach[0] == "regex":
|
||||
# The incident's misuse, deterministic: the regex-shaped
|
||||
# pattern (it can never match a fixed-substring grep).
|
||||
stream = _tool_call_stream(
|
||||
"grep", {"pattern": GREP_TEACH_PATTERN}, "call_0"
|
||||
)
|
||||
elif grep_teach[0] == "plain":
|
||||
# The one-round correction: the plain-form retry (the
|
||||
# teaching line handed over exactly this hint).
|
||||
stream = _tool_call_stream(
|
||||
"grep", {"pattern": GREP_TEACH_PLAIN}, "call_1"
|
||||
)
|
||||
elif grep_teach[0] == "read":
|
||||
stream = _tool_call_stream(
|
||||
"read", {"path": grep_teach[1]}, grep_teach[2]
|
||||
)
|
||||
elif grep_teach[0] == "nomatch":
|
||||
stream = _sse_stream(
|
||||
_apply_max_tokens(
|
||||
"No matches — the knowledge base has no such text.",
|
||||
body.get("max_tokens"),
|
||||
),
|
||||
0.0,
|
||||
)
|
||||
else: # "answer" — quote the read document (first 80 chars)
|
||||
stream = _sse_stream(
|
||||
_apply_max_tokens(
|
||||
f"Read {grep_teach[1]}. {grep_teach[2][:80]}",
|
||||
body.get("max_tokens"),
|
||||
),
|
||||
0.0,
|
||||
)
|
||||
return StreamingResponse(
|
||||
stream,
|
||||
media_type="text/event-stream",
|
||||
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
|
||||
)
|
||||
# Phase 68 (search tool): the deterministic search marker flow —
|
||||
# checked BEFORE the phase-37 tool flow (the more specific
|
||||
# trigger phrase wins, same convention as THINK_PARAS_TRIGGER).
|
||||
|
||||
@@ -0,0 +1,485 @@
|
||||
"""The 2026-09-05 incident E2E (Playwright, mock-only): the
|
||||
grep-regex-teaching self-correction loop through the real UI.
|
||||
|
||||
Context: the sample question "What are the correct llama.cpp arguments
|
||||
for Qwen 3.8?" failed repeatedly against the live KB. The harness
|
||||
prior is that ``grep(pattern)`` takes a REGEX (pi.dev's grep, ripgrep,
|
||||
grep itself); this app's grep is a case-insensitive fixed substring
|
||||
(owner-locked A5 — the contract does not change). A regex-shaped
|
||||
first grep (``qwen.*3\\.8``) can therefore NEVER match, and the bare
|
||||
"no matches" result made the turbo model trust the miss and end the
|
||||
turn with a wrong "I searched the entire knowledge base" refusal.
|
||||
The fix under test (``app.rag.agent``): a no-match for a regex-shaped
|
||||
pattern is the TEACHING line (``NO_MATCHES_REGEX`` /
|
||||
``NO_MATCHES_REGEX_SCOPED``) — the plain-substring contract stated,
|
||||
the ``plain_form`` retry hint handed over — so the model self-corrects
|
||||
in one round.
|
||||
|
||||
Run in isolation (DB must be up: ``podman compose up -d db``):
|
||||
|
||||
uv run pytest tests/e2e/test_grep_regex_teaching.py -v --no-cov
|
||||
|
||||
MOCK-ONLY suite: ``E2E_REAL_LLM=1`` is not supported — the gate is the
|
||||
deterministic GREP-TEACH flow in ``tests/e2e/mock_llm.py``
|
||||
(``GREP_TEACH_TRIGGER`` — "what are the correct llama.cpp arguments"
|
||||
— + the HIGH prompt's ``<tools>`` section): the incident's regex-
|
||||
shaped first grep (``grep`` with ``{"pattern": "qwen.*3\\.8"}``, id
|
||||
``call_0``) → the agent's TEACHING no-match line (the mock's plain
|
||||
step keys on the marker ``grep matches a plain substring``) → the
|
||||
plain-form retry (``grep`` with ``{"pattern": "qwen3.8"}``, id
|
||||
``call_1``) → the match → the ``read`` of the first match line's
|
||||
document (id ``call_2``) → the deterministic echo answer.
|
||||
|
||||
KB fixture (TRUNCATE-then-seed, house pattern): ONE source with TWO
|
||||
documents of known ``source``/``path``/``title`` (catalog order =
|
||||
``(source, path)``, so the first match line is deterministic):
|
||||
|
||||
* ``Homelab/llama-server-args.md`` — the CATALOG-FIRST document,
|
||||
indexed WITHOUT chunks (catalog-only; never in the retrieval
|
||||
context, so the flow's ``read`` of it is NOT deduped as
|
||||
already-in-context). It carries the literal line the plain grep
|
||||
finds (``qwen3.8-27b`` — NOT the regex-shaped ``qwen.*3\\.8``,
|
||||
which can never match a fixed-substring grep) and the search-flow
|
||||
sentinel line (the no-regression turn). Its FIRST line is longer
|
||||
than 80 chars, so the mock's first-80-chars quote stays
|
||||
newline-free.
|
||||
* ``Homelab/ai-stack-notes.md`` — the retrievable document: one chunk
|
||||
whose embedding is the mock's own bag-of-words vector (the trigger
|
||||
question cosines well past the E2E 0.30 threshold → grounded, the
|
||||
``<tools>`` section rides along). It carries NO ``qwen3.8`` line
|
||||
(the plain grep's match is the catalog-first document alone) and
|
||||
its name carries no name-hit token (it stays the vector seed only).
|
||||
|
||||
Test → phase mapping (Playwright Mapping Rule):
|
||||
1. ``test_regex_grep_self_corrects_to_plain_form`` — the grounded
|
||||
GREP-TEACH turn: the turn settles (composer re-enables, ``done``
|
||||
observed), the answer bubble carries the read document's echo
|
||||
(``Read Homelab/llama-server-args.md. <first 80 chars>``), the UI
|
||||
shows the three tool lines (two ``Searching for`` lines + the
|
||||
``Reading`` line), and no error banner. Wire level: the ``tool``
|
||||
frames arrive in order — ``grep`` ``qwen.*3\\.8`` → ``grep``
|
||||
``qwen3.8`` → ``read`` the combined identity — and there is NO
|
||||
fourth ``tool`` frame (the loop ended in one correction, not at
|
||||
the round cap).
|
||||
2. ``test_plain_search_flow_not_swallowed_by_new_trigger`` — in the
|
||||
SAME session, the GREP-TEACH turn settles and a follow-up question
|
||||
carrying ``SEARCH_TRIGGER`` still settles with the search flow's
|
||||
``Found …`` answer (the new flow did not swallow the existing
|
||||
trigger).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
from playwright.sync_api import Page, expect
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.db import SessionLocal
|
||||
from app.models import Chunk, Document
|
||||
from tests.e2e.mock_llm import (
|
||||
GREP_TEACH_MARKER,
|
||||
GREP_TEACH_PATTERN,
|
||||
GREP_TEACH_PLAIN,
|
||||
GREP_TEACH_TRIGGER,
|
||||
SEARCH_PATTERN,
|
||||
SEARCH_TRIGGER,
|
||||
embed_text,
|
||||
)
|
||||
|
||||
REPO = Path(__file__).resolve().parents[2]
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# The one-source, two-document fixture (see the module docstring)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
SEED_SOURCE = "Homelab"
|
||||
DOC1_PATH = "llama-server-args.md"
|
||||
DOC1_TITLE = "Llama Server Args"
|
||||
DOC1_SP = f"{SEED_SOURCE}/{DOC1_PATH}"
|
||||
|
||||
DOC2_PATH = "ai-stack-notes.md"
|
||||
DOC2_TITLE = "AI Stack Notes"
|
||||
DOC2_SP = f"{SEED_SOURCE}/{DOC2_PATH}"
|
||||
|
||||
#: The catalog-first document (catalog order = (source, path) — DOC1
|
||||
#: sorts before DOC2): the plain grep's ONLY match, and the document
|
||||
#: the flow reads (so it must NOT be the seed — a seed read dedupes to
|
||||
#: "Already in your context.", which the mock flow does not model).
|
||||
#: Indexed WITHOUT chunks: catalog-only, never in the retrieval
|
||||
#: context. Line 1 (the read quote) is >80 chars and newline-free; it
|
||||
#: carries the literal ``qwen3.8-27b`` text (NEVER the regex-shaped
|
||||
#: ``qwen.*3\.8`` — that is the whole point of the incident).
|
||||
DOC1_CONTENT = (
|
||||
"The llama.cpp server launch line for the qwen3.8-27b juggernaut "
|
||||
"deployment pins the sampling and speculative-decoding flags.\n"
|
||||
"Server command (verbatim): --port 8000 -ctk q8_0 -ctv q8_0 "
|
||||
"--kv-unified -fa on --n-gpu-layers all --jinja.\n"
|
||||
"Model file: /models/qwen3.8-27b/Qwen3.8-27B-UD-Q6_K.gguf with "
|
||||
"the mmproj-BF16.gguf projector and the custom jinja template.\n"
|
||||
f"Regression sentinel line: {SEARCH_PATTERN} must stay findable "
|
||||
"by the plain search flow.\n"
|
||||
)
|
||||
assert "\n" not in DOC1_CONTENT[:80] # the read quote stays one line
|
||||
assert GREP_TEACH_PLAIN in DOC1_CONTENT # the plain grep matches DOC1
|
||||
assert GREP_TEACH_PATTERN not in DOC1_CONTENT # the regex never matches
|
||||
assert GREP_TEACH_MARKER not in DOC1_CONTENT # the marker stays tool-side
|
||||
|
||||
#: The retrievable document (the grounded seed context): repeated lines
|
||||
#: carry the trigger question's key tokens (llama, cpp, arguments,
|
||||
#: qwen) — well past the E2E 0.30 cosine threshold — but NO literal
|
||||
#: ``qwen3.8`` line (the plain grep's match stays DOC1 alone) and the
|
||||
#: name carries no name-hit token (the seed stays the vector side
|
||||
#: only). FIRST line >80 chars, newline-free (the seed context stays
|
||||
#: one clean line).
|
||||
DOC2_CONTENT = (
|
||||
"Notes on the self-hosted ai stack: the llama cpp server arguments "
|
||||
"for every qwen model are kept next to the quadlet files.\n"
|
||||
+ (
|
||||
"The llama cpp server arguments — sampling, context, kv cache "
|
||||
"quantization — are documented per model in the quadlet notes.\n"
|
||||
)
|
||||
* 10
|
||||
+ "\n## Server notes\n\n"
|
||||
"Every qwen deployment shares the same llama cpp sampling "
|
||||
"defaults; the per-model file overrides the speculative flags.\n"
|
||||
)
|
||||
assert "\n" not in DOC2_CONTENT[:80]
|
||||
assert GREP_TEACH_PLAIN not in DOC2_CONTENT # the match stays DOC1 alone
|
||||
|
||||
#: Carries ``GREP_TEACH_TRIGGER`` (the incident's sample question,
|
||||
#: near-verbatim) and nothing else — no other mock marker.
|
||||
GREP_TEACH_QUESTION = (
|
||||
"What are the correct llama.cpp arguments for Qwen 3.8? Show me "
|
||||
"the exact server launch line from my notes."
|
||||
)
|
||||
assert GREP_TEACH_TRIGGER in GREP_TEACH_QUESTION.lower()
|
||||
for _other in (
|
||||
"use your tools",
|
||||
"read two documents",
|
||||
"search your documents",
|
||||
"list the files in this directory",
|
||||
"emit raw tool markup",
|
||||
"always emit raw tool markup",
|
||||
"show me a table",
|
||||
"think in paragraphs",
|
||||
"think out loud then hesitate",
|
||||
"think out loud",
|
||||
"show the end of your notes",
|
||||
"write a long answer",
|
||||
"fail then answer",
|
||||
"always fail",
|
||||
"embed fail once",
|
||||
"pretend to think slowly",
|
||||
):
|
||||
assert _other not in GREP_TEACH_QUESTION.lower(), _other
|
||||
|
||||
#: Carries ``SEARCH_TRIGGER`` (the phase-68 search flow) and nothing
|
||||
#: else — the no-regression follow-up question in the same session.
|
||||
SEARCH_QUESTION = (
|
||||
"Search your documents for the reese-sentinel-42 marker and tell "
|
||||
"me the line that carries it."
|
||||
)
|
||||
assert SEARCH_TRIGGER in SEARCH_QUESTION.lower()
|
||||
for _other in (
|
||||
GREP_TEACH_TRIGGER,
|
||||
"use your tools",
|
||||
"read two documents",
|
||||
"list the files in this directory",
|
||||
"emit raw tool markup",
|
||||
"always emit raw tool markup",
|
||||
"show me a table",
|
||||
"think in paragraphs",
|
||||
"think out loud then hesitate",
|
||||
"think out loud",
|
||||
"show the end of your notes",
|
||||
"write a long answer",
|
||||
"fail then answer",
|
||||
"always fail",
|
||||
"embed fail once",
|
||||
"pretend to think slowly",
|
||||
):
|
||||
assert _other not in SEARCH_QUESTION.lower(), _other
|
||||
|
||||
#: The mock's deterministic read echo (the read document reached the
|
||||
#: model and landed in the answer) — the flow reads DOC1 (the plain
|
||||
#: grep's first — only — match line).
|
||||
READ_ANSWER_PREFIX = f"Read {DOC1_SP}."
|
||||
READ_ANSWER_QUOTE = DOC1_CONTENT[:80]
|
||||
|
||||
|
||||
def _seed_fixture(db: Session) -> None:
|
||||
"""The one-source, two-document fixture (see the module docstring).
|
||||
|
||||
DOC1 (catalog-first, the grep match, the read target) is indexed
|
||||
WITHOUT chunks; DOC2 carries the single chunk (the mock's own
|
||||
embedding → the trigger question cosines well past the E2E 0.30
|
||||
threshold → grounded, the ``<tools>`` section rides along).
|
||||
"""
|
||||
db.add(
|
||||
Document(
|
||||
source=SEED_SOURCE,
|
||||
path=DOC1_PATH,
|
||||
full_path=f"/tmp/{DOC1_PATH}",
|
||||
title=DOC1_TITLE,
|
||||
content=DOC1_CONTENT,
|
||||
content_hash=hashlib.sha256(DOC1_CONTENT.encode()).hexdigest(),
|
||||
indexed_at=datetime.now(UTC),
|
||||
)
|
||||
)
|
||||
doc2 = Document(
|
||||
source=SEED_SOURCE,
|
||||
path=DOC2_PATH,
|
||||
full_path=f"/tmp/{DOC2_PATH}",
|
||||
title=DOC2_TITLE,
|
||||
content=DOC2_CONTENT,
|
||||
content_hash=hashlib.sha256(DOC2_CONTENT.encode()).hexdigest(),
|
||||
indexed_at=datetime.now(UTC),
|
||||
)
|
||||
db.add(doc2)
|
||||
db.flush()
|
||||
db.add(
|
||||
Chunk(
|
||||
document_id=doc2.id,
|
||||
position=0,
|
||||
content=DOC2_CONTENT,
|
||||
embedding=embed_text(DOC2_CONTENT),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _reset_db_fixture() -> None:
|
||||
"""Truncate the KB (plus the prompt-shaping tables), then seed the
|
||||
one-source, two-document fixture. ``steering_notes`` /
|
||||
``kb_overview`` are truncated too, so the HIGH prompt is exactly
|
||||
``<relevance>`` + ``<documents>`` + ``<tools>`` — byte-stable
|
||||
prompts, byte-stable answers."""
|
||||
with SessionLocal() as db:
|
||||
db.execute(
|
||||
text("TRUNCATE chunks, documents, query_log, steering_notes, kb_overview")
|
||||
)
|
||||
db.commit()
|
||||
_seed_fixture(db)
|
||||
db.commit()
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Page helpers (the house pattern — cf. test_tool_path_teaching.py)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
#: Captures the raw SSE ``data:`` payloads of the /api/chat stream
|
||||
#: (a response clone read in the background) — wire-level assertions
|
||||
#: for the ``tool`` frames, independent of the UI rendering.
|
||||
SSE_HOOK = """
|
||||
() => {
|
||||
if (window.__sseInstalled) return;
|
||||
window.__sseInstalled = true;
|
||||
window.__sseFrames = [];
|
||||
const origFetch = window.fetch;
|
||||
window.fetch = async function (...args) {
|
||||
const res = await origFetch.apply(this, args);
|
||||
try {
|
||||
const url = typeof args[0] === 'string' ? args[0] : args[0].url;
|
||||
if (url.includes('/api/chat')) {
|
||||
res.clone().text().then((bodyText) => {
|
||||
for (const block of bodyText.split('\\n\\n')) {
|
||||
const line = block.trim();
|
||||
if (line.startsWith('data: ')) {
|
||||
window.__sseFrames.push(line.slice(6));
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
} catch (e) { /* non-clonable responses: ignored */ }
|
||||
return res;
|
||||
};
|
||||
}
|
||||
"""
|
||||
|
||||
|
||||
def _install_sse_hook(page: Page) -> None:
|
||||
page.evaluate(SSE_HOOK)
|
||||
|
||||
|
||||
def _drain_frames(page: Page) -> list[dict]:
|
||||
"""One turn's SSE frames: wait for that turn's ``done`` frame, then
|
||||
return EVERY frame captured since the last drain (the hook's
|
||||
background read appends the whole stream at once after it closes, so
|
||||
clearing-and-reading is race-free per turn)."""
|
||||
deadline = time.monotonic() + 10.0
|
||||
while True:
|
||||
raw = page.evaluate(
|
||||
"() => { const f = window.__sseFrames || []; "
|
||||
"window.__sseFrames = []; return f; }"
|
||||
)
|
||||
parsed = [json.loads(line) for line in raw if line]
|
||||
if any(f.get("type") == "done" for f in parsed):
|
||||
return parsed
|
||||
if time.monotonic() > deadline:
|
||||
raise AssertionError(
|
||||
f"SSE hook captured no `done` frame (frames so far: "
|
||||
f"{len(parsed)}) — hook install failed?"
|
||||
)
|
||||
time.sleep(0.05)
|
||||
|
||||
|
||||
def _tool_frames(frames: list[dict]) -> list[dict]:
|
||||
return [f for f in frames if f.get("type") == "tool"]
|
||||
|
||||
|
||||
def _submit(page: Page, question: str) -> None:
|
||||
page.fill("#message-input", question)
|
||||
page.click("#send-btn")
|
||||
expect(page.locator(".msg.user .bubble").last).to_contain_text(question)
|
||||
|
||||
|
||||
def _wait_settled(page: Page) -> None:
|
||||
"""The turn is complete: answer text in the bubble, button recovered.
|
||||
|
||||
Phase 48: the label assertion carries the settle wait with an
|
||||
explicit timeout — the in-flight button is the enabled Stop control
|
||||
(never disabled), so ``to_be_enabled`` no longer blocks until the
|
||||
turn settles."""
|
||||
expect(page.locator(".msg.brain .bubble").last).not_to_have_text("", timeout=30_000)
|
||||
expect(page.locator("#send-btn")).to_be_enabled(timeout=30_000)
|
||||
expect(page.locator("#send-label")).to_have_text("Send", timeout=30_000)
|
||||
|
||||
|
||||
def _assert_no_error_banner(page: Page) -> None:
|
||||
"""The turn settled through the normal done path — never the red
|
||||
role=alert error banner (the KB-offline banner is a separate,
|
||||
health-driven state the db_ready fixture keeps away)."""
|
||||
banner = page.locator("#kb-banner")
|
||||
expect(banner).to_be_hidden()
|
||||
expect(banner).not_to_have_attribute("role", "alert")
|
||||
expect(banner).not_to_have_class(re.compile(r"is-error"))
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 1. The grounded GREP-TEACH turn: the incident's regex-shaped first
|
||||
# grep → the teaching no-match line → the plain-form retry → the
|
||||
# match → the read → the echo answer — the loop settles in ONE
|
||||
# correction (three tool rounds), pinned on the SSE wire
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_regex_grep_self_corrects_to_plain_form(
|
||||
page: Page, app_url: str, mock_llm: int, db_ready: None
|
||||
) -> None:
|
||||
page.set_default_timeout(30_000)
|
||||
_reset_db_fixture()
|
||||
page.goto(app_url)
|
||||
_install_sse_hook(page)
|
||||
|
||||
_submit(page, GREP_TEACH_QUESTION)
|
||||
_wait_settled(page)
|
||||
|
||||
# Self-correction: the answer quotes the READ document — the plain
|
||||
# grep's match was located, the document reached the model, and the
|
||||
# echo landed in the answer (the wrong-deflection end state — no
|
||||
# tool success, an "I searched everything" refusal — is impossible
|
||||
# on this wire: the done frame below is not deflected).
|
||||
bubble = page.locator(".msg.brain .bubble").last
|
||||
expect(bubble).to_contain_text(READ_ANSWER_PREFIX)
|
||||
expect(bubble).to_contain_text(READ_ANSWER_QUOTE)
|
||||
_assert_no_error_banner(page)
|
||||
|
||||
# The UI shows the three tool lines in order: the regex-shaped
|
||||
# first grep (Searching for <code>qwen.*3\.8</code>), the
|
||||
# plain-form retry (Searching for <code>qwen3.8</code>), the read
|
||||
# of the combined identity.
|
||||
lines = page.locator(".msg.brain .tool-call")
|
||||
expect(lines).to_have_count(3)
|
||||
expect(lines.nth(0)).to_contain_text("Searching for")
|
||||
expect(lines.nth(0).locator("code")).to_have_text(GREP_TEACH_PATTERN)
|
||||
expect(lines.nth(1)).to_contain_text("Searching for")
|
||||
expect(lines.nth(1).locator("code")).to_have_text(GREP_TEACH_PLAIN)
|
||||
expect(lines.nth(2)).to_contain_text("Reading")
|
||||
expect(lines.nth(2).locator("code")).to_have_text(DOC1_SP)
|
||||
|
||||
# Three rounds on the wire: the tool frames arrive in order —
|
||||
# grep qwen.*3\.8 (the incident's regex-shaped first call), grep
|
||||
# qwen3.8 (the plain-form correction the teaching line triggered),
|
||||
# read the combined identity — and there is NO fourth tool frame:
|
||||
# the loop ended in one correction, not at the round cap.
|
||||
frames = _drain_frames(page)
|
||||
assert _tool_frames(frames) == [
|
||||
{"type": "tool", "name": "grep", "argument": GREP_TEACH_PATTERN},
|
||||
{"type": "tool", "name": "grep", "argument": GREP_TEACH_PLAIN},
|
||||
{"type": "tool", "name": "read", "argument": DOC1_SP},
|
||||
]
|
||||
first_delta = next(i for i, f in enumerate(frames) if f.get("type") == "delta")
|
||||
assert all(
|
||||
i < first_delta for i, f in enumerate(frames) if f.get("type") == "tool"
|
||||
)
|
||||
done = next(f for f in frames if f.get("type") == "done")
|
||||
assert done["deflected"] is False
|
||||
assert not [f for f in frames if f.get("type") == "error"]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 2. No regression to the plain search flow — the SAME session: after
|
||||
# the GREP-TEACH turn, the SEARCH_TRIGGER follow-up (the phase-68
|
||||
# search flow) still settles with the "Found …" answer
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_plain_search_flow_not_swallowed_by_new_trigger(
|
||||
page: Page, app_url: str, mock_llm: int, db_ready: None
|
||||
) -> None:
|
||||
page.set_default_timeout(30_000)
|
||||
_reset_db_fixture()
|
||||
page.goto(app_url)
|
||||
_install_sse_hook(page)
|
||||
|
||||
# Turn 1 — the GREP-TEACH flow (the incident's regex-shaped first
|
||||
# grep → the teaching line → the plain-form retry → the read → the
|
||||
# echo answer).
|
||||
_submit(page, GREP_TEACH_QUESTION)
|
||||
_wait_settled(page)
|
||||
teach_frames = _drain_frames(page)
|
||||
assert _tool_frames(teach_frames) == [
|
||||
{"type": "tool", "name": "grep", "argument": GREP_TEACH_PATTERN},
|
||||
{"type": "tool", "name": "grep", "argument": GREP_TEACH_PLAIN},
|
||||
{"type": "tool", "name": "read", "argument": DOC1_SP},
|
||||
]
|
||||
expect(
|
||||
page.locator(".msg.brain .bubble").last
|
||||
).to_contain_text(READ_ANSWER_PREFIX)
|
||||
|
||||
# Turn 2 — the SAME session: the phase-68 search flow on
|
||||
# SEARCH_TRIGGER. The new flow must not have swallowed the existing
|
||||
# trigger: the follow-up settles with the search flow's answer
|
||||
# (grep the sentinel → "Found <first matched line>").
|
||||
_submit(page, SEARCH_QUESTION)
|
||||
_wait_settled(page)
|
||||
|
||||
second_msg = page.locator(".msg.brain").last
|
||||
lines = second_msg.locator(".tool-call")
|
||||
expect(lines).to_have_count(1)
|
||||
expect(lines.nth(0)).to_contain_text("Searching for")
|
||||
expect(lines.nth(0).locator("code")).to_have_text(SEARCH_PATTERN)
|
||||
|
||||
# The answer is the search flow's echo: "Found <first matched
|
||||
# line's content up to 80 chars>" — the sentinel line from DOC1.
|
||||
sentinel_line = next(
|
||||
line for line in DOC1_CONTENT.splitlines() if SEARCH_PATTERN in line
|
||||
)
|
||||
bubble = second_msg.locator(".bubble").last
|
||||
expect(bubble).to_contain_text(f"Found {sentinel_line[:80]}")
|
||||
_assert_no_error_banner(page)
|
||||
|
||||
# Wire level for the follow-up: one grep of the sentinel pattern —
|
||||
# the search flow, unchanged.
|
||||
frames = _drain_frames(page)
|
||||
assert _tool_frames(frames) == [
|
||||
{"type": "tool", "name": "grep", "argument": SEARCH_PATTERN},
|
||||
]
|
||||
done = next(f for f in frames if f.get("type") == "done")
|
||||
assert done["deflected"] is False
|
||||
assert not [f for f in frames if f.get("type") == "error"]
|
||||
@@ -0,0 +1,207 @@
|
||||
"""Integration: the name-hit lexical signal against real Postgres (the
|
||||
2026-09-05 "Qwen 3.8" incident).
|
||||
|
||||
The unit suite (``tests/unit/test_retriever.py``) covers the pure
|
||||
mapping with fake rows; this suite covers the SQL side on real
|
||||
Postgres: the document-projection scan, the LATERAL representative-
|
||||
chunk fetch (the ``is_summary`` chunk wins, chunk 0 otherwise, and a
|
||||
chunk-less name match is EXCLUDED — the ``c.id IS NOT NULL`` guard),
|
||||
the (count, length, catalog) ranking, the name-hits-lead-the-lexical-
|
||||
list union with the FTS rows (chunk-id dedup), and the full
|
||||
``retrieve()`` → ``select_documents()`` path putting the versioned-
|
||||
name document into the seeded top-N.
|
||||
|
||||
Requires: ``podman compose up -d db``.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from collections.abc import Iterator
|
||||
from datetime import UTC, datetime
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.models import Chunk, Document
|
||||
from app.rag.retriever import (
|
||||
NAME_HIT_LIMIT,
|
||||
_lexical_candidates,
|
||||
_name_hit_chunks,
|
||||
retrieve,
|
||||
select_documents,
|
||||
)
|
||||
|
||||
INCIDENT_QUESTION = "What are the correct llama.cpp arguments for Qwen 3.8?"
|
||||
|
||||
#: 768-dim test vectors (the pgvector column's dimension) — axis unit
|
||||
#: vectors so the cosines are exact (1.0 parallel, 0.0 orthogonal,
|
||||
#: 0.7071 half-parallel).
|
||||
D = 768
|
||||
|
||||
|
||||
def _vec(axis: int, second: bool = False) -> list[float]:
|
||||
v = [0.0] * D
|
||||
v[axis] = 1.0
|
||||
if second:
|
||||
v[axis + 1] = 1.0
|
||||
return v
|
||||
|
||||
|
||||
def _doc(db: Session, source: str, path: str, title: str, content: str) -> Document:
|
||||
doc = Document(
|
||||
id=uuid.uuid4(),
|
||||
source=source,
|
||||
path=path,
|
||||
full_path=f"/tmp/{source}/{path}",
|
||||
title=title,
|
||||
content=content,
|
||||
content_hash="0" * 64,
|
||||
indexed_at=datetime.now(UTC),
|
||||
)
|
||||
db.add(doc)
|
||||
return doc
|
||||
|
||||
|
||||
def _chunk(
|
||||
db: Session, doc: Document, position: int, content: str, is_summary: bool = False
|
||||
) -> Chunk:
|
||||
chunk = Chunk(
|
||||
id=uuid.uuid4(),
|
||||
document_id=doc.id,
|
||||
position=position,
|
||||
content=content,
|
||||
is_summary=is_summary,
|
||||
)
|
||||
db.add(chunk)
|
||||
return chunk
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def kb(db) -> Iterator[None]:
|
||||
"""A fresh KB with the incident shape: the qwen3.8 quadlet (the
|
||||
name hit, with a summary chunk + an ordinary chunk), a qwen3.6
|
||||
quadlet (same family, different version — NOT a hit), an
|
||||
unrelated document (FTS-only candidate), and a chunk-less document
|
||||
whose name DOES carry the token (the exclusion guard)."""
|
||||
db.execute(text("TRUNCATE chunks, documents"))
|
||||
db.commit()
|
||||
|
||||
q38 = _doc(
|
||||
db,
|
||||
"deploy",
|
||||
"reeseapps/ai/deployments/juggernaut/quadlets/qwen3.8-27b-juggernaut-vulkan.container",
|
||||
"qwen3.8-27b-juggernaut-vulkan",
|
||||
"# llama.cpp juggernaut\nExec=--port 8000 -ctk q8_0 -ctv q8_0 --jinja\n"
|
||||
"-m /models/qwen3.8-27b/Qwen3.8-27B-UD-Q6_K.gguf\n",
|
||||
)
|
||||
_chunk(
|
||||
db,
|
||||
q38,
|
||||
-1,
|
||||
"Podman quadlet: llama.cpp server for Qwen 3.8 27B (juggernaut).",
|
||||
is_summary=True,
|
||||
)
|
||||
_chunk(db, q38, 0, "# llama.cpp juggernaut\nExec=--port 8000 -ctk q8_0")
|
||||
db.flush()
|
||||
# A vector the question vector (below) cosines with — non-NULL so
|
||||
# the chunk is eligible for the vector list too.
|
||||
for c in q38.chunks:
|
||||
c.embedding = _vec(1)
|
||||
db.commit()
|
||||
|
||||
q36 = _doc(
|
||||
db,
|
||||
"deploy",
|
||||
"reeseapps/ai/deployments/juggernaut/quadlets/qwen3.6-27b-juggernaut-vulkan.container",
|
||||
"qwen3.6-27b-juggernaut-vulkan",
|
||||
"# llama.cpp juggernaut\n-m /models/qwen3.6-27b/model.gguf\n",
|
||||
)
|
||||
c36 = _chunk(db, q36, 0, "# llama.cpp juggernaut\n-m /models/qwen3.6-27b/model.gguf")
|
||||
c36.embedding = _vec(0) # orthogonal to the question vector
|
||||
db.commit()
|
||||
|
||||
other = _doc(
|
||||
db, "homelab", "notes/llama.cpp.md", "llama.cpp notes", "llama cpp server arguments notes\n"
|
||||
)
|
||||
c_other = _chunk(db, other, 0, "llama cpp server arguments notes")
|
||||
c_other.embedding = _vec(1, second=True) # half-parallel to the question
|
||||
db.commit()
|
||||
|
||||
# Name carries the token, ZERO chunks — the exclusion guard.
|
||||
_doc(db, "deploy", "qwen3.8-empty.container", "qwen3.8-empty", "(empty file)")
|
||||
db.commit()
|
||||
yield
|
||||
db.execute(text("TRUNCATE chunks, documents"))
|
||||
db.commit()
|
||||
|
||||
|
||||
def test_name_hit_chunks_real_sql(kb, db) -> None:
|
||||
"""Real Postgres: the projection scan finds exactly the qwen3.8
|
||||
quadlet (the qwen3.6 sibling and the chunk-less name match are
|
||||
excluded), and the LATERAL fetch hands back the SUMMARY chunk as
|
||||
the representative (position −1, is_summary)."""
|
||||
out = _name_hit_chunks(db, INCIDENT_QUESTION)
|
||||
assert [rc.document.path for rc in out] == [
|
||||
"reeseapps/ai/deployments/juggernaut/quadlets/qwen3.8-27b-juggernaut-vulkan.container"
|
||||
]
|
||||
rc = out[0]
|
||||
assert rc.position == -1 # the summary chunk wins the LATERAL order
|
||||
assert rc.is_summary is True
|
||||
assert rc.fts_hit is True # the lexical signal — the A8 gate answers
|
||||
assert rc.cosine == 0.0 # no vector rank on the name-hit row
|
||||
assert "qwen3.8-empty.container" not in [r.document.path for r in out] # chunk-less guard
|
||||
|
||||
|
||||
def test_lexical_candidates_name_hit_leads_real_sql(kb, db) -> None:
|
||||
"""The full lexical list on real Postgres: the name hit leads, the
|
||||
FTS rows follow (the qwen3.6 and llama.cpp docs both match the
|
||||
OR-tsquery on llama|cpp|arguments|… — the pre-incident pollution —
|
||||
but the name hit still ranks them behind it)."""
|
||||
out = _lexical_candidates(db, INCIDENT_QUESTION, limit=30)
|
||||
paths = [rc.document.path for rc in out]
|
||||
assert paths[0] == (
|
||||
"reeseapps/ai/deployments/juggernaut/quadlets/qwen3.8-27b-juggernaut-vulkan.container"
|
||||
)
|
||||
# The FTS pollution is still present (the incident's shape) — but
|
||||
# behind the name hit, no longer ahead of it.
|
||||
assert (
|
||||
"reeseapps/ai/deployments/juggernaut/quadlets/qwen3.6-27b-juggernaut-vulkan.container"
|
||||
in paths
|
||||
)
|
||||
assert all(rc.fts_hit is True for rc in out)
|
||||
|
||||
|
||||
def test_retrieve_selects_name_hit_doc_into_top_n(kb, db) -> None:
|
||||
"""The product path: hybrid ``retrieve()`` (vector ∪ lexical, RRF
|
||||
fused) → ``select_documents`` puts the qwen3.8 quadlet in the
|
||||
seeded top-N — the incident's seed miss (the two overview docs
|
||||
only) is fixed. The question vector is parallel to the q38 chunk
|
||||
embeddings (cosine 1.0), orthogonal to q36 (0.0)."""
|
||||
question_vec = _vec(1)
|
||||
chunks = retrieve(db, INCIDENT_QUESTION, question_vec)
|
||||
docs = select_documents(chunks, n=2)
|
||||
assert [d.path for d in docs] == [
|
||||
"reeseapps/ai/deployments/juggernaut/quadlets/qwen3.8-27b-juggernaut-vulkan.container",
|
||||
"notes/llama.cpp.md",
|
||||
]
|
||||
|
||||
|
||||
def test_name_hit_limit_real_sql(db) -> None:
|
||||
"""Twelve identical (1, 6) name hits — the LATERAL fetch (and the
|
||||
output) carries exactly ``NAME_HIT_LIMIT`` winners, catalog order."""
|
||||
db.execute(text("TRUNCATE chunks, documents"))
|
||||
db.commit()
|
||||
for i in range(12):
|
||||
doc = _doc(
|
||||
db, "S", f"quadlets/m{i:02d}-qwen38.container", f"m{i:02d}-qwen38", "llama cpp qwen38\n"
|
||||
)
|
||||
db.flush()
|
||||
c = _chunk(db, doc, 0, f"llama cpp qwen38 doc {i}")
|
||||
c.embedding = _vec(2)
|
||||
db.commit()
|
||||
out = _name_hit_chunks(db, "what are the llama.cpp arguments for qwen 3.8")
|
||||
assert len(out) == NAME_HIT_LIMIT
|
||||
assert [rc.document.path for rc in out] == [
|
||||
f"quadlets/m{i:02d}-qwen38.container" for i in range(NAME_HIT_LIMIT)
|
||||
]
|
||||
+196
-8
@@ -209,17 +209,26 @@ def test_agent_tools_names_and_parameters() -> None:
|
||||
# ("pass ONLY `pattern`") + the source-name-is-not-a-document
|
||||
# clause (the model kept scoping grep with an ls-style source name
|
||||
# — the 2026-09-03 incident loop shape, but on grep) plus the
|
||||
# one-call-at-a-time discipline clause.
|
||||
# one-call-at-a-time discipline clause. The 2026-09-05 incident
|
||||
# (the "Qwen 3.8" sample question — the harness prior is that grep
|
||||
# takes a REGEX; this grep is a fixed substring, owner-locked A5):
|
||||
# the plain-substring-never-a-regex clause states the contract up
|
||||
# front, so the regex-shaped first grep that does fire gets the
|
||||
# teaching no-match line instead of a trusted miss.
|
||||
assert grep["description"] == (
|
||||
"Search the indexed documents for an exact string "
|
||||
"(case-insensitive) and return up to 20 matching lines "
|
||||
"as `source/path:line: text` — a locator, not a "
|
||||
"context-adder: read the winner with `read`. For a "
|
||||
"normal search pass ONLY `pattern` — it searches every "
|
||||
"document and that is how you search the knowledge "
|
||||
"base; never pass a source name as `path` (a source "
|
||||
"name is not a document). Call one tool at a time — "
|
||||
"wait for this result before your next call."
|
||||
"context-adder: read the winner with `read`. The "
|
||||
"pattern is a plain substring, NEVER a regex — if a "
|
||||
"pattern with regex syntax (like '.*' or '\\.') comes "
|
||||
"back with no matches, retry with the plain text you "
|
||||
"expect to see. For a normal search pass ONLY `pattern` "
|
||||
"— it searches every document and that is how you "
|
||||
"search the knowledge base; never pass a source name "
|
||||
"as `path` (a source name is not a document). Call one "
|
||||
"tool at a time — wait for this result before your "
|
||||
"next call."
|
||||
)
|
||||
grep_params = grep["parameters"]
|
||||
assert grep_params["type"] == "object"
|
||||
@@ -227,7 +236,8 @@ def test_agent_tools_names_and_parameters() -> None:
|
||||
assert set(grep_params["properties"]) == {"pattern", "path"}
|
||||
assert all(p["type"] == "string" for p in grep_params["properties"].values())
|
||||
assert grep_params["properties"]["pattern"]["description"] == (
|
||||
"The exact text to search for (a plain substring, not a regex)"
|
||||
"The exact text to search for (a plain substring, "
|
||||
"not a regex — no '.*', no '\\.', no character classes)"
|
||||
)
|
||||
# Phase 72 (task 02): the bare-path contract is stated up front;
|
||||
# task 05 (live gate iterations 1-8): the one-known-document clause
|
||||
@@ -1164,6 +1174,184 @@ def test_grep_truncates_match_lines_at_200_chars(monkeypatch: pytest.MonkeyPatch
|
||||
assert holder.tool_calls == 1
|
||||
|
||||
|
||||
# ---------- grep no-match teaching: the regex-shaped pattern
|
||||
# (the 2026-09-05 "Qwen 3.8" incident — the harness prior is that
|
||||
# grep takes a REGEX; this grep is a fixed substring, owner-locked
|
||||
# A5, and the contract does not change) ----------
|
||||
|
||||
|
||||
def test_plain_form_reduces_regex_to_literal_text() -> None:
|
||||
"""The plain-form hint: the pattern reduced to literal text — the
|
||||
incident's exact recovery (``qwen.*3\\.8`` → ``qwen3.8``) plus the
|
||||
edge cases (raw ``.*`` runs dropped before unescape, so an escaped
|
||||
dot survives; first alternative only; classes/quantifiers/parens/
|
||||
anchors gone; whitespace preserved; pure metacharacters → ``""``).
|
||||
"""
|
||||
assert agent.plain_form(r"qwen.*3\.8") == "qwen3.8" # the incident
|
||||
assert agent.plain_form(r"qwen 3\.8") == "qwen 3.8"
|
||||
assert agent.plain_form(r"Qwen 3\.8") == "Qwen 3.8" # case kept
|
||||
assert agent.plain_form(r"qwen3\.8") == "qwen3.8"
|
||||
assert agent.plain_form(r"llama\.cpp") == "llama.cpp" # escaped dot kept
|
||||
assert agent.plain_form(r"qwen[0-9]+") == "qwen" # class + quantifier
|
||||
assert agent.plain_form("a|b") == "a" # first alternative only
|
||||
assert agent.plain_form(r"\d+") == "" # no literal text — no hint
|
||||
assert agent.plain_form(r".*") == "" # pure wildcard — no hint
|
||||
assert agent.plain_form(r"(qwen)3\.8") == "qwen3.8" # group contents kept
|
||||
assert agent.plain_form("a{2,3}b") == "ab"
|
||||
assert agent.plain_form(r"^qwen$") == "qwen" # anchors dropped
|
||||
assert agent.plain_form(r"a\.b") == "a.b" # escaped dot is a literal
|
||||
assert agent.plain_form("plain") == "plain" # identity for plain text
|
||||
|
||||
|
||||
def test_looks_like_regex_detection() -> None:
|
||||
"""One metacharacter anywhere marks the pattern regex-shaped; a
|
||||
plain substring (even with a space) does not."""
|
||||
for p in (
|
||||
r"qwen.*3\.8", r"qwen 3\.8", "qwen+", "a?b", "x|y", "(a)", "[a-z]", "a^b", "b$c", "a{2}"
|
||||
):
|
||||
assert agent.looks_like_regex(p) is True, p
|
||||
for p in ("qwen 3.8", "qwen3.8", "plain substring", ""):
|
||||
assert agent.looks_like_regex(p) is False, p
|
||||
|
||||
|
||||
def test_grep_no_match_regex_pattern_gets_teaching_line(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""The incident's shape: a regex-shaped pattern that (necessarily)
|
||||
misses gets the TEACHING no-match line — the plain-substring
|
||||
contract stated, the plain-form retry hint handed over. Still a
|
||||
counted result; the context is untouched (locked A5)."""
|
||||
d1 = _doc("Alpha", "a/one.md", "One", "qwen3.8-27b juggernaut\nno regex text")
|
||||
monkeypatch.setattr(agent, "all_documents", lambda db: [d1])
|
||||
holder = AgentHolder()
|
||||
llm = ScriptedLLM(
|
||||
[
|
||||
ToolCallPiece(
|
||||
id="call_1", name="grep", arguments={"pattern": r"qwen.*3\.8"}
|
||||
)
|
||||
],
|
||||
[StreamPiece("content", "ans")],
|
||||
)
|
||||
asyncio.run(_run(llm, holder, _settings()))
|
||||
assert llm.requests[1][0][3]["content"] == agent.NO_MATCHES_REGEX.format(
|
||||
pattern=r"qwen.*3\.8", plain="qwen3.8"
|
||||
)
|
||||
assert llm.requests[1][0][3]["content"] == (
|
||||
"No matches for 'qwen.*3\\.8'. grep matches a plain substring "
|
||||
"(case-insensitive), not a regex — '.*', '\\.' and the like are "
|
||||
"literal text here, so that pattern can never match. Retry with "
|
||||
"the plain text you expect to see (e.g. 'qwen3.8')."
|
||||
)
|
||||
assert holder.tool_calls == 1 # a no-match with teaching is still a result
|
||||
assert holder.read_docs == [] # locked A5: a grep adds no context
|
||||
|
||||
|
||||
def test_grep_no_match_regex_scoped_gets_teaching_line(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""The scoped teaching variant: the resolved identity is echoed, the
|
||||
hint handed over."""
|
||||
d1 = _doc("Alpha", "a/one.md", "One", "nothing regex-shaped here")
|
||||
|
||||
def _find(db: Any, source: str, path: str) -> Document | None:
|
||||
return d1 if (source, path) == ("Alpha", "a/one.md") else None
|
||||
|
||||
monkeypatch.setattr(agent, "find_document", _find)
|
||||
holder = AgentHolder()
|
||||
llm = ScriptedLLM(
|
||||
[
|
||||
ToolCallPiece(
|
||||
id="call_1",
|
||||
name="grep",
|
||||
arguments={"pattern": r"qwen 3\.8", "path": "Alpha/a/one.md"},
|
||||
)
|
||||
],
|
||||
[StreamPiece("content", "ans")],
|
||||
)
|
||||
asyncio.run(_run(llm, holder, _settings()))
|
||||
assert llm.requests[1][0][3]["content"] == (
|
||||
"No matches for 'qwen 3\\.8' in Alpha/a/one.md. grep matches a "
|
||||
"plain substring (case-insensitive), not a regex — retry with "
|
||||
"the plain text you expect to see (e.g. 'qwen 3.8')."
|
||||
)
|
||||
assert holder.tool_calls == 1
|
||||
assert holder.read_docs == []
|
||||
|
||||
|
||||
def test_grep_no_match_plain_pattern_keeps_ordinary_line(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""A no-match for a PLAIN pattern (no metacharacters — "qwen 3.8" with
|
||||
the space included) keeps the ordinary line byte-identical: the
|
||||
teaching never fires for a well-formed pattern (the retrieval side —
|
||||
the name-hit lexical signal — is what covers that case)."""
|
||||
d1 = _doc("Alpha", "a/one.md", "One", "qwen3.8-27b juggernaut")
|
||||
monkeypatch.setattr(agent, "all_documents", lambda db: [d1])
|
||||
holder = AgentHolder()
|
||||
llm = ScriptedLLM(
|
||||
[ToolCallPiece(id="call_1", name="grep", arguments={"pattern": "qwen 3.8"})],
|
||||
[StreamPiece("content", "ans")],
|
||||
)
|
||||
asyncio.run(_run(llm, holder, _settings()))
|
||||
assert llm.requests[1][0][3]["content"] == (
|
||||
"No matches for 'qwen 3.8' in the knowledge base."
|
||||
)
|
||||
assert holder.tool_calls == 1
|
||||
|
||||
|
||||
def test_grep_matched_regex_pattern_returns_matches_not_teaching(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""A pattern with metacharacters that MATCHES literally gets the
|
||||
ordinary match output — the teaching can never suppress a real hit
|
||||
(the detection keys on a NO-MATCH only)."""
|
||||
d1 = _doc("S", "a.md", "A", "the C++ compiler is here")
|
||||
monkeypatch.setattr(agent, "all_documents", lambda db: [d1])
|
||||
holder = AgentHolder()
|
||||
llm = ScriptedLLM(
|
||||
[ToolCallPiece(id="call_1", name="grep", arguments={"pattern": "C++"})],
|
||||
[StreamPiece("content", "ans")],
|
||||
)
|
||||
asyncio.run(_run(llm, holder, _settings()))
|
||||
assert llm.requests[1][0][3]["content"] == "S/a.md:1: the C++ compiler is here"
|
||||
assert holder.tool_calls == 1
|
||||
|
||||
|
||||
def test_grep_no_match_regex_reducing_to_empty_falls_back(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""A regex-shaped pattern with no literal text left after the
|
||||
reduction (``.*``) gets the ORDINARY line — no empty hint."""
|
||||
d1 = _doc("Alpha", "a/one.md", "One", "any text at all")
|
||||
monkeypatch.setattr(agent, "all_documents", lambda db: [d1])
|
||||
holder = AgentHolder()
|
||||
llm = ScriptedLLM(
|
||||
[ToolCallPiece(id="call_1", name="grep", arguments={"pattern": r".*"})],
|
||||
[StreamPiece("content", "ans")],
|
||||
)
|
||||
asyncio.run(_run(llm, holder, _settings()))
|
||||
assert llm.requests[1][0][3]["content"] == (
|
||||
"No matches for '.*' in the knowledge base."
|
||||
)
|
||||
assert holder.tool_calls == 1
|
||||
|
||||
|
||||
def test_no_matches_regex_templates_pin() -> None:
|
||||
"""The teaching templates are verbatim pins (the model-facing copy —
|
||||
the mock E2E keys off the plain-substring clause)."""
|
||||
assert agent.NO_MATCHES_REGEX == (
|
||||
"No matches for '{pattern}'. grep matches a plain substring "
|
||||
"(case-insensitive), not a regex — '.*', '\\.' and the like are "
|
||||
"literal text here, so that pattern can never match. Retry with "
|
||||
"the plain text you expect to see (e.g. '{plain}')."
|
||||
)
|
||||
assert agent.NO_MATCHES_REGEX_SCOPED == (
|
||||
"No matches for '{pattern}' in {source}/{path}. grep matches a "
|
||||
"plain substring (case-insensitive), not a regex — retry with "
|
||||
"the plain text you expect to see (e.g. '{plain}')."
|
||||
)
|
||||
|
||||
|
||||
def test_grep_scoped_to_one_document(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Scoped grep: only the named document is loaded (find_document on
|
||||
the first-slash split), ``all_documents`` never runs, and the match
|
||||
|
||||
@@ -133,6 +133,20 @@ def test_lexical_tsquery_stopwords_left_to_postgres() -> None:
|
||||
assert lexical_tsquery("how do i") == "how | do | i"
|
||||
|
||||
|
||||
def test_lexical_tsquery_dotted_tokens_kept_whole() -> None:
|
||||
"""The 2026-09-05 incident: the default parser lexes dotted words
|
||||
as ONE lexeme ("llama.cpp" → 'llama.cpp', "Qwen 3.8" → '3.8'), so
|
||||
the query carries them whole — split tokens (llama | cpp) can never
|
||||
match the document side."""
|
||||
assert lexical_tsquery(
|
||||
"What are the correct llama.cpp arguments for Qwen 3.8?"
|
||||
) == "what | are | the | correct | llama.cpp | arguments | for | qwen | 3.8"
|
||||
# The dash still splits (only dots group): ai | internal.network.
|
||||
assert lexical_tsquery("how did I set up ai-internal.network?") == (
|
||||
"how | did | i | set | up | ai | internal.network"
|
||||
)
|
||||
|
||||
|
||||
def test_fuse_combines_both_lists_for_double_hits() -> None:
|
||||
v1 = _rc("a.md", cosine=0.9)
|
||||
v2 = _rc("b.md", cosine=0.5)
|
||||
@@ -196,7 +210,14 @@ def test_fuse_empty_lists() -> None:
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
from app.models import Chunk # noqa: E402
|
||||
from app.rag.retriever import _lexical_candidates, _vector_candidates # noqa: E402
|
||||
from app.rag.retriever import ( # noqa: E402
|
||||
NAME_HIT_LIMIT,
|
||||
_lexical_candidates,
|
||||
_name_hit_chunks,
|
||||
_normalize_name,
|
||||
_vector_candidates,
|
||||
name_hit_tokens,
|
||||
)
|
||||
|
||||
|
||||
class _FakeResult:
|
||||
@@ -210,15 +231,29 @@ class _FakeResult:
|
||||
|
||||
|
||||
class _FakeSession:
|
||||
"""Returns canned rows from ``execute`` without touching Postgres."""
|
||||
"""Returns canned rows from ``execute`` without touching Postgres.
|
||||
|
||||
def __init__(self, rows: list) -> None:
|
||||
self._rows = rows
|
||||
One list of rows (legacy form) is returned for EVERY call; several
|
||||
lists (one per successive ``execute``) model a query sequence — the
|
||||
name-hit lexical path (2026-09-05) issues the document-projection
|
||||
query and, when hits exist, the LATERAL chunk query, BEFORE the FTS
|
||||
query.
|
||||
"""
|
||||
|
||||
def __init__(self, *rowsets: list) -> None:
|
||||
if len(rowsets) == 1 and not (
|
||||
rowsets[0] and isinstance(rowsets[0][0], list)
|
||||
):
|
||||
rowsets = (rowsets[0],) # the single-rowset legacy form
|
||||
self._rowsets = rowsets
|
||||
self._call = 0
|
||||
self.statements: list = []
|
||||
|
||||
def execute(self, stmt, params: dict | None = None) -> _FakeResult:
|
||||
self.statements.append((stmt, params))
|
||||
return _FakeResult(self._rows)
|
||||
rows = self._rowsets[min(self._call, len(self._rowsets) - 1)]
|
||||
self._call += 1
|
||||
return _FakeResult(rows)
|
||||
|
||||
|
||||
def _chunk_row(is_summary: bool) -> Chunk:
|
||||
@@ -286,9 +321,17 @@ def _lexical_row(is_summary: bool, doc_path: str) -> object:
|
||||
|
||||
|
||||
def test_lexical_candidates_carry_is_summary_flag() -> None:
|
||||
"""The lexical list reads ``c.is_summary`` from the raw row."""
|
||||
"""The lexical list reads ``c.is_summary`` from the raw row.
|
||||
|
||||
The question carries no digit-bearing name token (no bare, no
|
||||
numeric-join), so the name-hit path issues NO queries at all — the
|
||||
single FTS rowset answers the only (FTS) call, and the list is the
|
||||
plain FTS rows: the pre-name-hit behavior, unchanged.
|
||||
"""
|
||||
rows = [_lexical_row(True, "summary-src.yaml"), _lexical_row(False, "other.md")]
|
||||
out = _lexical_candidates(_FakeSession(rows), "how do i configure the thing", limit=10) # pyright: ignore[reportArgumentType]
|
||||
out = _lexical_candidates(
|
||||
_FakeSession(rows), "how do i configure the thing", limit=10 # pyright: ignore[reportArgumentType]
|
||||
)
|
||||
assert len(out) == 2
|
||||
by_path = {rc.document.path: rc for rc in out}
|
||||
assert by_path["summary-src.yaml"].is_summary is True
|
||||
@@ -323,3 +366,180 @@ def test_fuse_default_is_summary_stays_false_for_legacy_chunks() -> None:
|
||||
out = fuse([_rc("a.md", cosine=0.8)], [_rc("b.md")], k=60)
|
||||
assert len(out) == 2
|
||||
assert all(rc.is_summary is False for rc in out)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Name-hit lexical signal (the 2026-09-05 incident — the versioned-name
|
||||
# case the default parser lexes incompatibly: "Qwen 3.8" → qwen/3/8 can
|
||||
# never match a document's qwen3/8/27b tokens)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
INCIDENT_QUESTION = "What are the correct llama.cpp arguments for Qwen 3.8?"
|
||||
|
||||
|
||||
def test_normalize_name() -> None:
|
||||
assert _normalize_name("Qwen 3.8") == "qwen38"
|
||||
assert _normalize_name("qwen3.8-27b-juggernaut-vulkan") == "qwen3827bjuggernautvulkan"
|
||||
assert _normalize_name("Mixed CASE-99") == "mixedcase99"
|
||||
assert _normalize_name("!!!") == ""
|
||||
|
||||
|
||||
def test_name_hit_tokens_incident_question() -> None:
|
||||
"""The incident question yields EXACTLY the versioned join
|
||||
``qwen38`` — the token the document names actually carry. Plain
|
||||
prose words (``what``, ``llamacpp``, ``arguments``, ``server`` —
|
||||
no digit) never name-match (the precision guard); the single
|
||||
digits ("3", "8") and the bare "38" are < 4 chars; the
|
||||
digit-leading ``38show`` boundary artifact is dropped."""
|
||||
tokens = name_hit_tokens(INCIDENT_QUESTION)
|
||||
assert tokens == ["qwen38"]
|
||||
for absent in ("what", "qwen", "llamacpp", "arguments", "3", "8", "38", "38show", "server"):
|
||||
assert absent not in tokens
|
||||
|
||||
|
||||
def test_name_hit_tokens_no_digit_question_returns_empty() -> None:
|
||||
"""A question with no digit-bearing token (bare or joined) yields
|
||||
no name candidates — prose joins like ``correctllama`` never count."""
|
||||
assert name_hit_tokens("what is the correct caddy config") == []
|
||||
assert name_hit_tokens("a e i o u 3 8") == []
|
||||
|
||||
|
||||
def test_name_hit_tokens_bare_digit_bearing_token() -> None:
|
||||
"""A single written token that carries a digit (``1panel``) is a
|
||||
name candidate on its own — no join needed."""
|
||||
tokens = name_hit_tokens("what is my 1panel dashboard setup")
|
||||
assert tokens == ["1panel"]
|
||||
|
||||
|
||||
def _name_row(doc: Document) -> tuple:
|
||||
"""One row of the name-hit document projection (catalog order)."""
|
||||
return (doc.id, doc.source, doc.path, doc.title)
|
||||
|
||||
|
||||
def _name_hit_lateral_row(doc: Document, is_summary: bool = False) -> SimpleNamespace:
|
||||
"""One row of the name-hit LATERAL chunk query."""
|
||||
return SimpleNamespace(
|
||||
doc_id=doc.id,
|
||||
source=doc.source,
|
||||
path=doc.path,
|
||||
full_path=doc.full_path,
|
||||
title=doc.title,
|
||||
doc_content=doc.content,
|
||||
content_hash=doc.content_hash,
|
||||
indexed_at=None,
|
||||
chunk_id=uuid.uuid4(),
|
||||
position=-1 if is_summary else 0,
|
||||
content="summary chunk" if is_summary else "content chunk",
|
||||
is_summary=is_summary,
|
||||
)
|
||||
|
||||
|
||||
def test_name_hit_chunks_no_tokens_skips_all_queries() -> None:
|
||||
"""A question with no name tokens issues no queries at all."""
|
||||
session = _FakeSession([]) # any call would surface a statement
|
||||
assert _name_hit_chunks(session, "a e i o u 3 8") == [] # pyright: ignore[reportArgumentType]
|
||||
assert session.statements == []
|
||||
|
||||
|
||||
def test_name_hit_chunks_no_matching_doc_returns_empty() -> None:
|
||||
"""Name tokens exist but no document name carries one: the
|
||||
projection runs, the LATERAL fetch does not."""
|
||||
doc = _doc("quadlets/other.container", "body")
|
||||
name_rows = [_name_row(doc)]
|
||||
session = _FakeSession(name_rows, [])
|
||||
assert _name_hit_chunks(session, INCIDENT_QUESTION) == [] # pyright: ignore[reportArgumentType]
|
||||
assert len(session.statements) == 1 # projection only — no LATERAL fetch
|
||||
|
||||
|
||||
def test_name_hit_chunks_ranked_by_count_length_catalog() -> None:
|
||||
"""A two-candidate question (``qwen38`` + ``1panel``): the document
|
||||
whose name carries BOTH (2 matches, 12 total chars) leads; the two
|
||||
single-match documents tie on (1, 6) and fall to catalog order
|
||||
(``dashboards/1panel-notes.md`` before ``quadlets/qwen3.8…``).
|
||||
Hits carry ``fts_hit=True`` (the A8 gate answers), ``cosine=0.0``,
|
||||
and the summary flag of their representative chunk."""
|
||||
both = _doc("dashboards/1panel-qwen3.8.md", "body", title="1Panel Qwen 3.8")
|
||||
panel = _doc("dashboards/1panel-notes.md", "body")
|
||||
q38 = _doc("quadlets/qwen3.8-27b-juggernaut-vulkan.container", "body")
|
||||
name_rows = [_name_row(d) for d in (panel, both, q38)] # catalog order
|
||||
question = "what are the correct llama.cpp arguments for qwen 3.8 and the 1panel dashboard?"
|
||||
lateral_rows = [
|
||||
_name_hit_lateral_row(q38, is_summary=True), # LATERAL may return any order
|
||||
_name_hit_lateral_row(both),
|
||||
_name_hit_lateral_row(panel),
|
||||
]
|
||||
session = _FakeSession(name_rows, lateral_rows)
|
||||
out = _name_hit_chunks(session, question) # pyright: ignore[reportArgumentType]
|
||||
assert [rc.document.path for rc in out] == [
|
||||
"dashboards/1panel-qwen3.8.md", # 2 matched tokens — leads
|
||||
"dashboards/1panel-notes.md", # (1, 6) — catalog order
|
||||
"quadlets/qwen3.8-27b-juggernaut-vulkan.container", # (1, 6) — after
|
||||
]
|
||||
assert all(rc.fts_hit is True for rc in out) # the lexical signal
|
||||
assert all(rc.cosine == 0.0 for rc in out) # no vector rank
|
||||
assert all(rc.score == 0.0 for rc in out) # fuse fills the score
|
||||
by_path = {rc.document.path: rc for rc in out}
|
||||
assert by_path["quadlets/qwen3.8-27b-juggernaut-vulkan.container"].is_summary is True
|
||||
assert by_path["quadlets/qwen3.8-27b-juggernaut-vulkan.container"].position == -1
|
||||
assert by_path["dashboards/1panel-notes.md"].is_summary is False
|
||||
|
||||
|
||||
def test_name_hit_chunks_capped_at_limit() -> None:
|
||||
"""Twelve tied name hits (one matched token each) yield exactly
|
||||
``NAME_HIT_LIMIT`` of them — catalog order (the deterministic
|
||||
tie-break)."""
|
||||
docs = [_doc(f"quadlets/m{i:02d}.container", "body") for i in range(12)]
|
||||
for d in docs: # give every document a name that carries the token
|
||||
d.title = "qwen38 model i"
|
||||
name_rows = [_name_row(d) for d in docs]
|
||||
# Only the ten winners (catalog order — the deterministic tie-break
|
||||
# of the twelve identical scores) reach the LATERAL fetch; the fake
|
||||
# answers with exactly those rows.
|
||||
lateral_rows = [_name_hit_lateral_row(d) for d in docs[:NAME_HIT_LIMIT]]
|
||||
session = _FakeSession(name_rows, lateral_rows)
|
||||
out = _name_hit_chunks(
|
||||
session, "tell me about the qwen 3.8 models" # pyright: ignore[reportArgumentType]
|
||||
)
|
||||
assert len(out) == NAME_HIT_LIMIT
|
||||
assert [rc.document.path for rc in out] == [f"quadlets/m{i:02d}.container" for i in range(10)]
|
||||
|
||||
|
||||
def test_lexical_candidates_name_hits_lead_and_dedupe_with_fts() -> None:
|
||||
"""The full lexical list: name hits LEAD (their representative
|
||||
chunks), the FTS rows follow, and an FTS row sharing the name hit's
|
||||
chunk id appears exactly once (deduped)."""
|
||||
q38 = _doc("quadlets/qwen3.8-27b-juggernaut-vulkan.container", "body")
|
||||
other = _doc("quadlets/qwen38-other.container", "body") # 1 matched token
|
||||
name_rows = [_name_row(q38), _name_row(other)]
|
||||
q38_chunk = uuid.uuid4()
|
||||
|
||||
def _lateral(doc: Document) -> SimpleNamespace:
|
||||
row = _name_hit_lateral_row(doc)
|
||||
if doc is q38:
|
||||
row.chunk_id = q38_chunk
|
||||
return row
|
||||
|
||||
lateral_rows = [_lateral(q38), _lateral(other)]
|
||||
fts_rows = [
|
||||
# an FTS hit on the SAME chunk as the q38 name hit (deduped away)
|
||||
SimpleNamespace(
|
||||
chunk_id=q38_chunk, position=1, content="c", doc_id=q38.id,
|
||||
source=q38.source, path=q38.path, full_path=q38.full_path,
|
||||
title=q38.title, doc_content=q38.content, content_hash=q38.content_hash,
|
||||
indexed_at=None, is_summary=False, rank=0.1,
|
||||
),
|
||||
# an FTS hit on a different chunk of the OTHER doc (kept)
|
||||
_lexical_row(False, "quadlets/qwen38-other.container"),
|
||||
]
|
||||
session = _FakeSession(name_rows, lateral_rows, fts_rows)
|
||||
out = _lexical_candidates(session, INCIDENT_QUESTION, limit=10) # pyright: ignore[reportArgumentType]
|
||||
assert len(out) == 3 # q38 (once), other (name hit), other (FTS chunk)
|
||||
# Both name hits tie on (1, 6) — catalog order: "qwen3." (ASCII 46)
|
||||
# sorts before "qwen38" (ASCII 56).
|
||||
assert out[0].document.path == "quadlets/qwen3.8-27b-juggernaut-vulkan.container"
|
||||
assert out[1].document.path == "quadlets/qwen38-other.container"
|
||||
assert out[0].chunk_id == q38_chunk # the name-hit representative row
|
||||
assert {
|
||||
rc.chunk_id for rc in out
|
||||
} == {q38_chunk, fts_rows[1].chunk_id, lateral_rows[1].chunk_id}
|
||||
assert all(rc.fts_hit is True for rc in out)
|
||||
|
||||
Reference in New Issue
Block a user