finally getting accurate answers

app/rag/retriever.py

@@ -3,8 +3,27 @@
 * **Vector list** — top-N chunks by cosine distance (``embedding <=> $1``),
   each carrying its cosine ``1 − distance`` (the honesty-gate input).
 * **Lexical list** — top-N chunks matching an OR-``tsquery`` over the
-  question's tokens, ordered by ``ts_rank``. This is what finds
-  name-your-tool questions ("gitlab") that vector similarity buries.
+  question's tokens, ordered by ``ts_rank``, UNION the name-hit list:
+  documents whose normalized name (title + path stem, alnum-only,
+  lowercased) contains a DIGIT-BEARING normalized question token of
+  length >= 4 — bare tokens (``1panel``) and the join of adjacent
+  question tokens starting with a letter ("Qwen 3.8" → ``qwen38``).
+  The digit requirement is the precision guard: plain prose words
+  ("server", "arguments" — 4+ chars but no digit) never name-match,
+  while versioned product names (the incident's whole point) always
+  carry one. The name-hit documents LEAD the lexical list (ranked by
+  match count, then total matched length, then catalog order; capped
+  at :data:`NAME_HIT_LIMIT`),
+  the FTS rows follow. This is what finds name-your-tool questions
+  ("gitlab") that vector similarity buries — and, the 2026-09-05
+  incident, the versioned-name case the default parser lexes
+  incompatibly (question "Qwen 3.8" → ``qwen``/``3``/``8`` can never
+  match a document's ``qwen3``/``8``/``27b`` tokens, while every
+  unrelated llama.cpp quadlet out-ranks the target on the shared
+  ``llama``/``cpp`` tokens). The name-hit rows carry ``fts_hit=True``
+  (they ARE the lexical signal — the A8 honesty gate then answers
+  instead of deflecting) and ``cosine=0.0``; the RRF fusion is
+  unchanged (same lists, same ``1/(k+rank)`` terms).
 * **Fusion** — Reciprocal Rank Fusion (``score = Σ 1/(k + rank)`` over the
   lists a chunk appears in; chunks hit by both lists get both terms). The
   fused score ranks; :meth:`select_documents` and :func:`weak_hit_titles`
@@ -26,6 +45,7 @@ import re
 import uuid
 from collections.abc import Sequence
 from dataclasses import dataclass, replace
+from pathlib import Path
 
 from sqlalchemy import select, text
 from sqlalchemy.orm import Session
@@ -38,8 +58,78 @@ from app.models import Chunk, Document
 #: path never truncates (A7 revised, owner permission 2026-08-24).
 TRUNCATION_MARKER = "[…truncated…]"
 
-#: Alphanumeric tokens of a question (``to_tsquery`` input, OR-joined).
-_TOKEN_RE = re.compile(r"[a-z0-9]+")
+#: Alphanumeric tokens of a question (``to_tsquery`` input, OR-joined),
+#: with DOTTED tokens kept whole (``llama.cpp`` → ``llama.cpp``). The
+#: default parser lexes a dotted word as ONE lexeme (``to_tsvector`` of
+#: "llama.cpp" → ``'llama.cpp'``; of "Qwen 3.8" → ``'3.8'``), so a split
+#: token (``llama`` | ``cpp``) can never match the document side — the
+#: 2026-09-05 incident's question phrase "llama.cpp" lexed
+#: incompatibly on both sides of the query. Dotted tokens are passed
+#: through as single ``to_tsquery`` lexemes (verified: Postgres accepts
+#: ``'llama.cpp'`` and ``'3.8'`` as lexemes).
+_TOKEN_RE = re.compile(r"[a-z0-9]+(?:\.[a-z0-9]+)*")
+
+#: Name-hit policy (the 2026-09-05 incident): a question token shorter
+#: than this (normalized) is too weak a name signal ("3", "8", "the") —
+#: it would match half the KB by accident.
+NAME_TOKEN_MIN_LEN = 4
+
+#: The name-hit list may contribute at most this many documents to the
+#: lexical side (before the FTS rows); the rest are dropped.
+NAME_HIT_LIMIT = 10
+
+#: Alphanumeric runs of a lowercased string (name normalization).
+_ALNUM_RE = re.compile(r"[a-z0-9]+")
+
+
+def _normalize_name(s: str) -> str:
+    """Lowercased, alnum-only form of *s* (``Qwen 3.8`` → ``qwen38``)."""
+    return "".join(_ALNUM_RE.findall(s.lower()))
+
+
+def name_hit_tokens(question: str) -> list[str]:
+    """The normalized name-match candidates of one question.
+
+    Only DIGIT-BEARING candidates count (the precision guard — a plain
+    prose word like "server" or "arguments" must never name-match a
+    document; a versioned product name always carries a digit):
+
+    * the :func:`_normalize_name` form of every whitespace token, kept
+      when at least :data:`NAME_TOKEN_MIN_LEN` chars (``1panel``,
+      ``qwen38`` from a single written token);
+    * the versioned-name JOIN — the normalized concatenation of every
+      ADJACENT token pair whose SECOND token is purely numeric (a
+      version number: ``"… for Qwen 3.8"`` → ``Qwen`` + ``3.8`` joins
+      to ``qwen38``; prose joins like ``correct`` + ``llama`` and the
+      word-after-version boundaries like ``3.8`` + ``show`` are
+      dropped). A purely-numeric second token means the join always
+      starts with the first token's text (a letter in practice), so no
+      digit-leading artifact (``38show``) can survive.
+
+    Order of first appearance, de-duplicated.
+    """
+    tokens = question.split()
+    out: list[str] = []
+    seen: set[str] = set()
+
+    def add(tok: str) -> None:
+        if (
+            len(tok) >= NAME_TOKEN_MIN_LEN
+            and any(ch.isdigit() for ch in tok)
+            and tok not in seen
+        ):
+            seen.add(tok)
+            out.append(tok)
+
+    for i, raw in enumerate(tokens):
+        norm = _normalize_name(raw)
+        add(norm)
+        if i + 1 < len(tokens):
+            next_norm = _normalize_name(tokens[i + 1])
+            if next_norm and next_norm.isdigit():
+                add(norm + next_norm)  # the versioned-name join
+    return out
+
 
 #: One row of the lexical candidate query (all fields needed to build a
 #: detached :class:`Document` plus the chunk fields and ``ts_rank``).
@@ -96,11 +186,14 @@ class RetrievedChunk:
 def lexical_tsquery(question: str) -> str | None:
     """OR-joined token string for ``to_tsquery('english', …)``, or ``None``.
 
-    Tokens are lowercased ``[a-z0-9]+`` runs, de-duplicated in order of
-    first appearance. Postgres does the lexing/stemming; a question whose
-    tokens are all stopwords lexes to an *empty* tsquery (which matches
-    nothing), so no special-casing is needed there. Pure-symbol questions
-    ("???", "🔧") yield no tokens → ``None`` → no lexical query at all.
+    Tokens are lowercased ``[a-z0-9]+`` runs with DOTTED words kept whole
+    (see :data:`_TOKEN_RE` — the default parser lexes ``llama.cpp`` as
+    one lexeme, so the query must carry it whole too), de-duplicated in
+    order of first appearance. Postgres does the lexing/stemming; a
+    question whose tokens are all stopwords lexes to an *empty* tsquery
+    (which matches nothing), so no special-casing is needed there.
+    Pure-symbol questions ("???", "🔧") yield no tokens → ``None`` → no
+    lexical query at all.
     """
     seen: set[str] = set()
     tokens: list[str] = []
@@ -181,21 +274,141 @@ def _vector_candidates(
     ]
 
 
-def _lexical_candidates(db: Session, question: str, limit: int) -> list[RetrievedChunk]:
-    """Top-*limit* chunks matching the question's OR-tsquery (A7).
-
-    Ordered by ``ts_rank`` (with ``d.path, c.position`` as the
-    deterministic tie-break); an empty tsquery (stopword-only question)
-    simply matches nothing.
+#: One name-hit row: the document fields (detached :class:`Document`)
+#: plus the representative chunk — the ``is_summary`` chunk when the
+#: document has one (its natural-language summary is the best chunk of
+#: a machine file, and the chunk the vector list is likeliest to have
+#: ranked too, so the RRF merge dedupes cleanly), else chunk 0.
+_NAME_HIT_SQL = text(
     """
+    SELECT d.id            AS doc_id,
+           d.source        AS source,
+           d.path          AS path,
+           d.full_path     AS full_path,
+           d.title         AS title,
+           d.content       AS doc_content,
+           d.content_hash  AS content_hash,
+           d.indexed_at    AS indexed_at,
+           c.id            AS chunk_id,
+           c.position      AS position,
+           c.content       AS content,
+           c.is_summary    AS is_summary
+    FROM documents d
+    LEFT JOIN LATERAL (
+        SELECT id, position, content, is_summary
+        FROM chunks
+        WHERE document_id = d.id
+        ORDER BY is_summary DESC, position ASC
+        LIMIT 1
+    ) c ON true
+    WHERE d.id = ANY(:ids) AND c.id IS NOT NULL
+    """
+)
+
+
+def _name_hit_chunks(db: Session, question: str) -> list[RetrievedChunk]:
+    """The documents whose NAME matches the question (the 2026-09-05
+    incident, the versioned-name signal).
+
+    A document is a name hit when its normalized name —
+    :func:`_normalize_name` of its ``title`` followed by the normalized
+    path stem (``qwen3.8-27b-juggernaut-vulkan.container`` →
+    ``qwen3827bjuggernautvulkan``) — contains at least one
+    :func:`name_hit_tokens` candidate (``qwen38``). Ranked by
+    (distinct matched tokens, total matched length, source, path) —
+    catalog order is the final deterministic tie-break — capped at
+    :data:`NAME_HIT_LIMIT`. Each hit becomes one lexical
+    :class:`RetrievedChunk` (its representative chunk, ``fts_hit=True``,
+    ``cosine=0.0``). Two lightweight queries: one projection over
+    (id, source, path, title) in catalog order, one LATERAL chunk
+    fetch for the ≤ :data:`NAME_HIT_LIMIT` winners (no full-content
+    load; the content joins in via the row fetch below).
+    """
+    tokens = name_hit_tokens(question)
+    if not tokens:
+        return []
+    rows = db.execute(
+        select(Document.id, Document.source, Document.path, Document.title).order_by(
+            Document.source, Document.path
+        )
+    ).all()
+    scored: list[tuple[int, int, uuid.UUID]] = []
+    by_id: dict[uuid.UUID, tuple[str, str]] = {}  # id -> (source, path)
+    for doc_id, source, path, title in rows:
+        by_id[doc_id] = (source, path)
+        name = _normalize_name(title) + _normalize_name(Path(path).stem)
+        matched = [t for t in tokens if t in name]
+        if matched:
+            scored.append((len(matched), sum(len(t) for t in matched), doc_id))
+    if not scored:
+        return []
+    # Ranked by (distinct matched tokens, total matched length), the
+    # deterministic catalog tie-break (source, path) last.
+    scored.sort(
+        key=lambda s: (-s[0], -s[1], by_id[s[2]][0], by_id[s[2]][1])
+    )
+    ids = [s[2] for s in scored[:NAME_HIT_LIMIT]]
+    hit_rows = list(db.execute(_NAME_HIT_SQL, {"ids": ids}).all())
+    # The LATERAL query returns winners in id order; re-order by the
+    # ranked order computed above so the lexical list is deterministic.
+    order = {doc_id: rank for rank, doc_id in enumerate(ids)}
+    hit_rows.sort(key=lambda row: order[row.doc_id])
+    out: list[RetrievedChunk] = []
+    for row in hit_rows:
+        doc = Document(
+            id=row.doc_id,
+            source=row.source,
+            path=row.path,
+            full_path=row.full_path,
+            title=row.title,
+            content=row.doc_content,
+            content_hash=row.content_hash,
+            indexed_at=row.indexed_at,
+        )
+        out.append(
+            RetrievedChunk(
+                chunk_id=row.chunk_id,
+                position=row.position,
+                content=row.content,
+                score=0.0,  # filled in by :func:`fuse`
+                document=doc,
+                cosine=0.0,  # no vector rank — name-only hit
+                fts_hit=True,  # the lexical signal — the A8 gate answers
+                is_summary=bool(row.is_summary),
+            )
+        )
+    return out
+
+
+def _lexical_candidates(db: Session, question: str, limit: int) -> list[RetrievedChunk]:
+    """The lexical candidate list: name hits, then FTS rows (A7).
+
+    The name-hit documents (:func:`_name_hit_chunks`, the versioned-name
+    signal) LEAD the list — they are the strongest lexical evidence for
+    a name-your-thing question — followed by the top-*limit* chunks
+    matching the question's OR-tsquery, ordered by ``ts_rank`` (with
+    ``d.path, c.position`` as the deterministic tie-break). Deduped by
+    chunk id (a name-hit representative chunk the FTS list also ranked
+    appears once). An empty tsquery (stopword-only question) simply
+    contributes no FTS rows; a question with no name tokens contributes
+    no name hits — both halves are independent.
+    """
+    out: list[RetrievedChunk] = []
+    seen: set[uuid.UUID] = set()
+    for rc in _name_hit_chunks(db, question):
+        if rc.chunk_id not in seen:
+            seen.add(rc.chunk_id)
+            out.append(rc)
     tsquery = lexical_tsquery(question)
     if tsquery is None:
-        return []
+        return out
     rows = db.execute(
         _LEXICAL_SQL, {"tsquery": tsquery, "limit": limit}
     ).all()
-    out: list[RetrievedChunk] = []
     for row in rows:
+        if row.chunk_id in seen:
+            continue
+        seen.add(row.chunk_id)
         doc = Document(
             id=row.doc_id,
             source=row.source,