finally getting accurate answers

tests/integration/test_name_hit_lexical.py

@@ -0,0 +1,207 @@
+"""Integration: the name-hit lexical signal against real Postgres (the
+2026-09-05 "Qwen 3.8" incident).
+
+The unit suite (``tests/unit/test_retriever.py``) covers the pure
+mapping with fake rows; this suite covers the SQL side on real
+Postgres: the document-projection scan, the LATERAL representative-
+chunk fetch (the ``is_summary`` chunk wins, chunk 0 otherwise, and a
+chunk-less name match is EXCLUDED — the ``c.id IS NOT NULL`` guard),
+the (count, length, catalog) ranking, the name-hits-lead-the-lexical-
+list union with the FTS rows (chunk-id dedup), and the full
+``retrieve()`` → ``select_documents()`` path putting the versioned-
+name document into the seeded top-N.
+
+Requires: ``podman compose up -d db``.
+"""
+from __future__ import annotations
+
+import uuid
+from collections.abc import Iterator
+from datetime import UTC, datetime
+
+import pytest
+from sqlalchemy import text
+from sqlalchemy.orm import Session
+
+from app.models import Chunk, Document
+from app.rag.retriever import (
+    NAME_HIT_LIMIT,
+    _lexical_candidates,
+    _name_hit_chunks,
+    retrieve,
+    select_documents,
+)
+
+INCIDENT_QUESTION = "What are the correct llama.cpp arguments for Qwen 3.8?"
+
+#: 768-dim test vectors (the pgvector column's dimension) — axis unit
+#: vectors so the cosines are exact (1.0 parallel, 0.0 orthogonal,
+#: 0.7071 half-parallel).
+D = 768
+
+
+def _vec(axis: int, second: bool = False) -> list[float]:
+    v = [0.0] * D
+    v[axis] = 1.0
+    if second:
+        v[axis + 1] = 1.0
+    return v
+
+
+def _doc(db: Session, source: str, path: str, title: str, content: str) -> Document:
+    doc = Document(
+        id=uuid.uuid4(),
+        source=source,
+        path=path,
+        full_path=f"/tmp/{source}/{path}",
+        title=title,
+        content=content,
+        content_hash="0" * 64,
+        indexed_at=datetime.now(UTC),
+    )
+    db.add(doc)
+    return doc
+
+
+def _chunk(
+    db: Session, doc: Document, position: int, content: str, is_summary: bool = False
+) -> Chunk:
+    chunk = Chunk(
+        id=uuid.uuid4(),
+        document_id=doc.id,
+        position=position,
+        content=content,
+        is_summary=is_summary,
+    )
+    db.add(chunk)
+    return chunk
+
+
+@pytest.fixture()
+def kb(db) -> Iterator[None]:
+    """A fresh KB with the incident shape: the qwen3.8 quadlet (the
+    name hit, with a summary chunk + an ordinary chunk), a qwen3.6
+    quadlet (same family, different version — NOT a hit), an
+    unrelated document (FTS-only candidate), and a chunk-less document
+    whose name DOES carry the token (the exclusion guard)."""
+    db.execute(text("TRUNCATE chunks, documents"))
+    db.commit()
+
+    q38 = _doc(
+        db,
+        "deploy",
+        "reeseapps/ai/deployments/juggernaut/quadlets/qwen3.8-27b-juggernaut-vulkan.container",
+        "qwen3.8-27b-juggernaut-vulkan",
+        "# llama.cpp juggernaut\nExec=--port 8000 -ctk q8_0 -ctv q8_0 --jinja\n"
+        "-m /models/qwen3.8-27b/Qwen3.8-27B-UD-Q6_K.gguf\n",
+    )
+    _chunk(
+        db,
+        q38,
+        -1,
+        "Podman quadlet: llama.cpp server for Qwen 3.8 27B (juggernaut).",
+        is_summary=True,
+    )
+    _chunk(db, q38, 0, "# llama.cpp juggernaut\nExec=--port 8000 -ctk q8_0")
+    db.flush()
+    # A vector the question vector (below) cosines with — non-NULL so
+    # the chunk is eligible for the vector list too.
+    for c in q38.chunks:
+        c.embedding = _vec(1)
+    db.commit()
+
+    q36 = _doc(
+        db,
+        "deploy",
+        "reeseapps/ai/deployments/juggernaut/quadlets/qwen3.6-27b-juggernaut-vulkan.container",
+        "qwen3.6-27b-juggernaut-vulkan",
+        "# llama.cpp juggernaut\n-m /models/qwen3.6-27b/model.gguf\n",
+    )
+    c36 = _chunk(db, q36, 0, "# llama.cpp juggernaut\n-m /models/qwen3.6-27b/model.gguf")
+    c36.embedding = _vec(0)  # orthogonal to the question vector
+    db.commit()
+
+    other = _doc(
+        db, "homelab", "notes/llama.cpp.md", "llama.cpp notes", "llama cpp server arguments notes\n"
+    )
+    c_other = _chunk(db, other, 0, "llama cpp server arguments notes")
+    c_other.embedding = _vec(1, second=True)  # half-parallel to the question
+    db.commit()
+
+    # Name carries the token, ZERO chunks — the exclusion guard.
+    _doc(db, "deploy", "qwen3.8-empty.container", "qwen3.8-empty", "(empty file)")
+    db.commit()
+    yield
+    db.execute(text("TRUNCATE chunks, documents"))
+    db.commit()
+
+
+def test_name_hit_chunks_real_sql(kb, db) -> None:
+    """Real Postgres: the projection scan finds exactly the qwen3.8
+    quadlet (the qwen3.6 sibling and the chunk-less name match are
+    excluded), and the LATERAL fetch hands back the SUMMARY chunk as
+    the representative (position −1, is_summary)."""
+    out = _name_hit_chunks(db, INCIDENT_QUESTION)
+    assert [rc.document.path for rc in out] == [
+        "reeseapps/ai/deployments/juggernaut/quadlets/qwen3.8-27b-juggernaut-vulkan.container"
+    ]
+    rc = out[0]
+    assert rc.position == -1  # the summary chunk wins the LATERAL order
+    assert rc.is_summary is True
+    assert rc.fts_hit is True  # the lexical signal — the A8 gate answers
+    assert rc.cosine == 0.0  # no vector rank on the name-hit row
+    assert "qwen3.8-empty.container" not in [r.document.path for r in out]  # chunk-less guard
+
+
+def test_lexical_candidates_name_hit_leads_real_sql(kb, db) -> None:
+    """The full lexical list on real Postgres: the name hit leads, the
+    FTS rows follow (the qwen3.6 and llama.cpp docs both match the
+    OR-tsquery on llama|cpp|arguments|… — the pre-incident pollution —
+    but the name hit still ranks them behind it)."""
+    out = _lexical_candidates(db, INCIDENT_QUESTION, limit=30)
+    paths = [rc.document.path for rc in out]
+    assert paths[0] == (
+        "reeseapps/ai/deployments/juggernaut/quadlets/qwen3.8-27b-juggernaut-vulkan.container"
+    )
+    # The FTS pollution is still present (the incident's shape) — but
+    # behind the name hit, no longer ahead of it.
+    assert (
+        "reeseapps/ai/deployments/juggernaut/quadlets/qwen3.6-27b-juggernaut-vulkan.container"
+        in paths
+    )
+    assert all(rc.fts_hit is True for rc in out)
+
+
+def test_retrieve_selects_name_hit_doc_into_top_n(kb, db) -> None:
+    """The product path: hybrid ``retrieve()`` (vector ∪ lexical, RRF
+    fused) → ``select_documents`` puts the qwen3.8 quadlet in the
+    seeded top-N — the incident's seed miss (the two overview docs
+    only) is fixed. The question vector is parallel to the q38 chunk
+    embeddings (cosine 1.0), orthogonal to q36 (0.0)."""
+    question_vec = _vec(1)
+    chunks = retrieve(db, INCIDENT_QUESTION, question_vec)
+    docs = select_documents(chunks, n=2)
+    assert [d.path for d in docs] == [
+        "reeseapps/ai/deployments/juggernaut/quadlets/qwen3.8-27b-juggernaut-vulkan.container",
+        "notes/llama.cpp.md",
+    ]
+
+
+def test_name_hit_limit_real_sql(db) -> None:
+    """Twelve identical (1, 6) name hits — the LATERAL fetch (and the
+    output) carries exactly ``NAME_HIT_LIMIT`` winners, catalog order."""
+    db.execute(text("TRUNCATE chunks, documents"))
+    db.commit()
+    for i in range(12):
+        doc = _doc(
+            db, "S", f"quadlets/m{i:02d}-qwen38.container", f"m{i:02d}-qwen38", "llama cpp qwen38\n"
+        )
+        db.flush()
+        c = _chunk(db, doc, 0, f"llama cpp qwen38 doc {i}")
+        c.embedding = _vec(2)
+    db.commit()
+    out = _name_hit_chunks(db, "what are the llama.cpp arguments for qwen 3.8")
+    assert len(out) == NAME_HIT_LIMIT
+    assert [rc.document.path for rc in out] == [
+        f"quadlets/m{i:02d}-qwen38.container" for i in range(NAME_HIT_LIMIT)
+    ]