brain-of-reese/app/rag/retriever.py

"""Hybrid retrieval: pgvector cosine ∪ Postgres FTS, RRF-fused (PLAN §6, A7).

* **Vector list** — top-N chunks by cosine distance (``embedding <=> $1``),
  each carrying its cosine ``1 − distance`` (the honesty-gate input).
* **Lexical list** — top-N chunks matching an OR-``tsquery`` over the
  question's tokens, ordered by ``ts_rank``, UNION the name-hit list:
  documents whose normalized name (title + path stem, alnum-only,
  lowercased) contains a DIGIT-BEARING normalized question token of
  length >= 4 — bare tokens (``1panel``) and the join of adjacent
  question tokens starting with a letter ("Qwen 3.8" → ``qwen38``).
  The digit requirement is the precision guard: plain prose words
  ("server", "arguments" — 4+ chars but no digit) never name-match,
  while versioned product names (the incident's whole point) always
  carry one. The name-hit documents LEAD the lexical list (ranked by
  match count, then total matched length, then catalog order; capped
  at :data:`NAME_HIT_LIMIT`),
  the FTS rows follow. This is what finds name-your-tool questions
  ("gitlab") that vector similarity buries — and, the 2026-09-05
  incident, the versioned-name case the default parser lexes
  incompatibly (question "Qwen 3.8" → ``qwen``/``3``/``8`` can never
  match a document's ``qwen3``/``8``/``27b`` tokens, while every
  unrelated llama.cpp quadlet out-ranks the target on the shared
  ``llama``/``cpp`` tokens). The name-hit rows carry ``fts_hit=True``
  (they ARE the lexical signal — the A8 honesty gate answers on them
  only when the best cosine clears ``BOR_LEXICAL_SUPPORT_FLOOR``,
  A8 revised 2026-09-14) and ``cosine=0.0``; the RRF fusion is
  unchanged (same lists, same ``1/(k+rank)`` terms).
* **Fusion** — Reciprocal Rank Fusion (``score = Σ 1/(k + rank)`` over the
  lists a chunk appears in; chunks hit by both lists get both terms). The
  fused score ranks; :func:`select_documents` and :func:`weak_hit_titles`
  keep working off ``score``. Phase 113: the document tiering
  (:func:`select_documents_tiered`) keeps the same rank order and adds the
  usefulness bar — a document earns the cited tier only when its best
  hit-chunk cosine clears ``BOR_SOURCE_USEFULNESS_FLOOR``; the rest of the
  ranked documents (up to ``BOR_RELATED_MAX_DOCS``) become the related
  tier.

The product requirement (A7, re-revised by the phase-118 owner directive,
LOCKED A6, 2026-09-15): the retrieval path seeds **summaries** — the
suggestion tier (:func:`select_suggested`, top-N distinct documents, no
cosine floor, LOCKED A3) whose summary blocks are the grounded prompt's
``<documents>`` starting points. The full text of a document enters the
context ONLY through the agent's capped ``read`` tool
(:mod:`app.rag.agent`; ``BOR_READ_MAX_CHARS`` + :data:`TRUNCATION_MARKER`),
never through the retrieval seeding. A document's content itself is still
carried on its rows byte-identical — the ``read`` tool serves it whole,
un-truncated up to its cap.

Deterministic tie-break for equal fused scores:
``(−fused, −cosine, document.path, chunk.position)``.
"""
from __future__ import annotations

import math
import re
import uuid
from collections.abc import Sequence
from dataclasses import dataclass, replace
from datetime import UTC, datetime
from pathlib import Path

from sqlalchemy import select, text
from sqlalchemy.orm import Session

from app.config import get_settings
from app.models import Chunk, Document

#: Shared overflow marker (phase 15; imported by ``app.rag.prompts``)
#: — used by the steering (<tuning>) section, the phase-118 NULL-summary
#: suggestion preview fallback (A5), and the capped agent ``read``
#: result. The seeded summary blocks and the ``read``-served document
#: content never truncate silently (A6 re-revised): full text enters the
#: context only through the capped ``read`` tool.
TRUNCATION_MARKER = "[…truncated…]"

#: Alphanumeric tokens of a question (``to_tsquery`` input, OR-joined),
#: with DOTTED tokens kept whole (``llama.cpp`` → ``llama.cpp``). The
#: default parser lexes a dotted word as ONE lexeme (``to_tsvector`` of
#: "llama.cpp" → ``'llama.cpp'``; of "Qwen 3.8" → ``'3.8'``), so a split
#: token (``llama`` | ``cpp``) can never match the document side — the
#: 2026-09-05 incident's question phrase "llama.cpp" lexed
#: incompatibly on both sides of the query. Dotted tokens are passed
#: through as single ``to_tsquery`` lexemes (verified: Postgres accepts
#: ``'llama.cpp'`` and ``'3.8'`` as lexemes).
_TOKEN_RE = re.compile(r"[a-z0-9]+(?:\.[a-z0-9]+)*")

#: Name-hit policy (the 2026-09-05 incident): a question token shorter
#: than this (normalized) is too weak a name signal ("3", "8", "the") —
#: it would match half the KB by accident.
NAME_TOKEN_MIN_LEN = 4

#: The name-hit list may contribute at most this many documents to the
#: lexical side (before the FTS rows); the rest are dropped.
NAME_HIT_LIMIT = 10

#: Alphanumeric runs of a lowercased string (name normalization).
_ALNUM_RE = re.compile(r"[a-z0-9]+")


def _normalize_name(s: str) -> str:
    """Lowercased, alnum-only form of *s* (``Qwen 3.8`` → ``qwen38``)."""
    return "".join(_ALNUM_RE.findall(s.lower()))


def name_hit_tokens(question: str) -> list[str]:
    """The normalized name-match candidates of one question.

    Only DIGIT-BEARING candidates count (the precision guard — a plain
    prose word like "server" or "arguments" must never name-match a
    document; a versioned product name always carries a digit):

    * the :func:`_normalize_name` form of every whitespace token, kept
      when at least :data:`NAME_TOKEN_MIN_LEN` chars (``1panel``,
      ``qwen38`` from a single written token);
    * the versioned-name JOIN — the normalized concatenation of every
      ADJACENT token pair whose SECOND token is purely numeric (a
      version number: ``"… for Qwen 3.8"`` → ``Qwen`` + ``3.8`` joins
      to ``qwen38``; prose joins like ``correct`` + ``llama`` and the
      word-after-version boundaries like ``3.8`` + ``show`` are
      dropped). A purely-numeric second token means the join always
      starts with the first token's text (a letter in practice), so no
      digit-leading artifact (``38show``) can survive.

    Order of first appearance, de-duplicated.
    """
    tokens = question.split()
    out: list[str] = []
    seen: set[str] = set()

    def add(tok: str) -> None:
        if (
            len(tok) >= NAME_TOKEN_MIN_LEN
            and any(ch.isdigit() for ch in tok)
            and tok not in seen
        ):
            seen.add(tok)
            out.append(tok)

    for i, raw in enumerate(tokens):
        norm = _normalize_name(raw)
        add(norm)
        if i + 1 < len(tokens):
            next_norm = _normalize_name(tokens[i + 1])
            if next_norm and next_norm.isdigit():
                add(norm + next_norm)  # the versioned-name join
    return out


#: One row of the lexical candidate query (all fields needed to build a
#: detached :class:`Document` plus the chunk fields and ``ts_rank``).
_LEXICAL_SQL = text(
    """
    SELECT c.id            AS chunk_id,
           c.position      AS position,
           c.content       AS content,
           d.id            AS doc_id,
           d.source        AS source,
           d.path          AS path,
           d.full_path     AS full_path,
           d.title         AS title,
           d.content       AS doc_content,
           d.content_hash  AS content_hash,
           d.indexed_at    AS indexed_at,
           d.created_at    AS created_at,
           c.is_summary    AS is_summary,
           ts_rank(c.tsv, to_tsquery('english', :tsquery)) AS rank
    FROM chunks c
    JOIN documents d ON d.id = c.document_id
    WHERE c.tsv @@ to_tsquery('english', :tsquery)
    ORDER BY rank DESC, d.path ASC, c.position ASC
    LIMIT :limit
    """
)


@dataclass
class RetrievedChunk:
    """One retrieval candidate: fused rank score + parent document row.

    * ``score`` — RRF fused score (the ranking key for document selection
      and weak-hit titles).
    * ``cosine`` — vector similarity ``1 − distance`` (the honesty-gate
      input; ``0.0`` for lexical-only hits that have no vector rank).
    * ``fts_hit`` — the chunk matched the question's OR-tsquery.
    * ``is_summary`` — True for the lite-model summary chunk (phase 30,
      position −1): its parent *is* the source document, so a summary hit
      resolves to the full source document through the unchanged
      chunk→document mapping (A7 revised). Default ``False`` keeps every
      ordinary content chunk valid.
    """

    chunk_id: uuid.UUID
    position: int
    content: str
    score: float
    document: Document
    cosine: float = 0.0
    fts_hit: bool = False
    is_summary: bool = False


def lexical_tsquery(question: str) -> str | None:
    """OR-joined token string for ``to_tsquery('english', …)``, or ``None``.

    Tokens are lowercased ``[a-z0-9]+`` runs with DOTTED words kept whole
    (see :data:`_TOKEN_RE` — the default parser lexes ``llama.cpp`` as
    one lexeme, so the query must carry it whole too), de-duplicated in
    order of first appearance. Postgres does the lexing/stemming; a
    question whose tokens are all stopwords lexes to an *empty* tsquery
    (which matches nothing), so no special-casing is needed there.
    Pure-symbol questions ("???", "🔧") yield no tokens → ``None`` → no
    lexical query at all.
    """
    seen: set[str] = set()
    tokens: list[str] = []
    for tok in _TOKEN_RE.findall(question.lower()):
        if tok not in seen:
            seen.add(tok)
            tokens.append(tok)
    return " | ".join(tokens) if tokens else None


def fuse(
    vector: Sequence[RetrievedChunk],
    lexical: Sequence[RetrievedChunk],
    k: int,
) -> list[RetrievedChunk]:
    """Reciprocal Rank Fusion over the two ranked candidate lists.

    ``score(chunk) = Σ 1/(k + rank)`` — one term per list the chunk appears
    in (ranks are 1-based; a chunk in both lists gets both terms). Returns
    the union ordered by ``(-score, -cosine, document.path, position)``.

    Lexical-only hits (no vector rank) enter with ``cosine=0.0`` and
    ``fts_hit=True``; vector chunks matched by the lexical list get
    ``fts_hit=True`` in place (the input objects are mutated — callers
    should not reuse them afterwards).
    """
    if k <= 0:
        raise ValueError("rrf k must be > 0")
    by_id: dict[uuid.UUID, RetrievedChunk] = {}
    fused: dict[uuid.UUID, float] = {}
    for rank, rc in enumerate(vector, start=1):
        by_id[rc.chunk_id] = rc
        fused[rc.chunk_id] = fused.get(rc.chunk_id, 0.0) + 1.0 / (k + rank)
    for rank, rc in enumerate(lexical, start=1):
        term = 1.0 / (k + rank)
        if rc.chunk_id in by_id:
            existing = by_id[rc.chunk_id]
            by_id[rc.chunk_id] = replace(existing, fts_hit=True)
            fused[rc.chunk_id] += term
        else:
            rc = replace(rc, fts_hit=True)
            by_id[rc.chunk_id] = rc
            fused[rc.chunk_id] = fused.get(rc.chunk_id, 0.0) + term
    out = [replace(rc, score=fused[rc.chunk_id]) for rc in by_id.values()]
    out.sort(key=lambda rc: (-rc.score, -rc.cosine, rc.document.path, rc.position))
    return out


def apply_recency_boost(
    chunks: Sequence[RetrievedChunk],
    *,
    now: datetime | None = None,
    weight: float | None = None,
    half_life_days: int | None = None,
) -> list[RetrievedChunk]:
    """Additive recency boost on the fused score (phase 106, D6).

    Each chunk's score becomes
    ``score + weight * exp(−age_days / half_life_days)`` where
    ``age_days = max(0, (now − document.created_at))`` in days — a
    zero-age document gets the full *weight* (the MAXIMUM additive
    score), each ``half_life_days`` of age multiplies the remaining
    boost by ``e**-1`` (≈0.37), and a FUTURE ``created_at`` clamps to
    age 0 (the document reads as brand-new — consistent with D3's
    today-folding in :mod:`app.rag.doc_dates`). Defaults: *weight* /
    *half_life_days* from :func:`get_settings` (``recency_boost`` /
    ``recency_half_life_days``), *now* from ``datetime.now(UTC)``.

    Magnitude rationale (the ``0.0007`` default, the k=60 RRF scale):
    rank 1 vs 2 in one list differs by ~0.00026 and rank 1 vs 10 by
    ~0.0021, so the full weight is a bounded 2-3 rank head start —
    enough to break near-ties toward the newer document, far below the
    fused gap between a document that answers and one that merely
    resembles (the phase-106 fine-line battery pins the measured
    margin: 0.00263 ≥ 3× the zero-age boost).

    Pure (the :func:`fuse` convention): the inputs are never mutated —
    every boosted chunk is a ``replace()`` copy — and the result is
    re-sorted with the EXISTING deterministic key
    ``(−score, −cosine, document.path, position)``; with ``weight=0``
    every score is untouched and an already-fused (already-sorted)
    input comes back byte-identical (the kill switch, pinned).

    Untouched by design: the A8 honesty gate and ``query_log.top_score``
    (both read the chunk's ``cosine``, which the boost never modifies),
    :func:`weak_hit_titles` (titles only), and the never-truncated
    top-N contract (:func:`select_documents` still feeds whole
    documents — the boost re-ranks WHICH documents, never truncates).
    SINGLE APPLY SITE: :func:`retrieve()` is the only caller in
    ``app/`` — the chat API and ``scripts/eval_retrieval.py`` inherit
    the boost through it; nothing else may apply it.
    """
    if weight is None or half_life_days is None:
        settings = get_settings()
        if weight is None:
            weight = settings.recency_boost
        if half_life_days is None:
            half_life_days = settings.recency_half_life_days
    if half_life_days <= 0:
        raise ValueError("half_life_days must be > 0")
    if now is None:
        now = datetime.now(UTC)
    boosted: list[RetrievedChunk] = []
    for rc in chunks:
        age_days = max(0.0, (now - rc.document.created_at).total_seconds() / 86400.0)
        boosted.append(
            replace(
                rc,
                score=rc.score + weight * math.exp(-age_days / half_life_days),
            )
        )
    boosted.sort(key=lambda rc: (-rc.score, -rc.cosine, rc.document.path, rc.position))
    return boosted


def _vector_candidates(
    db: Session, question_embedding: list[float], limit: int
) -> list[RetrievedChunk]:
    """Top-*limit* chunks by pgvector cosine distance (``<=>``).

    ``cosine = 1 − distance``. Chunks whose embedding is still NULL
    (two-phase import in progress) are skipped. Each candidate carries
    its ``Chunk.is_summary`` flag (phase 30) so a summary hit stays
    identifiable after fusion.
    """
    distance = Chunk.embedding.cosine_distance(question_embedding)
    rows = db.execute(
        select(Chunk, distance.label("distance"), Document)
        .join(Document, Chunk.document_id == Document.id)
        .where(Chunk.embedding.is_not(None))
        .order_by(distance)
        .limit(limit)
    ).all()
    return [
        RetrievedChunk(
            chunk_id=chunk.id,
            position=chunk.position,
            content=chunk.content,
            score=0.0,  # fused score is filled in by :func:`fuse`
            document=doc,
            cosine=round(1.0 - float(dist), 6),
            is_summary=chunk.is_summary,
        )
        for chunk, dist, doc in rows
    ]


#: One name-hit row: the document fields (detached :class:`Document`)
#: plus the representative chunk — the ``is_summary`` chunk when the
#: document has one (its natural-language summary is the best chunk of
#: a machine file, and the chunk the vector list is likeliest to have
#: ranked too, so the RRF merge dedupes cleanly), else chunk 0.
_NAME_HIT_SQL = text(
    """
    SELECT d.id            AS doc_id,
           d.source        AS source,
           d.path          AS path,
           d.full_path     AS full_path,
           d.title         AS title,
           d.content       AS doc_content,
           d.content_hash  AS content_hash,
           d.indexed_at    AS indexed_at,
           d.created_at    AS created_at,
           c.id            AS chunk_id,
           c.position      AS position,
           c.content       AS content,
           c.is_summary    AS is_summary
    FROM documents d
    LEFT JOIN LATERAL (
        SELECT id, position, content, is_summary
        FROM chunks
        WHERE document_id = d.id
        ORDER BY is_summary DESC, position ASC
        LIMIT 1
    ) c ON true
    WHERE d.id = ANY(:ids) AND c.id IS NOT NULL
    """
)


def _name_hit_chunks(db: Session, question: str) -> list[RetrievedChunk]:
    """The documents whose NAME matches the question (the 2026-09-05
    incident, the versioned-name signal).

    A document is a name hit when its normalized name —
    :func:`_normalize_name` of its ``title`` followed by the normalized
    path stem (``qwen3.8-27b-juggernaut-vulkan.container`` →
    ``qwen3827bjuggernautvulkan``) — contains at least one
    :func:`name_hit_tokens` candidate (``qwen38``). Ranked by
    (distinct matched tokens, total matched length, source, path) —
    catalog order is the final deterministic tie-break — capped at
    :data:`NAME_HIT_LIMIT`. Each hit becomes one lexical
    :class:`RetrievedChunk` (its representative chunk, ``fts_hit=True``,
    ``cosine=0.0``). Two lightweight queries: one projection over
    (id, source, path, title) in catalog order, one LATERAL chunk
    fetch for the ≤ :data:`NAME_HIT_LIMIT` winners (no full-content
    load; the content joins in via the row fetch below).
    """
    tokens = name_hit_tokens(question)
    if not tokens:
        return []
    rows = db.execute(
        select(Document.id, Document.source, Document.path, Document.title).order_by(
            Document.source, Document.path
        )
    ).all()
    scored: list[tuple[int, int, uuid.UUID]] = []
    by_id: dict[uuid.UUID, tuple[str, str]] = {}  # id -> (source, path)
    for doc_id, source, path, title in rows:
        by_id[doc_id] = (source, path)
        name = _normalize_name(title) + _normalize_name(Path(path).stem)
        matched = [t for t in tokens if t in name]
        if matched:
            scored.append((len(matched), sum(len(t) for t in matched), doc_id))
    if not scored:
        return []
    # Ranked by (distinct matched tokens, total matched length), the
    # deterministic catalog tie-break (source, path) last.
    scored.sort(
        key=lambda s: (-s[0], -s[1], by_id[s[2]][0], by_id[s[2]][1])
    )
    ids = [s[2] for s in scored[:NAME_HIT_LIMIT]]
    hit_rows = list(db.execute(_NAME_HIT_SQL, {"ids": ids}).all())
    # The LATERAL query returns winners in id order; re-order by the
    # ranked order computed above so the lexical list is deterministic.
    order = {doc_id: rank for rank, doc_id in enumerate(ids)}
    hit_rows.sort(key=lambda row: order[row.doc_id])
    out: list[RetrievedChunk] = []
    for row in hit_rows:
        doc = Document(
            id=row.doc_id,
            source=row.source,
            path=row.path,
            full_path=row.full_path,
            title=row.title,
            content=row.doc_content,
            content_hash=row.content_hash,
            indexed_at=row.indexed_at,
            created_at=row.created_at,
        )
        out.append(
            RetrievedChunk(
                chunk_id=row.chunk_id,
                position=row.position,
                content=row.content,
                score=0.0,  # filled in by :func:`fuse`
                document=doc,
                cosine=0.0,  # no vector rank — name-only hit
                fts_hit=True,  # lexical signal — A8 answers if cosine corroborates
                is_summary=bool(row.is_summary),
            )
        )
    return out


def _lexical_candidates(db: Session, question: str, limit: int) -> list[RetrievedChunk]:
    """The lexical candidate list: name hits, then FTS rows (A7).

    The name-hit documents (:func:`_name_hit_chunks`, the versioned-name
    signal) LEAD the list — they are the strongest lexical evidence for
    a name-your-thing question — followed by the top-*limit* chunks
    matching the question's OR-tsquery, ordered by ``ts_rank`` (with
    ``d.path, c.position`` as the deterministic tie-break). Deduped by
    chunk id (a name-hit representative chunk the FTS list also ranked
    appears once). An empty tsquery (stopword-only question) simply
    contributes no FTS rows; a question with no name tokens contributes
    no name hits — both halves are independent.
    """
    out: list[RetrievedChunk] = []
    seen: set[uuid.UUID] = set()
    for rc in _name_hit_chunks(db, question):
        if rc.chunk_id not in seen:
            seen.add(rc.chunk_id)
            out.append(rc)
    tsquery = lexical_tsquery(question)
    if tsquery is None:
        return out
    rows = db.execute(
        _LEXICAL_SQL, {"tsquery": tsquery, "limit": limit}
    ).all()
    for row in rows:
        if row.chunk_id in seen:
            continue
        seen.add(row.chunk_id)
        doc = Document(
            id=row.doc_id,
            source=row.source,
            path=row.path,
            full_path=row.full_path,
            title=row.title,
            content=row.doc_content,
            content_hash=row.content_hash,
            indexed_at=row.indexed_at,
            created_at=row.created_at,
        )
        out.append(
            RetrievedChunk(
                chunk_id=row.chunk_id,
                position=row.position,
                content=row.content,
                score=0.0,  # filled in by :func:`fuse`
                document=doc,
                cosine=0.0,  # no vector rank — lexical-only hit
                fts_hit=True,
                is_summary=row.is_summary,
            )
        )
    return out


def retrieve(
    db: Session,
    question: str,
    question_embedding: list[float],
    vector_candidates: int | None = None,
    lexical_candidates: int | None = None,
) -> list[RetrievedChunk]:
    """Hybrid retrieval (A7): vector top-N ∪ FTS top-N, RRF-fused.

    Returns the fused candidate list in rank order (best first). Each
    :class:`RetrievedChunk` carries the fused ``score`` (ranking), the
    ``cosine`` similarity (honesty gate) and the ``fts_hit`` flag.
    """
    settings = get_settings()
    v_n = (
        settings.hybrid_vector_candidates if vector_candidates is None else vector_candidates
    )
    l_n = (
        settings.hybrid_lexical_candidates if lexical_candidates is None else lexical_candidates
    )
    if v_n <= 0:
        raise ValueError("vector_candidates must be >= 1")
    if l_n <= 0:
        raise ValueError("lexical_candidates must be >= 1")
    vector = _vector_candidates(db, question_embedding, v_n)
    lexical = _lexical_candidates(db, question, l_n)
    fused = fuse(vector, lexical, settings.rrf_k)
    if settings.recency_boost > 0:
        # Phase 106, D6 — the SINGLE recency-boost apply site: an
        # additive post-fusion re-rank (see :func:`apply_recency_boost`).
        # ``0`` = off: the pre-phase ranking returns byte-identical (the
        # kill switch) and weight-0 callers pay nothing.
        return apply_recency_boost(fused)
    return fused


def weak_hit_titles(chunks: Sequence[RetrievedChunk]) -> list[str]:
    """Distinct parent-document titles of *chunks*, best fused score first.

    Deflection mode (PLAN §6, A8) is built from these *titles only* — the
    LOW prompt and the "Maybe try" chips never see document content.
    """
    titles: list[str] = []
    seen: set[uuid.UUID] = set()
    for rc in sorted(chunks, key=lambda c: c.score, reverse=True):
        if rc.document.id in seen:
            continue
        seen.add(rc.document.id)
        titles.append(rc.document.title)
    return titles


def select_documents_tiered(
    chunks: Sequence[RetrievedChunk],
    n: int | None = None,
    floor: float = 0.0,
    related_cap: int = 0,
) -> tuple[list[Document], list[Document]]:
    """Tier chunk hits into the cited and the related parent documents
    (phase 113, LOCKED A2/A4 — the usefulness bar).

    Phase 118 retired the full-text seeding role (A6); the suggested
    tier (:func:`select_suggested`) seeds the prompt now — this helper
    stays as a dormant public helper (env back-compat for the settings
    it was calibrated by).

    Distinct parent documents are ranked exactly like :func:`select_documents`
    (best fused score first — the same stable score-descending walk, so a
    document's rank position is fixed by its FIRST seen chunk) and each
    document's **best hit-chunk cosine** is tracked across all of its
    chunks. The tiers are then cut in that rank order:

    * **cited** — the documents whose best-chunk cosine clears *floor*,
      up to *n* (default ``BOR_TOP_N_DOCS``). The ceiling, never a quota:
      a single strong document yields one cited document, and documents
      whose cosine stays below the bar are skipped (the next-ranked
      clearing document takes their slot — the bar filters, it does not
      backfill). The bar is on the **cosine**, not the RRF fused score:
      the fused ``score`` is a rank key, not a similarity, and a
      lexical-only hit has cosine 0.0 — vector-unsupported by definition
      (LOCKED A2, consistent with the A8 honesty gate).
    * **related** — the next distinct documents in the same rank order
      that are not already cited (any cosine, including 0.0 lexical-only
      hits), up to *related_cap* (default 0). Never overlaps the cited
      list. The done frame carries them in the secondary ``related``
      tier — the UI's de-emphasized "nearby docs" row, never a citation
      chip (LOCKED A4).

    With ``floor=0.0`` (no bar — a zero floor admits every scored
    document, so the legacy "any score, top-N" selection holds exactly)
    and ``related_cap=0`` the tiering degenerates to the legacy behavior:
    :func:`select_documents` is a thin wrapper on that.

    The returned rows carry the full document content, byte-identical —
    a matched parent document is **never truncated** (A7 revised, owner
    permission 2026-08-24; A6 re-revised 2026-09-15: the retrieval path
    seeds SUMMARIES — the cited tier's full texts no longer ride the
    grounded prompt, full text enters the context only through the
    capped ``read`` tool; the rows themselves still carry the whole
    content).
    """
    top_n = n if n is not None else get_settings().top_n_docs
    no_bar = floor <= 0.0

    order: list[Document] = []
    best_cosine: dict[uuid.UUID, float] = {}
    for rc in sorted(chunks, key=lambda c: c.score, reverse=True):
        doc = rc.document
        if doc.id in best_cosine:
            if rc.cosine > best_cosine[doc.id]:
                best_cosine[doc.id] = rc.cosine
            continue
        best_cosine[doc.id] = rc.cosine
        order.append(doc)

    cited: list[Document] = []
    for doc in order:
        if len(cited) >= top_n:
            break
        if no_bar or best_cosine[doc.id] >= floor:
            cited.append(doc)

    cited_ids = {doc.id for doc in cited}
    related: list[Document] = []
    for doc in order:
        if len(related) >= related_cap:
            break
        if doc.id not in cited_ids:
            related.append(doc)
    return cited, related


def select_documents(
    chunks: Sequence[RetrievedChunk],
    n: int | None = None,
) -> list[Document]:
    """Map chunk hits to distinct parent documents, ranked by best fused score.

    Phase 118 retired the full-text seeding role (A6); the suggested
    tier (:func:`select_suggested`) seeds the prompt now — this helper
    stays as a dormant public helper (env back-compat for the settings
    it was calibrated by).

    At most *n* documents are returned (default ``BOR_TOP_N_DOCS``). The
    returned rows carry the full document content, byte-identical — a
    matched parent document is **never truncated** (A7 revised, owner
    permission 2026-08-24; A6 re-revised 2026-09-15: the seeded prompt
    now carries SUMMARIES — the full text reaches the context only
    through the capped ``read`` tool, not through this selection). There
    is deliberately no context budget: an oversized prompt must fail
    loudly through the ``LLMError`` → SSE ``error`` path, never arrive
    as silent partial context.

    Phase 113: a thin wrapper on :func:`select_documents_tiered` — the
    legacy "any score, top-N" behavior is the cited tier with a zero
    floor (no bar) and an empty related tier, byte-identical for all
    existing callers.
    """
    cited, _ = select_documents_tiered(chunks, n, 0.0, 0)
    return cited


def select_suggested(
    chunks: Sequence[RetrievedChunk],
    n: int | None = None,
) -> list[Document]:
    """Top-N distinct parent documents in fused rank order — the phase-118
    "start here" suggestion tier (LOCKED A3), with NO cosine floor.

    Distinct parent documents are walked in the SAME stable score-
    descending order as :func:`select_documents_tiered` (a document's rank
    position is fixed by its FIRST seen chunk; dedupe by ``document.id``),
    and at most *n* of them are returned (default the
    ``BOR_SUGGESTED_DOCS`` setting, 5). Unlike the phase-113 cited tier,
    the usefulness bar NEVER filters here: a lexical-only hit with
    cosine 0.0 is suggested when it ranks. Suggestions are opt-in
    starting points, not citations — the seeded prompt carries the
    document's summary, and the LLM decides whether to extend its context
    by reading the document's full text.

    The returned rows carry the full document content, byte-identical —
    the content is what the agent's ``read`` tool serves later (never
    truncated; A6 re-revises A7: full text enters the context only through
    the capped ``read`` tool).
    """
    top_n = n if n is not None else get_settings().suggested_docs

    order: list[Document] = []
    seen: set[uuid.UUID] = set()
    for rc in sorted(chunks, key=lambda c: c.score, reverse=True):
        if len(order) >= top_n:
            break
        doc = rc.document
        if doc.id in seen:
            continue
        seen.add(doc.id)
        order.append(doc)
    return order


def select_related(
    chunks: Sequence[RetrievedChunk],
    excluded_ids: set[uuid.UUID],
    cap: int,
) -> list[Document]:
    """The documents ranked AFTER *excluded_ids* — the phase-118 related
    tier (rank 6+ for the contiguous top-5 suggestion set), up to *cap*
    (``BOR_RELATED_MAX_DOCS``).

    The SAME stable score-descending walk as
    :func:`select_documents_tiered` / :func:`select_suggested` (a
    document's rank position is fixed by its FIRST seen chunk; dedupe by
    ``document.id``), skipping every document whose id is in
    *excluded_ids* and admitting at most *cap* documents. There is NO
    cosine floor: the related tier is the ranked remainder (a lexical-
    only cosine 0.0 hit is included) — its job on the ``done`` frame is
    visibility (the UI's de-emphasized "nearby docs" row), not
    citation. With the turn wiring's exclusion — exactly the suggested
    tier's document ids (LOCKED A3: a contiguous top-N, no floor) —
    "excluding the suggested" is exactly "rank 6+".

    The returned rows carry the full document content, byte-identical
    (the tier is metadata for the ``done`` frame and the durable
    record; the prompt and ``read`` contract are untouched).
    """
    out: list[Document] = []
    seen: set[uuid.UUID] = set()
    for rc in sorted(chunks, key=lambda c: c.score, reverse=True):
        if len(out) >= cap:
            break
        doc = rc.document
        if doc.id in seen or doc.id in excluded_ids:
            continue
        seen.add(doc.id)
        out.append(doc)
    return out