brain-of-reese/tests/integration/test_recency_boost.py

"""Integration: the D6 recency boost against real Postgres (phase 106,
task 07 — the "fine line" battery, the owner's warning pinned
permanently).

The owner's scenario (2026-09-13): "make sure to test with documents
that have the correct answer but are older against documents that are
similar and newer but don't quit correctly answer the question."
Deterministic axis vectors (the ``test_name_hit_lexical.py`` idiom —
exact cosines) pin every fused score to a known rank pair, so the
margins below are exact floats, not flaky measurements.

**Measured geometry (recorded per task step 4/5):**

* Owner scenario — A (``backups/retention.md``, created 2020-01-01,
  the exact answer, cosine 1.0) lands at vector rank 1 + lexical rank
  3 (fused 0.03226646); B (``backups/retention-draft.md``, created
  yesterday, the "under review, no decision yet" draft, cosine
  0.707107) lands at vector rank 10 + lexical rank 1 (fused
  0.03067915). Pre-boost fused margin **A−B = 1/63 − 1/70 =
  0.00158730** — re-pinned by phase 119 (task 01): the two-class
  name-hit rule (LOCKED A2) makes the digitless question token
  "retention" exact-match the stem sub-component of all three
  ``retention*`` docs, so the lexical list now LEADS with those name
  hits in catalog order (draft, review, retention) and B takes the
  rank-1 lexical slot (pre-119 it was a plain FTS rank-1 / rank-3
  hit). The flip-safety bar the margin is pinned against — margin >
  the zero-age boost, so a zero-age B's full boost cannot close it
  (A's own boost being ≈ 0 at its 2020 age) — still holds, at 2.27×
  the default.
* Twin near-tie — C (``twin/c-older.md``, 2019) and D
  (``twin/d-newer.md``, yesterday) with IDENTICAL chunk text and
  near-identical vectors (cosine 1.0 vs 0.9999 — a literal identical
  vector ties the vector list's ``ORDER BY distance``, which Postgres
  resolves arbitrarily, and a permanent pin may not depend on that)
  sit one adjacent rank step apart in BOTH lists: base gap
  **C−D = 2/61 − 2/62 = 0.00052882** — a true near-tie on the RRF
  scale.
* The DEFAULT was tuned from the design starting point (0.001) down to
  **0.0007** (task step 5: "tune the DEFAULTS … until old-correct wins
  comfortably"): on the k=60 scale the owner scenario's margin was
  0.00262816 < 3×0.001, and a 0.001 zero-age boost (+0.000997 for a
  yesterday doc) would have FLIPPED the pinned scenario. 0.0007 keeps
  the flip margin comfortable (0.000698 > 0.00052882, lead
  +0.000169). Phase 119 (task 01) re-pinned the owner margin to
  0.00158730 (above): still above the zero-age boost (2.27×), so the
  worst-case flip — a zero-age B against A's ≈ 0 boost — keeps A
  first; the twin-flip figures (gap 0.00052882, boost 0.000698) are
  untouched (the twin KB has no name hits). The owner re-tunes live
  via ``BOR_RECENCY_BOOST``.

Requires: ``podman compose up -d db``.
"""
from __future__ import annotations

import math
import uuid
from collections.abc import Iterator
from datetime import UTC, datetime, timedelta
from typing import Any

import pytest
from sqlalchemy import text
from sqlalchemy.orm import Session

from app.config import Settings, get_settings
from app.models import Chunk, Document
from app.rag import retriever
from app.rag.retriever import (
    _lexical_candidates,
    _vector_candidates,
    fuse,
    retrieve,
    select_documents,
)

QUESTION = "How did I configure the backup retention policy?"

#: 768-dim test vectors (the pgvector column's dimension) — axis unit
#: vectors so the cosines are exact (1.0 parallel, 0.7071 half-parallel,
#: and constructed unit vectors with exact cosine ``q``).
D = 768

def _vec(axis: int, second: bool = False) -> list[float]:
    v = [0.0] * D
    v[axis] = 1.0
    if second:
        v[axis + 1] = 1.0
    return v


def _cos_vec(axis: int, q: float, side: int | None = None) -> list[float]:
    """A unit vector with EXACT cosine ``q`` against the axis unit vector."""
    v = [0.0] * D
    v[axis] = q
    v[axis + (side if side is not None else 1)] = math.sqrt(max(0.0, 1.0 - q * q))
    return v


A_TEXT = (
    "The backup retention policy: I configured restic on the homelab NAS "
    "with 35 daily, 12 weekly and 12 monthly backups kept. The backup "
    "retention policy was configured in /etc/retention.conf and the "
    "configured schedule is reviewed every quarter."
)
#: Similar-but-wrong: shares the topic tokens, NO answer (no "configured").
B_TEXT = "Draft: the backup retention policy is under review, no decision yet."
#: The FTS rank-2 decoy: the topic tokens at a higher ts_rank than B.
REVIEW_TEXT = (
    "backup retention policy review: the backup retention policy needs a "
    "refresh, backup retention policy discussion notes, backup retention "
    "policy follow-up planned."
)
NOTE_TEXT = "backup note {i}: a single word of shared vocabulary."
F2_TEXT = "nfs snapshot notes: the policy for nfs shares is to snapshot nightly."

#: The twins' IDENTICAL chunk body (both match the question's tsquery).
TWIN_TEXT = (
    "Twin document for the recency battery: the backup retention policy is "
    "configured the same way here."
)


def _seed(
    db: Session,
    path: str,
    title: str,
    content: str,
    created_at: datetime,
    embedding: list[float],
    source: str = "Homelab",
) -> None:
    doc = Document(
        id=uuid.uuid4(),
        source=source,
        path=path,
        full_path=f"/tmp/{path}",
        title=title,
        content=content,
        content_hash="0" * 64,
        indexed_at=datetime.now(UTC),
        created_at=created_at,
    )
    db.add(doc)
    db.flush()
    chunk = Chunk(
        id=uuid.uuid4(), document_id=doc.id, position=0, content=content
    )
    db.add(chunk)
    db.flush()
    chunk.embedding = embedding


#: The question's vector (synthetic — ``retrieve`` takes it as an arg):
#: the axis unit vector, so the seeded cosines are exact.
QUESTION_VEC = _vec(5)


def _boost_settings(**overrides: Any) -> Settings:
    """The live settings with the recency knobs overridden (the house
    settings-override pattern — ``Settings(_env_file=None, …)``)."""
    live = get_settings()
    kwargs: dict[str, Any] = {
        "recency_boost": live.recency_boost,
        "recency_half_life_days": live.recency_half_life_days,
    }
    kwargs.update(overrides)
    return Settings(_env_file=None, **kwargs)  # pyright: ignore[reportCallIssue]


def _boost_off(monkeypatch: pytest.MonkeyPatch) -> None:
    """Patch the retriever's settings to the kill switch (``0`` = off)."""
    monkeypatch.setattr(
        retriever, "get_settings", lambda: _boost_settings(recency_boost=0.0)
    )


@pytest.fixture()
def owner_kb(db) -> Iterator[None]:
    """THE owner scenario: the older doc that ANSWERS (A, 2020) vs the
    newer doc that merely resembles the topic (B, yesterday) — plus the
    KB of similar-but-not-answering backup docs that push B to vector
    rank 10 while keeping it a solid FTS hit at rank 3 (the task's
    described shape)."""
    db.execute(text("TRUNCATE chunks, documents"))
    db.commit()
    _seed(
        db, "backups/retention.md", "backup retention", A_TEXT,
        datetime(2020, 1, 1, tzinfo=UTC), _vec(5),
    )
    for i, q in enumerate((0.99, 0.98, 0.97, 0.96, 0.95, 0.94, 0.93, 0.92)):
        _seed(
            db, f"backups/notes/n{i:02d}.md", f"backup note {i}",
            NOTE_TEXT.format(i=i), datetime(2021 + i % 3, 1, 1 + i, tzinfo=UTC),
            _cos_vec(5, q),
        )
    _seed(
        db, "backups/retention-review.md", "retention review", REVIEW_TEXT,
        datetime(2021, 3, 5, tzinfo=UTC), _cos_vec(5, 0.5),
    )
    _seed(
        db, "backups/retention-draft.md", "retention draft", B_TEXT,
        datetime.now(UTC) - timedelta(days=1), _vec(5, second=True),
    )
    _seed(
        db, "backups/nfs-snapshots.md", "nfs snapshots", F2_TEXT,
        datetime(2022, 6, 10, tzinfo=UTC), _cos_vec(5, 0.4),
    )
    db.commit()
    yield
    db.execute(text("TRUNCATE chunks, documents"))
    db.commit()


def _seed_twins(db: Session, d_created_at: datetime) -> None:
    db.execute(text("TRUNCATE chunks, documents"))
    db.commit()
    _seed(
        db, "twin/c-older.md", "twin c", TWIN_TEXT,
        datetime(2019, 6, 1, tzinfo=UTC), _vec(5), source="twin",
    )
    _seed(
        db, "twin/d-newer.md", "twin d", TWIN_TEXT, d_created_at,
        _cos_vec(5, 0.9999), source="twin",
    )
    db.commit()


@pytest.fixture()
def twins(db) -> Iterator[None]:
    """The near-tie pair: IDENTICAL text, near-identical vectors, C
    (2019) older and base-ranked first, D (yesterday) newer."""
    _seed_twins(db, datetime.now(UTC) - timedelta(days=1))
    yield
    db.execute(text("TRUNCATE chunks, documents"))
    db.commit()


@pytest.fixture()
def twins_aged(db) -> Iterator[None]:
    """The same pair with D aged to ``half_life + 365`` days (730 at the
    default — two half-lives, the boost decayed to ``e**-2`` ≈ 0.135 of
    the full weight)."""
    half_life = get_settings().recency_half_life_days
    _seed_twins(db, datetime.now(UTC) - timedelta(days=half_life + 365))
    yield
    db.execute(text("TRUNCATE chunks, documents"))
    db.commit()


def test_owner_scenario_old_correct_beats_new_similar(
    owner_kb, db, monkeypatch: pytest.MonkeyPatch
) -> None:
    """THE owner scenario, pinned at the DEFAULTS: the older doc that
    answers ranks above the newer similar one — AND the pre-boost
    fused margin exceeds the zero-age boost (the flip-safety bar; the
    phase-119-re-pinned measured 0.00158730 vs the 0.0007 default is
    recorded in the module docstring). Re-pinned with the boost OFF:
    relevance alone already ordered them (no regression — the boost is
    not what makes A win)."""
    chunks = retrieve(db, QUESTION, QUESTION_VEC)
    assert select_documents(chunks, n=2)[0].path == "backups/retention.md"

    # The pre-boost fused scores, computed via ``fuse`` directly.
    s = get_settings()
    vector = _vector_candidates(db, QUESTION_VEC, s.hybrid_vector_candidates)
    lexical = _lexical_candidates(db, QUESTION, s.hybrid_lexical_candidates)
    fused = fuse(vector, lexical, s.rrf_k)
    by_path = {rc.document.path: rc.score for rc in fused}
    margin = by_path["backups/retention.md"] - by_path["backups/retention-draft.md"]
    # Phase 119 (task 01) re-pinned geometry: the three ``retention*``
    # docs are name hits (the digitless token "retention" exact-matches
    # the stem sub-component), ranked in catalog order (draft, review,
    # retention) — so A sits at vector rank 1 + lexical rank 3 and B
    # at vector rank 10 + lexical rank 1: the margin is exactly
    # 1/63 − 1/70. The flip-safety bar: a zero-age B's full boost
    # cannot close it (A's own boost is ≈ 0 at its 2020 age).
    assert margin == pytest.approx(1 / 63 - 1 / 70)
    assert margin > s.recency_boost  # 2.27× at the default

    # The kill switch: A still first (relevance alone), and the
    # weight-0 scores are the pre-phase fused scores byte-identical.
    _boost_off(monkeypatch)
    chunks_off = retrieve(db, QUESTION, QUESTION_VEC)
    assert select_documents(chunks_off, n=2)[0].path == "backups/retention.md"
    assert {rc.chunk_id: rc.score for rc in chunks_off} == {
        rc.chunk_id: rc.score for rc in fused
    }


def test_near_tie_flips_toward_the_newer_with_the_boost(
    twins, db, monkeypatch: pytest.MonkeyPatch
) -> None:
    """The boost is REAL: a near-tie (one RRF rank step apart in both
    lists, base gap 0.00052882 favoring the OLDER document) flips
    toward the NEWER one with the boost on (D's yesterday boost
    0.000698 > the gap), and the pre-phase order (C first) stands with
    the boost off — proving the boost, not drift, is the
    differentiator."""
    chunks = retrieve(db, QUESTION, QUESTION_VEC)
    assert [d.path for d in select_documents(chunks, n=2)] == [
        "twin/d-newer.md",
        "twin/c-older.md",
    ]
    _boost_off(monkeypatch)
    chunks_off = retrieve(db, QUESTION, QUESTION_VEC)
    assert [d.path for d in select_documents(chunks_off, n=2)] == [
        "twin/c-older.md",
        "twin/d-newer.md",
    ]


def test_the_boost_fades_with_age_end_to_end(twins_aged, db) -> None:
    """Decay end to end: the same pair with D aged to two half-lives
    (730 days) — D's boost decays to ``weight·e**-2`` ≈ 0.135×weight
    (the task's e^-3 figure assumed three half-lives; 730/365 = 2),
    which is BELOW the base gap — C (older) is first again. Recency is
    an age signal, not a binary: the faded boost still shows in D's
    effective score (pinned to the analytic decay), it just no longer
    overcomes a real (near-)tie."""
    chunks = retrieve(db, QUESTION, QUESTION_VEC)
    assert [d.path for d in select_documents(chunks, n=2)] == [
        "twin/c-older.md",
        "twin/d-newer.md",
    ]
    # Magnitude pin: D's observed boost == the analytic decayed weight
    # (the fixture ages D by exactly half_life + 365 days; the
    # retrieve()-time drift is microseconds, far inside the tolerance).
    s = get_settings()
    vector = _vector_candidates(db, QUESTION_VEC, s.hybrid_vector_candidates)
    lexical = _lexical_candidates(db, QUESTION, s.hybrid_lexical_candidates)
    fused = {rc.document.path: rc.score for rc in fuse(vector, lexical, s.rrf_k)}
    boosted = {rc.document.path: rc.score for rc in chunks}
    age_days = s.recency_half_life_days + 365
    observed = boosted["twin/d-newer.md"] - fused["twin/d-newer.md"]
    assert observed == pytest.approx(
        s.recency_boost * math.exp(-age_days / s.recency_half_life_days),
        rel=1e-3,
    )
    # And the faded boost is far below the full weight (e^-2 ≈ 0.135).
    assert observed < 0.2 * s.recency_boost


def test_the_a8_cosine_gate_input_is_untouched_by_the_boost(
    owner_kb, db, monkeypatch: pytest.MonkeyPatch
) -> None:
    """The boost is score-side ONLY: every chunk's ``cosine`` — the A8
    honesty-gate input and the ``query_log.top_score`` source — is
    byte-identical with the boost on vs off (asserted per chunk)."""
    chunks_on = retrieve(db, QUESTION, QUESTION_VEC)
    cosines_on = {rc.chunk_id: rc.cosine for rc in chunks_on}
    _boost_off(monkeypatch)
    chunks_off = retrieve(db, QUESTION, QUESTION_VEC)
    cosines_off = {rc.chunk_id: rc.cosine for rc in chunks_off}
    assert cosines_on == cosines_off
    # The gate input for this question: the answering document's exact
    # axis (cosine 1.0) — unchanged by the re-rank.
    assert max(cosines_on.values()) == 1.0