phase: 106_document_dates
Everything is verified green. Final report: **Phase 106 — Document dates (final verification pass; all 10 tasks already complete)** - Verified all phase artifacts: alembic `0020` (dev DB at `0020`), `doc_dates.py`, git `file_commit_dates`, importer `doc_dates_by_root`/`dates_updated`, both entry-point wirings, date APIs + tree `created_at`/`updated_at`, LLM surfaces (prompt block, `read` line 2, appended `ls` field), `apply_recency_boost` in `retrieve()`, UI columns/badge, admin editor, mock-LLM regex — all present and correct; no defects found, no fixes needed. - `uv run pytest --cov=app --cov-report=term-missing` → **2299 passed, TOTAL 99%** (>90% ✓) - `uv run pytest tests/e2e/test_document_dates.py -v --no-cov` → **6/6 passed** in isolation (DB up) - 12 regression E2E suites (retrieval_quality, whole_document_context, agent_document_tools, ls_tree_drilldown, read_truncation_cap, kb_tree, kb_tree_nav, document_viewer, edit_summaries, import_documents, sync_button, hidden_folders_toggle, smoke) → **all green in isolation** - `uv run ruff check .` → clean; `uv run pyright` → **0 errors, 0 warnings** **Completion criteria:** 1) non-null `created_at` + 0020 upgrade/downgrade on dev DB ✓ (real-Alembic integration tests) 2) sync refresh/older/manual-persists/content-reset/no sources_meta bump ✓ 3) zip/tar mtime + future→today ✓ 4) LLM date surfaces + cross-check ✓ 5) UI Created/Updated/badge positions ✓ 6) admin editor set+revert round-trip ✓ 7) old-correct-beats-new-similar (defaults & boost-off) + near-tie + `BOR_RECENCY_BOOST=0` byte-identical ✓ 8) full gate ✓ 9) commit/phase-move — left to harness per instructions. - **Notable:** recency default tuned 0.001 → **0.0007** (task 07 step 5 explicitly permits; measured margins recorded in `test_recency_boost.py` docstring). - **Next pending phase:** none — `todo/` holds only this phase.
This commit is contained in:
@@ -0,0 +1,200 @@
|
||||
"""Session-level DB self-heal for the integration suite (phase 106, task 02).
|
||||
|
||||
Incident 2026-09-13: the dev DB's ``documents`` table hit PostgreSQL's
|
||||
1600-attribute hard limit and every ``ALTER TABLE … ADD COLUMN`` failed
|
||||
with ``TooManyColumns``, red-lining the full suite. Cause: the house
|
||||
migration-test pattern (A13 — every migration exercises a real
|
||||
downgrade, then repairs back to head) leaks *dropped-column
|
||||
placeholder* attributes on every downgrade→upgrade round-trip
|
||||
(``pg_attribute`` rows with ``attisdropped=true``). ``VACUUM (FULL)``
|
||||
does NOT reclaim them (verified on PG 17.11) — only a table rewrite
|
||||
does — and at ~95 leaks per full-suite run the shared dev DB bricks
|
||||
every ~17 runs (faster when two runs race, which is how the incident
|
||||
triggered).
|
||||
|
||||
This session fixture rebuilds any ``public`` table whose dropped-
|
||||
attribute count exceeds :data:`DROPPED_ATTR_LIMIT` **before** the first
|
||||
integration test of the session runs: rename + ``CREATE TABLE
|
||||
(LIKE … INCLUDING ALL)`` + row copy + FK rewiring (both directions,
|
||||
original constraint names and actions preserved). The normal case costs
|
||||
one small catalog query per session; the heal path only fires while a
|
||||
table is far below the 1600 cap (the limit is 200 — a tenth of the
|
||||
headroom), so the migration tests never run on a near-bricked DB.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import uuid
|
||||
from collections.abc import Iterator
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import text
|
||||
|
||||
from app.db import db_available, engine
|
||||
|
||||
logger = logging.getLogger("bor.integration.self_heal")
|
||||
|
||||
#: Rebuild a table once its *dropped* attribute count passes this.
|
||||
#: PostgreSQL's hard cap is 1600 TOTAL attributes (dropped included),
|
||||
#: and the migration round-trips leak ~95 per full-suite run.
|
||||
DROPPED_ATTR_LIMIT = 200
|
||||
|
||||
_BLOATED_SQL = text(
|
||||
"SELECT c.relname FROM pg_class c"
|
||||
" JOIN pg_namespace n ON n.oid = c.relnamespace"
|
||||
" WHERE c.relkind = 'r' AND n.nspname = 'public'"
|
||||
" AND (SELECT count(*) FROM pg_attribute a"
|
||||
" WHERE a.attrelid = c.oid AND a.attnum > 0 AND a.attisdropped)"
|
||||
" > :limit"
|
||||
" ORDER BY 1"
|
||||
)
|
||||
|
||||
_FKS_SQL = text(
|
||||
"SELECT c.conname,"
|
||||
" c.conrelid::regclass::text AS child,"
|
||||
" c.confrelid::regclass::text AS parent,"
|
||||
" c.confdeltype, c.confupdtype, c.confmatchtype AS matchtype,"
|
||||
" (SELECT string_agg(ca.attname, ', ' ORDER BY ck.ord)"
|
||||
" FROM unnest(c.conkey) WITH ORDINALITY ck(attnum, ord)"
|
||||
" JOIN pg_attribute ca ON ca.attrelid = c.conrelid AND ca.attnum = ck.attnum)"
|
||||
" AS child_cols,"
|
||||
" (SELECT string_agg(pa.attname, ', ' ORDER BY pk.ord)"
|
||||
" FROM unnest(c.confkey) WITH ORDINALITY pk(attnum, ord)"
|
||||
" JOIN pg_attribute pa ON pa.attrelid = c.confrelid AND pa.attnum = pk.attnum)"
|
||||
" AS parent_cols"
|
||||
" FROM pg_constraint c"
|
||||
" WHERE c.contype = 'f'"
|
||||
" AND (:t = c.conrelid::regclass::text OR :t = c.confrelid::regclass::text)"
|
||||
)
|
||||
|
||||
#: pg_constraint confdeltype/confupdtype codes → the DDL clause (``None``
|
||||
#: = NO ACTION, the default — the clause is omitted).
|
||||
_FK_ACTION: dict[str, str | None] = {
|
||||
"a": None, # NO ACTION — the default, the clause is omitted
|
||||
"r": "RESTRICT",
|
||||
"c": "CASCADE",
|
||||
"n": "SET NULL",
|
||||
"d": "SET DEFAULT",
|
||||
}
|
||||
_FK_MATCH: dict[str, str] = {"f": "MATCH FULL", "p": "MATCH PARTIAL"}
|
||||
|
||||
|
||||
def _fk_clause(fk) -> str:
|
||||
"""The trailing ``MATCH …/ON DELETE …/ON UPDATE …`` of an FK."""
|
||||
parts = [
|
||||
_FK_MATCH.get(fk.matchtype, ""),
|
||||
f"ON DELETE {_FK_ACTION[fk.confdeltype]}" if _FK_ACTION[fk.confdeltype] else "",
|
||||
f"ON UPDATE {_FK_ACTION[fk.confupdtype]}" if _FK_ACTION[fk.confupdtype] else "",
|
||||
]
|
||||
return " ".join(p for p in parts if p)
|
||||
|
||||
|
||||
def _rebuild_table(table: str) -> None:
|
||||
"""Rewrite *table* to purge its dropped-column placeholders.
|
||||
|
||||
Rename + ``LIKE … INCLUDING ALL`` (live columns, constraints,
|
||||
indexes, defaults) + row copy + FK rewiring (incoming AND outgoing,
|
||||
original constraint names/actions). One transaction — a failure
|
||||
rolls the whole table's surgery back and fails the session loudly
|
||||
(a half-healed DB must never feed the migration tests).
|
||||
|
||||
The staging names carry a per-run suffix: a previous (interrupted or
|
||||
repeated) heal may still own the plain names, and a collision would
|
||||
make PG auto-suffix the LIKE-copied constraint names (…``_pkey1``)
|
||||
and defeat the PK rename below.
|
||||
|
||||
Note: the PK is renamed back to its conventional ``<table>_pkey``;
|
||||
other LIKE-copied objects keep PG's auto-generated names (nothing in
|
||||
the repo references constraint/index names by name — DDL is
|
||||
alembic-only, the ORM never issues DDL).
|
||||
"""
|
||||
new_name = f"{table}_heal_new_{uuid.uuid4().hex[:8]}"
|
||||
old_name = f"{table}_heal_old_{uuid.uuid4().hex[:8]}"
|
||||
with engine.begin() as conn:
|
||||
fks = conn.execute(_FKS_SQL, {"t": table}).fetchall()
|
||||
for fk in fks:
|
||||
conn.execute(
|
||||
text(f'ALTER TABLE "{fk.child}" DROP CONSTRAINT "{fk.conname}"')
|
||||
)
|
||||
conn.execute(
|
||||
text(f'CREATE TABLE "{new_name}" (LIKE "{table}" INCLUDING ALL)')
|
||||
)
|
||||
# Explicit non-generated column list (attnum order): ``SELECT *``
|
||||
# cannot be used — chunks.tsv is a STORED generated column, and
|
||||
# generated columns refuse explicit values (it recomputes them).
|
||||
cols = conn.execute(
|
||||
text(
|
||||
"SELECT string_agg('\"' || a.attname || '\"', ', '"
|
||||
" ORDER BY a.attnum)"
|
||||
" FROM pg_attribute a JOIN pg_class tc ON tc.oid = a.attrelid"
|
||||
" WHERE tc.relname = :t AND a.attnum > 0"
|
||||
" AND NOT a.attisdropped AND a.attgenerated NOT IN ('s', 'v')"
|
||||
),
|
||||
{"t": table},
|
||||
).scalar()
|
||||
conn.execute(
|
||||
text(f'INSERT INTO "{new_name}" ({cols}) SELECT {cols} FROM "{table}"')
|
||||
)
|
||||
conn.execute(text(f'ALTER TABLE "{table}" RENAME TO "{old_name}"'))
|
||||
# Drop the old table BEFORE the staging table takes its name: the
|
||||
# old table's index/constraint names (including ``<table>_pkey``
|
||||
# from a previous heal) live in the schema namespace until the
|
||||
# DROP, and the PK rename below needs that name free.
|
||||
conn.execute(text(f'DROP TABLE "{old_name}"'))
|
||||
conn.execute(text(f'ALTER TABLE "{new_name}" RENAME TO "{table}"'))
|
||||
# The rename above does NOT follow to LIKE-copied objects: put the
|
||||
# PK back on its conventional name (every migration here auto-names
|
||||
# PKs ``<table>_pkey`` — the one name tools/scripts reference).
|
||||
has_auto_pkey = conn.execute(
|
||||
text(
|
||||
"SELECT 1 FROM pg_constraint c"
|
||||
" JOIN pg_class tc ON tc.oid = c.conrelid"
|
||||
" WHERE tc.relname = :t AND c.conname = :n AND c.contype = 'p'"
|
||||
),
|
||||
{"t": table, "n": f"{new_name}_pkey"},
|
||||
).fetchone()
|
||||
if has_auto_pkey:
|
||||
conn.execute(
|
||||
text(
|
||||
f'ALTER TABLE "{table}" RENAME CONSTRAINT'
|
||||
f' "{new_name}_pkey" TO "{table}_pkey"'
|
||||
)
|
||||
)
|
||||
# FK rewiring LAST: only now does ``<table>`` refer to the rebuilt
|
||||
# table with all staging names gone.
|
||||
for fk in fks:
|
||||
clause = _fk_clause(fk)
|
||||
conn.execute(
|
||||
text(
|
||||
f'ALTER TABLE "{fk.child}" ADD CONSTRAINT "{fk.conname}"'
|
||||
f" FOREIGN KEY ({fk.child_cols})"
|
||||
f' REFERENCES "{fk.parent}" ({fk.parent_cols})'
|
||||
+ (f" {clause}" if clause else "")
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True, scope="session")
|
||||
def heal_bloated_tables() -> Iterator[None]:
|
||||
"""Rebuild bloated ``public`` tables before the session's first test.
|
||||
|
||||
One cheap catalog query per session while healthy (the normal
|
||||
case); the rebuild path only fires when a table's dropped-attribute
|
||||
count passes :data:`DROPPED_ATTR_LIMIT` (see the module docstring
|
||||
for the 2026-09-13 incident this exists to outlive).
|
||||
"""
|
||||
if db_available():
|
||||
with engine.connect() as conn:
|
||||
bloated = [
|
||||
row[0]
|
||||
for row in conn.execute(_BLOATED_SQL, {"limit": DROPPED_ATTR_LIMIT})
|
||||
]
|
||||
for table in bloated:
|
||||
logger.warning(
|
||||
"integration self-heal: rebuilding %r (dropped attributes"
|
||||
" > %d — migration round-trip placeholders)",
|
||||
table,
|
||||
DROPPED_ATTR_LIMIT,
|
||||
)
|
||||
_rebuild_table(table)
|
||||
yield
|
||||
@@ -22,7 +22,11 @@ unknown folder → NOT-A_FOLDER with the parent's subfolders).
|
||||
(first-slash split; a bare source name and an unknown identity get the
|
||||
no-document refusal), and ``grep`` (``all_documents`` for a whole-KB
|
||||
search, ``find_document`` for a scoped one) — both byte-identical
|
||||
across the phase-94 change.
|
||||
across the phase-94 change. Phase 106 (D5): the ``ls`` FILE line ends
|
||||
with the appended `` | date: YYYY-MM-DD`` field and the ``read``
|
||||
result carries the ``date: YYYY-MM-DD`` second line (first line
|
||||
byte-identical) — the fixture documents carry a fixed ``created_at``
|
||||
so the pins stay deterministic.
|
||||
|
||||
Requires: podman compose up -d db
|
||||
"""
|
||||
@@ -32,6 +36,7 @@ import asyncio
|
||||
import uuid
|
||||
from collections.abc import AsyncIterator, Iterator
|
||||
from copy import deepcopy
|
||||
from datetime import UTC, datetime
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
import pytest
|
||||
@@ -53,6 +58,11 @@ from app.rag.llm import (
|
||||
if TYPE_CHECKING:
|
||||
from app.rag.scaffolding import ScaffoldingFilter
|
||||
|
||||
#: The fixture documents' fixed creation date (phase 106, D5) — the
|
||||
#: ``ls`` file line and the ``read`` second line format its UTC date
|
||||
#: part; a fixed value keeps the pins deterministic.
|
||||
_FIXTURE_CREATED_AT = datetime(2024, 6, 15, 12, 0, 0, tzinfo=UTC)
|
||||
|
||||
|
||||
def _doc(db: Session, source: str, path: str, title: str, content: str) -> Document:
|
||||
doc = Document(
|
||||
@@ -63,6 +73,7 @@ def _doc(db: Session, source: str, path: str, title: str, content: str) -> Docum
|
||||
title=title,
|
||||
content=content,
|
||||
content_hash="0" * 64,
|
||||
created_at=_FIXTURE_CREATED_AT,
|
||||
)
|
||||
db.add(doc)
|
||||
return doc
|
||||
@@ -121,8 +132,8 @@ def test_source_document_rows_order_by_path_within_the_source(kb, db) -> None:
|
||||
db.commit()
|
||||
|
||||
assert agent._source_document_rows(db, "Zeta") == [
|
||||
("a/first.md", "Zeta A"),
|
||||
("b/second.md", "Zeta B"),
|
||||
("a/first.md", "Zeta A", "2024-06-15"),
|
||||
("b/second.md", "Zeta B", "2024-06-15"),
|
||||
]
|
||||
|
||||
|
||||
@@ -355,7 +366,7 @@ def test_ls_source_scope_lists_root_folder_through_run_agent(kb, registry, db) -
|
||||
" backups/ — 2 documents: Backup notes.\n"
|
||||
" networking/ — 1 documents\n"
|
||||
"\n"
|
||||
"source: Homelab | path: readme.md | title: Readme"
|
||||
"source: Homelab | path: readme.md | title: Readme | date: 2024-06-15"
|
||||
)
|
||||
assert holder.tool_calls == 1
|
||||
assert holder.read_docs == []
|
||||
@@ -393,8 +404,8 @@ def test_ls_nested_folder_scope_drills_one_level_through_run_agent(
|
||||
assert llm2.requests[1][0][3]["content"] == (
|
||||
"Homelab/networking/lan — 2 documents, 0 folders:\n"
|
||||
"\n"
|
||||
"source: Homelab | path: networking/lan/a.md | title: A\n"
|
||||
"source: Homelab | path: networking/lan/b.md | title: B"
|
||||
"source: Homelab | path: networking/lan/a.md | title: A | date: 2024-06-15\n"
|
||||
"source: Homelab | path: networking/lan/b.md | title: B | date: 2024-06-15"
|
||||
)
|
||||
assert holder2.tool_calls == 1
|
||||
|
||||
@@ -411,8 +422,12 @@ def test_ls_folder_file_cap_through_run_agent(kb, registry, db) -> None:
|
||||
content = llm.requests[1][0][3]["content"]
|
||||
lines = content.splitlines()
|
||||
assert lines[0] == "Homelab/big — 51 documents, 0 folders:"
|
||||
assert lines[2] == "source: Homelab | path: big/f000.md | title: T0"
|
||||
assert lines[51] == "source: Homelab | path: big/f049.md | title: T49"
|
||||
assert lines[2] == (
|
||||
"source: Homelab | path: big/f000.md | title: T0 | date: 2024-06-15"
|
||||
)
|
||||
assert lines[51] == (
|
||||
"source: Homelab | path: big/f049.md | title: T49 | date: 2024-06-15"
|
||||
)
|
||||
assert lines[52] == (
|
||||
"…and 1 more documents in this folder — use grep (pattern) to "
|
||||
"find a specific one."
|
||||
@@ -491,8 +506,12 @@ def test_read_combined_path_through_run_agent(kb, db) -> None:
|
||||
|
||||
holder, llm = _run_call(db, "read", {"path": "Alpha/deep/nested/doc.md"})
|
||||
|
||||
# Phase 106 (D5): the date rides every read — the SECOND line (the
|
||||
# first line stays the byte-identical header).
|
||||
assert llm.requests[1][0][3]["content"] == (
|
||||
"Document Alpha/deep/nested/doc.md:\nFULL-TEXT"
|
||||
"Document Alpha/deep/nested/doc.md:\n"
|
||||
"date: 2024-06-15\n"
|
||||
"FULL-TEXT"
|
||||
)
|
||||
assert holder.tool_calls == 1
|
||||
assert holder.read_docs == [created]
|
||||
@@ -563,9 +582,12 @@ def test_read_bare_path_single_source_suggestion_then_corrected_read(kb, db) ->
|
||||
)
|
||||
assert llm.requests[1][1] == AGENT_TOOLS
|
||||
# Round 2: the corrected combined identity succeeds — the full
|
||||
# content, the holder records the row, and it counts.
|
||||
# content (plus the phase-106 D5 date line), the holder records the
|
||||
# row, and it counts.
|
||||
assert llm.requests[2][0][5]["content"] == (
|
||||
"Document Alpha/deep/nested/doc.md:\nFULL-TEXT"
|
||||
"Document Alpha/deep/nested/doc.md:\n"
|
||||
"date: 2024-06-15\n"
|
||||
"FULL-TEXT"
|
||||
)
|
||||
assert llm.requests[2][1] == AGENT_TOOLS
|
||||
assert holder.read_docs == [created]
|
||||
@@ -598,7 +620,9 @@ def test_read_bare_path_two_sources_one_of_suggestion_then_corrected_read(
|
||||
"No document at 'shared/x.md' — did you mean one of: "
|
||||
"'Alpha/shared/x.md', 'Beta/shared/x.md'?"
|
||||
)
|
||||
assert llm.requests[2][0][5]["content"] == "Document Alpha/shared/x.md:\nA-TEXT"
|
||||
assert llm.requests[2][0][5]["content"] == (
|
||||
"Document Alpha/shared/x.md:\ndate: 2024-06-15\nA-TEXT"
|
||||
)
|
||||
assert holder.read_docs == [a]
|
||||
assert holder.tool_calls == 1 # only the corrected read executed
|
||||
|
||||
|
||||
@@ -0,0 +1,251 @@
|
||||
"""Integration: the phase-106 D5 date surfaces against REAL Postgres
|
||||
rows (task 06).
|
||||
|
||||
The two tool surfaces the model reads carry the document's creation
|
||||
date: the ``read`` result's SECOND line (``date: YYYY-MM-DD`` — the
|
||||
FIRST line stays the byte-identical ``Document {source}/{path}:``
|
||||
header the E2E mock's ``_READ_RESULT_PREFIX`` contract keys on) and
|
||||
the ``ls`` FILE line's APPENDED `` | date: YYYY-MM-DD`` field (the
|
||||
mock's ``_CATALOG_LINE_RE`` ``title: .+$`` tail absorbs it). The rows
|
||||
carry DISTINCT fixed ``created_at`` values, so the pins prove the date
|
||||
is the ROW's date (per row), not a constant.
|
||||
|
||||
Requires: podman compose up -d db
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import uuid
|
||||
from collections.abc import AsyncIterator, Iterator
|
||||
from copy import deepcopy
|
||||
from datetime import UTC, datetime
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import delete, text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import Settings
|
||||
from app.models import Document, GitSource
|
||||
from app.rag.agent import AgentHolder, run_agent
|
||||
from app.rag.llm import LLMClient, RetryPiece, StreamPiece, ToolCallPiece, ToolResultPiece
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.rag.scaffolding import ScaffoldingFilter
|
||||
|
||||
#: DISTINCT fixed creation dates — the per-row date pins (task 06):
|
||||
#: each document renders ITS OWN row's UTC date part.
|
||||
D1 = datetime(2019, 6, 15, 3, 4, 6, tzinfo=UTC) # "2019-06-15"
|
||||
D2 = datetime(2020, 1, 2, 5, 0, 0, tzinfo=UTC) # "2020-01-02"
|
||||
D3 = datetime(2024, 6, 15, 23, 59, 59, tzinfo=UTC) # "2024-06-15" (late UTC instant)
|
||||
|
||||
D1_STR, D2_STR, D3_STR = "2019-06-15", "2020-01-02", "2024-06-15"
|
||||
|
||||
|
||||
def _doc(
|
||||
db: Session,
|
||||
source: str,
|
||||
path: str,
|
||||
title: str,
|
||||
content: str,
|
||||
created_at: datetime,
|
||||
) -> Document:
|
||||
doc = Document(
|
||||
id=uuid.uuid4(),
|
||||
source=source,
|
||||
path=path,
|
||||
full_path=f"/tmp/{source}/{path}",
|
||||
title=title,
|
||||
content=content,
|
||||
content_hash="0" * 64,
|
||||
created_at=created_at, # D1: explicit — the pins prove per-row dates
|
||||
)
|
||||
db.add(doc)
|
||||
return doc
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def kb(db) -> Iterator[None]:
|
||||
"""Fresh documents table (chunks first — the FK)."""
|
||||
db.execute(text("TRUNCATE chunks, documents"))
|
||||
db.commit()
|
||||
yield
|
||||
db.execute(text("TRUNCATE chunks, documents"))
|
||||
db.commit()
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def src(db) -> Iterator[GitSource]:
|
||||
"""One registered git source — the scoped ``ls`` source-name check
|
||||
reads the real registry (``repo_name`` resolves the URL to
|
||||
``Homelab``)."""
|
||||
row = GitSource(url="https://github.com/reese/Homelab.git", kind="git")
|
||||
db.add(row)
|
||||
db.commit()
|
||||
yield row
|
||||
db.execute(delete(GitSource).where(GitSource.id == row.id))
|
||||
db.commit()
|
||||
|
||||
|
||||
class ScriptedToolLLM:
|
||||
"""One scripted tool-call stream, then one canned answer stream.
|
||||
Records every ``chat_stream`` request's messages and tools."""
|
||||
|
||||
def __init__(self, call: ToolCallPiece) -> None:
|
||||
self.call = call
|
||||
self.requests: list[
|
||||
tuple[list[dict[str, Any]], list[dict[str, Any]] | None]
|
||||
] = []
|
||||
|
||||
async def chat_stream(
|
||||
self,
|
||||
messages: list[dict[str, str]],
|
||||
tools: list[dict[str, Any]] | None = None,
|
||||
scaffolding: ScaffoldingFilter | None = None,
|
||||
) -> AsyncIterator[StreamPiece | ToolCallPiece]:
|
||||
self.requests.append((deepcopy(messages), deepcopy(tools)))
|
||||
if len(self.requests) == 1:
|
||||
yield self.call
|
||||
else:
|
||||
yield StreamPiece("content", "ans")
|
||||
|
||||
|
||||
def _settings(**kwargs: Any) -> Settings:
|
||||
kwargs.setdefault("_env_file", None)
|
||||
return Settings(**kwargs) # pyright: ignore[reportCallIssue]
|
||||
|
||||
|
||||
def _run_call(
|
||||
db: Session, name: str, arguments: dict[str, Any]
|
||||
) -> tuple[AgentHolder, ScriptedToolLLM]:
|
||||
"""Drive one scripted tool call through ``run_agent``."""
|
||||
holder = AgentHolder()
|
||||
llm = ScriptedToolLLM(ToolCallPiece(id="call_1", name=name, arguments=arguments))
|
||||
asyncio.run(_consume(cast("LLMClient", llm), db, holder))
|
||||
return holder, llm
|
||||
|
||||
|
||||
async def _consume(
|
||||
llm: LLMClient, db: Session, holder: AgentHolder
|
||||
) -> list[StreamPiece | ToolCallPiece | RetryPiece | ToolResultPiece]:
|
||||
out: list[StreamPiece | ToolCallPiece | RetryPiece | ToolResultPiece] = []
|
||||
async for piece in run_agent(
|
||||
llm,
|
||||
db,
|
||||
system_prompt="SYSTEM_PROMPT",
|
||||
user_message="QUESTION",
|
||||
seed_docs=[],
|
||||
settings=_settings(),
|
||||
holder=holder,
|
||||
):
|
||||
out.append(piece)
|
||||
return out
|
||||
|
||||
|
||||
# --------------------------------------------------------------------
|
||||
# read — the date is the stored row's date, on the SECOND line
|
||||
# --------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_read_result_second_line_is_stored_date(kb, db) -> None:
|
||||
"""A real row (distinct ``created_at``): the ``read`` result's
|
||||
SECOND line is the stored date (the UTC date part), the FIRST line
|
||||
stays the byte-identical header, and the content follows whole."""
|
||||
created = _doc(
|
||||
db, "Alpha", "deep/nested/doc.md", "The Doc", "FULL-TEXT", D2
|
||||
)
|
||||
db.commit()
|
||||
|
||||
holder, llm = _run_call(db, "read", {"path": "Alpha/deep/nested/doc.md"})
|
||||
|
||||
content = llm.requests[1][0][3]["content"]
|
||||
lines = content.splitlines()
|
||||
assert lines[0] == "Document Alpha/deep/nested/doc.md:" # byte-identical header
|
||||
assert lines[1] == f"date: {D2_STR}" # the STORED date (row's UTC date part)
|
||||
assert lines[2:] == ["FULL-TEXT"]
|
||||
assert holder.read_docs == [created]
|
||||
assert holder.tool_calls == 1
|
||||
|
||||
|
||||
def test_read_result_date_is_the_row_date_not_a_constant(kb, db) -> None:
|
||||
"""Two rows with DISTINCT dates: each ``read`` renders its OWN
|
||||
row's date (a late-UTC instant renders its date part, no time)."""
|
||||
a = _doc(db, "Alpha", "a.md", "A", "A-TEXT", D1)
|
||||
b = _doc(db, "Alpha", "b.md", "B", "B-TEXT", D3)
|
||||
db.commit()
|
||||
|
||||
holder_a, llm_a = _run_call(db, "read", {"path": "Alpha/a.md"})
|
||||
assert llm_a.requests[1][0][3]["content"] == (
|
||||
f"Document Alpha/a.md:\ndate: {D1_STR}\nA-TEXT"
|
||||
)
|
||||
holder_b, llm_b = _run_call(db, "read", {"path": "Alpha/b.md"})
|
||||
assert llm_b.requests[1][0][3]["content"] == (
|
||||
f"Document Alpha/b.md:\ndate: {D3_STR}\nB-TEXT"
|
||||
)
|
||||
assert holder_a.read_docs == [a] and holder_b.read_docs == [b]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------
|
||||
# ls — every FILE line carries its date in the appended field
|
||||
# --------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_ls_drill_file_lines_carry_their_dates(kb, src, db) -> None:
|
||||
"""A source drill against real rows: EVERY file line ends with the
|
||||
appended `` | date: YYYY-MM-DD`` field — each row's OWN stored date
|
||||
— while the header and subfolder lines stay date-free."""
|
||||
_doc(db, "Homelab", "backups/cron.md", "Cron", "CRON", D1)
|
||||
_doc(db, "Homelab", "backups/restic.md", "Restic", "RESTIC", D2)
|
||||
_doc(db, "Homelab", "networking/lan.md", "LAN", "LAN", D3)
|
||||
_doc(db, "Homelab", "readme.md", "Readme", "README", D1)
|
||||
db.commit()
|
||||
|
||||
holder, llm = _run_call(db, "ls", {"path": "Homelab"})
|
||||
|
||||
content = llm.requests[1][0][3]["content"]
|
||||
lines = content.splitlines()
|
||||
# The root level: one direct file (readme.md — D1) + the two
|
||||
# subfolder lines (date-free) + the date-free header.
|
||||
assert lines[0] == "Homelab — 1 documents, 2 folders:"
|
||||
assert lines[2] == " backups/ — 2 documents" # subfolder: no date
|
||||
assert lines[3] == " networking/ — 1 documents" # subfolder: no date
|
||||
assert lines[5] == (
|
||||
f"source: Homelab | path: readme.md | title: Readme | date: {D1_STR}"
|
||||
)
|
||||
assert holder.tool_calls == 1
|
||||
|
||||
# Drill into backups: BOTH files list, each with its OWN date.
|
||||
holder2, llm2 = _run_call(db, "ls", {"path": "Homelab/backups"})
|
||||
lines2 = llm2.requests[1][0][3]["content"].splitlines()
|
||||
assert lines2[0] == "Homelab/backups — 2 documents, 0 folders:"
|
||||
assert lines2[2] == (
|
||||
f"source: Homelab | path: backups/cron.md | title: Cron | date: {D1_STR}"
|
||||
)
|
||||
assert lines2[3] == (
|
||||
f"source: Homelab | path: backups/restic.md | title: Restic | date: {D2_STR}"
|
||||
)
|
||||
assert holder2.tool_calls == 1
|
||||
|
||||
|
||||
def test_ls_top_level_source_lines_carry_no_date(kb, db) -> None:
|
||||
"""The top level (source lines) is UNCHANGED in shape — sources are
|
||||
not documents, so no date rides them (only FILE lines do). The
|
||||
registry is FRESH (truncated + the one source re-registered), so
|
||||
the top level is exactly the one source block."""
|
||||
db.execute(text("TRUNCATE git_sources"))
|
||||
db.commit()
|
||||
db.add(GitSource(url="https://github.com/reese/Homelab.git", kind="git"))
|
||||
db.commit()
|
||||
try:
|
||||
_doc(db, "Homelab", "a.md", "A", "A-TEXT", D1)
|
||||
db.commit()
|
||||
|
||||
holder, llm = _run_call(db, "ls", {})
|
||||
|
||||
content = llm.requests[1][0][3]["content"]
|
||||
assert content == "1 sources:\n\nHomelab — 1 documents"
|
||||
assert "date" not in content
|
||||
assert holder.tool_calls == 1
|
||||
finally:
|
||||
db.execute(text("TRUNCATE git_sources"))
|
||||
db.commit()
|
||||
@@ -424,6 +424,7 @@ def test_document_content_admin_contract(client: TestClient, db) -> None:
|
||||
"title",
|
||||
"format",
|
||||
"summary", # nullable field added in phase 36 (null here — markdown)
|
||||
"created_at", # added in phase 106 (task 05)
|
||||
"content",
|
||||
"indexed_at",
|
||||
"chunks",
|
||||
|
||||
@@ -18,6 +18,7 @@ import math
|
||||
import re
|
||||
import uuid
|
||||
from collections.abc import Iterator
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
@@ -40,6 +41,13 @@ from app.rag.retriever import TRUNCATION_MARKER
|
||||
from app.schemas import ChatDoneEvent, SourceRef
|
||||
from tests.conftest import ADMIN_PASSWORD
|
||||
|
||||
#: The fixture documents' fixed creation date (phase 106, D5): the
|
||||
#: ``read`` result's second line is the row's ``created_at`` UTC date
|
||||
#: part — a fixed value keeps the read-result pins deterministic
|
||||
#: (instead of the ``now()`` server default of a bare insert).
|
||||
_FIXTURE_CREATED_AT = datetime(2024, 6, 15, 12, 0, 0, tzinfo=UTC)
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.rag.scaffolding import ScaffoldingFilter
|
||||
|
||||
@@ -663,6 +671,7 @@ pins the agent-loop yield order on the real prompt path."""
|
||||
title="Big Doc",
|
||||
content=content,
|
||||
content_hash="1" * 64,
|
||||
created_at=_FIXTURE_CREATED_AT,
|
||||
)
|
||||
db.add(doc)
|
||||
db.commit()
|
||||
@@ -695,7 +704,11 @@ pins the agent-loop yield order on the real prompt path."""
|
||||
]
|
||||
assert tool_msgs, "the executed read must be appended as a tool message"
|
||||
body = tool_msgs[-1]["content"]
|
||||
assert body.startswith("Document docs/big.md:\n" + content[:cap])
|
||||
# Phase 106, D5: the date rides every read — the SECOND line
|
||||
# (first line byte-identical — the mock's header contract).
|
||||
assert body.startswith(
|
||||
"Document docs/big.md:\ndate: 2024-06-15\n" + content[:cap]
|
||||
)
|
||||
assert TRUNCATION_MARKER in body
|
||||
assert (
|
||||
READ_TRUNCATION_NOTICE.format(shown=cap, total=len(content)) in body
|
||||
@@ -749,6 +762,7 @@ marker in the model's context."""
|
||||
title="Fits Doc",
|
||||
content=content,
|
||||
content_hash="2" * 64,
|
||||
created_at=_FIXTURE_CREATED_AT,
|
||||
)
|
||||
db.add(doc)
|
||||
db.commit()
|
||||
@@ -770,14 +784,18 @@ marker in the model's context."""
|
||||
# No ToolResultPiece, no holder entry.
|
||||
assert not any(isinstance(p, ToolResultPiece) for p in pieces)
|
||||
assert holder.read_truncations == []
|
||||
# The model's context is the whole document, byte-identical to
|
||||
# the pre-phase-95 read result (no marker, no notice). (The fake
|
||||
# aliases the mutated messages list, so take the last tool msg.)
|
||||
# The model's context is the whole document, the pre-phase-95
|
||||
# read result plus the phase-106 D5 date line (no marker, no
|
||||
# notice). (The fake aliases the mutated messages list, so
|
||||
# take the last tool msg.)
|
||||
tool_msgs = [
|
||||
m for r in scripted.seen_messages for m in r if m.get("role") == "tool"
|
||||
]
|
||||
assert tool_msgs, "the executed read must be appended as a tool message"
|
||||
assert tool_msgs[-1]["content"] == "Document docs/fits.md:\n" + content
|
||||
assert (
|
||||
tool_msgs[-1]["content"]
|
||||
== "Document docs/fits.md:\ndate: 2024-06-15\n" + content
|
||||
)
|
||||
assert TRUNCATION_MARKER not in tool_msgs[-1]["content"]
|
||||
# Still a successful read.
|
||||
assert holder.tool_calls == 1
|
||||
@@ -800,6 +818,7 @@ def _insert_big_doc(db, content: str) -> Document:
|
||||
title="Big Read Doc",
|
||||
content=content,
|
||||
content_hash="3" * 64,
|
||||
created_at=_FIXTURE_CREATED_AT,
|
||||
)
|
||||
db.add(doc)
|
||||
db.commit()
|
||||
@@ -890,7 +909,8 @@ def test_truncated_read_streams_tool_result_frame_after_tool_frame(
|
||||
]
|
||||
assert tool_msgs
|
||||
body = tool_msgs[-1]["content"]
|
||||
assert body.startswith(f"Document docs/big-read.md:\n{content[:cap]}")
|
||||
# Phase 106, D5: the date rides every read — the SECOND line.
|
||||
assert body.startswith(f"Document docs/big-read.md:\ndate: 2024-06-15\n{content[:cap]}")
|
||||
assert TRUNCATION_MARKER in body
|
||||
assert READ_TRUNCATION_NOTICE.format(shown=cap, total=len(content)) in body
|
||||
# The truncated read is still a SUCCESSFUL call — cited in done.
|
||||
@@ -956,7 +976,10 @@ def test_untruncated_read_streams_no_tool_result_frame(
|
||||
m for r in scripted.seen_messages for m in r if m.get("role") == "tool"
|
||||
]
|
||||
assert tool_msgs
|
||||
assert tool_msgs[-1]["content"] == "Document docs/big-read.md:\n" + content
|
||||
assert (
|
||||
tool_msgs[-1]["content"]
|
||||
== "Document docs/big-read.md:\ndate: 2024-06-15\n" + content
|
||||
)
|
||||
assert TRUNCATION_MARKER not in tool_msgs[-1]["content"]
|
||||
|
||||
|
||||
|
||||
@@ -148,8 +148,14 @@ def test_docs_response_matches_schema_shape(admin_client, db) -> None:
|
||||
body = r.json()
|
||||
assert set(body) == {"documents"}
|
||||
for d in body["documents"]:
|
||||
assert set(d) == {"id", "source", "path", "title", "chunks", "indexed_at"}
|
||||
# Wire-additive (phase 106, task 05): the pre-date keys are all
|
||||
# still there, joined by ``created_at`` (the document's creation
|
||||
# date — the RAG view's ``Created`` column).
|
||||
assert set(d) == {
|
||||
"id", "source", "path", "title", "chunks", "created_at", "indexed_at"
|
||||
}
|
||||
assert isinstance(d["chunks"], int) and d["chunks"] >= 0
|
||||
datetime.fromisoformat(d["created_at"]) # raises if not ISO-8601
|
||||
|
||||
|
||||
# --------------------------------------------------------------------
|
||||
@@ -198,8 +204,11 @@ def test_docs_tree_populated_shape_order_counts_summaries(admin_client, db) -> N
|
||||
|
||||
homelab, deployments = sources
|
||||
# Wire-additive (phase 98, task 03): the pre-pending keys are all
|
||||
# still there, joined by ``summary_pending``.
|
||||
assert set(homelab) == {"name", "documents", "summary", "summary_pending", "children"}
|
||||
# still there, joined by ``summary_pending`` — and (phase 106,
|
||||
# task 05) by ``updated_at`` (the subtree's max document date, D9).
|
||||
assert set(homelab) == {
|
||||
"name", "documents", "updated_at", "summary", "summary_pending", "children"
|
||||
}
|
||||
assert homelab["documents"] == 4 # the whole recursive count
|
||||
assert homelab["summary"] == "Homelab docs." # the (source, "") row
|
||||
assert homelab["summary_pending"] is False # the stored root row covers it
|
||||
@@ -338,7 +347,10 @@ def test_docs_tree_summary_pending_on_source_and_folder_nodes(admin_client, db)
|
||||
r = admin_client.get("/api/docs/tree")
|
||||
assert r.status_code == 200
|
||||
(homelab,) = r.json()["sources"]
|
||||
assert set(homelab) == {"name", "documents", "summary", "summary_pending", "children"}
|
||||
# Phase 106 (task 05): ``updated_at`` joins the source node keys.
|
||||
assert set(homelab) == {
|
||||
"name", "documents", "updated_at", "summary", "summary_pending", "children"
|
||||
}
|
||||
assert homelab["summary"] == "Homelab docs."
|
||||
assert homelab["summary_pending"] is False
|
||||
# Direct subfolders in path order: k8s < wiki.
|
||||
|
||||
@@ -0,0 +1,455 @@
|
||||
"""Integration: the phase-106 date API surface (task 05, D7/D8/D9).
|
||||
|
||||
``GET /api/docs`` and ``GET /api/documents/content`` serve the
|
||||
document's ``created_at``; ``GET /api/docs/tree`` serves the file
|
||||
``created_at`` verbatim plus the DERIVED subtree-max ``updated_at`` on
|
||||
folder/source nodes (``null`` for a 0-document registered source); and
|
||||
the admin-only ``PATCH /api/documents/date`` matrix — set (ISO date or
|
||||
full ISO datetime, future folds to today, ``created_at_manual``
|
||||
flagged), clear (null/absent → flag drops, stored date stands),
|
||||
malformed 422, unknown-pair 404, anonymous/token-user 403 — the
|
||||
phase-57 split intact (the viewer stays user-gated, the edit is
|
||||
admin-gated).
|
||||
|
||||
Uses the real compose Postgres (``db`` fixture) and FastAPI's
|
||||
TestClient, mirroring ``test_docs_api.py``.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
import uuid
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlalchemy import select, text
|
||||
|
||||
import app.api.docs as docs_api
|
||||
from app.core import tokens as token_service
|
||||
from app.main import app as fastapi_app
|
||||
from app.models import Chunk, Document, GitSource
|
||||
|
||||
_TRUNCATE = "chunks, documents, git_sources"
|
||||
|
||||
#: Deliberately distinct creation stamps (the D9 max fixture) — kept
|
||||
#: separate from ``indexed_at`` so a confused-column pin fails loudly.
|
||||
C0 = datetime(2020, 1, 1, 0, 0, 0, tzinfo=UTC)
|
||||
C1 = datetime(2021, 6, 15, 12, 0, 0, tzinfo=UTC)
|
||||
C2 = datetime(2022, 3, 1, 6, 0, 0, tzinfo=UTC)
|
||||
C3 = datetime(2023, 11, 30, 23, 59, 59, tzinfo=UTC)
|
||||
|
||||
|
||||
def _truncate(db) -> None:
|
||||
db.execute(text(f"TRUNCATE {_TRUNCATE}"))
|
||||
db.commit()
|
||||
|
||||
|
||||
def _seed_doc(
|
||||
db,
|
||||
source: str,
|
||||
path: str,
|
||||
title: str,
|
||||
n_chunks: int,
|
||||
indexed_at: datetime,
|
||||
created_at: datetime,
|
||||
) -> Document:
|
||||
"""One indexed document with an explicit creation date (D8)."""
|
||||
doc = Document(
|
||||
source=source,
|
||||
path=path,
|
||||
full_path=f"/tmp/{source}/{path}",
|
||||
title=title,
|
||||
content=f"# {title}\n\nBody.",
|
||||
content_hash=uuid.uuid4().hex, # unique per row (no real sha needed)
|
||||
indexed_at=indexed_at,
|
||||
created_at=created_at,
|
||||
)
|
||||
db.add(doc)
|
||||
db.flush()
|
||||
if n_chunks:
|
||||
db.add_all(
|
||||
Chunk(document_id=doc.id, position=i, content=f"chunk {i}", embedding=[0.01] * 768)
|
||||
for i in range(n_chunks)
|
||||
)
|
||||
db.commit()
|
||||
return doc
|
||||
|
||||
|
||||
def _tree_node(sources: list[dict], name: str) -> dict:
|
||||
return next(s for s in sources if s["name"] == name)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Reads — ``created_at`` on the two flat surfaces + the tree (D8/D9).
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_docs_list_reports_created_at_per_row(admin_client, db) -> None:
|
||||
"""``GET /api/docs`` gains ``created_at`` per row (ISO-8601,
|
||||
verbatim from the row) — ``indexed_at`` untouched (a different
|
||||
concept: the index time)."""
|
||||
_truncate(db)
|
||||
base = datetime.now(UTC)
|
||||
_seed_doc(db, "Homelab", "a/top.md", "Top", 1, base, C2)
|
||||
_seed_doc(db, "Homelab", "a/b/deep.md", "Deep", 2, base + timedelta(hours=1), C3)
|
||||
try:
|
||||
r = admin_client.get("/api/docs")
|
||||
assert r.status_code == 200
|
||||
body = r.json()
|
||||
# (source, path) order is unchanged by the new column
|
||||
# ("a/b/deep.md" < "a/top.md" — the slash sorts before the 't').
|
||||
assert [d["path"] for d in body["documents"]] == ["a/b/deep.md", "a/top.md"]
|
||||
by_path = {d["path"]: d for d in body["documents"]}
|
||||
assert by_path["a/top.md"]["created_at"] == C2.isoformat()
|
||||
assert by_path["a/b/deep.md"]["created_at"] == C3.isoformat()
|
||||
# ``indexed_at`` still reports the (different) index stamp —
|
||||
# the two concepts never get confused on the wire.
|
||||
assert abs(
|
||||
datetime.fromisoformat(by_path["a/top.md"]["indexed_at"]) - base
|
||||
).total_seconds() < 5
|
||||
assert by_path["a/top.md"]["indexed_at"] != by_path["a/top.md"]["created_at"]
|
||||
finally:
|
||||
_truncate(db)
|
||||
|
||||
|
||||
def test_document_content_carries_created_at(admin_client, db) -> None:
|
||||
"""``GET /api/documents/content`` gains ``created_at`` (the viewer's
|
||||
top meta row renders the ``Created`` badge from it, task 08)."""
|
||||
_truncate(db)
|
||||
base = datetime.now(UTC)
|
||||
_seed_doc(db, "Homelab", "k8s.md", "K8s", 1, base, C1)
|
||||
try:
|
||||
r = admin_client.get(
|
||||
"/api/documents/content", params={"source": "Homelab", "path": "k8s.md"}
|
||||
)
|
||||
assert r.status_code == 200
|
||||
body = r.json()
|
||||
assert "created_at" in body
|
||||
assert body["created_at"] == C1.isoformat()
|
||||
# The two stamps are distinct concepts and both ride the shape.
|
||||
assert body["indexed_at"] != body["created_at"]
|
||||
finally:
|
||||
_truncate(db)
|
||||
|
||||
|
||||
def test_tree_file_dates_and_derived_subtree_max(admin_client, db) -> None:
|
||||
"""``GET /api/docs/tree``: file ``created_at`` verbatim; folder and
|
||||
source ``updated_at`` = the subtree's MAX document ``created_at``
|
||||
(D9 — derived in the pure builder, never stored); a registered
|
||||
0-document source reports ``updated_at: null`` with no children."""
|
||||
_truncate(db)
|
||||
base = datetime.now(UTC)
|
||||
db.add(GitSource(url="https://github.com/reese/Homelab.git", kind="git", added_at=base))
|
||||
db.add(
|
||||
GitSource(
|
||||
url="https://github.com/reese/Empty.git",
|
||||
kind="git",
|
||||
added_at=base + timedelta(hours=1),
|
||||
)
|
||||
)
|
||||
# The D9 max fixture: the deepest file holds the newest date, so a
|
||||
# folder that only LOOKS recent at its own level still reports the
|
||||
# deeper date (deeper beats shallower at every level here).
|
||||
_seed_doc(db, "Homelab", "a/b/c/deep.md", "Deep", 1, base, C3)
|
||||
_seed_doc(db, "Homelab", "a/b/shallow.md", "Shallow", 0, base, C1)
|
||||
_seed_doc(db, "Homelab", "a/top.md", "Top", 1, base, C2)
|
||||
_seed_doc(db, "Homelab", "root.md", "Root", 0, base, C0)
|
||||
try:
|
||||
r = admin_client.get("/api/docs/tree")
|
||||
assert r.status_code == 200
|
||||
sources = r.json()["sources"]
|
||||
# Registry order leads; the 0-document registered source lists.
|
||||
assert [s["name"] for s in sources] == ["Homelab", "Empty"]
|
||||
homelab, empty = sources
|
||||
assert (empty["documents"], empty["children"], empty["updated_at"]) == (0, [], None)
|
||||
|
||||
assert homelab["documents"] == 4
|
||||
assert homelab["updated_at"] == C3.isoformat()
|
||||
a = next(c for c in homelab["children"] if c["kind"] == "folder")
|
||||
root_md = next(c for c in homelab["children"] if c["kind"] == "file")
|
||||
assert a["path"] == "a"
|
||||
assert a["documents"] == 3
|
||||
assert a["updated_at"] == C3.isoformat() # deep C3 beats the direct C2
|
||||
assert root_md["created_at"] == C0.isoformat()
|
||||
a_b = next(c for c in a["children"] if c["kind"] == "folder")
|
||||
top = next(c for c in a["children"] if c["kind"] == "file")
|
||||
assert a_b["path"] == "a/b"
|
||||
assert a_b["documents"] == 2
|
||||
assert a_b["updated_at"] == C3.isoformat() # deep C3 beats the direct C1
|
||||
assert top["created_at"] == C2.isoformat()
|
||||
# a/b's children: the subfolder first, then its ONE direct file.
|
||||
a_b_c, shallow = a_b["children"]
|
||||
assert (a_b_c["kind"], a_b_c["path"], a_b_c["documents"]) == ("folder", "a/b/c", 1)
|
||||
assert a_b_c["updated_at"] == C3.isoformat()
|
||||
assert (shallow["kind"], shallow["path"], shallow["created_at"]) == (
|
||||
"file",
|
||||
"a/b/shallow.md",
|
||||
C1.isoformat(),
|
||||
)
|
||||
(deep,) = a_b_c["children"]
|
||||
assert (deep["kind"], deep["path"], deep["created_at"]) == (
|
||||
"file",
|
||||
"a/b/c/deep.md",
|
||||
C3.isoformat(),
|
||||
)
|
||||
# File nodes carry their own date only — no ``updated_at`` key.
|
||||
for node in (root_md, top, deep, shallow):
|
||||
assert "updated_at" not in node
|
||||
finally:
|
||||
_truncate(db)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# PATCH /api/documents/date (D7) — the admin document-date editor.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_patch_date_set_round_trips_and_flags_manual(admin_client, db) -> None:
|
||||
"""Set a bare ``YYYY-MM-DD``: the parse (midnight UTC) is stored
|
||||
verbatim, ``created_at_manual`` flips to true, the response echoes
|
||||
the stored state, and BOTH read surfaces confirm on re-GET."""
|
||||
_truncate(db)
|
||||
base = datetime.now(UTC)
|
||||
_seed_doc(db, "Homelab", "k8s.md", "K8s", 0, base, C0)
|
||||
try:
|
||||
r = admin_client.patch(
|
||||
"/api/documents/date",
|
||||
json={"source": "Homelab", "path": "k8s.md", "date": "2020-01-02"},
|
||||
)
|
||||
assert r.status_code == 200, r.text
|
||||
assert set(r.json()) == {"source", "path", "created_at", "created_at_manual"}
|
||||
assert r.json() == {
|
||||
"source": "Homelab",
|
||||
"path": "k8s.md",
|
||||
"created_at": "2020-01-02T00:00:00+00:00",
|
||||
"created_at_manual": True,
|
||||
}
|
||||
# The viewer's re-render source (the echo) and both GETs agree.
|
||||
content = admin_client.get(
|
||||
"/api/documents/content", params={"source": "Homelab", "path": "k8s.md"}
|
||||
).json()
|
||||
assert content["created_at"] == "2020-01-02T00:00:00+00:00"
|
||||
list_row = next(
|
||||
d
|
||||
for d in admin_client.get("/api/docs").json()["documents"]
|
||||
if d["path"] == "k8s.md"
|
||||
)
|
||||
assert list_row["created_at"] == "2020-01-02T00:00:00+00:00"
|
||||
# DB state: the flag is set (the sync-time importer then skips
|
||||
# this row — D1/D4).
|
||||
db.expire_all()
|
||||
row = db.scalar(
|
||||
select(Document).where(Document.source == "Homelab", Document.path == "k8s.md")
|
||||
)
|
||||
assert row is not None
|
||||
assert row.created_at == datetime(2020, 1, 2, tzinfo=UTC)
|
||||
assert row.created_at_manual is True
|
||||
finally:
|
||||
_truncate(db)
|
||||
|
||||
|
||||
def test_patch_date_full_iso_datetime_is_converted_to_utc(admin_client, db) -> None:
|
||||
"""A full ISO datetime with a NON-UTC offset is converted to UTC
|
||||
before storing (D3 — the single normalization choke point)."""
|
||||
_truncate(db)
|
||||
base = datetime.now(UTC)
|
||||
_seed_doc(db, "Homelab", "k8s.md", "K8s", 0, base, C0)
|
||||
try:
|
||||
r = admin_client.patch(
|
||||
"/api/documents/date",
|
||||
json={"source": "Homelab", "path": "k8s.md", "date": "2021-06-15T12:30:00+02:00"},
|
||||
)
|
||||
assert r.status_code == 200, r.text
|
||||
assert r.json()["created_at"] == "2021-06-15T10:30:00+00:00"
|
||||
assert r.json()["created_at_manual"] is True
|
||||
finally:
|
||||
_truncate(db)
|
||||
|
||||
|
||||
def test_patch_date_malformed_422_and_leaves_row_untouched(admin_client, db) -> None:
|
||||
"""A malformed non-null value 422s in the HANDLER (the model field
|
||||
is an unconstrained ``str | None`` on purpose, so the detail can
|
||||
name the field) — and the row is left untouched."""
|
||||
_truncate(db)
|
||||
base = datetime.now(UTC)
|
||||
_seed_doc(db, "Homelab", "k8s.md", "K8s", 0, base, C0)
|
||||
try:
|
||||
r = admin_client.patch(
|
||||
"/api/documents/date",
|
||||
json={"source": "Homelab", "path": "k8s.md", "date": "not-a-date"},
|
||||
)
|
||||
assert r.status_code == 422
|
||||
assert r.json() == {
|
||||
"detail": "date must be an ISO date or datetime (e.g. 2024-06-15)"
|
||||
}
|
||||
db.expire_all()
|
||||
row = db.scalar(
|
||||
select(Document).where(Document.source == "Homelab", Document.path == "k8s.md")
|
||||
)
|
||||
assert row is not None
|
||||
assert row.created_at == C0 # untouched
|
||||
assert row.created_at_manual is False # untouched
|
||||
finally:
|
||||
_truncate(db)
|
||||
|
||||
|
||||
def test_patch_date_null_clear_drops_flag_and_keeps_date(admin_client, db) -> None:
|
||||
"""The CLEAR (D7): ``date: null`` drops ``created_at_manual`` ONLY —
|
||||
the stored date stands until the next sync refreshes it (the API is
|
||||
DB-only). An ABSENT ``date`` key is the same operation."""
|
||||
_truncate(db)
|
||||
base = datetime.now(UTC)
|
||||
_seed_doc(db, "Homelab", "k8s.md", "K8s", 0, base, C0)
|
||||
try:
|
||||
# Set first, so there is a correction to clear.
|
||||
r = admin_client.patch(
|
||||
"/api/documents/date",
|
||||
json={"source": "Homelab", "path": "k8s.md", "date": "2020-01-02"},
|
||||
)
|
||||
assert r.status_code == 200 and r.json()["created_at_manual"] is True
|
||||
|
||||
r = admin_client.patch(
|
||||
"/api/documents/date",
|
||||
json={"source": "Homelab", "path": "k8s.md", "date": None},
|
||||
)
|
||||
assert r.status_code == 200, r.text
|
||||
assert r.json() == {
|
||||
"source": "Homelab",
|
||||
"path": "k8s.md",
|
||||
"created_at": "2020-01-02T00:00:00+00:00", # the date STOOD
|
||||
"created_at_manual": False,
|
||||
}
|
||||
|
||||
# Re-set, then clear with the key ABSENT — same operation.
|
||||
r = admin_client.patch(
|
||||
"/api/documents/date",
|
||||
json={"source": "Homelab", "path": "k8s.md", "date": "2020-01-02"},
|
||||
)
|
||||
assert r.status_code == 200 and r.json()["created_at_manual"] is True
|
||||
r = admin_client.patch(
|
||||
"/api/documents/date",
|
||||
json={"source": "Homelab", "path": "k8s.md"},
|
||||
)
|
||||
assert r.status_code == 200, r.text
|
||||
assert r.json()["created_at_manual"] is False
|
||||
assert r.json()["created_at"] == "2020-01-02T00:00:00+00:00"
|
||||
|
||||
db.expire_all()
|
||||
row = db.scalar(
|
||||
select(Document).where(Document.source == "Homelab", Document.path == "k8s.md")
|
||||
)
|
||||
assert row is not None
|
||||
assert row.created_at == datetime(2020, 1, 2, tzinfo=UTC) # still standing
|
||||
assert row.created_at_manual is False
|
||||
finally:
|
||||
_truncate(db)
|
||||
|
||||
|
||||
def test_patch_date_future_folds_to_today(admin_client, db) -> None:
|
||||
"""A manually set FUTURE date also folds to today (D3 — the same
|
||||
normalization choke point as the sourced path), and it is STILL
|
||||
flagged manual (the owner's correction survives syncs until
|
||||
cleared)."""
|
||||
_truncate(db)
|
||||
base = datetime.now(UTC)
|
||||
_seed_doc(db, "Homelab", "k8s.md", "K8s", 0, base, C0)
|
||||
try:
|
||||
r = admin_client.patch(
|
||||
"/api/documents/date",
|
||||
json={"source": "Homelab", "path": "k8s.md", "date": "2999-01-01"},
|
||||
)
|
||||
assert r.status_code == 200, r.text
|
||||
stored = datetime.fromisoformat(r.json()["created_at"])
|
||||
assert abs((stored - datetime.now(UTC)).total_seconds()) < 300 # ≈ today
|
||||
assert r.json()["created_at_manual"] is True
|
||||
db.expire_all()
|
||||
row = db.scalar(
|
||||
select(Document).where(Document.source == "Homelab", Document.path == "k8s.md")
|
||||
)
|
||||
assert row is not None
|
||||
assert row.created_at_manual is True
|
||||
assert row.created_at != datetime(2999, 1, 1, tzinfo=UTC)
|
||||
finally:
|
||||
_truncate(db)
|
||||
|
||||
|
||||
def test_patch_date_404_unknown_pair_and_traversal(admin_client, db) -> None:
|
||||
"""Row-lookup semantics (the ``/documents/content`` rule): unknown
|
||||
pairs — including traversal strings — are simply not rows → 404
|
||||
``document not found``; nothing is written."""
|
||||
_truncate(db)
|
||||
base = datetime.now(UTC)
|
||||
_seed_doc(db, "Homelab", "real.md", "Real", 0, base, C0)
|
||||
try:
|
||||
for source, path in (
|
||||
("Ghost", "x.md"), # unknown source
|
||||
("Homelab", "nope.md"), # known source, unknown path
|
||||
("Homelab", "../../etc/passwd"), # traversal: not a row
|
||||
):
|
||||
r = admin_client.patch(
|
||||
"/api/documents/date",
|
||||
json={"source": source, "path": path, "date": "2020-01-02"},
|
||||
)
|
||||
assert r.status_code == 404, (source, path, r.status_code)
|
||||
assert r.json() == {"detail": "document not found"}, (source, path)
|
||||
db.expire_all()
|
||||
row = db.scalar(
|
||||
select(Document).where(Document.source == "Homelab", Document.path == "real.md")
|
||||
)
|
||||
assert row is not None
|
||||
assert row.created_at_manual is False # nothing written
|
||||
finally:
|
||||
_truncate(db)
|
||||
|
||||
|
||||
def test_patch_date_403_anonymous_and_token_user(client, db) -> None:
|
||||
"""The ``require_admin`` gate (the phase-57 split): an anonymous
|
||||
caller AND a live access-token user who is not the admin both get
|
||||
403 ``admin only`` — while the viewer content itself stays
|
||||
user-gated (a token holder could read the document, just not edit
|
||||
its date)."""
|
||||
_truncate(db)
|
||||
base = datetime.now(UTC)
|
||||
_seed_doc(db, "Homelab", "k8s.md", "K8s", 0, base, C0)
|
||||
db.execute(text("TRUNCATE api_tokens"))
|
||||
db.commit()
|
||||
body = {"source": "Homelab", "path": "k8s.md", "date": "2020-01-02"}
|
||||
try:
|
||||
# Anonymous (the shared ``client`` is unsigned in this module).
|
||||
r = client.patch("/api/documents/date", json=body)
|
||||
assert r.status_code == 403
|
||||
assert r.json() == {"detail": "admin only"}
|
||||
|
||||
# A live access-token user who is not the admin (phase 79).
|
||||
_row, plaintext = token_service.create_token(db, "pin-holder")
|
||||
db.commit()
|
||||
holder = TestClient(fastapi_app)
|
||||
s = holder.post("/api/token-auth", json={"token": plaintext})
|
||||
assert s.status_code == 204, s.text
|
||||
r = holder.patch("/api/documents/date", json=body)
|
||||
assert r.status_code == 403
|
||||
assert r.json() == {"detail": "admin only"}
|
||||
# …and the SPLIT: the same holder CAN read the document (the
|
||||
# viewer stays user-gated, phase 79).
|
||||
got = holder.get(
|
||||
"/api/documents/content", params={"source": "Homelab", "path": "k8s.md"}
|
||||
)
|
||||
assert got.status_code == 200
|
||||
db.expire_all()
|
||||
row = db.scalar(
|
||||
select(Document).where(Document.source == "Homelab", Document.path == "k8s.md")
|
||||
)
|
||||
assert row is not None
|
||||
assert row.created_at_manual is False # the rejected edits wrote nothing
|
||||
finally:
|
||||
db.execute(text("TRUNCATE api_tokens"))
|
||||
db.commit()
|
||||
_truncate(db)
|
||||
|
||||
|
||||
def test_patch_date_handler_is_db_only_and_never_constructs_an_llm_client() -> None:
|
||||
"""Source pin (the house pattern of the phase-97 folder-summary
|
||||
editor): a date is NEVER embedded — no chunk, no retrieval role —
|
||||
so the handler must never touch the LLM client (the deliberate
|
||||
contrast with the phase-57 ``is_summary`` re-embed)."""
|
||||
src = inspect.getsource(docs_api.update_document_date)
|
||||
assert "LLMClient" not in src
|
||||
@@ -79,6 +79,10 @@ def _seed_doc(
|
||||
content=content,
|
||||
content_hash="a" * 64,
|
||||
indexed_at=datetime.now(UTC),
|
||||
# Phase 106: pin the creation date explicitly — the endpoint
|
||||
# serves it verbatim (the column is NOT NULL; the server
|
||||
# default would make the pin time-dependent).
|
||||
created_at=datetime(2020, 5, 4, 8, 30, 0, tzinfo=UTC),
|
||||
summary=summary,
|
||||
)
|
||||
db.add(doc)
|
||||
@@ -417,9 +421,13 @@ def test_content_200_all_fields(client, db) -> None:
|
||||
)
|
||||
assert r.status_code == 200
|
||||
body = r.json()
|
||||
# Wire-additive (phase 106, task 05): ``created_at`` joins the
|
||||
# content shape (after ``summary``, before ``content``).
|
||||
assert set(body) == {
|
||||
"source", "path", "title", "format", "summary", "content", "indexed_at", "chunks"
|
||||
"source", "path", "title", "format", "summary", "created_at",
|
||||
"content", "indexed_at", "chunks",
|
||||
}
|
||||
datetime.fromisoformat(body["created_at"]) # raises if not ISO-8601
|
||||
assert body["source"] == "Homelab"
|
||||
assert body["path"] == "kubernetes.md"
|
||||
assert body["title"] == "Kubernetes Homelab Cluster"
|
||||
|
||||
@@ -0,0 +1,248 @@
|
||||
"""Integration: phase 106 task 03 — ``file_commit_dates`` against real
|
||||
git scratch repos (D2/D10).
|
||||
|
||||
Builds scratch repositories with controlled ``GIT_COMMITTER_DATE``s
|
||||
(the 2026-09-13 verification recipe: file ``a.md`` committed once in
|
||||
2020, file ``b.md`` committed in 2020 and touched again in 2024, a
|
||||
``docs/deep.md`` subdirectory file committed once in 2020) and pins
|
||||
the VERIFIED checkout behavior:
|
||||
|
||||
* a LOCAL-PATH ``clone_or_pull`` keeps FULL history (git's own
|
||||
"--depth is ignored in local clones" warning — the ``--depth 1``
|
||||
flag stays, D10) → TRUE per-file last-commit dates (first-sighting
|
||||
wins: ``a.md`` 2020, ``b.md`` 2024, ``docs/deep.md`` 2020);
|
||||
* a shallow URL-transport clone (``file://``, made directly in this
|
||||
test — the test harness, not ``clone_or_pull``, makes this one) →
|
||||
the TIP commit's date for EVERY working-tree file (the
|
||||
shallow-boundary property, D10: uniform per repo, real across
|
||||
repos);
|
||||
* fail-soft: a directory without ``.git``, an empty repo (no
|
||||
commits), a git failure, and a malformed log output all yield
|
||||
``{}`` — a date walk must never break a sync (the importer, task
|
||||
04, falls back to file mtimes).
|
||||
|
||||
DB-free by design: ``file_commit_dates`` takes a path, no session.
|
||||
Skipped (not failed) on a machine without the git CLI (the
|
||||
``test_doc_drafts_api.py`` guard).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from scripts.git_sync import (
|
||||
GitSyncError,
|
||||
_parse_commit_dates, # pyright: ignore[reportPrivateUsage]
|
||||
clone_or_pull,
|
||||
file_commit_dates,
|
||||
)
|
||||
|
||||
|
||||
def _git_available() -> bool:
|
||||
try:
|
||||
proc = subprocess.run(["git", "--version"], capture_output=True, check=False)
|
||||
return proc.returncode == 0
|
||||
except (FileNotFoundError, OSError):
|
||||
return False
|
||||
|
||||
|
||||
#: Real ``git`` in the test environment — skipped cleanly without it
|
||||
#: (the ``test_doc_drafts_api.py`` house pattern).
|
||||
GIT = _git_available()
|
||||
|
||||
pytestmark = pytest.mark.skipif(not GIT, reason="git CLI not available")
|
||||
|
||||
#: The two controlled commit dates (the 2026-09-13 verification recipe).
|
||||
DATE_A = datetime(2020, 1, 2, 3, 4, 6, tzinfo=UTC) # commit one (2020)
|
||||
DATE_B = datetime(2024, 6, 15, 10, 0, 0, tzinfo=UTC) # commit two = the tip (2024)
|
||||
|
||||
|
||||
def _git(cwd: Path, *argv: str, when: datetime | None = None) -> None:
|
||||
"""Run one git command for the test harness (fixture setup); a
|
||||
non-zero exit fails the fixture, not the test under test."""
|
||||
env = os.environ.copy()
|
||||
if when is not None:
|
||||
iso = when.isoformat()
|
||||
env["GIT_AUTHOR_DATE"] = iso
|
||||
env["GIT_COMMITTER_DATE"] = iso
|
||||
env["GIT_AUTHOR_NAME"] = "T"
|
||||
env["GIT_AUTHOR_EMAIL"] = "t@example.com"
|
||||
env["GIT_COMMITTER_NAME"] = "T"
|
||||
env["GIT_COMMITTER_EMAIL"] = "t@example.com"
|
||||
proc = subprocess.run(
|
||||
["git", *argv], cwd=cwd, env=env, capture_output=True, text=True, check=False
|
||||
)
|
||||
assert proc.returncode == 0, f"git {' '.join(argv)} failed: {proc.stderr}"
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def scratch_repo(tmp_path: Path) -> Path:
|
||||
"""The 2026-09-13 recipe: commit one (2020-01-02) adds ``a.md``,
|
||||
``b.md``, ``docs/deep.md``; commit two (2024-06-15, the tip)
|
||||
touches ONLY ``b.md``."""
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
_git(repo, "init", "-q")
|
||||
_git(repo, "config", "user.email", "t@example.com")
|
||||
_git(repo, "config", "user.name", "T")
|
||||
_git(repo, "config", "commit.gpgsign", "false")
|
||||
(repo / "docs").mkdir()
|
||||
(repo / "a.md").write_text("# A\nstable since 2020\n", encoding="utf-8")
|
||||
(repo / "b.md").write_text("# B\nfirst version\n", encoding="utf-8")
|
||||
(repo / "docs" / "deep.md").write_text("# Deep\nalso 2020\n", encoding="utf-8")
|
||||
_git(repo, "add", "-A", when=DATE_A)
|
||||
_git(repo, "commit", "-qm", "one", when=DATE_A)
|
||||
(repo / "b.md").write_text("# B\nupdated 2024\n", encoding="utf-8")
|
||||
_git(repo, "add", "-A", when=DATE_B)
|
||||
_git(repo, "commit", "-qm", "two", when=DATE_B)
|
||||
return repo
|
||||
|
||||
|
||||
def test_local_clone_yields_true_per_file_dates(scratch_repo: Path, tmp_path: Path) -> None:
|
||||
"""(a) LOCAL-PATH ``clone_or_pull`` → full history (git warns
|
||||
"--depth is ignored in local clones" and does not shallow) → TRUE
|
||||
per-file last-commit dates: the first (newest) sighting of each
|
||||
path wins — ``b.md`` the 2024 touch, the rest the 2020 commit."""
|
||||
dest = tmp_path / "local"
|
||||
clone_or_pull(str(scratch_repo), dest)
|
||||
assert (dest / ".git").exists() # a real checkout
|
||||
assert file_commit_dates(dest) == {
|
||||
"a.md": DATE_A,
|
||||
"b.md": DATE_B, # touched again by the tip commit
|
||||
"docs/deep.md": DATE_A,
|
||||
}
|
||||
|
||||
|
||||
def test_shallow_file_clone_yields_tip_date_for_every_file(
|
||||
scratch_repo: Path, tmp_path: Path
|
||||
) -> None:
|
||||
"""(b) SHALLOW URL-TRANSPORT clone (``file://``, made directly in
|
||||
the test — D10): in a shallow clone git reports the TIP commit as
|
||||
every existing file's last commit (the shallow boundary is each
|
||||
file's history root) → EVERY working-tree file carries the tip
|
||||
date, uniform within the repo."""
|
||||
dest = tmp_path / "shallow"
|
||||
_git(tmp_path, "clone", "-q", "--depth", "1", f"file://{scratch_repo}", str(dest))
|
||||
assert file_commit_dates(dest) == {
|
||||
"a.md": DATE_B,
|
||||
"b.md": DATE_B,
|
||||
"docs/deep.md": DATE_B,
|
||||
}
|
||||
|
||||
|
||||
def test_directory_without_dotgit_fails_soft(tmp_path: Path) -> None:
|
||||
"""(c) a plain directory (no ``.git``) → ``git log`` exits
|
||||
non-zero → ``{}`` (fail-soft, no raise) — the importer falls back
|
||||
to file mtimes."""
|
||||
plain = tmp_path / "notarepo"
|
||||
plain.mkdir()
|
||||
(plain / "a.md").write_text("# A\nnot a git repo\n", encoding="utf-8")
|
||||
assert file_commit_dates(plain) == {}
|
||||
|
||||
|
||||
def test_nonexistent_directory_fails_soft(tmp_path: Path) -> None:
|
||||
"""(c) a missing checkout directory → ``{}`` without even
|
||||
invoking git (no raise)."""
|
||||
assert file_commit_dates(tmp_path / "gone") == {}
|
||||
|
||||
|
||||
def test_empty_repo_fails_soft(tmp_path: Path) -> None:
|
||||
"""(c) an initialized repo with NO commits → ``git log`` fails
|
||||
(nothing to log) → ``{}`` (a cloned-but-empty source must not
|
||||
break the sync)."""
|
||||
empty = tmp_path / "emptyrepo"
|
||||
empty.mkdir()
|
||||
_git(empty, "init", "-q")
|
||||
_git(empty, "config", "commit.gpgsign", "false")
|
||||
assert file_commit_dates(empty) == {}
|
||||
|
||||
|
||||
def test_git_error_fails_soft(
|
||||
scratch_repo: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
"""(c) a ``GitSyncError`` from the walk (git missing/failed) →
|
||||
``{}`` + a logged warning naming the fallback — the fail-soft
|
||||
contract (pinned)."""
|
||||
|
||||
def boom(argv: list[str], cwd: Path) -> str:
|
||||
raise GitSyncError("git log failed (exit 128): fatal: bad object")
|
||||
|
||||
monkeypatch.setattr("scripts.git_sync.run_git", boom)
|
||||
with caplog.at_level("WARNING"):
|
||||
assert file_commit_dates(scratch_repo) == {}
|
||||
assert any("file_commit_dates" in record.message for record in caplog.records)
|
||||
|
||||
|
||||
def test_malformed_log_output_fails_soft(
|
||||
scratch_repo: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""(c) ANY parse anomaly (a commit date ``fromisoformat`` cannot
|
||||
read) → ``{}`` (fail-soft) — the walk is all-or-nothing: a
|
||||
partially parsed date map would be worse than none."""
|
||||
monkeypatch.setattr(
|
||||
"scripts.git_sync.run_git",
|
||||
lambda argv, cwd: "@@not-a-date\nb.md\n",
|
||||
)
|
||||
assert file_commit_dates(scratch_repo) == {}
|
||||
|
||||
|
||||
# --- the pure parser (canned git output — no git, no DB) -------------------
|
||||
|
||||
|
||||
def test_parser_first_sighting_wins_newest_first() -> None:
|
||||
"""The walk is newest-first, so the FIRST sighting of a path is
|
||||
its last-commit date: ``b.md`` appears under both commits and keeps
|
||||
the 2024 (newest) date; the 2020 ``a.md`` keeps 2020. Blank lines
|
||||
(git's commit separators) are skipped."""
|
||||
output = "\n".join(
|
||||
[
|
||||
"@@2024-06-15T10:00:00+00:00",
|
||||
"",
|
||||
"b.md",
|
||||
"@@2020-01-02T03:04:06+00:00",
|
||||
"",
|
||||
"a.md",
|
||||
"b.md",
|
||||
]
|
||||
)
|
||||
assert _parse_commit_dates(output) == {"a.md": DATE_A, "b.md": DATE_B}
|
||||
|
||||
|
||||
def test_parser_normalizes_paths() -> None:
|
||||
"""Path lines are whitespace-split (defensively - git's name-only
|
||||
output is one path per line), backslash-normalized to ``/``, and a
|
||||
leading ``/`` is stripped (repo-relative POSIX keys); the line is
|
||||
stripped first."""
|
||||
output = "\n".join(
|
||||
[
|
||||
"@@2024-06-15T10:00:00+00:00",
|
||||
"",
|
||||
"docs\\deep.md",
|
||||
"/rooted.md",
|
||||
" padded.md ",
|
||||
]
|
||||
)
|
||||
assert _parse_commit_dates(output) == {
|
||||
"docs/deep.md": DATE_B,
|
||||
"rooted.md": DATE_B,
|
||||
"padded.md": DATE_B,
|
||||
}
|
||||
|
||||
|
||||
def test_parser_rejects_path_before_header() -> None:
|
||||
"""A path line before ANY commit header is a malformed walk →
|
||||
``ValueError`` (the caller's fail-soft path turns it into
|
||||
``{}``)."""
|
||||
with pytest.raises(ValueError, match="before any commit header"):
|
||||
_parse_commit_dates("stray.md\n@@2024-06-15T10:00:00+00:00\n")
|
||||
|
||||
|
||||
def test_parser_rejects_bad_date() -> None:
|
||||
"""A commit date ``fromisoformat`` cannot read → ``ValueError``
|
||||
(ISO-strict ``%cI`` always parses — this is the anomaly guard)."""
|
||||
with pytest.raises(ValueError):
|
||||
_parse_commit_dates("@@yesterday\nb.md\n")
|
||||
@@ -28,6 +28,7 @@ from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
@@ -93,11 +94,13 @@ class FakeImportSources:
|
||||
limit: int | None = None,
|
||||
ignore_by_root: dict[str, list[str]] | None = None, # phase 89
|
||||
include_hidden_by_root: dict[str, bool] | None = None, # phase 105
|
||||
doc_dates_by_root: dict[str, dict[str, datetime]] | None = None, # phase 106
|
||||
) -> ImportSummary:
|
||||
self.calls.append(
|
||||
{"sources": list(sources), "prune": prune, "limit": limit,
|
||||
"ignore_by_root": ignore_by_root,
|
||||
"include_hidden_by_root": include_hidden_by_root}
|
||||
"include_hidden_by_root": include_hidden_by_root,
|
||||
"doc_dates_by_root": doc_dates_by_root}
|
||||
)
|
||||
return ImportSummary(files=1, added=1)
|
||||
|
||||
@@ -194,7 +197,7 @@ def test_resolve_sources_git_urls_cloned_into_sources_dir(
|
||||
sources_dir=str(tmp_path / "bor"),
|
||||
)
|
||||
|
||||
sources, ignore_map, hidden_map = import_docs._resolve_sources(None, settings)
|
||||
sources, ignore_map, hidden_map, date_map = import_docs._resolve_sources(None, settings)
|
||||
|
||||
assert sources == [tmp_path / "bor" / "homelab", tmp_path / "bor" / "deploy"]
|
||||
assert ignore_map == {} # phase 89: no row carries a list → empty map
|
||||
@@ -204,6 +207,13 @@ def test_resolve_sources_git_urls_cloned_into_sources_dir(
|
||||
str(tmp_path / "bor" / "homelab"): False,
|
||||
str(tmp_path / "bor" / "deploy"): False,
|
||||
}
|
||||
# Phase 106: git rows are listed with their checkout's date walk —
|
||||
# the fake checkouts are not git repos, so the walk fails soft to
|
||||
# ``{}`` (the importer would then take the mtime fallback).
|
||||
assert date_map == {
|
||||
str(tmp_path / "bor" / "homelab"): {},
|
||||
str(tmp_path / "bor" / "deploy"): {},
|
||||
}
|
||||
assert calls == [
|
||||
("https://host/a/homelab.git", tmp_path / "bor" / "homelab"),
|
||||
("git@host:user/deploy.git", tmp_path / "bor" / "deploy"),
|
||||
@@ -218,11 +228,14 @@ def test_resolve_sources_cli_source_wins(
|
||||
settings = _settings(git_sources="https://host/a/repo.git")
|
||||
manual = tmp_path / "Manual"
|
||||
|
||||
sources, ignore_map, hidden_map = import_docs._resolve_sources([manual], settings)
|
||||
sources, ignore_map, hidden_map, date_map = import_docs._resolve_sources([manual], settings)
|
||||
|
||||
assert sources == [manual]
|
||||
assert ignore_map == {} # phase 89: manual dirs have no rows → no ignore
|
||||
assert hidden_map == {} # phase 105: manual dirs have no rows → hidden skipped
|
||||
# Phase 106: manual dirs have no rows (no clone) → no date map
|
||||
# entries (the importer's mtime fallback applies).
|
||||
assert date_map == {}
|
||||
assert calls == [] # git is never touched when --source is given
|
||||
|
||||
|
||||
@@ -243,12 +256,14 @@ def test_resolve_sources_db_rows_win_over_env(
|
||||
sources_dir=str(tmp_path / "bor"),
|
||||
)
|
||||
|
||||
sources, ignore_map, hidden_map = import_docs._resolve_sources(None, settings)
|
||||
sources, ignore_map, hidden_map, date_map = import_docs._resolve_sources(None, settings)
|
||||
|
||||
assert sources == [tmp_path / "bor" / "only"]
|
||||
assert ignore_map == {} # phase 89: no row carries a list → empty map
|
||||
# Phase 105: the default-flag row contributes its root with False (A4).
|
||||
assert hidden_map == {str(tmp_path / "bor" / "only"): False}
|
||||
# Phase 106: the git row's (fake, non-repo) checkout fails soft → {}.
|
||||
assert date_map == {str(tmp_path / "bor" / "only"): {}}
|
||||
assert calls == [("https://db.example/only.git", tmp_path / "bor" / "only")]
|
||||
|
||||
|
||||
@@ -257,10 +272,13 @@ def test_resolve_sources_defaults_when_nothing_configured(
|
||||
) -> None:
|
||||
# Both origins empty (the resolver's ``([], "env")``) → legacy dirs.
|
||||
monkeypatch.setattr(import_docs, "effective_sources", lambda db: ([], "env"))
|
||||
sources, ignore_map, hidden_map = import_docs._resolve_sources(None, _settings())
|
||||
sources, ignore_map, hidden_map, date_map = import_docs._resolve_sources(None, _settings())
|
||||
assert sources == [p.expanduser() for p in import_docs.DEFAULT_SOURCES]
|
||||
assert ignore_map == {} # phase 89: the legacy fallback has no rows
|
||||
assert hidden_map == {} # phase 105: the legacy fallback has no rows
|
||||
# Phase 106: the legacy fallback has no rows (no clone) → mtime
|
||||
# fallback for every file.
|
||||
assert date_map == {}
|
||||
|
||||
|
||||
def test_resolve_sources_rows_branch_builds_ignore_map(
|
||||
@@ -286,13 +304,16 @@ def test_resolve_sources_rows_branch_builds_ignore_map(
|
||||
)
|
||||
settings = _settings(sources_dir=str(tmp_path / "bor"))
|
||||
|
||||
sources, ignore_map, hidden_map = import_docs._resolve_sources(None, settings)
|
||||
sources, ignore_map, hidden_map, date_map = import_docs._resolve_sources(None, settings)
|
||||
|
||||
assert sources == [tmp_path / "bor" / "only", local_dir]
|
||||
# Keyed by the SAME string the importer sees (the root, not the name).
|
||||
assert ignore_map == {str(local_dir): ["ignore/"]}
|
||||
# Phase 105: both rows are flag-off → per-root False entries (A4).
|
||||
assert hidden_map == {str(tmp_path / "bor" / "only"): False, str(local_dir): False}
|
||||
# Phase 106: ONLY the git row is listed (local rows take the mtime
|
||||
# fallback); the fake checkout's date walk fails soft to ``{}``.
|
||||
assert date_map == {str(tmp_path / "bor" / "only"): {}}
|
||||
|
||||
|
||||
def test_resolve_sources_two_rows_sharing_root_string_extend(
|
||||
@@ -317,7 +338,7 @@ def test_resolve_sources_two_rows_sharing_root_string_extend(
|
||||
)
|
||||
settings = _settings(sources_dir=str(tmp_path / "bor"))
|
||||
|
||||
sources, ignore_map, hidden_map = import_docs._resolve_sources(None, settings)
|
||||
sources, ignore_map, hidden_map, date_map = import_docs._resolve_sources(None, settings)
|
||||
|
||||
shared = str(tmp_path / "bor" / "shared")
|
||||
assert sources == [tmp_path / "bor" / "shared", tmp_path / "bor" / "shared"]
|
||||
@@ -325,6 +346,10 @@ def test_resolve_sources_two_rows_sharing_root_string_extend(
|
||||
# Phase 105 collision: the shared root gets the OR of the flags —
|
||||
# both rows off here, so one False entry for the one root string.
|
||||
assert hidden_map == {shared: False}
|
||||
# Phase 106 collision: both git rows resolve to the SAME root — one
|
||||
# date walk for the one root string (last row's walk wins, both
|
||||
# fail soft to ``{}`` for the fake checkout).
|
||||
assert date_map == {shared: {}}
|
||||
|
||||
|
||||
def test_main_rows_branch_passes_ignore_map_to_import(
|
||||
@@ -362,6 +387,9 @@ def test_main_rows_branch_passes_ignore_map_to_import(
|
||||
# Phase 105: the default-flag row passes the per-root map too — a
|
||||
# False entry, not an absent key (the importer reads it per root).
|
||||
assert call["include_hidden_by_root"] == {str(local_dir): False}
|
||||
# Phase 106: the local-only resolution contributes no date map
|
||||
# (no clone — the importer's mtime fallback applies).
|
||||
assert call["doc_dates_by_root"] == {}
|
||||
assert call["prune"] is False # the CLI's no-prune default is unchanged
|
||||
|
||||
|
||||
@@ -539,6 +567,12 @@ def test_main_git_sources_clone_then_import(
|
||||
tmp_path / "bor" / "homelab",
|
||||
tmp_path / "bor" / "deploy",
|
||||
]
|
||||
# Phase 106: the git rows' (fake, non-repo) checkouts fail soft to
|
||||
# ``{}`` — but the roots ARE listed (the CLI feeds the map).
|
||||
assert fake_import.calls[0]["doc_dates_by_root"] == {
|
||||
str(tmp_path / "bor" / "homelab"): {},
|
||||
str(tmp_path / "bor" / "deploy"): {},
|
||||
}
|
||||
for dest in (tmp_path / "bor" / "homelab", tmp_path / "bor" / "deploy"):
|
||||
assert (dest / "notes.md").is_file()
|
||||
# The final summary print reflects the import (added > 0).
|
||||
@@ -574,6 +608,9 @@ def test_main_cli_source_still_imports_manual_dir(
|
||||
assert rc == 0
|
||||
assert calls == []
|
||||
assert fake_import.calls[0]["sources"] == [manual]
|
||||
# Phase 106: manual --source has no rows (no clone) → no date map
|
||||
# (the importer's mtime fallback applies).
|
||||
assert fake_import.calls[0]["doc_dates_by_root"] == {}
|
||||
assert fake_import.calls[0]["prune"] is False
|
||||
# Phase 53: a manual --source run that changes the KB bumps exactly
|
||||
# once (the CLI is the other canonical sync path).
|
||||
@@ -605,12 +642,15 @@ def test_resolve_sources_mixed_git_and_local(
|
||||
sources_dir=str(tmp_path / "bor"),
|
||||
)
|
||||
|
||||
sources, ignore_map, hidden_map = import_docs._resolve_sources(None, settings)
|
||||
sources, ignore_map, hidden_map, date_map = import_docs._resolve_sources(None, settings)
|
||||
|
||||
assert sources == [tmp_path / "bor" / "only", local_dir]
|
||||
assert ignore_map == {} # phase 89: neither row carries a list
|
||||
# Phase 105: both rows default-flag → per-root False entries (A4).
|
||||
assert hidden_map == {str(tmp_path / "bor" / "only"): False, str(local_dir): False}
|
||||
# Phase 106: only the git row is listed (local takes the mtime
|
||||
# fallback); the fake checkout's date walk fails soft to ``{}``.
|
||||
assert date_map == {str(tmp_path / "bor" / "only"): {}}
|
||||
assert calls == [("https://db.example/only.git", tmp_path / "bor" / "only")]
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
"""Integration: phase 106 (task 04) — the importer's date semantics on
|
||||
real Postgres.
|
||||
|
||||
The backfill-correction case (D4, the owner's "on sync, update the
|
||||
date"): a row first imported with a "today" mtime (the shape every
|
||||
pre-phase-106 deployment has after the migration's
|
||||
``server_default=now()`` backfill) gets its REAL — older — date on the
|
||||
next sync even though the content did not change; the refresh is a
|
||||
date-only ``unchanged``, so the ``sources_meta`` generation the sync
|
||||
paths gate their bump on (``added + updated + pruned > 0``) stays put
|
||||
(seeded before, read after). The D1 manual lock and the prune
|
||||
interaction are pinned across the real DB boundary too.
|
||||
|
||||
Deterministic in-process :class:`~tests.fakes.FakeEmbedder` — no
|
||||
network, no live model (the ``test_importer_e2e.py`` pattern).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from collections.abc import Iterator
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select, text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.models import Document
|
||||
from app.rag.importer import import_sources
|
||||
from app.rag.sources_meta import current_sources_version
|
||||
from tests.fakes import FakeEmbedder
|
||||
|
||||
#: mtime granularity tolerance (os.utime + stat round-trip).
|
||||
_TOL = timedelta(milliseconds=50)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clean_kb(db: Session) -> Iterator[None]:
|
||||
"""Global KB state — truncated around every test (the
|
||||
``test_importer_e2e.py`` shape)."""
|
||||
db.execute(text("TRUNCATE chunks, documents, query_log"))
|
||||
db.commit()
|
||||
yield
|
||||
db.execute(text("TRUNCATE chunks, documents, query_log"))
|
||||
db.commit()
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _pin_sources_version(db: Session) -> Iterator[None]:
|
||||
"""The single-row ``sources_meta`` generation is global mutable
|
||||
state — pin it to a known, non-zero value around every test so the
|
||||
no-bump assertion proves the gate, not the seed."""
|
||||
db.execute(text("UPDATE sources_meta SET version = 7 WHERE id = 1"))
|
||||
db.commit()
|
||||
yield
|
||||
db.execute(text("UPDATE sources_meta SET version = 0 WHERE id = 1"))
|
||||
db.commit()
|
||||
|
||||
|
||||
def _doc(db: Session, source: str, rel: str) -> Document:
|
||||
doc = db.scalar(select(Document).where(Document.source == source, Document.path == rel))
|
||||
assert doc is not None, f"no documents row for ({source!r}, {rel!r})"
|
||||
return doc
|
||||
|
||||
|
||||
def test_unchanged_reimport_refreshes_backfilled_date_without_version_bump(
|
||||
db: Session, tmp_path: Path
|
||||
) -> None:
|
||||
"""The backfill-correction case (D4), end to end on real Postgres.
|
||||
|
||||
Run 1: the file's mtime is "now" (the migration backfill shape) →
|
||||
the row stores a today-date. The file is then ``os.utime``'d back
|
||||
to 2019 with IDENTICAL content. Run 2: the row stores the 2019
|
||||
date — ``added/updated/pruned`` all 0 (it is still ``unchanged``)
|
||||
and ``dates_updated == 1``. Because the content counts did not
|
||||
move, the ``sources_meta`` generation the sync paths bump on a
|
||||
KB change stays exactly where it was seeded (7).
|
||||
"""
|
||||
root = tmp_path / "Backfill"
|
||||
root.mkdir()
|
||||
file = root / "note.md"
|
||||
file.write_text("# Note\n\nthe content never changes\n", encoding="utf-8")
|
||||
assert current_sources_version(db) == 7 # the seeded generation
|
||||
|
||||
llm = FakeEmbedder()
|
||||
first = asyncio.run(import_sources([root], llm, session=db))
|
||||
assert (first.added, first.unchanged, first.dates_updated) == (1, 0, 0)
|
||||
# The backfill shape: the stored date is the "today" mtime.
|
||||
stored = _doc(db, root.name, "note.md").created_at
|
||||
assert abs(stored - datetime.now(UTC)) < timedelta(seconds=60)
|
||||
|
||||
real = datetime(2019, 3, 4, 8, 0, 0, tzinfo=UTC)
|
||||
os.utime(file, (real.timestamp(), real.timestamp())) # content identical
|
||||
|
||||
second = asyncio.run(import_sources([root], llm, session=db))
|
||||
# A date-only refresh: unchanged for every content gate.
|
||||
assert (second.added, second.updated, second.pruned, second.unchanged) == (0, 0, 0, 1)
|
||||
assert second.dates_updated == 1
|
||||
db.expire_all()
|
||||
doc = _doc(db, root.name, "note.md")
|
||||
assert abs(doc.created_at - real) <= _TOL # the real (OLDER) date stored
|
||||
assert doc.created_at_manual is False
|
||||
# The date-only refresh left the generation untouched — the
|
||||
# ``added + updated + pruned > 0`` gate the sync paths use never
|
||||
# fired (D4: no sources_meta bump, no regeneration).
|
||||
assert current_sources_version(db) == 7
|
||||
|
||||
|
||||
def test_manual_date_survives_unchanged_reimport_on_real_db(
|
||||
db: Session, tmp_path: Path
|
||||
) -> None:
|
||||
"""D1 across the DB boundary: the owner's correction
|
||||
(``created_at_manual`` — task 05's API writes it) survives an
|
||||
unchanged re-sync whose source date moved; the generation stays
|
||||
put too (no write happened at all on that row)."""
|
||||
root = tmp_path / "ManualKeep"
|
||||
root.mkdir()
|
||||
file = root / "note.md"
|
||||
file.write_text("# Note\n\ncorrected by the owner\n", encoding="utf-8")
|
||||
llm = FakeEmbedder()
|
||||
first = asyncio.run(import_sources([root], llm, session=db))
|
||||
assert first.added == 1
|
||||
correction = datetime(2024, 11, 30, 15, 45, 0, tzinfo=UTC)
|
||||
doc = _doc(db, root.name, "note.md")
|
||||
doc.created_at = correction
|
||||
doc.created_at_manual = True
|
||||
db.commit()
|
||||
ts = datetime(2018, 1, 1, 0, 0, 0, tzinfo=UTC).timestamp()
|
||||
os.utime(file, (ts, ts))
|
||||
|
||||
second = asyncio.run(import_sources([root], llm, session=db))
|
||||
assert (second.added, second.updated, second.unchanged) == (0, 0, 1)
|
||||
assert second.dates_updated == 0 # the correction was NOT refreshed
|
||||
db.expire_all()
|
||||
doc = _doc(db, root.name, "note.md")
|
||||
assert doc.created_at == correction # byte-identical (no rewrite)
|
||||
assert doc.created_at_manual is True
|
||||
assert current_sources_version(db) == 7
|
||||
|
||||
|
||||
def test_date_only_refresh_coexists_with_prune_on_real_db(
|
||||
db: Session, tmp_path: Path
|
||||
) -> None:
|
||||
"""The matrix in one ``prune=True`` run (the sync button's shape):
|
||||
a manual row survives untouched, a non-manual unchanged row gets
|
||||
its date refreshed (counted in ``dates_updated`` only), and a
|
||||
deleted file is still pruned — the content gates and the date
|
||||
refresh compose without interfering."""
|
||||
root = tmp_path / "Matrix"
|
||||
root.mkdir()
|
||||
kept_manual = root / "manual.md"
|
||||
kept_manual.write_text("# Manual\n\nowner-corrected\n", encoding="utf-8")
|
||||
kept_plain = root / "plain.md"
|
||||
kept_plain.write_text("# Plain\n\nrefreshes\n", encoding="utf-8")
|
||||
gone = root / "gone.md"
|
||||
gone.write_text("# Gone\n\ndeleted upstream\n", encoding="utf-8")
|
||||
llm = FakeEmbedder()
|
||||
first = asyncio.run(import_sources([root], llm, session=db, prune=True))
|
||||
assert first.added == 3
|
||||
|
||||
correction = datetime(2022, 7, 1, 10, 0, 0, tzinfo=UTC)
|
||||
doc = _doc(db, root.name, "manual.md")
|
||||
doc.created_at = correction
|
||||
doc.created_at_manual = True
|
||||
db.commit()
|
||||
moved = datetime(2017, 9, 9, 9, 9, 9, tzinfo=UTC)
|
||||
os.utime(kept_plain, (moved.timestamp(), moved.timestamp()))
|
||||
gone.unlink() # deleted upstream
|
||||
|
||||
second = asyncio.run(import_sources([root], llm, session=db, prune=True))
|
||||
# The refresh counts ONLY in dates_updated; the prune is a content
|
||||
# count (so this run DOES advance the generation — the gate is on
|
||||
# pruned, not on dates_updated).
|
||||
assert (second.added, second.updated, second.unchanged, second.pruned) == (0, 0, 2, 1)
|
||||
assert second.dates_updated == 1
|
||||
db.expire_all()
|
||||
manual = _doc(db, root.name, "manual.md")
|
||||
assert manual.created_at == correction and manual.created_at_manual is True
|
||||
plain = _doc(db, root.name, "plain.md")
|
||||
assert abs(plain.created_at - moved) <= _TOL and plain.created_at_manual is False
|
||||
# The pruned row is gone (the content gate did its job alongside the
|
||||
# date refresh).
|
||||
assert (
|
||||
db.scalar(select(Document).where(Document.source == root.name, Document.path == "gone.md"))
|
||||
is None
|
||||
)
|
||||
# The version the sync paths gate on was seeded, not advanced — this
|
||||
# suite only runs ``import_sources`` (the bump lives in the entry
|
||||
# points, which this run's ``pruned=1`` would trigger).
|
||||
assert current_sources_version(db) == 7
|
||||
@@ -0,0 +1,359 @@
|
||||
"""Integration: migration 0020 (documents.created_at / created_at_manual)
|
||||
schema contract (phase 106, task 01).
|
||||
|
||||
Drives the **real Alembic engine** against the live dev database
|
||||
(``podman compose up -d db``), mirroring the house pattern of
|
||||
``test_migration_0019.py`` (information_schema assertions on the state the
|
||||
migration must leave). The tests target the 0019 → 0020 step explicitly so
|
||||
later migrations cannot break the pins:
|
||||
|
||||
* upgrade 0019 → 0020 → both columns exist with the full contract —
|
||||
``created_at`` TIMESTAMP WITH TIME ZONE NOT NULL, server default
|
||||
``now()``; ``created_at_manual`` BOOLEAN NOT NULL, server default
|
||||
``false`` — while the 0019 ``documents`` schema (incl. ``indexed_at``,
|
||||
``summary``) survives;
|
||||
* a ``documents`` row inserted while the DB is at 0019 backfills
|
||||
``created_at ≈ now()`` (the D1 backfill-to-today, within a few seconds
|
||||
of the upgrade moment) and ``created_at_manual is False``; a row written
|
||||
after the upgrade without the columns takes both server defaults;
|
||||
* the ORM contract agrees: a freshly inserted ``Document`` (nothing passed)
|
||||
reads ``created_at_manual is False`` + non-null ``created_at``, and an
|
||||
explicit ``created_at`` + ``created_at_manual=True`` round-trips through
|
||||
a fresh session;
|
||||
* downgrade to 0019 → both columns GONE (A13) while the row + its content
|
||||
survive; upgrade back to 0020 → both columns back (round-trip).
|
||||
|
||||
The ``alembic`` fixture guarantees the DB ends at head even if a test
|
||||
fails or the process is interrupted.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import uuid
|
||||
from collections.abc import Iterator
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from alembic.config import Config
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from alembic import command
|
||||
from app.db import SessionLocal, db_available
|
||||
from app.models import Document
|
||||
|
||||
SOURCE = "mig0020"
|
||||
CONTENT = "content here"
|
||||
CONTENT_HASH = hashlib.sha256(CONTENT.encode()).hexdigest()
|
||||
# The backfill is evaluated by the ALTER at the upgrade moment; the 5 s
|
||||
# slack each side absorbs test-process scheduling without weakening the
|
||||
# "≈ now()" pin (the DB and the test share the host clock).
|
||||
SLACK = timedelta(seconds=5)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def alembic(db: Session) -> Iterator[Config]:
|
||||
"""Real Alembic config bound to the dev DB (URL from app settings).
|
||||
|
||||
Starts at head (repairs an interrupted earlier run); teardown upgrades
|
||||
to head no matter what happened, so the dev DB is never left below
|
||||
head.
|
||||
"""
|
||||
if not db_available():
|
||||
pytest.skip("Postgres not reachable — run `podman compose up -d db` first")
|
||||
cfg = Config() # no alembic.ini file — env.py gets the URL from app config
|
||||
cfg.set_main_option("script_location", "alembic")
|
||||
command.upgrade(cfg, "head")
|
||||
try:
|
||||
yield cfg
|
||||
finally:
|
||||
# Release the test session's open transaction BEFORE the repair
|
||||
# DDL: an idle-in-transaction SELECT holds an ACCESS SHARE lock
|
||||
# on ``documents``, which would deadlock the repair's
|
||||
# ``ALTER TABLE`` (0020) forever.
|
||||
db.rollback()
|
||||
command.upgrade(cfg, "head")
|
||||
|
||||
|
||||
def _version(db: Session) -> str | None:
|
||||
return db.execute(text("SELECT version_num FROM alembic_version")).scalar()
|
||||
|
||||
|
||||
def _column(db: Session, column: str) -> tuple[Any, ...] | None:
|
||||
"""(data_type, is_nullable, column_default) for one documents column."""
|
||||
row = db.execute(
|
||||
text(
|
||||
"SELECT data_type, is_nullable, column_default"
|
||||
" FROM information_schema.columns"
|
||||
" WHERE table_name = 'documents' AND column_name = :c"
|
||||
),
|
||||
{"c": column},
|
||||
).fetchone()
|
||||
return tuple(row) if row is not None else None
|
||||
|
||||
|
||||
def _row_dates(db: Session, path: str) -> tuple[Any, Any]:
|
||||
"""(created_at, created_at_manual) for one documents row."""
|
||||
row = db.execute(
|
||||
text("SELECT created_at, created_at_manual FROM documents WHERE path = :p"),
|
||||
{"p": path},
|
||||
).fetchone()
|
||||
assert row is not None, f"documents row {path!r} must exist"
|
||||
return row[0], row[1]
|
||||
|
||||
|
||||
def _insert_sql(db: Session, path: str) -> uuid.UUID:
|
||||
"""Insert one documents row (the pre-0020 column shape — the new
|
||||
columns, when present, are omitted so the server defaults apply)."""
|
||||
doc_id = uuid.uuid4()
|
||||
db.execute(
|
||||
text(
|
||||
"INSERT INTO documents (id, source, path, full_path, title, content,"
|
||||
" content_hash) VALUES (:id, :s, :p, :fp, :t, :c, :h)"
|
||||
),
|
||||
{
|
||||
"id": doc_id,
|
||||
"s": SOURCE,
|
||||
"p": path,
|
||||
"fp": f"/tmp/{path}",
|
||||
"t": f"Doc {path}",
|
||||
"c": CONTENT,
|
||||
"h": CONTENT_HASH,
|
||||
},
|
||||
)
|
||||
db.commit()
|
||||
return doc_id
|
||||
|
||||
|
||||
def _delete_by_path(db: Session, path: str) -> None:
|
||||
db.execute(text("DELETE FROM documents WHERE path = :p"), {"p": path})
|
||||
db.commit()
|
||||
|
||||
|
||||
def test_upgrade_to_0020_adds_created_at(db: Session, alembic: Config) -> None:
|
||||
"""Upgrade 0019 → 0020: both columns exist with the full contract
|
||||
(``created_at`` TIMESTAMP WITH TIME ZONE NOT NULL default ``now()``;
|
||||
``created_at_manual`` BOOLEAN NOT NULL default ``false``), are ABSENT
|
||||
at 0019, a pre-0020 row backfills ``created_at ≈ now()`` + the
|
||||
manual flag ``false`` (D1), a new row without the columns takes both
|
||||
server defaults, and an explicit ``true`` round-trips — while the
|
||||
0019 table contract (``indexed_at``, ``summary``) survives."""
|
||||
command.downgrade(alembic, "0019") # start from the pre-0020 state
|
||||
assert _version(db) == "0019"
|
||||
assert _column(db, "created_at") is None, "created_at must be absent at 0019"
|
||||
assert _column(db, "created_at_manual") is None, (
|
||||
"created_at_manual must be absent at 0019"
|
||||
)
|
||||
|
||||
path_pre = "pre-existing.md"
|
||||
_insert_sql(db, path_pre) # no created_at* columns exist at 0019
|
||||
try:
|
||||
window_start = datetime.now(UTC)
|
||||
command.upgrade(alembic, "0020")
|
||||
window_end = datetime.now(UTC)
|
||||
assert _version(db) == "0020", "alembic_version must be at 0020"
|
||||
|
||||
created = _column(db, "created_at")
|
||||
assert created is not None, "documents.created_at is missing"
|
||||
assert created[0] == "timestamp with time zone", (
|
||||
"created_at must be TIMESTAMP WITH TIME ZONE"
|
||||
)
|
||||
assert created[1] == "NO", "created_at must be NOT NULL"
|
||||
assert created[2] is not None and "now()" in str(created[2]), (
|
||||
"created_at must carry the `now()` server default"
|
||||
)
|
||||
|
||||
manual = _column(db, "created_at_manual")
|
||||
assert manual is not None, "documents.created_at_manual is missing"
|
||||
assert manual[0] == "boolean", "created_at_manual must be BOOLEAN"
|
||||
assert manual[1] == "NO", "created_at_manual must be NOT NULL"
|
||||
assert manual[2] is not None and "false" in str(manual[2]), (
|
||||
"created_at_manual must carry the `false` server default"
|
||||
)
|
||||
|
||||
# The pre-0020 row backfilled created_at ≈ now() (D1 — the owner's
|
||||
# "set it to today's date during the migration") + flag false.
|
||||
backfilled, manual_pre = _row_dates(db, path_pre)
|
||||
assert backfilled is not None, "the backfilled created_at must be non-null"
|
||||
assert backfilled.tzinfo is not None, "created_at must be tz-aware"
|
||||
backfilled_utc = backfilled.astimezone(UTC)
|
||||
assert window_start - SLACK <= backfilled_utc <= window_end + SLACK, (
|
||||
f"the backfill must be ≈ the upgrade moment (got {backfilled_utc})"
|
||||
)
|
||||
assert manual_pre is False, (
|
||||
"the backfilled row must read created_at_manual is False"
|
||||
)
|
||||
|
||||
# A row written without the columns takes both server defaults.
|
||||
path_new = "new-row.md"
|
||||
_insert_sql(db, path_new)
|
||||
try:
|
||||
created_new, manual_new = _row_dates(db, path_new)
|
||||
assert created_new is not None
|
||||
created_new_utc = created_new.astimezone(UTC)
|
||||
assert window_end - SLACK <= created_new_utc <= datetime.now(UTC) + SLACK, (
|
||||
f"an omitted created_at takes the `now()` server default"
|
||||
f" (got {created_new_utc})"
|
||||
)
|
||||
assert manual_new is False, (
|
||||
"an omitted flag takes the `false` server default"
|
||||
)
|
||||
|
||||
# The flag round-trips through an explicit ``true``.
|
||||
db.execute(
|
||||
text("UPDATE documents SET created_at_manual = true WHERE path = :p"),
|
||||
{"p": path_new},
|
||||
)
|
||||
db.commit()
|
||||
assert _row_dates(db, path_new)[1] is True, (
|
||||
"created_at_manual = true must round-trip"
|
||||
)
|
||||
finally:
|
||||
_delete_by_path(db, path_new)
|
||||
|
||||
# The 0019 schema survives the additive upgrade.
|
||||
indexed = _column(db, "indexed_at")
|
||||
assert indexed is not None, "documents.indexed_at (0001) must survive the upgrade"
|
||||
assert indexed[0] == "timestamp with time zone" and indexed[1] == "NO", (
|
||||
"documents.indexed_at (0001) must keep its 0019 contract after the upgrade"
|
||||
)
|
||||
summary = _column(db, "summary")
|
||||
assert summary is not None and summary[0] == "text" and summary[1] == "YES", (
|
||||
"documents.summary (0004) must survive the upgrade"
|
||||
)
|
||||
finally:
|
||||
_delete_by_path(db, path_pre)
|
||||
|
||||
|
||||
def test_orm_fresh_row_defaults_and_explicit_round_trips(
|
||||
db: Session, alembic: Config
|
||||
) -> None:
|
||||
"""The ORM contract agrees with the column contract: a freshly
|
||||
inserted ``Document`` (nothing passed for the new columns) reads
|
||||
``created_at_manual is False`` + non-null ``created_at`` (the server
|
||||
default took effect — D1), and an explicit ``created_at`` +
|
||||
``created_at_manual=True`` round-trips through a fresh session."""
|
||||
command.upgrade(alembic, "head")
|
||||
path_default = "orm-default.md"
|
||||
path_explicit = "orm-explicit.md"
|
||||
try:
|
||||
# Fresh row, both new columns omitted → server/Python defaults.
|
||||
row_default = Document(
|
||||
source=SOURCE,
|
||||
path=path_default,
|
||||
full_path=f"/tmp/{path_default}",
|
||||
title="Default",
|
||||
content=CONTENT,
|
||||
content_hash=CONTENT_HASH,
|
||||
)
|
||||
db.add(row_default)
|
||||
db.commit()
|
||||
db.expire_all()
|
||||
reloaded_default = db.get(Document, row_default.id)
|
||||
assert reloaded_default is not None, "the fresh row must be readable"
|
||||
assert reloaded_default.created_at is not None, (
|
||||
"a fresh row must read a non-null created_at (server default)"
|
||||
)
|
||||
assert reloaded_default.created_at_manual is False, (
|
||||
"a fresh row must read created_at_manual is False"
|
||||
)
|
||||
|
||||
# Explicit created_at + created_at_manual=True round-trip through
|
||||
# a FRESH session.
|
||||
explicit = datetime(2020, 6, 15, 12, 30, 45, 123456, tzinfo=UTC)
|
||||
row_explicit = Document(
|
||||
source=SOURCE,
|
||||
path=path_explicit,
|
||||
full_path=f"/tmp/{path_explicit}",
|
||||
title="Explicit",
|
||||
content=CONTENT,
|
||||
content_hash=CONTENT_HASH,
|
||||
created_at=explicit,
|
||||
created_at_manual=True,
|
||||
)
|
||||
db.add(row_explicit)
|
||||
db.commit()
|
||||
with SessionLocal() as fresh:
|
||||
reloaded = fresh.get(Document, row_explicit.id)
|
||||
assert reloaded is not None, "the row must exist in a fresh session"
|
||||
assert reloaded.created_at is not None
|
||||
assert reloaded.created_at.astimezone(UTC) == explicit, (
|
||||
"the explicit created_at must round-trip through the DB"
|
||||
)
|
||||
assert reloaded.created_at_manual is True, (
|
||||
"created_at_manual=True must round-trip through the DB"
|
||||
)
|
||||
finally:
|
||||
_delete_by_path(db, path_default)
|
||||
_delete_by_path(db, path_explicit)
|
||||
|
||||
|
||||
def test_downgrade_to_0019_drops_the_columns(db: Session, alembic: Config) -> None:
|
||||
"""Downgrade 0020 → 0019: both columns are gone (A13 — fully
|
||||
reversible) while the row + its content survive, and the rest of the
|
||||
0019 table contract (``indexed_at``) is intact."""
|
||||
command.upgrade(alembic, "head")
|
||||
path = "survivor.md"
|
||||
_insert_sql(db, path)
|
||||
try:
|
||||
command.downgrade(alembic, "0019")
|
||||
assert _version(db) == "0019"
|
||||
assert _column(db, "created_at") is None, "created_at must be dropped"
|
||||
assert _column(db, "created_at_manual") is None, (
|
||||
"created_at_manual must be dropped"
|
||||
)
|
||||
row = db.execute(
|
||||
text(
|
||||
"SELECT path, title, content, content_hash, indexed_at"
|
||||
" FROM documents WHERE path = :p"
|
||||
),
|
||||
{"p": path},
|
||||
).fetchone()
|
||||
assert row is not None and row[0] == path, (
|
||||
"the row must survive the column drops"
|
||||
)
|
||||
assert row[1] == "Doc survivor.md" and row[2] == CONTENT, (
|
||||
"title + content must survive the column drops"
|
||||
)
|
||||
assert row[3] == CONTENT_HASH, "the content hash must survive the drop"
|
||||
assert row[4] is not None, "indexed_at must survive the column drops"
|
||||
|
||||
indexed = _column(db, "indexed_at")
|
||||
assert indexed is not None and indexed[0] == "timestamp with time zone", (
|
||||
"documents.indexed_at must survive the downgrade"
|
||||
)
|
||||
finally:
|
||||
_delete_by_path(db, path)
|
||||
# Repair: the fixture teardown re-upgrades to head.
|
||||
|
||||
|
||||
def test_upgrade_round_trip_restores_the_columns(db: Session, alembic: Config) -> None:
|
||||
"""Downgrade to 0019, then upgrade back to 0020: both columns are
|
||||
back with the full contract (TIMESTAMPTZ NOT NULL default ``now()``;
|
||||
BOOLEAN NOT NULL default ``false``)."""
|
||||
command.downgrade(alembic, "0019")
|
||||
command.upgrade(alembic, "0020")
|
||||
assert _version(db) == "0020", "round-trip upgrade must land at 0020"
|
||||
|
||||
created = _column(db, "created_at")
|
||||
assert created is not None, "documents.created_at must be back"
|
||||
assert created[0] == "timestamp with time zone", (
|
||||
"created_at must be TIMESTAMP WITH TIME ZONE after the round-trip"
|
||||
)
|
||||
assert created[1] == "NO", "created_at must be NOT NULL after the round-trip"
|
||||
assert created[2] is not None and "now()" in str(created[2]), (
|
||||
"the `now()` server default must survive the round-trip"
|
||||
)
|
||||
|
||||
manual = _column(db, "created_at_manual")
|
||||
assert manual is not None, "documents.created_at_manual must be back"
|
||||
assert manual[0] == "boolean", (
|
||||
"created_at_manual must be BOOLEAN after the round-trip"
|
||||
)
|
||||
assert manual[1] == "NO", (
|
||||
"created_at_manual must be NOT NULL after the round-trip"
|
||||
)
|
||||
assert manual[2] is not None and "false" in str(manual[2]), (
|
||||
"the `false` server default must survive the round-trip"
|
||||
)
|
||||
@@ -0,0 +1,336 @@
|
||||
"""Integration: the D6 recency boost against real Postgres (phase 106,
|
||||
task 07 — the "fine line" battery, the owner's warning pinned
|
||||
permanently).
|
||||
|
||||
The owner's scenario (2026-09-13): "make sure to test with documents
|
||||
that have the correct answer but are older against documents that are
|
||||
similar and newer but don't quit correctly answer the question."
|
||||
Deterministic axis vectors (the ``test_name_hit_lexical.py`` idiom —
|
||||
exact cosines) pin every fused score to a known rank pair, so the
|
||||
margins below are exact floats, not flaky measurements.
|
||||
|
||||
**Measured geometry (recorded per task step 4/5):**
|
||||
|
||||
* Owner scenario — A (``backups/retention.md``, created 2020-01-01,
|
||||
the exact answer, cosine 1.0) lands at vector rank 1 + FTS rank 1
|
||||
(fused 0.03278689); B (``backups/retention-draft.md``, created
|
||||
yesterday, the "under review, no decision yet" draft, cosine
|
||||
0.707107) lands at vector rank 10 + FTS rank 3 (fused 0.03015873 —
|
||||
a solid FTS hit at rank 3, as the task describes). Pre-boost fused
|
||||
margin **A−B = 0.00262816** (asserted ≥ 3× the zero-age boost =
|
||||
0.002100 at the default → ratio 1.25, the "comfortable margin").
|
||||
* Twin near-tie — C (``twin/c-older.md``, 2019) and D
|
||||
(``twin/d-newer.md``, yesterday) with IDENTICAL chunk text and
|
||||
near-identical vectors (cosine 1.0 vs 0.9999 — a literal identical
|
||||
vector ties the vector list's ``ORDER BY distance``, which Postgres
|
||||
resolves arbitrarily, and a permanent pin may not depend on that)
|
||||
sit one adjacent rank step apart in BOTH lists: base gap
|
||||
**C−D = 2/61 − 2/62 = 0.00052882** — a true near-tie on the RRF
|
||||
scale.
|
||||
* The DEFAULT was tuned from the design starting point (0.001) down to
|
||||
**0.0007** (task step 5: "tune the DEFAULTS … until old-correct wins
|
||||
comfortably"): on the k=60 scale the owner scenario's margin is
|
||||
0.00262816 < 3×0.001, and a 0.001 zero-age boost (+0.000997 for a
|
||||
yesterday doc) would have FLIPPED the pinned scenario. 0.0007 keeps
|
||||
the flip margin comfortable (0.000698 > 0.00052882, lead
|
||||
+0.000169) while staying 1.25× under the 3×-boost margin bar. The
|
||||
owner re-tunes live via ``BOR_RECENCY_BOOST``.
|
||||
|
||||
Requires: ``podman compose up -d db``.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import uuid
|
||||
from collections.abc import Iterator
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import Settings, get_settings
|
||||
from app.models import Chunk, Document
|
||||
from app.rag import retriever
|
||||
from app.rag.retriever import (
|
||||
_lexical_candidates,
|
||||
_vector_candidates,
|
||||
fuse,
|
||||
retrieve,
|
||||
select_documents,
|
||||
)
|
||||
|
||||
QUESTION = "How did I configure the backup retention policy?"
|
||||
|
||||
#: 768-dim test vectors (the pgvector column's dimension) — axis unit
|
||||
#: vectors so the cosines are exact (1.0 parallel, 0.7071 half-parallel,
|
||||
#: and constructed unit vectors with exact cosine ``q``).
|
||||
D = 768
|
||||
|
||||
def _vec(axis: int, second: bool = False) -> list[float]:
|
||||
v = [0.0] * D
|
||||
v[axis] = 1.0
|
||||
if second:
|
||||
v[axis + 1] = 1.0
|
||||
return v
|
||||
|
||||
|
||||
def _cos_vec(axis: int, q: float, side: int | None = None) -> list[float]:
|
||||
"""A unit vector with EXACT cosine ``q`` against the axis unit vector."""
|
||||
v = [0.0] * D
|
||||
v[axis] = q
|
||||
v[axis + (side if side is not None else 1)] = math.sqrt(max(0.0, 1.0 - q * q))
|
||||
return v
|
||||
|
||||
|
||||
A_TEXT = (
|
||||
"The backup retention policy: I configured restic on the homelab NAS "
|
||||
"with 35 daily, 12 weekly and 12 monthly backups kept. The backup "
|
||||
"retention policy was configured in /etc/retention.conf and the "
|
||||
"configured schedule is reviewed every quarter."
|
||||
)
|
||||
#: Similar-but-wrong: shares the topic tokens, NO answer (no "configured").
|
||||
B_TEXT = "Draft: the backup retention policy is under review, no decision yet."
|
||||
#: The FTS rank-2 decoy: the topic tokens at a higher ts_rank than B.
|
||||
REVIEW_TEXT = (
|
||||
"backup retention policy review: the backup retention policy needs a "
|
||||
"refresh, backup retention policy discussion notes, backup retention "
|
||||
"policy follow-up planned."
|
||||
)
|
||||
NOTE_TEXT = "backup note {i}: a single word of shared vocabulary."
|
||||
F2_TEXT = "nfs snapshot notes: the policy for nfs shares is to snapshot nightly."
|
||||
|
||||
#: The twins' IDENTICAL chunk body (both match the question's tsquery).
|
||||
TWIN_TEXT = (
|
||||
"Twin document for the recency battery: the backup retention policy is "
|
||||
"configured the same way here."
|
||||
)
|
||||
|
||||
|
||||
def _seed(
|
||||
db: Session,
|
||||
path: str,
|
||||
title: str,
|
||||
content: str,
|
||||
created_at: datetime,
|
||||
embedding: list[float],
|
||||
source: str = "Homelab",
|
||||
) -> None:
|
||||
doc = Document(
|
||||
id=uuid.uuid4(),
|
||||
source=source,
|
||||
path=path,
|
||||
full_path=f"/tmp/{path}",
|
||||
title=title,
|
||||
content=content,
|
||||
content_hash="0" * 64,
|
||||
indexed_at=datetime.now(UTC),
|
||||
created_at=created_at,
|
||||
)
|
||||
db.add(doc)
|
||||
db.flush()
|
||||
chunk = Chunk(
|
||||
id=uuid.uuid4(), document_id=doc.id, position=0, content=content
|
||||
)
|
||||
db.add(chunk)
|
||||
db.flush()
|
||||
chunk.embedding = embedding
|
||||
|
||||
|
||||
#: The question's vector (synthetic — ``retrieve`` takes it as an arg):
|
||||
#: the axis unit vector, so the seeded cosines are exact.
|
||||
QUESTION_VEC = _vec(5)
|
||||
|
||||
|
||||
def _boost_settings(**overrides: Any) -> Settings:
|
||||
"""The live settings with the recency knobs overridden (the house
|
||||
settings-override pattern — ``Settings(_env_file=None, …)``)."""
|
||||
live = get_settings()
|
||||
kwargs: dict[str, Any] = {
|
||||
"recency_boost": live.recency_boost,
|
||||
"recency_half_life_days": live.recency_half_life_days,
|
||||
}
|
||||
kwargs.update(overrides)
|
||||
return Settings(_env_file=None, **kwargs) # pyright: ignore[reportCallIssue]
|
||||
|
||||
|
||||
def _boost_off(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Patch the retriever's settings to the kill switch (``0`` = off)."""
|
||||
monkeypatch.setattr(
|
||||
retriever, "get_settings", lambda: _boost_settings(recency_boost=0.0)
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def owner_kb(db) -> Iterator[None]:
|
||||
"""THE owner scenario: the older doc that ANSWERS (A, 2020) vs the
|
||||
newer doc that merely resembles the topic (B, yesterday) — plus the
|
||||
KB of similar-but-not-answering backup docs that push B to vector
|
||||
rank 10 while keeping it a solid FTS hit at rank 3 (the task's
|
||||
described shape)."""
|
||||
db.execute(text("TRUNCATE chunks, documents"))
|
||||
db.commit()
|
||||
_seed(
|
||||
db, "backups/retention.md", "backup retention", A_TEXT,
|
||||
datetime(2020, 1, 1, tzinfo=UTC), _vec(5),
|
||||
)
|
||||
for i, q in enumerate((0.99, 0.98, 0.97, 0.96, 0.95, 0.94, 0.93, 0.92)):
|
||||
_seed(
|
||||
db, f"backups/notes/n{i:02d}.md", f"backup note {i}",
|
||||
NOTE_TEXT.format(i=i), datetime(2021 + i % 3, 1, 1 + i, tzinfo=UTC),
|
||||
_cos_vec(5, q),
|
||||
)
|
||||
_seed(
|
||||
db, "backups/retention-review.md", "retention review", REVIEW_TEXT,
|
||||
datetime(2021, 3, 5, tzinfo=UTC), _cos_vec(5, 0.5),
|
||||
)
|
||||
_seed(
|
||||
db, "backups/retention-draft.md", "retention draft", B_TEXT,
|
||||
datetime.now(UTC) - timedelta(days=1), _vec(5, second=True),
|
||||
)
|
||||
_seed(
|
||||
db, "backups/nfs-snapshots.md", "nfs snapshots", F2_TEXT,
|
||||
datetime(2022, 6, 10, tzinfo=UTC), _cos_vec(5, 0.4),
|
||||
)
|
||||
db.commit()
|
||||
yield
|
||||
db.execute(text("TRUNCATE chunks, documents"))
|
||||
db.commit()
|
||||
|
||||
|
||||
def _seed_twins(db: Session, d_created_at: datetime) -> None:
|
||||
db.execute(text("TRUNCATE chunks, documents"))
|
||||
db.commit()
|
||||
_seed(
|
||||
db, "twin/c-older.md", "twin c", TWIN_TEXT,
|
||||
datetime(2019, 6, 1, tzinfo=UTC), _vec(5), source="twin",
|
||||
)
|
||||
_seed(
|
||||
db, "twin/d-newer.md", "twin d", TWIN_TEXT, d_created_at,
|
||||
_cos_vec(5, 0.9999), source="twin",
|
||||
)
|
||||
db.commit()
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def twins(db) -> Iterator[None]:
|
||||
"""The near-tie pair: IDENTICAL text, near-identical vectors, C
|
||||
(2019) older and base-ranked first, D (yesterday) newer."""
|
||||
_seed_twins(db, datetime.now(UTC) - timedelta(days=1))
|
||||
yield
|
||||
db.execute(text("TRUNCATE chunks, documents"))
|
||||
db.commit()
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def twins_aged(db) -> Iterator[None]:
|
||||
"""The same pair with D aged to ``half_life + 365`` days (730 at the
|
||||
default — two half-lives, the boost decayed to ``e**-2`` ≈ 0.135 of
|
||||
the full weight)."""
|
||||
half_life = get_settings().recency_half_life_days
|
||||
_seed_twins(db, datetime.now(UTC) - timedelta(days=half_life + 365))
|
||||
yield
|
||||
db.execute(text("TRUNCATE chunks, documents"))
|
||||
db.commit()
|
||||
|
||||
|
||||
def test_owner_scenario_old_correct_beats_new_similar(
|
||||
owner_kb, db, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""THE owner scenario, pinned at the DEFAULTS: the older doc that
|
||||
answers ranks above the newer similar one — AND the pre-boost fused
|
||||
margin is ≥ 3× the zero-age boost (the "comfortable margin"; the
|
||||
measured 0.00262816 vs the 0.0021 bar is recorded in the module
|
||||
docstring). Re-pinned with the boost OFF: relevance alone already
|
||||
ordered them (no regression — the boost is not what makes A win)."""
|
||||
chunks = retrieve(db, QUESTION, QUESTION_VEC)
|
||||
assert select_documents(chunks, n=2)[0].path == "backups/retention.md"
|
||||
|
||||
# The pre-boost fused scores, computed via ``fuse`` directly.
|
||||
s = get_settings()
|
||||
vector = _vector_candidates(db, QUESTION_VEC, s.hybrid_vector_candidates)
|
||||
lexical = _lexical_candidates(db, QUESTION, s.hybrid_lexical_candidates)
|
||||
fused = fuse(vector, lexical, s.rrf_k)
|
||||
by_path = {rc.document.path: rc.score for rc in fused}
|
||||
margin = by_path["backups/retention.md"] - by_path["backups/retention-draft.md"]
|
||||
assert margin >= 3 * s.recency_boost
|
||||
|
||||
# The kill switch: A still first (relevance alone), and the
|
||||
# weight-0 scores are the pre-phase fused scores byte-identical.
|
||||
_boost_off(monkeypatch)
|
||||
chunks_off = retrieve(db, QUESTION, QUESTION_VEC)
|
||||
assert select_documents(chunks_off, n=2)[0].path == "backups/retention.md"
|
||||
assert {rc.chunk_id: rc.score for rc in chunks_off} == {
|
||||
rc.chunk_id: rc.score for rc in fused
|
||||
}
|
||||
|
||||
|
||||
def test_near_tie_flips_toward_the_newer_with_the_boost(
|
||||
twins, db, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""The boost is REAL: a near-tie (one RRF rank step apart in both
|
||||
lists, base gap 0.00052882 favoring the OLDER document) flips
|
||||
toward the NEWER one with the boost on (D's yesterday boost
|
||||
0.000698 > the gap), and the pre-phase order (C first) stands with
|
||||
the boost off — proving the boost, not drift, is the
|
||||
differentiator."""
|
||||
chunks = retrieve(db, QUESTION, QUESTION_VEC)
|
||||
assert [d.path for d in select_documents(chunks, n=2)] == [
|
||||
"twin/d-newer.md",
|
||||
"twin/c-older.md",
|
||||
]
|
||||
_boost_off(monkeypatch)
|
||||
chunks_off = retrieve(db, QUESTION, QUESTION_VEC)
|
||||
assert [d.path for d in select_documents(chunks_off, n=2)] == [
|
||||
"twin/c-older.md",
|
||||
"twin/d-newer.md",
|
||||
]
|
||||
|
||||
|
||||
def test_the_boost_fades_with_age_end_to_end(twins_aged, db) -> None:
|
||||
"""Decay end to end: the same pair with D aged to two half-lives
|
||||
(730 days) — D's boost decays to ``weight·e**-2`` ≈ 0.135×weight
|
||||
(the task's e^-3 figure assumed three half-lives; 730/365 = 2),
|
||||
which is BELOW the base gap — C (older) is first again. Recency is
|
||||
an age signal, not a binary: the faded boost still shows in D's
|
||||
effective score (pinned to the analytic decay), it just no longer
|
||||
overcomes a real (near-)tie."""
|
||||
chunks = retrieve(db, QUESTION, QUESTION_VEC)
|
||||
assert [d.path for d in select_documents(chunks, n=2)] == [
|
||||
"twin/c-older.md",
|
||||
"twin/d-newer.md",
|
||||
]
|
||||
# Magnitude pin: D's observed boost == the analytic decayed weight
|
||||
# (the fixture ages D by exactly half_life + 365 days; the
|
||||
# retrieve()-time drift is microseconds, far inside the tolerance).
|
||||
s = get_settings()
|
||||
vector = _vector_candidates(db, QUESTION_VEC, s.hybrid_vector_candidates)
|
||||
lexical = _lexical_candidates(db, QUESTION, s.hybrid_lexical_candidates)
|
||||
fused = {rc.document.path: rc.score for rc in fuse(vector, lexical, s.rrf_k)}
|
||||
boosted = {rc.document.path: rc.score for rc in chunks}
|
||||
age_days = s.recency_half_life_days + 365
|
||||
observed = boosted["twin/d-newer.md"] - fused["twin/d-newer.md"]
|
||||
assert observed == pytest.approx(
|
||||
s.recency_boost * math.exp(-age_days / s.recency_half_life_days),
|
||||
rel=1e-3,
|
||||
)
|
||||
# And the faded boost is far below the full weight (e^-2 ≈ 0.135).
|
||||
assert observed < 0.2 * s.recency_boost
|
||||
|
||||
|
||||
def test_the_a8_cosine_gate_input_is_untouched_by_the_boost(
|
||||
owner_kb, db, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""The boost is score-side ONLY: every chunk's ``cosine`` — the A8
|
||||
honesty-gate input and the ``query_log.top_score`` source — is
|
||||
byte-identical with the boost on vs off (asserted per chunk)."""
|
||||
chunks_on = retrieve(db, QUESTION, QUESTION_VEC)
|
||||
cosines_on = {rc.chunk_id: rc.cosine for rc in chunks_on}
|
||||
_boost_off(monkeypatch)
|
||||
chunks_off = retrieve(db, QUESTION, QUESTION_VEC)
|
||||
cosines_off = {rc.chunk_id: rc.cosine for rc in chunks_off}
|
||||
assert cosines_on == cosines_off
|
||||
# The gate input for this question: the answering document's exact
|
||||
# axis (cosine 1.0) — unchanged by the re-rank.
|
||||
assert max(cosines_on.values()) == 1.0
|
||||
@@ -268,6 +268,11 @@ class FakeImportSources:
|
||||
# keying; a shared-root collision ORs — if either row says
|
||||
# "index hidden", the root does).
|
||||
self.include_hidden_maps: list[dict[str, bool]] = []
|
||||
# Phase 106: the per-root source-date map the runner builds
|
||||
# from ``file_commit_dates`` after each git clone (git rows
|
||||
# only, same root-string keying; local rows contribute
|
||||
# nothing).
|
||||
self.doc_dates_maps: list[dict[str, dict[str, datetime]]] = []
|
||||
|
||||
async def __call__(
|
||||
self,
|
||||
@@ -280,6 +285,7 @@ class FakeImportSources:
|
||||
progress: Callable[[str, str, int, int], None] | None = None,
|
||||
ignore_by_root: dict[str, list[str]] | None = None,
|
||||
include_hidden_by_root: dict[str, bool] | None = None,
|
||||
doc_dates_by_root: dict[str, dict[str, datetime]] | None = None, # phase 106
|
||||
) -> ImportSummary:
|
||||
self.sources.append(list(sources))
|
||||
self.llms.append(llm)
|
||||
@@ -287,6 +293,7 @@ class FakeImportSources:
|
||||
self.progress_hooks.append(progress)
|
||||
self.ignore_maps.append(ignore_by_root or {})
|
||||
self.include_hidden_maps.append(include_hidden_by_root or {})
|
||||
self.doc_dates_maps.append(doc_dates_by_root or {})
|
||||
if self.delay:
|
||||
await asyncio.sleep(self.delay)
|
||||
return self.summary
|
||||
@@ -453,9 +460,14 @@ def test_admin_sync_success_reports_full_detail(
|
||||
assert body["detail"] == {
|
||||
"files": 5, "added": 1, "updated": 2, "unchanged": 2, "pruned": 3,
|
||||
"errors": 0, "chunks": 11, "summaries": 1, "summary_errors": 0,
|
||||
"dates_updated": 0, # phase 106: additive key, after summary_errors
|
||||
"overview": True,
|
||||
"sources_version": 1, # phase 53: changed KB → exactly one bump (0 → 1)
|
||||
}
|
||||
# Phase 106: the runner feeds the per-root date map — the fake
|
||||
# checkout is not a git repo, so ``file_commit_dates`` fails soft
|
||||
# to ``{}`` for the one git row (root-keyed).
|
||||
assert fake_import.doc_dates_maps == [{str(tmp_path / "bor" / "repo"): {}}]
|
||||
# The bump committed: the counter advanced exactly once, not twice.
|
||||
assert current_sources_version(db) == 1
|
||||
# Git: the configured repo was cloned into BOR_SOURCES_DIR/<repo-name>/.
|
||||
@@ -607,6 +619,7 @@ class _GatedImport:
|
||||
progress: Callable[[str, str, int, int], None] | None = None,
|
||||
ignore_by_root: dict[str, list[str]] | None = None,
|
||||
include_hidden_by_root: dict[str, bool] | None = None,
|
||||
doc_dates_by_root: dict[str, dict[str, datetime]] | None = None, # phase 106
|
||||
) -> ImportSummary:
|
||||
if progress is not None:
|
||||
progress("repo", "notes/deep.md", 1, 3)
|
||||
@@ -1148,6 +1161,7 @@ def test_import_error_is_reported_with_credentials_masked(
|
||||
progress: Callable[[str, str, int, int], None] | None = None,
|
||||
ignore_by_root: dict[str, list[str]] | None = None,
|
||||
include_hidden_by_root: dict[str, bool] | None = None,
|
||||
doc_dates_by_root: dict[str, dict[str, datetime]] | None = None, # phase 106
|
||||
) -> ImportSummary:
|
||||
raise EmbeddingError(
|
||||
"embeddings request to https://user:secret@aipi.reeseapps.com/v1 "
|
||||
|
||||
@@ -722,7 +722,7 @@ def test_api_changed_sync_generates_folder_rows(
|
||||
# No new sync-status surface: the detail keeps its exact key set.
|
||||
assert set(body["detail"]) == {
|
||||
"files", "added", "updated", "unchanged", "pruned", "errors",
|
||||
"chunks", "summaries", "summary_errors", "overview",
|
||||
"chunks", "summaries", "summary_errors", "dates_updated", "overview",
|
||||
"sources_version",
|
||||
}
|
||||
assert body["detail"]["files"] == 2
|
||||
@@ -862,7 +862,7 @@ def test_api_unchanged_resync_with_gap_fills_only_the_missing_row(
|
||||
# No new sync-status surface: the detail keeps its exact key set.
|
||||
assert set(body["detail"]) == {
|
||||
"files", "added", "updated", "unchanged", "pruned", "errors",
|
||||
"chunks", "summaries", "summary_errors", "overview",
|
||||
"chunks", "summaries", "summary_errors", "dates_updated", "overview",
|
||||
"sources_version",
|
||||
}
|
||||
# Phase 98 (task 01): the progress hook fired on the UNCHANGED-KB
|
||||
|
||||
Reference in New Issue
Block a user