phase: 106_document_dates
Everything is verified green. Final report: **Phase 106 — Document dates (final verification pass; all 10 tasks already complete)** - Verified all phase artifacts: alembic `0020` (dev DB at `0020`), `doc_dates.py`, git `file_commit_dates`, importer `doc_dates_by_root`/`dates_updated`, both entry-point wirings, date APIs + tree `created_at`/`updated_at`, LLM surfaces (prompt block, `read` line 2, appended `ls` field), `apply_recency_boost` in `retrieve()`, UI columns/badge, admin editor, mock-LLM regex — all present and correct; no defects found, no fixes needed. - `uv run pytest --cov=app --cov-report=term-missing` → **2299 passed, TOTAL 99%** (>90% ✓) - `uv run pytest tests/e2e/test_document_dates.py -v --no-cov` → **6/6 passed** in isolation (DB up) - 12 regression E2E suites (retrieval_quality, whole_document_context, agent_document_tools, ls_tree_drilldown, read_truncation_cap, kb_tree, kb_tree_nav, document_viewer, edit_summaries, import_documents, sync_button, hidden_folders_toggle, smoke) → **all green in isolation** - `uv run ruff check .` → clean; `uv run pyright` → **0 errors, 0 warnings** **Completion criteria:** 1) non-null `created_at` + 0020 upgrade/downgrade on dev DB ✓ (real-Alembic integration tests) 2) sync refresh/older/manual-persists/content-reset/no sources_meta bump ✓ 3) zip/tar mtime + future→today ✓ 4) LLM date surfaces + cross-check ✓ 5) UI Created/Updated/badge positions ✓ 6) admin editor set+revert round-trip ✓ 7) old-correct-beats-new-similar (defaults & boost-off) + near-tie + `BOR_RECENCY_BOOST=0` byte-identical ✓ 8) full gate ✓ 9) commit/phase-move — left to harness per instructions. - **Notable:** recency default tuned 0.001 → **0.0007** (task 07 step 5 explicitly permits; measured margins recorded in `test_recency_boost.py` docstring). - **Next pending phase:** none — `todo/` holds only this phase.
This commit is contained in:
@@ -0,0 +1,235 @@
|
||||
"""Unit: phase 106 (task 04) — the importer's document-date semantics.
|
||||
|
||||
The D2/D4 matrix against the fake LLM + the house ``db`` session
|
||||
(real compose Postgres, the ``tests/unit/test_importer.py`` pattern
|
||||
for importer tests): added files store their source date (the
|
||||
``doc_dates_by_root`` map entry when present, else the file mtime —
|
||||
both normalized by :func:`app.rag.doc_dates.normalize_doc_date`,
|
||||
D3); the unchanged path REFRESHES the stored date from the same
|
||||
source and counts it in ``dates_updated`` (content counts preserved);
|
||||
manual rows (``created_at_manual``) are skipped (the D1 lock); a
|
||||
content change re-sources the date AND clears the manual flag; a
|
||||
future (beyond-skew) mtime folds to today through the importer.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.models import Document
|
||||
from app.rag.importer import import_sources
|
||||
from tests.fakes import FakeEmbedder
|
||||
|
||||
#: mtime granularity tolerance — ``os.utime`` + ``stat`` round-trip on
|
||||
#: the test filesystem (the archive-date suite uses ±1 s; 50 ms is far
|
||||
#: tighter and still filesystem-agnostic).
|
||||
_TOL = timedelta(milliseconds=50)
|
||||
|
||||
OLD_2020 = datetime(2020, 1, 2, 3, 4, 5, 123456, tzinfo=UTC)
|
||||
OLD_2021 = datetime(2021, 6, 1, 12, 0, 0, tzinfo=UTC)
|
||||
|
||||
|
||||
def _utime(path: Path, when: datetime) -> None:
|
||||
ts = when.timestamp()
|
||||
os.utime(path, (ts, ts))
|
||||
|
||||
|
||||
def _doc(db, source: str, rel: str) -> Document:
|
||||
doc = db.scalar(select(Document).where(Document.source == source, Document.path == rel))
|
||||
assert doc is not None, f"no documents row for ({source!r}, {rel!r})"
|
||||
return doc
|
||||
|
||||
|
||||
def _cleanup_source(db, source: str) -> None:
|
||||
for doc in db.scalars(select(Document).where(Document.source == source)).all():
|
||||
db.delete(doc)
|
||||
db.commit()
|
||||
|
||||
|
||||
def _make_root(tmp_path: Path, name: str) -> tuple[Path, Path]:
|
||||
root = tmp_path / name
|
||||
root.mkdir()
|
||||
file = root / "note.md"
|
||||
file.write_text("# Note\n\ncontent for the KB\n", encoding="utf-8")
|
||||
return root, file
|
||||
|
||||
|
||||
# --- added: the source date lands on the new row -----------------------------
|
||||
|
||||
|
||||
def test_added_file_stores_its_mtime_as_created_at(db, tmp_path: Path) -> None:
|
||||
"""D2 fallback: an unmapped file's mtime IS its source date — a
|
||||
file ``os.utime``'d to 2020-01-02 imports with that ``created_at``
|
||||
(the added branch, D3 normalized), and ``created_at_manual`` stays
|
||||
the column default (False)."""
|
||||
root, file = _make_root(tmp_path, "DateAdded")
|
||||
_utime(file, OLD_2020)
|
||||
llm = FakeEmbedder()
|
||||
try:
|
||||
summary = asyncio.run(import_sources([root], llm, session=db))
|
||||
assert (summary.added, summary.unchanged, summary.dates_updated) == (1, 0, 0)
|
||||
doc = _doc(db, root.name, "note.md")
|
||||
assert abs(doc.created_at - OLD_2020) <= _TOL
|
||||
assert doc.created_at_manual is False
|
||||
finally:
|
||||
_cleanup_source(db, root.name)
|
||||
|
||||
|
||||
def test_doc_dates_map_entry_beats_mtime(db, tmp_path: Path) -> None:
|
||||
"""D2 git case: a ``doc_dates_by_root`` entry (keyed by ``str(root)``
|
||||
— the root string exactly as passed in ``sources``) names the file's
|
||||
raw date and BEATS the file's mtime (the map says 2020, the mtime
|
||||
says now → 2020 stored). A path missing from its root's map takes
|
||||
the mtime fallback (≈ now) in the same run."""
|
||||
root = tmp_path / "DateMap"
|
||||
root.mkdir()
|
||||
mapped = root / "git_file.md"
|
||||
mapped.write_text("# Git\n\nfrom the repo\n", encoding="utf-8")
|
||||
unmapped = root / "local_file.md"
|
||||
unmapped.write_text("# Local\n\nnot in the map\n", encoding="utf-8")
|
||||
now_before = datetime.now(UTC)
|
||||
llm = FakeEmbedder()
|
||||
try:
|
||||
summary = asyncio.run(
|
||||
import_sources(
|
||||
[root], llm, session=db,
|
||||
doc_dates_by_root={str(root): {"git_file.md": OLD_2020}},
|
||||
)
|
||||
)
|
||||
assert summary.added == 2
|
||||
git_doc = _doc(db, root.name, "git_file.md")
|
||||
assert abs(git_doc.created_at - OLD_2020) <= _TOL # the map, not the mtime
|
||||
local_doc = _doc(db, root.name, "local_file.md")
|
||||
# The unmapped file fell back to its mtime (written just now).
|
||||
assert now_before - _TOL <= local_doc.created_at <= datetime.now(UTC) + _TOL
|
||||
# Unchanged re-import with the SAME map: both dates already
|
||||
# stored → no refresh (the map hit is stable, not a rewrite).
|
||||
s2 = asyncio.run(
|
||||
import_sources(
|
||||
[root], llm, session=db,
|
||||
doc_dates_by_root={str(root): {"git_file.md": OLD_2020}},
|
||||
)
|
||||
)
|
||||
assert (s2.added, s2.updated, s2.unchanged) == (0, 0, 2)
|
||||
assert s2.dates_updated == 0
|
||||
finally:
|
||||
_cleanup_source(db, root.name)
|
||||
|
||||
|
||||
def test_future_mtime_folds_to_today_through_importer(db, tmp_path: Path) -> None:
|
||||
"""D3 through the importer: a mtime YEARS in the future (beyond the
|
||||
1-day clock-skew tolerance) folds to the import moment (today), not
|
||||
the raw future value."""
|
||||
root, file = _make_root(tmp_path, "DateFuture")
|
||||
_utime(file, datetime(2030, 1, 1, 0, 0, 0, tzinfo=UTC))
|
||||
llm = FakeEmbedder()
|
||||
try:
|
||||
before = datetime.now(UTC)
|
||||
summary = asyncio.run(import_sources([root], llm, session=db))
|
||||
after = datetime.now(UTC)
|
||||
assert summary.added == 1
|
||||
doc = _doc(db, root.name, "note.md")
|
||||
# The folded date is the normalization moment — between the run's
|
||||
# bounds (a hair of slack on each side).
|
||||
assert before - _TOL <= doc.created_at <= after + _TOL
|
||||
assert doc.created_at.year == before.year # 2030 never stored
|
||||
finally:
|
||||
_cleanup_source(db, root.name)
|
||||
|
||||
|
||||
# --- unchanged: the date refresh (D4) ----------------------------------------
|
||||
|
||||
|
||||
def test_unchanged_reimport_refreshes_date_when_mtime_moves(db, tmp_path: Path) -> None:
|
||||
"""D4: an unchanged file whose source date moved gets the new date
|
||||
(it may go OLDER — no monotonic guard) and is counted in
|
||||
``dates_updated`` — ``added/updated/pruned`` stay 0 (content counts
|
||||
preserved)."""
|
||||
root, file = _make_root(tmp_path, "DateRefresh")
|
||||
llm = FakeEmbedder()
|
||||
try:
|
||||
first = asyncio.run(import_sources([root], llm, session=db))
|
||||
assert first.added == 1 and first.dates_updated == 0
|
||||
_utime(file, OLD_2021) # the source date moved; content identical
|
||||
second = asyncio.run(import_sources([root], llm, session=db))
|
||||
assert (second.added, second.updated, second.pruned, second.unchanged) == (0, 0, 0, 1)
|
||||
assert second.dates_updated == 1
|
||||
doc = _doc(db, root.name, "note.md")
|
||||
assert abs(doc.created_at - OLD_2021) <= _TOL
|
||||
finally:
|
||||
_cleanup_source(db, root.name)
|
||||
|
||||
|
||||
def test_unchanged_reimport_same_date_no_refresh(db, tmp_path: Path) -> None:
|
||||
"""D4's no-op case: the source date is unchanged → no write,
|
||||
``dates_updated`` stays 0 (an unchanged re-sync is byte-identical)."""
|
||||
root, _file = _make_root(tmp_path, "DateSame")
|
||||
llm = FakeEmbedder()
|
||||
try:
|
||||
first = asyncio.run(import_sources([root], llm, session=db))
|
||||
assert first.added == 1 and first.dates_updated == 0
|
||||
second = asyncio.run(import_sources([root], llm, session=db))
|
||||
assert (second.added, second.updated, second.pruned, second.unchanged) == (0, 0, 0, 1)
|
||||
assert second.dates_updated == 0
|
||||
finally:
|
||||
_cleanup_source(db, root.name)
|
||||
|
||||
|
||||
# --- the D1 manual lock --------------------------------------------------------
|
||||
|
||||
|
||||
def test_manual_row_date_survives_unchanged_reimport(db, tmp_path: Path) -> None:
|
||||
"""D1: a row carrying the owner's correction (``created_at_manual``)
|
||||
is left ENTIRELY alone on the unchanged path — the moved mtime does
|
||||
not refresh it and ``dates_updated`` stays 0 (the sibling of the
|
||||
phase-97 ``manually_edited`` precedent)."""
|
||||
root, file = _make_root(tmp_path, "DateManual")
|
||||
llm = FakeEmbedder()
|
||||
correction = datetime(2023, 5, 5, 9, 30, 0, tzinfo=UTC)
|
||||
try:
|
||||
first = asyncio.run(import_sources([root], llm, session=db))
|
||||
assert first.added == 1
|
||||
doc = _doc(db, root.name, "note.md")
|
||||
doc.created_at = correction # the owner's correction (task 05's API)
|
||||
doc.created_at_manual = True
|
||||
db.commit()
|
||||
_utime(file, OLD_2021) # the source moved to a DIFFERENT date
|
||||
second = asyncio.run(import_sources([root], llm, session=db))
|
||||
assert (second.added, second.updated, second.pruned, second.unchanged) == (0, 0, 0, 1)
|
||||
assert second.dates_updated == 0
|
||||
db.expire_all()
|
||||
doc = _doc(db, root.name, "note.md")
|
||||
assert doc.created_at == correction # the correction survived
|
||||
assert doc.created_at_manual is True
|
||||
finally:
|
||||
_cleanup_source(db, root.name)
|
||||
|
||||
|
||||
def test_content_change_resets_date_and_manual_flag(db, tmp_path: Path) -> None:
|
||||
"""D4: a content change is a new document version — the date is
|
||||
re-sourced from the file AND the manual flag is reset (the
|
||||
correction referred to the old content)."""
|
||||
root, file = _make_root(tmp_path, "DateReset")
|
||||
llm = FakeEmbedder()
|
||||
correction = datetime(2023, 5, 5, 9, 30, 0, tzinfo=UTC)
|
||||
try:
|
||||
first = asyncio.run(import_sources([root], llm, session=db))
|
||||
assert first.added == 1
|
||||
doc = _doc(db, root.name, "note.md")
|
||||
doc.created_at = correction
|
||||
doc.created_at_manual = True
|
||||
db.commit()
|
||||
file.write_text("# Note\n\nNEW content — a new version\n", encoding="utf-8")
|
||||
_utime(file, OLD_2021)
|
||||
second = asyncio.run(import_sources([root], llm, session=db))
|
||||
assert (second.added, second.updated, second.unchanged) == (0, 1, 0)
|
||||
assert second.dates_updated == 0 # an update is not a date-only refresh
|
||||
doc = _doc(db, root.name, "note.md")
|
||||
assert abs(doc.created_at - OLD_2021) <= _TOL # re-sourced
|
||||
assert doc.created_at_manual is False # reset
|
||||
finally:
|
||||
_cleanup_source(db, root.name)
|
||||
Reference in New Issue
Block a user