"""Unit: phase 106 (task 04) — the importer's document-date semantics. The D2/D4 matrix against the fake LLM + the house ``db`` session (real compose Postgres, the ``tests/unit/test_importer.py`` pattern for importer tests): added files store their source date (the ``doc_dates_by_root`` map entry when present, else the file mtime — both normalized by :func:`app.rag.doc_dates.normalize_doc_date`, D3); the unchanged path REFRESHES the stored date from the same source and counts it in ``dates_updated`` (content counts preserved); manual rows (``created_at_manual``) are skipped (the D1 lock); a content change re-sources the date AND clears the manual flag; a future (beyond-skew) mtime folds to today through the importer. """ from __future__ import annotations import asyncio import os from datetime import UTC, datetime, timedelta from pathlib import Path from sqlalchemy import select from app.models import Document from app.rag.importer import import_sources from tests.fakes import FakeEmbedder #: mtime granularity tolerance — ``os.utime`` + ``stat`` round-trip on #: the test filesystem (the archive-date suite uses ±1 s; 50 ms is far #: tighter and still filesystem-agnostic). _TOL = timedelta(milliseconds=50) OLD_2020 = datetime(2020, 1, 2, 3, 4, 5, 123456, tzinfo=UTC) OLD_2021 = datetime(2021, 6, 1, 12, 0, 0, tzinfo=UTC) def _utime(path: Path, when: datetime) -> None: ts = when.timestamp() os.utime(path, (ts, ts)) def _doc(db, source: str, rel: str) -> Document: doc = db.scalar(select(Document).where(Document.source == source, Document.path == rel)) assert doc is not None, f"no documents row for ({source!r}, {rel!r})" return doc def _cleanup_source(db, source: str) -> None: for doc in db.scalars(select(Document).where(Document.source == source)).all(): db.delete(doc) db.commit() def _make_root(tmp_path: Path, name: str) -> tuple[Path, Path]: root = tmp_path / name root.mkdir() file = root / "note.md" file.write_text("# Note\n\ncontent for the KB\n", encoding="utf-8") return root, file # --- added: the source date lands on the new row ----------------------------- def test_added_file_stores_its_mtime_as_created_at(db, tmp_path: Path) -> None: """D2 fallback: an unmapped file's mtime IS its source date — a file ``os.utime``'d to 2020-01-02 imports with that ``created_at`` (the added branch, D3 normalized), and ``created_at_manual`` stays the column default (False).""" root, file = _make_root(tmp_path, "DateAdded") _utime(file, OLD_2020) llm = FakeEmbedder() try: summary = asyncio.run(import_sources([root], llm, session=db)) assert (summary.added, summary.unchanged, summary.dates_updated) == (1, 0, 0) doc = _doc(db, root.name, "note.md") assert abs(doc.created_at - OLD_2020) <= _TOL assert doc.created_at_manual is False finally: _cleanup_source(db, root.name) def test_doc_dates_map_entry_beats_mtime(db, tmp_path: Path) -> None: """D2 git case: a ``doc_dates_by_root`` entry (keyed by ``str(root)`` — the root string exactly as passed in ``sources``) names the file's raw date and BEATS the file's mtime (the map says 2020, the mtime says now → 2020 stored). A path missing from its root's map takes the mtime fallback (≈ now) in the same run.""" root = tmp_path / "DateMap" root.mkdir() mapped = root / "git_file.md" mapped.write_text("# Git\n\nfrom the repo\n", encoding="utf-8") unmapped = root / "local_file.md" unmapped.write_text("# Local\n\nnot in the map\n", encoding="utf-8") now_before = datetime.now(UTC) llm = FakeEmbedder() try: summary = asyncio.run( import_sources( [root], llm, session=db, doc_dates_by_root={str(root): {"git_file.md": OLD_2020}}, ) ) assert summary.added == 2 git_doc = _doc(db, root.name, "git_file.md") assert abs(git_doc.created_at - OLD_2020) <= _TOL # the map, not the mtime local_doc = _doc(db, root.name, "local_file.md") # The unmapped file fell back to its mtime (written just now). assert now_before - _TOL <= local_doc.created_at <= datetime.now(UTC) + _TOL # Unchanged re-import with the SAME map: both dates already # stored → no refresh (the map hit is stable, not a rewrite). s2 = asyncio.run( import_sources( [root], llm, session=db, doc_dates_by_root={str(root): {"git_file.md": OLD_2020}}, ) ) assert (s2.added, s2.updated, s2.unchanged) == (0, 0, 2) assert s2.dates_updated == 0 finally: _cleanup_source(db, root.name) def test_future_mtime_folds_to_today_through_importer(db, tmp_path: Path) -> None: """D3 through the importer: a mtime YEARS in the future (beyond the 1-day clock-skew tolerance) folds to the import moment (today), not the raw future value.""" root, file = _make_root(tmp_path, "DateFuture") _utime(file, datetime(2030, 1, 1, 0, 0, 0, tzinfo=UTC)) llm = FakeEmbedder() try: before = datetime.now(UTC) summary = asyncio.run(import_sources([root], llm, session=db)) after = datetime.now(UTC) assert summary.added == 1 doc = _doc(db, root.name, "note.md") # The folded date is the normalization moment — between the run's # bounds (a hair of slack on each side). assert before - _TOL <= doc.created_at <= after + _TOL assert doc.created_at.year == before.year # 2030 never stored finally: _cleanup_source(db, root.name) # --- unchanged: the date refresh (D4) ---------------------------------------- def test_unchanged_reimport_refreshes_date_when_mtime_moves(db, tmp_path: Path) -> None: """D4: an unchanged file whose source date moved gets the new date (it may go OLDER — no monotonic guard) and is counted in ``dates_updated`` — ``added/updated/pruned`` stay 0 (content counts preserved).""" root, file = _make_root(tmp_path, "DateRefresh") llm = FakeEmbedder() try: first = asyncio.run(import_sources([root], llm, session=db)) assert first.added == 1 and first.dates_updated == 0 _utime(file, OLD_2021) # the source date moved; content identical second = asyncio.run(import_sources([root], llm, session=db)) assert (second.added, second.updated, second.pruned, second.unchanged) == (0, 0, 0, 1) assert second.dates_updated == 1 doc = _doc(db, root.name, "note.md") assert abs(doc.created_at - OLD_2021) <= _TOL finally: _cleanup_source(db, root.name) def test_unchanged_reimport_same_date_no_refresh(db, tmp_path: Path) -> None: """D4's no-op case: the source date is unchanged → no write, ``dates_updated`` stays 0 (an unchanged re-sync is byte-identical).""" root, _file = _make_root(tmp_path, "DateSame") llm = FakeEmbedder() try: first = asyncio.run(import_sources([root], llm, session=db)) assert first.added == 1 and first.dates_updated == 0 second = asyncio.run(import_sources([root], llm, session=db)) assert (second.added, second.updated, second.pruned, second.unchanged) == (0, 0, 0, 1) assert second.dates_updated == 0 finally: _cleanup_source(db, root.name) # --- the D1 manual lock -------------------------------------------------------- def test_manual_row_date_survives_unchanged_reimport(db, tmp_path: Path) -> None: """D1: a row carrying the owner's correction (``created_at_manual``) is left ENTIRELY alone on the unchanged path — the moved mtime does not refresh it and ``dates_updated`` stays 0 (the sibling of the phase-97 ``manually_edited`` precedent).""" root, file = _make_root(tmp_path, "DateManual") llm = FakeEmbedder() correction = datetime(2023, 5, 5, 9, 30, 0, tzinfo=UTC) try: first = asyncio.run(import_sources([root], llm, session=db)) assert first.added == 1 doc = _doc(db, root.name, "note.md") doc.created_at = correction # the owner's correction (task 05's API) doc.created_at_manual = True db.commit() _utime(file, OLD_2021) # the source moved to a DIFFERENT date second = asyncio.run(import_sources([root], llm, session=db)) assert (second.added, second.updated, second.pruned, second.unchanged) == (0, 0, 0, 1) assert second.dates_updated == 0 db.expire_all() doc = _doc(db, root.name, "note.md") assert doc.created_at == correction # the correction survived assert doc.created_at_manual is True finally: _cleanup_source(db, root.name) def test_content_change_resets_date_and_manual_flag(db, tmp_path: Path) -> None: """D4: a content change is a new document version — the date is re-sourced from the file AND the manual flag is reset (the correction referred to the old content).""" root, file = _make_root(tmp_path, "DateReset") llm = FakeEmbedder() correction = datetime(2023, 5, 5, 9, 30, 0, tzinfo=UTC) try: first = asyncio.run(import_sources([root], llm, session=db)) assert first.added == 1 doc = _doc(db, root.name, "note.md") doc.created_at = correction doc.created_at_manual = True db.commit() file.write_text("# Note\n\nNEW content — a new version\n", encoding="utf-8") _utime(file, OLD_2021) second = asyncio.run(import_sources([root], llm, session=db)) assert (second.added, second.updated, second.unchanged) == (0, 1, 0) assert second.dates_updated == 0 # an update is not a date-only refresh doc = _doc(db, root.name, "note.md") assert abs(doc.created_at - OLD_2021) <= _TOL # re-sourced assert doc.created_at_manual is False # reset finally: _cleanup_source(db, root.name)