brain-of-reese/tests/unit/test_importer_dates.py

"""Unit: phase 106 (task 04) — the importer's document-date semantics.

The D2/D4 matrix against the fake LLM + the house ``db`` session
(real compose Postgres, the ``tests/unit/test_importer.py`` pattern
for importer tests): added files store their source date (the
``doc_dates_by_root`` map entry when present, else the file mtime —
both normalized by :func:`app.rag.doc_dates.normalize_doc_date`,
D3); the unchanged path REFRESHES the stored date from the same
source and counts it in ``dates_updated`` (content counts preserved);
manual rows (``created_at_manual``) are skipped (the D1 lock); a
content change re-sources the date AND clears the manual flag; a
future (beyond-skew) mtime folds to today through the importer.
"""
from __future__ import annotations

import asyncio
import os
from datetime import UTC, datetime, timedelta
from pathlib import Path

from sqlalchemy import select

from app.models import Document
from app.rag.importer import import_sources
from tests.fakes import FakeEmbedder

#: mtime granularity tolerance — ``os.utime`` + ``stat`` round-trip on
#: the test filesystem (the archive-date suite uses ±1 s; 50 ms is far
#: tighter and still filesystem-agnostic).
_TOL = timedelta(milliseconds=50)

OLD_2020 = datetime(2020, 1, 2, 3, 4, 5, 123456, tzinfo=UTC)
OLD_2021 = datetime(2021, 6, 1, 12, 0, 0, tzinfo=UTC)


def _utime(path: Path, when: datetime) -> None:
    ts = when.timestamp()
    os.utime(path, (ts, ts))


def _doc(db, source: str, rel: str) -> Document:
    doc = db.scalar(select(Document).where(Document.source == source, Document.path == rel))
    assert doc is not None, f"no documents row for ({source!r}, {rel!r})"
    return doc


def _cleanup_source(db, source: str) -> None:
    for doc in db.scalars(select(Document).where(Document.source == source)).all():
        db.delete(doc)
    db.commit()


def _make_root(tmp_path: Path, name: str) -> tuple[Path, Path]:
    root = tmp_path / name
    root.mkdir()
    file = root / "note.md"
    file.write_text("# Note\n\ncontent for the KB\n", encoding="utf-8")
    return root, file


# --- added: the source date lands on the new row -----------------------------


def test_added_file_stores_its_mtime_as_created_at(db, tmp_path: Path) -> None:
    """D2 fallback: an unmapped file's mtime IS its source date — a
    file ``os.utime``'d to 2020-01-02 imports with that ``created_at``
    (the added branch, D3 normalized), and ``created_at_manual`` stays
    the column default (False)."""
    root, file = _make_root(tmp_path, "DateAdded")
    _utime(file, OLD_2020)
    llm = FakeEmbedder()
    try:
        summary = asyncio.run(import_sources([root], llm, session=db))
        assert (summary.added, summary.unchanged, summary.dates_updated) == (1, 0, 0)
        doc = _doc(db, root.name, "note.md")
        assert abs(doc.created_at - OLD_2020) <= _TOL
        assert doc.created_at_manual is False
    finally:
        _cleanup_source(db, root.name)


def test_doc_dates_map_entry_beats_mtime(db, tmp_path: Path) -> None:
    """D2 git case: a ``doc_dates_by_root`` entry (keyed by ``str(root)``
    — the root string exactly as passed in ``sources``) names the file's
    raw date and BEATS the file's mtime (the map says 2020, the mtime
    says now → 2020 stored). A path missing from its root's map takes
    the mtime fallback (≈ now) in the same run."""
    root = tmp_path / "DateMap"
    root.mkdir()
    mapped = root / "git_file.md"
    mapped.write_text("# Git\n\nfrom the repo\n", encoding="utf-8")
    unmapped = root / "local_file.md"
    unmapped.write_text("# Local\n\nnot in the map\n", encoding="utf-8")
    now_before = datetime.now(UTC)
    llm = FakeEmbedder()
    try:
        summary = asyncio.run(
            import_sources(
                [root], llm, session=db,
                doc_dates_by_root={str(root): {"git_file.md": OLD_2020}},
            )
        )
        assert summary.added == 2
        git_doc = _doc(db, root.name, "git_file.md")
        assert abs(git_doc.created_at - OLD_2020) <= _TOL  # the map, not the mtime
        local_doc = _doc(db, root.name, "local_file.md")
        # The unmapped file fell back to its mtime (written just now).
        assert now_before - _TOL <= local_doc.created_at <= datetime.now(UTC) + _TOL
        # Unchanged re-import with the SAME map: both dates already
        # stored → no refresh (the map hit is stable, not a rewrite).
        s2 = asyncio.run(
            import_sources(
                [root], llm, session=db,
                doc_dates_by_root={str(root): {"git_file.md": OLD_2020}},
            )
        )
        assert (s2.added, s2.updated, s2.unchanged) == (0, 0, 2)
        assert s2.dates_updated == 0
    finally:
        _cleanup_source(db, root.name)


def test_future_mtime_folds_to_today_through_importer(db, tmp_path: Path) -> None:
    """D3 through the importer: a mtime YEARS in the future (beyond the
    1-day clock-skew tolerance) folds to the import moment (today), not
    the raw future value."""
    root, file = _make_root(tmp_path, "DateFuture")
    _utime(file, datetime(2030, 1, 1, 0, 0, 0, tzinfo=UTC))
    llm = FakeEmbedder()
    try:
        before = datetime.now(UTC)
        summary = asyncio.run(import_sources([root], llm, session=db))
        after = datetime.now(UTC)
        assert summary.added == 1
        doc = _doc(db, root.name, "note.md")
        # The folded date is the normalization moment — between the run's
        # bounds (a hair of slack on each side).
        assert before - _TOL <= doc.created_at <= after + _TOL
        assert doc.created_at.year == before.year  # 2030 never stored
    finally:
        _cleanup_source(db, root.name)


# --- unchanged: the date refresh (D4) ----------------------------------------


def test_unchanged_reimport_refreshes_date_when_mtime_moves(db, tmp_path: Path) -> None:
    """D4: an unchanged file whose source date moved gets the new date
    (it may go OLDER — no monotonic guard) and is counted in
    ``dates_updated`` — ``added/updated/pruned`` stay 0 (content counts
    preserved)."""
    root, file = _make_root(tmp_path, "DateRefresh")
    llm = FakeEmbedder()
    try:
        first = asyncio.run(import_sources([root], llm, session=db))
        assert first.added == 1 and first.dates_updated == 0
        _utime(file, OLD_2021)  # the source date moved; content identical
        second = asyncio.run(import_sources([root], llm, session=db))
        assert (second.added, second.updated, second.pruned, second.unchanged) == (0, 0, 0, 1)
        assert second.dates_updated == 1
        doc = _doc(db, root.name, "note.md")
        assert abs(doc.created_at - OLD_2021) <= _TOL
    finally:
        _cleanup_source(db, root.name)


def test_unchanged_reimport_same_date_no_refresh(db, tmp_path: Path) -> None:
    """D4's no-op case: the source date is unchanged → no write,
    ``dates_updated`` stays 0 (an unchanged re-sync is byte-identical)."""
    root, _file = _make_root(tmp_path, "DateSame")
    llm = FakeEmbedder()
    try:
        first = asyncio.run(import_sources([root], llm, session=db))
        assert first.added == 1 and first.dates_updated == 0
        second = asyncio.run(import_sources([root], llm, session=db))
        assert (second.added, second.updated, second.pruned, second.unchanged) == (0, 0, 0, 1)
        assert second.dates_updated == 0
    finally:
        _cleanup_source(db, root.name)


# --- the D1 manual lock --------------------------------------------------------


def test_manual_row_date_survives_unchanged_reimport(db, tmp_path: Path) -> None:
    """D1: a row carrying the owner's correction (``created_at_manual``)
    is left ENTIRELY alone on the unchanged path — the moved mtime does
    not refresh it and ``dates_updated`` stays 0 (the sibling of the
    phase-97 ``manually_edited`` precedent)."""
    root, file = _make_root(tmp_path, "DateManual")
    llm = FakeEmbedder()
    correction = datetime(2023, 5, 5, 9, 30, 0, tzinfo=UTC)
    try:
        first = asyncio.run(import_sources([root], llm, session=db))
        assert first.added == 1
        doc = _doc(db, root.name, "note.md")
        doc.created_at = correction  # the owner's correction (task 05's API)
        doc.created_at_manual = True
        db.commit()
        _utime(file, OLD_2021)  # the source moved to a DIFFERENT date
        second = asyncio.run(import_sources([root], llm, session=db))
        assert (second.added, second.updated, second.pruned, second.unchanged) == (0, 0, 0, 1)
        assert second.dates_updated == 0
        db.expire_all()
        doc = _doc(db, root.name, "note.md")
        assert doc.created_at == correction  # the correction survived
        assert doc.created_at_manual is True
    finally:
        _cleanup_source(db, root.name)


def test_content_change_resets_date_and_manual_flag(db, tmp_path: Path) -> None:
    """D4: a content change is a new document version — the date is
    re-sourced from the file AND the manual flag is reset (the
    correction referred to the old content)."""
    root, file = _make_root(tmp_path, "DateReset")
    llm = FakeEmbedder()
    correction = datetime(2023, 5, 5, 9, 30, 0, tzinfo=UTC)
    try:
        first = asyncio.run(import_sources([root], llm, session=db))
        assert first.added == 1
        doc = _doc(db, root.name, "note.md")
        doc.created_at = correction
        doc.created_at_manual = True
        db.commit()
        file.write_text("# Note\n\nNEW content — a new version\n", encoding="utf-8")
        _utime(file, OLD_2021)
        second = asyncio.run(import_sources([root], llm, session=db))
        assert (second.added, second.updated, second.unchanged) == (0, 1, 0)
        assert second.dates_updated == 0  # an update is not a date-only refresh
        doc = _doc(db, root.name, "note.md")
        assert abs(doc.created_at - OLD_2021) <= _TOL  # re-sourced
        assert doc.created_at_manual is False  # reset
    finally:
        _cleanup_source(db, root.name)