"""Phase 194, task 01 — the importer's pre-scan + the pending-count switch. The owner report that opened the phase (2026-10-19): *"I just synced the prod brain of reese and it said '1216 remaining' when it only had like 10 files to sync."* The arithmetic of the phase-193 label was fine; its INPUTS were not — ``total`` was every importable file (prod: ~10,000) and ``done`` was the position in that walk, so the label counted the WALK instead of the WORK. Task 01 moves the pair to the PREDICTED change set: ``import_sources`` pre-scans (one streamed sha256 + one scalar ``documents.file_hash`` read per importable file, off the event loop) and reports ``done``/``total`` as the position in the pending set, with ``BOR_SYNC_PENDING_COUNT`` as the kill switch (``0`` = the exact phase-193 walk position, negative = refuses startup). Pinned here (the behaviour under a real run — the pending-set parity, the zero-hydration pin, the off-loop span and the cancel — lives in ``tests/integration/test_sync_pending_count.py``): * the GATE: no ``progress`` caller means no scan at all (the callback never fires and the scalar statement is never built — the phase-64 skip rule, extended), and the switch at ``0`` on a progress caller runs the phase-64 stats pre-walk instead (a source marker pin: that old code IS the ``0`` branch, and a behaviour pin on the pair it reports); * the switch is read ONCE per run (the phase-184 read-once discipline), on the worker-thread side of ``asyncio.to_thread`` (the phase-179 rule); * the scalar statement's projection is EXACTLY ``documents.file_hash`` — one scalar column, no entity, no ``defer``/``load_only`` row (the phase-185 column-list style); :func:`_doc_row_stmt` stays the importer's only ROW load; * the pending PREDICTATE: fresh (no row), changed (digest mismatch) and a pre-phase ``NULL`` row are pending, a hash match is not, a file the ignore rule hides is never even walked, and a file that cannot be read is predicted changed rather than fatal; * ``on_scan_progress``: once per file, the 1-based position against the combined importable total, and NOTHING for an empty KB (the phase-64 ``total === 0`` contract); * ``done`` as the pending position: a 3-file KB with 2 predicted reports ``1``, ``1``, ``2`` against ``total = 2`` — the file that is not predicted holds the position while the walk moves past it. """ from __future__ import annotations import ast import asyncio import inspect import re import threading import uuid from pathlib import Path from typing import Any import pytest from sqlalchemy import text from sqlalchemy.dialects import postgresql from sqlalchemy.orm import Session import app.rag.importer as importer from app.config import Settings from app.models import Document from app.rag.importer import ( _pending_hash_stmt, _prescan_pending, import_sources, ) from tests.fakes import FakeEmbedder # -------------------------------------------------------------------------- # Helpers # -------------------------------------------------------------------------- def _code_only(fn: Any) -> str: """A function's source WITHOUT its docstring. The marker pins below count occurrences of names in the code; the docstrings argue about those same names by name (the house convention), so counting the raw source would pin prose. The docstring is located through the AST and sliced out by its own end line. """ src = inspect.getsource(fn) lines = src.splitlines() node = ast.parse(src).body[0] # `import_sources` is an ASYNC def, so it parses to AsyncFunctionDef — # narrowing to FunctionDef alone would silently skip the docstring strip # and pin prose. if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): return src first = node.body[0] if ( isinstance(first, ast.Expr) and isinstance(first.value, ast.Constant) and isinstance(first.value.value, str) ): return "\n".join(lines[first.end_lineno or 1 :]) return src def _settings(**overrides: Any) -> Settings: """Fresh settings, never the developer's ``.env`` (the house pattern).""" return Settings(_env_file=None, **overrides) # pyright: ignore[reportCallIssue] def _root(tmp_path: Path, name: str, files: dict[str, str]) -> Path: """One source root with *files* (relative path → content) under it.""" root = tmp_path / name for rel, body in files.items(): target = root / rel target.parent.mkdir(parents=True, exist_ok=True) target.write_text(body, encoding="utf-8") return root def _prescan( *sources: Path, session: Session, ignore_by_root: dict[str, list[str]] | None = None, include_hidden_by_root: dict[str, bool] | None = None, on_scan_progress: Any = None, image_extensions: frozenset[str] = frozenset(), ) -> tuple[int, dict[tuple[str, str], int]]: """:func:`_prescan_pending` with the shipped import extensions and an explicit session, so the predicate is testable without a full run.""" return _prescan_pending( list(sources), _settings().import_extension_set, image_extensions, ignore_by_root, include_hidden_by_root, on_scan_progress, session, ) def _seed_row( db: Session, source: str, rel: str, *, file_hash: str | None ) -> None: """One ``documents`` row with a KNOWN ``file_hash`` (the pre-scan reads that one column, so this is the whole fixture: raw SQL, no importer, no chunks).""" db.execute( text( "INSERT INTO documents (id, source, path, full_path, title, content, " "content_hash, file_hash, indexed_at, created_at, created_at_manual, " "summary, is_image) VALUES (:id, :source, :path, :full_path, :title, " ":content, :content_hash, :file_hash, now(), now(), false, :summary, " "false)" ), { "id": uuid.uuid4(), "source": source, "path": rel, "full_path": f"/tmp/{source}/{rel}", "title": rel, "content": "stored markdown", "content_hash": "content-digest", "file_hash": file_hash, "summary": "stored summary", }, ) db.commit() @pytest.fixture(autouse=True) def _clean_probe_rows(db: Session) -> Any: """The probe's own source-name prefix, removed around every test (the tables are global state — the ``test_importer.py`` hygiene pattern).""" _drop(db) yield _drop(db) def _drop(db: Session) -> None: db.execute( text("DELETE FROM documents WHERE source LIKE 'phase194-%'"), ) db.commit() db.expunge_all() # -------------------------------------------------------------------------- # The gate: who pays for the scan # -------------------------------------------------------------------------- def test_no_progress_caller_runs_no_scan_at_all( db: Session, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """THE skip rule (phase 64), extended: ``progress=None`` → no pre-scan. The CLI (``scripts/import_docs.py``) and the upload job pass no ``progress``, so they must not pay one extra read + hash + SELECT per importable file. A recording stand-in for the span proves the gate itself: the CLI-shaped call imports the file and never reaches the scan, and an ``on_scan_progress`` hook handed to a ``progress=None`` caller fires never (the scan exists only to serve the running counts). """ calls: list[tuple[Any, ...]] = [] scan_events: list[tuple[int, int]] = [] def _spy(*args: Any, **kwargs: Any) -> tuple[int, dict[tuple[str, str], int]]: calls.append(args) return 0, {} monkeypatch.setattr(importer, "_prescan_pending", _spy) root = _root(tmp_path, "phase194-gate-cli", {"a.md": "# A\n\na\n"}) try: summary = asyncio.run(import_sources([root], FakeEmbedder(), session=db)) assert summary.files == 1 and summary.added == 1 assert calls == [], "a progress=None caller ran the pre-scan" # The scan hook alone does not open the gate either. summary2 = asyncio.run( import_sources( [root], FakeEmbedder(), session=db, on_scan_progress=lambda done, total: scan_events.append((done, total)), ) ) assert summary2.files == 1 assert calls == [] assert scan_events == [] finally: db.execute( text("DELETE FROM documents WHERE source = :s"), {"s": root.name} ) db.commit() def test_switch_zero_runs_the_phase_193_stats_prewalk( db: Session, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """The rollback branch, from both sides: the marker (the old stats pre-walk code IS the ``0`` branch) and the reported pair. With the switch at ``0`` a progress caller never reaches the scan span, and the hook reports the phase-193 walk position byte-for-byte — every importable file in ``total``, the 1-based index across all sources in ``done`` — on a run where nothing is pending. That last clause is the point of the arm: the SAME unchanged fixture reports ``(0, 0)`` with the switch on and ``(1, 1)``, ``(2, 2)`` with it off. """ calls: list[tuple[Any, ...]] = [] def _spy(*args: Any, **kwargs: Any) -> tuple[int, dict[tuple[str, str], int]]: calls.append(args) return 0, {} monkeypatch.setattr(importer, "_prescan_pending", _spy) root = _root( tmp_path, "phase194-gate-zero", {"a.md": "# A\n\nalpha\n", "b.md": "# B\n\nbravo\n"}, ) llm = FakeEmbedder() llm.settings = _settings(sync_pending_count=0) events: list[tuple[str, str, int, int]] = [] try: # Seed the KB (a walk-position run of its own), so nothing pending. asyncio.run(import_sources([root], llm, session=db)) assert calls == [] events.clear() summary = asyncio.run( import_sources( [root], llm, session=db, progress=lambda s, r, d, t: events.append((s, r, d, t)), ) ) assert calls == [], "the switch was 0 but the pre-scan ran" assert (summary.files, summary.unchanged) == (2, 2) assert events == [ (root.name, "a.md", 1, 2), (root.name, "b.md", 2, 2), ], "the 0 branch must report the phase-193 walk position exactly" finally: db.execute(text("DELETE FROM documents WHERE source = :s"), {"s": root.name}) db.commit() # The marker: the stats-only pre-walk still EXISTS in the importer, and # it lives in the ``0`` branch (a third walk would mean the replacement # was not a replacement). src = _code_only(importer.import_sources) assert "BOR_SYNC_PENDING_COUNT=0" in src assert src.count("iter_importable_files(") == 2, ( "the importer walks the sources for the count more than once besides " "the pre-scan — the pre-scan REPLACES the stats pre-walk (phase 194 " "D1: no third walk)" ) def test_switch_is_read_once_before_the_loop_and_off_the_loop() -> None: """The phase-184 read-once discipline + the phase-179 off-loop rule. ``llm.settings.sync_pending_count`` is read exactly ONCE, before the source loop, so a settings overlay saved mid-run cannot flip the meaning of ``done``/``total`` between two files of the same walk; and the span is entered through ``asyncio.to_thread``, so a 10,000-file hash+SELECT pass never parks the single uvicorn loop. """ src = _code_only(importer.import_sources) assert src.count("llm.settings.sync_pending_count") == 1, ( "sync_pending_count is read more than once per run — a mid-run " "settings save would change the pair's meaning between two files " "(phase 184's read-once discipline)" ) read_at = src.index("llm.settings.sync_pending_count") loop_at = src.index("for root in sources:") assert read_at < loop_at, "the switch must be read BEFORE the source loop" assert "await asyncio.to_thread(\n _prescan_pending" in src, ( "the pre-scan is no longer entered through asyncio.to_thread — a long " "scan would park the event loop (the phase-179 rule, " "test_sync_git_freeze.py)" ) # The scan itself is a plain sync function (it runs in the worker). assert not inspect.iscoroutinefunction(importer._prescan_pending) # -------------------------------------------------------------------------- # The scalar statement's shape (the phase-185 column-list style) # -------------------------------------------------------------------------- def test_scalar_statement_projects_exactly_file_hash() -> None: """The pre-scan reads ONE column, so it loads no entity and keeps nothing. The compiled projection names ``documents.file_hash`` and no other ``documents`` column; the statement resolves to exactly one scalar column (``column_descriptions``), so ``session.scalar`` returns a string and no ORM object exists to land in an identity map. ``_doc_row_stmt`` stays the importer's only ``(source, path)`` ROW load (phase 185). """ compiled = str( _pending_hash_stmt("the-source", "notes/a.md").compile( dialect=postgresql.dialect() ) ) projection = re.split(r"\bFROM\b", compiled, maxsplit=1)[0] columns = set(re.findall(r"documents\.([a-z_]+)", projection)) assert columns == {"file_hash"}, ( f"the pre-scan statement projects {sorted(columns)} — it must project " "documents.file_hash and nothing else (one light metadata read per " "file, no entity hydration)" ) # The predicate is the unique (source, path) constraint, like the fast path. predicate = re.split(r"\bFROM\b", compiled, maxsplit=1)[1] assert "documents.source" in predicate and "documents.path" in predicate stmt = _pending_hash_stmt("s", "p") assert len(stmt.column_descriptions) == 1 assert stmt.column_descriptions[0]["type"] == Document.file_hash.type assert "Document" not in { str(desc.get("entity")) for desc in stmt.column_descriptions }, "the pre-scan statement selects an ENTITY — it must select the column" # The span is the only consumer, and the fast path's row load is intact. span_src = inspect.getsource(importer._prescan_pending) assert "_pending_hash_stmt(" in span_src assert "select(Document)" not in span_src assert "defer(" not in span_src and "load_only(" not in span_src, ( "the pre-scan picked up a loader option — there is no ROW to defer: " "it selects one column (phase 185 keeps _doc_row_stmt as the only " "row load)" ) # -------------------------------------------------------------------------- # The pending predicate # -------------------------------------------------------------------------- def test_pending_predicate_fresh_unchanged_changed_and_null_row( db: Session, tmp_path: Path ) -> None: """The four stored-hash states, and the position the set assigns. * no row at all (a first sync) → pending; * a row whose ``file_hash`` equals the file's digest → NOT pending; * a row whose ``file_hash`` differs → pending; * a pre-phase-147 row (``file_hash`` NULL) → pending — the same call the fast path makes, so the prediction and the loop agree and the loop stamps the hash on the way through. The positions are the 1-based walk order of the PREDICTED files only, and the scan total is every importable file (3 here, 3 predicted). """ root = _root( tmp_path, "phase194-predicate", { "fresh.md": "# Fresh\n\nnever indexed\n", "same.md": "# Same\n\nindexed as-is\n", "moved.md": "# Moved\n\nindexed differently now\n", }, ) from app.rag.importer import _file_hash _seed_row(db, root.name, "same.md", file_hash=_file_hash(root / "same.md")) _seed_row(db, root.name, "moved.md", file_hash="0" * 64) db.expunge_all() scan_total, pending = _prescan(root, session=db) assert scan_total == 3 assert pending == { (root.name, "fresh.md"): 1, (root.name, "moved.md"): 2, }, ( "the pending set is wrong — no row and a NULL row are pending, a " "digest match is not, a mismatch is (the fast path's own predicate)" ) # The scan reads metadata only: nothing was hydrated into the session. assert len(db.identity_map) == 0 def test_pending_predicate_ignores_what_the_walk_ignores( db: Session, tmp_path: Path ) -> None: """A file the ignore rule hides is never walked, so it is never pending. The scan uses the EXACT per-root ``ignore`` / ``include_hidden`` rules of the processing loop: a hidden dot-directory is out by default and in when the root's flag admits it, and an ignored prefix is out even with the flag on. A predicted file the loop can never see would be a count that never reaches zero, so this is the parity edge, not a nicety. """ root = _root( tmp_path, "phase194-ignore", { "keep.md": "# Keep\n\nbody\n", "skipme/hidden-by-rule.md": "# Skip\n\nbody\n", ".dot/secret.md": "# Dot\n\nbody\n", }, ) ignore = {str(root): ["skipme"]} scan_total, pending = _prescan(root, session=db, ignore_by_root=ignore) assert scan_total == 1 assert list(pending) == [(root.name, "keep.md")] # With the hidden-folders flag ON the dot-dir file joins the walk (and # the scan total with it) — still no pending entry for the ignored one. scan_total, pending = _prescan( root, session=db, ignore_by_root=ignore, include_hidden_by_root={str(root): True}, ) assert scan_total == 2 assert sorted(pending) == [ (root.name, ".dot/secret.md"), (root.name, "keep.md"), ] def test_unreadable_file_is_predicted_changed_and_the_scan_survives( db: Session, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, ) -> None: """A file the digest cannot reach is a PREDICTION of change, not a failure. The window is the phase's own: the scan walks the tree, then hashes it seconds later, and a live source can lose a file in between. Failing the whole sync for that would be the worse bug — the pre-scan only informs a label — so the file lands in the pending set (the loop's own walk decides whether it still exists at all) and one warning names it. """ root = _root( tmp_path, "phase194-unreadable", {"a.md": "# A\n\na\n", "b.md": "# B\n\nb\n"}, ) real_hash = importer._file_hash def flaky(path: Path) -> str: if path.name == "b.md": raise OSError(2, "No such file or directory", str(path)) return real_hash(path) monkeypatch.setattr(importer, "_file_hash", flaky) with caplog.at_level("WARNING", logger="app.importer"): scan_total, pending = _prescan(root, session=db) assert scan_total == 2 assert sorted(pending) == [ (root.name, "a.md"), (root.name, "b.md"), ] lines = [r.getMessage() for r in caplog.records if r.name == "app.importer"] assert any( line.startswith("import: pre-scan unreadable") and f"source={root.name}" in line and "path=b.md" in line for line in lines ), f"the skipped file was not named in a warning: {lines}" # -------------------------------------------------------------------------- # on_scan_progress + the empty KB # -------------------------------------------------------------------------- def test_scan_progress_reports_every_file_across_roots( db: Session, tmp_path: Path ) -> None: """The scan pair is ``(1-based position, combined importable total)``. The denominator is the number the phase-64 stats pre-walk used to compute (every importable file, across every root, in *sources* order), because the label is about the SCAN — how far the prediction has got — and the events cover every file, changed or not. """ root_a = _root( tmp_path, "phase194-scan-a", {"a1.md": "# A1\n\na\n", "a2.md": "# A2\n\nb\n"} ) root_b = _root(tmp_path, "phase194-scan-b", {"sub/b1.md": "# B1\n\nc\n"}) events: list[tuple[int, int]] = [] scan_total, pending = _prescan( root_a, root_b, session=db, on_scan_progress=lambda p, t: events.append((p, t)) ) assert scan_total == 3 assert events == [(1, 3), (2, 3), (3, 3)] assert len(pending) == 3 def test_scan_progress_reports_nothing_for_an_empty_kb( db: Session, tmp_path: Path ) -> None: """The degenerate KB: scan total 0, no callback, empty pending set. The phase-64 ``total === 0`` contract, extended to the scan: with nothing importable there is no position to report, so the UI's scan label renders bare (``Scanning…``, no count part) instead of ``0/0``. """ empty = _root(tmp_path, "phase194-empty", {}) missing = tmp_path / "phase194-not-a-dir" events: list[tuple[int, int]] = [] scan_total, pending = _prescan( empty, missing, session=db, on_scan_progress=lambda p, t: events.append((p, t)) ) assert scan_total == 0 assert pending == {} assert events == [] def test_scan_hook_is_called_from_the_worker_thread( tmp_path: Path, db: Session ) -> None: """The documented threading contract of ``on_scan_progress``. The span runs in an ``asyncio.to_thread`` worker, so the hook fires on THAT thread — which is only safe because the house hook rule is "assign a field", a single GIL-atomic store. The pin records the thread identity from inside the callback and from the awaiting coroutine. """ root = _root(tmp_path, "phase194-thread", {"a.md": "# A\n\na\n"}) seen: dict[str, int] = {} def hook(position: int, total: int) -> None: # noqa: ARG001 seen["scan"] = threading.get_ident() async def main() -> None: seen["loop"] = threading.get_ident() await asyncio.to_thread( importer._prescan_pending, [root], _settings().import_extension_set, frozenset(), None, None, hook, ) asyncio.run(main()) assert seen["scan"] != seen["loop"], ( "the pre-scan hook ran on the event-loop thread — the span must live " "in a worker (phase 179)" ) # -------------------------------------------------------------------------- # done = the position in the pending set # -------------------------------------------------------------------------- def test_done_is_the_pending_position_through_import_sources( db: Session, tmp_path: Path ) -> None: """3 importable files, 2 of them pending: the pair is ``1/2``, ``1/2``, ``2/2``. ``stable.md`` is indexed and unchanged, so the scan does not predict it and the walk moving past it does NOT advance the count — the label holds its position (this is the shape the owner's prod case takes: the count stops climbing across a run of unchanged files). The two changed files advance it in walk order, and the denominator is the pending set's size, never the 3-file walk. """ root = _root( tmp_path, "phase194-position", { "a.md": "# A\n\nalpha\n", "stable.md": "# Stable\n\nunchanged\n", "z.md": "# Z\n\nzulu\n", }, ) llm = FakeEmbedder() events: list[tuple[str, str, int, int]] = [] try: # Index everything, then change two files. asyncio.run(import_sources([root], llm, session=db)) (root / "a.md").write_text("# A\n\nalpha edited\n", encoding="utf-8") (root / "z.md").write_text("# Z\n\nzulu edited\n", encoding="utf-8") events.clear() summary = asyncio.run( import_sources( [root], llm, session=db, progress=lambda s, r, d, t: events.append((s, r, d, t)), ) ) assert (summary.files, summary.updated, summary.unchanged) == (3, 2, 1) assert events == [ (root.name, "a.md", 1, 2), (root.name, "stable.md", 1, 2), # not predicted: the position holds (root.name, "z.md", 2, 2), ] # The parity the UI depends on: the denominator IS the work. assert summary.updated + summary.added == events[-1][3] finally: db.execute(text("DELETE FROM documents WHERE source = :s"), {"s": root.name}) db.commit() def test_empty_kb_reports_no_count_part_through_import_sources( db: Session, tmp_path: Path ) -> None: """An empty source with the hook on: no file, no pair, ``files_total`` 0. The import is a no-op and the hook never fires, so the status keeps the zero denominator the UI's ``total === 0`` guard hides the count part for (the phase-64 contract, arriving through the pre-scan path rather than the stats pre-walk). """ root = _root(tmp_path, "phase194-empty-run", {}) events: list[tuple[str, str, int, int]] = [] scan_events: list[tuple[int, int]] = [] summary = asyncio.run( import_sources( [root], FakeEmbedder(), session=db, progress=lambda s, r, d, t: events.append((s, r, d, t)), on_scan_progress=lambda p, t: scan_events.append((p, t)), ) ) assert (summary.files, summary.added, summary.errors) == (0, 0, 0) assert events == [] assert scan_events == [] def test_cli_caller_is_untouched_by_the_pending_semantics( db: Session, tmp_path: Path ) -> None: """The CLI-shaped run (no hook at all) on an unchanged KB is the pre-phase run: one file processed, nothing counted, one walk. The byte-identity of the non-UI caller is the phase's cheapest promise — ``scripts/import_docs.py`` passes no ``progress``, so it neither pays the scan nor sees its numbers. """ root = _root(tmp_path, "phase194-cli", {"a.md": "# A\n\na\n"}) llm = FakeEmbedder() try: first = asyncio.run(import_sources([root], llm, session=db)) assert (first.files, first.added) == (1, 1) second = asyncio.run(import_sources([root], llm, session=db)) assert (second.files, second.added, second.unchanged) == (1, 0, 1) finally: db.execute(text("DELETE FROM documents WHERE source = :s"), {"s": root.name}) db.commit()