"""Phase 185, task 02 — the ONE fast-path row statement (unit pins). ``app/rag/importer.py::_doc_row_stmt`` is the single place the importer's unchanged fast path (phase 147) loads a ``Document`` row, and it defers exactly ONE column: ``Document.content``. That shape is load-bearing twice over — it is what keeps a full re-sync's memory bound to row METADATA instead of to the corpus (the 2026-10-15 incident: 17,407 rows × avg 15,217 chars = 252.6 MB of markdown hydrated for a run that re-indexed 217 files), AND it is what must NOT grow into a lazy-load landmine six phases from now. Pinned here (the behaviour itself — the captured SQL, the one-SELECT-per-file count and the backfill's single lazy read — lives in ``tests/integration/test_import_fast_path_columns.py``): * the statement defers ``content`` and NOTHING ELSE: its projection names every other mapped column of ``Document``, so a column a later phase adds keeps today's whole-row semantics (LOCKED A1 — no ``load_only`` allowlist, which would silently defer every future column and resurface as an unexplained N+1); * the statement is the ``(source, path)`` lookup the fast path has always used — the deferral changed the projection, never the predicate; * all FOUR fast-path row loads call the helper (``_index_file``, ``_index_pdf_file``, ``_index_one_embedded_image``, ``_index_image_file``) and none of them builds its own ``select(Document)`` — the four sites cannot drift; * the importer contains no ``raiseload``/``noload`` (LOCKED A2): the phase-118 summary-NULL backfill genuinely reads ``doc.content`` and must pay one SELECT, not crash; * reading the deferred column through a real session still yields the stored text (the honest "plain ``defer`` keeps today's behaviour" claim). """ from __future__ import annotations import ast import inspect import re import uuid from collections.abc import Iterator import pytest from sqlalchemy import inspect as sa_inspect from sqlalchemy import text from sqlalchemy.dialects import postgresql from sqlalchemy.orm import Session from app.models import Document from app.rag import importer from app.rag.importer import _doc_row_stmt #: The four loads that share the phase-147 unchanged fast path (phase 185 #: D1) — the ONLY call sites of :func:`_doc_row_stmt`. FAST_PATH_FUNCTIONS = ( "_index_file", "_index_pdf_file", "_index_one_embedded_image", "_index_image_file", ) #: The helper call as it appears at an importer call site (``session.scalar( #: _doc_row_stmt(source, rel))``) — counted, not trusted. _HELPER_CALL = "_doc_row_stmt(" #: ``documents.`` inside a compiled projection (everything the #: statement SELECTs before ``FROM``). The loader options are invisible in #: the ORM statement's own column list — only the COMPILED projection shows #: what actually travels to the database, which is what this phase is about. _PROJECTION_COLUMN_RE = re.compile(r"documents\.([a-z_]+)") def _projected_columns() -> set[str]: """The ``documents`` columns the fast-path statement actually SELECTs.""" compiled = str(_doc_row_stmt("s", "p").compile(dialect=postgresql.dialect())) projection = compiled.split("\nFROM")[0].split(" FROM ")[0] return set(_PROJECTION_COLUMN_RE.findall(projection)) def test_helper_defers_content_and_nothing_else() -> None: """``content`` is the ONLY column missing from the fast-path projection. The compiled projection is compared against the mapper's FULL column set, so a future ``Document`` column must appear there — if someone swaps the ``defer`` for a ``load_only`` allowlist, the new column goes missing and this test fails (which is exactly the drift LOCKED A1 forbids). """ mapped = {c.name for c in sa_inspect(Document).columns} projected = _projected_columns() assert "content" in mapped, "Document.content vanished — this pin needs a new shape" assert projected, "the fast-path statement projects no documents columns at all" assert "content" not in projected, ( "the fast-path row statement no longer defers Document.content — a " "full walk would hydrate the corpus markdown again (252.6 MB on the " "incident box)" ) assert projected == mapped - {"content"}, ( "the fast-path row statement defers more (or other) columns than " f"content: missing={sorted(mapped - projected - {'content'})} " f"unexpected={sorted(projected - mapped)} — only Document.content " "may be deferred (LOCKED A1)" ) def test_helper_keeps_the_fast_path_predicate() -> None: """The predicate stays the ``(source, path)`` lookup of phase 147.""" compiled = str(_doc_row_stmt("the-source", "the/path.md").compile()).lower() assert "documents.source" in compiled assert "documents.path" in compiled assert "where" in compiled # The bind names the integration probe attributes statements by. assert "source_1" in compiled and "path_1" in compiled def test_every_fast_path_load_calls_the_helper() -> None: """All four fast-path row loads go through ONE statement (phase 185 D1). Source-level on purpose: the deferral must be impossible to bypass, and a fifth whole-entity load would silently undo the phase for one document class. The two per-source post-pass walks (``_enrich_pending_images``, ``_backfill_image_contexts``) and the prune (``load_only`` of six light columns, phase 184) are NOT fast-path loads and are deliberately excluded — see the module docstring of ``tests/integration/test_import_fast_path_columns.py``. """ for name in FAST_PATH_FUNCTIONS: fn = getattr(importer, name) src = inspect.getsource(fn) assert _HELPER_CALL in src, ( f"{name} no longer loads its row through importer.{_HELPER_CALL} — " "the four fast-path row loads must share the ONE deferring " "statement so they cannot drift" ) assert "select(Document)" not in src, ( f"{name} builds its own select(Document) row again — a whole-entity " "fast-path load re-hydrates Document.content (phase 185)" ) def test_helper_has_exactly_the_four_fast_path_call_sites() -> None: """The importer calls the helper exactly four times (one per fast path). A fifth ``session.scalar(_doc_row_stmt(…)`` would mean a new consumer was wired to the fast-path statement (it is the ``(source, path)`` deferring lookup, not a general-purpose loader), and a surviving whole-entity ``(source, path)`` ``select(Document)`` would mean a bypass. """ src = inspect.getsource(importer) calls = src.count(f"session.scalar({_HELPER_CALL}") assert calls == len(FAST_PATH_FUNCTIONS), ( f"importer.py calls session.scalar({_HELPER_CALL}…) {calls} times — " f"expected exactly {len(FAST_PATH_FUNCTIONS)} (the four fast-path " "row loads)" ) # No fast-path-shaped whole-entity load survives next to the helper. assert src.count("select(Document).where(Document.source ==") == 0, ( "a (source, path) whole-entity Document load is back — route it " "through _doc_row_stmt or give it its own explicit projection" ) def test_no_raiseload_or_noload_anywhere_in_the_importer() -> None: """LOCKED A2: the deferred column must QUERY, never raise or come back None. ``raiseload``/``noload`` would turn the phase-118 summary-NULL backfill (``_unchanged_doc`` step 3, ``content=doc.content``) into a crash or an empty summary — a behaviour regression on the one branch that genuinely needs the stored markdown. The accepted cost is one SELECT for that row. Checked through the AST, not the text: the module's own docstrings argue against both options by NAME, so a substring scan would fail on the prose. Only real call/import names count here. """ tree = ast.parse(inspect.getsource(importer)) called: set[str] = set() imported: set[str] = set() for node in ast.walk(tree): if isinstance(node, ast.Call): if isinstance(node.func, ast.Name): called.add(node.func.id) elif isinstance(node.func, ast.Attribute): called.add(node.func.attr) elif isinstance(node, ast.ImportFrom | ast.Import): imported.update(alias.name for alias in node.names) for forbidden in ("raiseload", "noload"): assert forbidden not in called | imported, ( f"importer.py calls/imports {forbidden} — phase 185 LOCKED A2 " "keeps plain defer() so the summary-NULL backfill reads the " "stored content instead of raising" ) assert "defer" in called, "the fast path no longer calls defer() at all" # The importer's ONLY load_only is _prune's six-light-column row # (phase 184) — it must never reach the fast path, whose shape is # "everything except content" (LOCKED A1). prune_src = inspect.getsource(importer._prune) assert "load_only" in inspect.getsource(importer) assert "defer" not in prune_src, ( "_prune picked up a defer() — its light row is an explicit " "load_only allowlist by phase 184, not a deferral" ) @pytest.fixture() def probe_row(db: Session) -> Iterator[tuple[str, str]]: """One seeded row (raw SQL — this file pins a SELECT, not the importer) removed again after the test.""" source = f"phase185-unit-{uuid.uuid4().hex[:8]}" rel = "notes/deferred.md" db.execute( text( "INSERT INTO documents (id, source, path, full_path, title, content, " "content_hash, file_hash, indexed_at, created_at, created_at_manual, " "summary, is_image) VALUES (:id, :source, :path, :full_path, :title, " ":content, :content_hash, :file_hash, now(), now(), false, :summary, " "false)" ), { "id": uuid.uuid4(), "source": source, "path": rel, "full_path": f"/tmp/{source}/{rel}", "title": "Deferred", "content": "the stored markdown body — phase 185", "content_hash": "ch", "file_hash": "fh", "summary": None, }, ) db.commit() db.expunge_all() try: yield source, rel finally: db.execute(text("DELETE FROM documents WHERE source = :s"), {"s": source}) db.commit() db.expunge_all() def test_deferred_content_still_reads_back(db: Session, probe_row: tuple[str, str]) -> None: """A deferred column is LAZY, not gone: the row reads back whole. The trade phase 185 accepted (D2): the metadata arrives with the row, and the markdown arrives on demand for the one reader that asks for it. With ``raiseload`` this would raise; with ``noload`` it would be ``None`` — both would break the phase-118 backfill, and both are pinned out by :func:`test_no_raiseload_or_noload_anywhere_in_the_importer`. """ source, rel = probe_row doc = db.scalar(_doc_row_stmt(source, rel)) assert doc is not None assert doc.file_hash == "fh" assert doc.summary is None assert doc.content == "the stored markdown body — phase 185" # And the second read is served from the identity map, not the database. assert doc.content == "the stored markdown body — phase 185"