"""Integration: phase-89 ignore paths through the real import pipeline. Phase 89 (TODO.md L3): per-source ignore lists — source-relative path prefixes that are never walked, hence never embedded and never summarized (A1), and previously indexed files that newly match a pattern are pruned on the next ``prune=True`` run (A2). Mirrors the fixture-tree + mock-LLM pattern of ``test_importer_e2e.py``: a ``tmp_path`` source dir named ``IgnoreFix`` run through the real ``import_sources`` into the compose Postgres, with :class:`tests.fakes.FakeEmbedder` as the deterministic LLM stand-in. """ from __future__ import annotations import asyncio from pathlib import Path from sqlalchemy import select, text from sqlalchemy.orm import Session from app.models import Chunk, Document from app.rag.importer import import_sources from tests.fakes import FakeEmbedder NAME = "IgnoreFix" def _write(root: Path, rel: str, content: str) -> None: path = root / rel path.parent.mkdir(parents=True, exist_ok=True) path.write_text(content, encoding="utf-8") def _tree(tmp_path: Path, name: str = NAME) -> Path: """The fixture tree: two kept files + one md and one txt under ``ignore/``.""" root = tmp_path / name _write(root, "keep.md", "# Keep\n\nkept body\n") _write(root, "ignore/secret.md", "# Secret\n\nSECRET-CONTENT\n") _write(root, "ignore/notes.txt", "IGNORED-TEXT-CONTENT\n") _write(root, "top.txt", "TOP-TEXT-CONTENT\n") return root def _reset(db: Session) -> None: # House cleanup pattern (tests/integration/test_importer_e2e.py). db.execute(text("TRUNCATE chunks, documents, query_log")) db.commit() def test_ignored_files_never_indexed(db: Session, tmp_path: Path) -> None: _reset(db) root = _tree(tmp_path) llm = FakeEmbedder() summary = asyncio.run( import_sources( [root], llm, session=db, ignore_by_root={str(root): ["ignore/"]} ) ) # Only the two kept files are walked — the ignore/ subtree is # invisible to the pipeline. assert summary.files == 2 assert summary.errors == 0 # The kept non-markdown file IS summarized; the ignored .txt is not. assert summary.summaries == 1 assert summary.summary_errors == 0 docs = db.scalars(select(Document)).all() assert {(d.source, d.path) for d in docs} == { (NAME, "keep.md"), (NAME, "top.txt"), } # No documents row for an ignored file — hence no chunks rows for it, # no embedding call, and no summary column value, by construction. for rel in ("ignore/secret.md", "ignore/notes.txt"): assert not any(d.path == rel for d in docs) assert not any("SECRET-CONTENT" in c.content for c in db.scalars(select(Chunk)).all()) for texts in llm.calls: # every embed batch assert not any( "SECRET-CONTENT" in t or "IGNORED-TEXT-CONTENT" in t for t in texts ) # Exactly one summary call (top.txt) — the ignored files never reached # the lite model. assert len(llm.chat_calls) == 1 user = next(m["content"] for m in llm.chat_calls[0] if m["role"] == "user") assert "TOP-TEXT-CONTENT" in user assert "SECRET-CONTENT" not in user and "IGNORED-TEXT-CONTENT" not in user top = next(d for d in docs if d.path == "top.txt") assert top.summary is not None _reset(db) def test_newly_ignored_file_pruned_on_next_prune_run(db: Session, tmp_path: Path) -> None: _reset(db) root = tmp_path / NAME _write(root, "keep.md", "# Keep\n\nkept body\n") _write(root, "top.txt", "TOP-TEXT-CONTENT\n") # Exactly ONE file under ignore/ so the A2 prune count pins it. _write(root, "ignore/secret.md", "# Secret\n\nSECRET-CONTENT\n") llm = FakeEmbedder() # First run — no map (omitted entirely): everything is indexed, # including ignore/secret.md. s1 = asyncio.run(import_sources([root], llm, session=db)) assert (s1.files, s1.added) == (3, 3) secret = db.scalar(select(Document).where(Document.path == "ignore/secret.md")) assert secret is not None # Second run — the owner adds "ignore" (no trailing slash: A1 # normalization) and prunes. The file newly matches, never enters # ``seen``, and leaves the index (A2 — the A9 junk-precedent). s2 = asyncio.run( import_sources( [root], llm, session=db, prune=True, ignore_by_root={str(root): ["ignore"]}, ) ) assert s2.files == 2 assert s2.pruned == 1 assert ( db.scalar(select(Document).where(Document.path == "ignore/secret.md")) is None ) _reset(db) def test_progress_total_excludes_ignored(db: Session, tmp_path: Path) -> None: _reset(db) root = _tree(tmp_path) llm = FakeEmbedder() calls: list[tuple[str, str, int, int]] = [] def progress(source: str, rel: str, done: int, total: int) -> None: calls.append((source, rel, done, total)) summary = asyncio.run( import_sources( [root], llm, session=db, progress=progress, ignore_by_root={str(root): ["ignore/"]}, ) ) assert summary.files == 2 # The phase-64 pre-walk uses the same per-root tuple as the loop: # ``total`` counts ONLY the non-ignored files, and the hook fired # exactly once per imported file. assert [c[2] for c in calls] == [1, 2] # done assert {c[3] for c in calls} == {2} # total — never counts ignored files assert {c[1] for c in calls} == {"keep.md", "top.txt"} _reset(db) def test_no_map_behavior_is_byte_identical(db: Session, tmp_path: Path) -> None: _reset(db) root = _tree(tmp_path) llm = FakeEmbedder() # ``ignore_by_root=None`` (the default): all four files import exactly # as pre-phase-89 callers see them. summary = asyncio.run(import_sources([root], llm, session=db, ignore_by_root=None)) assert summary.files == 4 docs = db.scalars(select(Document)).all() assert {(d.source, d.path) for d in docs} == { (NAME, "keep.md"), (NAME, "top.txt"), (NAME, "ignore/secret.md"), (NAME, "ignore/notes.txt"), } _reset(db) def test_unlisted_source_unaffected(db: Session, tmp_path: Path) -> None: _reset(db) root_a = _tree(tmp_path, name="IgnoreFixA") root_b = tmp_path / "IgnoreFixB" _write(root_b, "one.md", "# One\n\none body\n") _write(root_b, "two.txt", "TWO-TEXT-CONTENT\n") llm = FakeEmbedder() # The map keys ONLY the first root — the second imports everything. summary = asyncio.run( import_sources( [root_a, root_b], llm, session=db, ignore_by_root={str(root_a): ["ignore"]}, ) ) # A: keep.md + top.txt (the whole ignore/ subtree is dropped) · B: one.md + two.txt assert summary.files == 4 docs = db.scalars(select(Document)).all() assert {(d.source, d.path) for d in docs} == { ("IgnoreFixA", "keep.md"), ("IgnoreFixA", "top.txt"), ("IgnoreFixB", "one.md"), ("IgnoreFixB", "two.txt"), } assert not any(d.path == "ignore/secret.md" for d in docs) _reset(db)