brain-of-reese/tests/integration/test_importer_ignore.py

"""Integration: phase-89 ignore paths through the real import pipeline.

Phase 89 (TODO.md L3): per-source ignore lists — source-relative path
prefixes that are never walked, hence never embedded and never
summarized (A1), and previously indexed files that newly match a
pattern are pruned on the next ``prune=True`` run (A2). Mirrors the
fixture-tree + mock-LLM pattern of ``test_importer_e2e.py``: a
``tmp_path`` source dir named ``IgnoreFix`` run through the real
``import_sources`` into the compose Postgres, with
:class:`tests.fakes.FakeEmbedder` as the deterministic LLM stand-in.
"""
from __future__ import annotations

import asyncio
from pathlib import Path

from sqlalchemy import select, text
from sqlalchemy.orm import Session

from app.models import Chunk, Document
from app.rag.importer import import_sources
from tests.fakes import FakeEmbedder

NAME = "IgnoreFix"


def _write(root: Path, rel: str, content: str) -> None:
    path = root / rel
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_text(content, encoding="utf-8")


def _tree(tmp_path: Path, name: str = NAME) -> Path:
    """The fixture tree: two kept files + one md and one txt under ``ignore/``."""
    root = tmp_path / name
    _write(root, "keep.md", "# Keep\n\nkept body\n")
    _write(root, "ignore/secret.md", "# Secret\n\nSECRET-CONTENT\n")
    _write(root, "ignore/notes.txt", "IGNORED-TEXT-CONTENT\n")
    _write(root, "top.txt", "TOP-TEXT-CONTENT\n")
    return root


def _reset(db: Session) -> None:
    # House cleanup pattern (tests/integration/test_importer_e2e.py).
    db.execute(text("TRUNCATE chunks, documents, query_log"))
    db.commit()


def test_ignored_files_never_indexed(db: Session, tmp_path: Path) -> None:
    _reset(db)
    root = _tree(tmp_path)
    llm = FakeEmbedder()

    summary = asyncio.run(
        import_sources(
            [root], llm, session=db, ignore_by_root={str(root): ["ignore/"]}
        )
    )
    # Only the two kept files are walked — the ignore/ subtree is
    # invisible to the pipeline.
    assert summary.files == 2
    assert summary.errors == 0
    # The kept non-markdown file IS summarized; the ignored .txt is not.
    assert summary.summaries == 1
    assert summary.summary_errors == 0

    docs = db.scalars(select(Document)).all()
    assert {(d.source, d.path) for d in docs} == {
        (NAME, "keep.md"),
        (NAME, "top.txt"),
    }
    # No documents row for an ignored file — hence no chunks rows for it,
    # no embedding call, and no summary column value, by construction.
    for rel in ("ignore/secret.md", "ignore/notes.txt"):
        assert not any(d.path == rel for d in docs)
    assert not any("SECRET-CONTENT" in c.content for c in db.scalars(select(Chunk)).all())
    for texts in llm.calls:  # every embed batch
        assert not any(
            "SECRET-CONTENT" in t or "IGNORED-TEXT-CONTENT" in t for t in texts
        )
    # Exactly one summary call (top.txt) — the ignored files never reached
    # the lite model.
    assert len(llm.chat_calls) == 1
    user = next(m["content"] for m in llm.chat_calls[0] if m["role"] == "user")
    assert "TOP-TEXT-CONTENT" in user
    assert "SECRET-CONTENT" not in user and "IGNORED-TEXT-CONTENT" not in user
    top = next(d for d in docs if d.path == "top.txt")
    assert top.summary is not None
    _reset(db)


def test_newly_ignored_file_pruned_on_next_prune_run(db: Session, tmp_path: Path) -> None:
    _reset(db)
    root = tmp_path / NAME
    _write(root, "keep.md", "# Keep\n\nkept body\n")
    _write(root, "top.txt", "TOP-TEXT-CONTENT\n")
    # Exactly ONE file under ignore/ so the A2 prune count pins it.
    _write(root, "ignore/secret.md", "# Secret\n\nSECRET-CONTENT\n")
    llm = FakeEmbedder()

    # First run — no map (omitted entirely): everything is indexed,
    # including ignore/secret.md.
    s1 = asyncio.run(import_sources([root], llm, session=db))
    assert (s1.files, s1.added) == (3, 3)
    secret = db.scalar(select(Document).where(Document.path == "ignore/secret.md"))
    assert secret is not None

    # Second run — the owner adds "ignore" (no trailing slash: A1
    # normalization) and prunes. The file newly matches, never enters
    # ``seen``, and leaves the index (A2 — the A9 junk-precedent).
    s2 = asyncio.run(
        import_sources(
            [root],
            llm,
            session=db,
            prune=True,
            ignore_by_root={str(root): ["ignore"]},
        )
    )
    assert s2.files == 2
    assert s2.pruned == 1
    assert (
        db.scalar(select(Document).where(Document.path == "ignore/secret.md")) is None
    )
    _reset(db)


def test_progress_total_excludes_ignored(db: Session, tmp_path: Path) -> None:
    _reset(db)
    root = _tree(tmp_path)
    llm = FakeEmbedder()
    calls: list[tuple[str, str, int, int]] = []

    def progress(source: str, rel: str, done: int, total: int) -> None:
        calls.append((source, rel, done, total))

    summary = asyncio.run(
        import_sources(
            [root],
            llm,
            session=db,
            progress=progress,
            ignore_by_root={str(root): ["ignore/"]},
        )
    )
    assert summary.files == 2
    # The phase-64 pre-walk uses the same per-root tuple as the loop:
    # ``total`` counts ONLY the non-ignored files, and the hook fired
    # exactly once per imported file.
    assert [c[2] for c in calls] == [1, 2]  # done
    assert {c[3] for c in calls} == {2}  # total — never counts ignored files
    assert {c[1] for c in calls} == {"keep.md", "top.txt"}
    _reset(db)


def test_no_map_behavior_is_byte_identical(db: Session, tmp_path: Path) -> None:
    _reset(db)
    root = _tree(tmp_path)
    llm = FakeEmbedder()

    # ``ignore_by_root=None`` (the default): all four files import exactly
    # as pre-phase-89 callers see them.
    summary = asyncio.run(import_sources([root], llm, session=db, ignore_by_root=None))
    assert summary.files == 4
    docs = db.scalars(select(Document)).all()
    assert {(d.source, d.path) for d in docs} == {
        (NAME, "keep.md"),
        (NAME, "top.txt"),
        (NAME, "ignore/secret.md"),
        (NAME, "ignore/notes.txt"),
    }
    _reset(db)


def test_unlisted_source_unaffected(db: Session, tmp_path: Path) -> None:
    _reset(db)
    root_a = _tree(tmp_path, name="IgnoreFixA")
    root_b = tmp_path / "IgnoreFixB"
    _write(root_b, "one.md", "# One\n\none body\n")
    _write(root_b, "two.txt", "TWO-TEXT-CONTENT\n")
    llm = FakeEmbedder()

    # The map keys ONLY the first root — the second imports everything.
    summary = asyncio.run(
        import_sources(
            [root_a, root_b],
            llm,
            session=db,
            ignore_by_root={str(root_a): ["ignore"]},
        )
    )
    # A: keep.md + top.txt (the whole ignore/ subtree is dropped) · B: one.md + two.txt
    assert summary.files == 4
    docs = db.scalars(select(Document)).all()
    assert {(d.source, d.path) for d in docs} == {
        ("IgnoreFixA", "keep.md"),
        ("IgnoreFixA", "top.txt"),
        ("IgnoreFixB", "one.md"),
        ("IgnoreFixB", "two.txt"),
    }
    assert not any(d.path == "ignore/secret.md" for d in docs)
    _reset(db)