"""Integration test: the phase-47 quadlet + j2 formats ride the import machinery unchanged (walk → delta → prune). The temp source tree holds byte-identical copies of the real fixture files (``tests/fixtures/docs/homelab/quadlet/*.container|.volume`` and ``templates/deploy.j2``), so this test tracks the fixtures even if their bytes change. Runs against the local compose Postgres (the ``db`` fixture from ``tests/conftest.py``) with the deterministic ``FakeEmbedder`` — same harness as ``test_importer_e2e.py``. A separate file (not an extension of that one) because the delta/prune lifecycle mutates the tree between runs, while the fixture e2e stays a single import + idempotent re-run over the shared fixture tree. Runs (DB must be up: ``podman compose up -d db``): uv run pytest tests/integration/test_import_quadlet_jinja.py -v """ from __future__ import annotations import asyncio from collections.abc import Iterator from pathlib import Path import pytest from sqlalchemy import func, select from sqlalchemy.orm import Session from app.models import Chunk, Document from app.rag.importer import import_sources from tests.fakes import FakeEmbedder FIXTURES = Path(__file__).resolve().parents[1] / "fixtures" / "docs" / "homelab" #: Fixture files copied into the temp source tree (a ``.container``, a #: ``.volume``, and a ``.j2``). QUADLET_RELS = ("quadlet/compose.container", "quadlet/cache.volume") JINJA_RELS = ("templates/deploy.j2",) ALL_RELS = QUADLET_RELS + JINJA_RELS def _cleanup_source(db: Session, source: str) -> None: for doc in db.scalars(select(Document).where(Document.source == source)).all(): db.delete(doc) db.commit() @pytest.fixture() def source_dir(db: Session, tmp_path: Path) -> Iterator[Path]: """A temp source tree seeded with copies of the real fixture files.""" root = tmp_path / "quadsrc" for rel in ALL_RELS: target = root / rel target.parent.mkdir(parents=True, exist_ok=True) target.write_bytes((FIXTURES / rel).read_bytes()) try: yield root finally: _cleanup_source(db, root.name) def test_quadlet_and_jinja_import_delta_and_prune(source_dir: Path, db: Session) -> None: llm = FakeEmbedder() # --- fresh import: all three docs land with stem titles + chunks --- s1 = asyncio.run(import_sources([source_dir], llm, prune=False, session=db)) assert (s1.files, s1.added, s1.unchanged, s1.updated, s1.pruned) == (3, 3, 0, 0, 0) assert s1.formats == {"container": 1, "volume": 1, "j2": 1} assert s1.errors == 0 docs = { d.path: d for d in db.scalars(select(Document).where(Document.source == source_dir.name)).all() } assert set(docs) == { "quadlet/compose.container", "quadlet/cache.volume", "templates/deploy.j2", } # Non-markdown titles come from the file stem (a leading ``#`` is a # comment in these formats, not a heading). assert docs["quadlet/compose.container"].title == "compose" assert docs["quadlet/cache.volume"].title == "cache" assert docs["templates/deploy.j2"].title == "deploy" for path, doc in docs.items(): content = [c for c in doc.chunks if not c.is_summary] assert content, f"{path} has no content chunks" assert all( c.embedding is not None and len(c.embedding) == 768 for c in content ), f"{path} content chunks not embedded" # Phase 30 parity: the new non-markdown formats also get their # lite summary + one ``is_summary`` chunk. assert doc.summary is not None, f"{path} should have a summary" schunks = [c for c in doc.chunks if c.is_summary] assert len(schunks) == 1 and schunks[0].position == -1 # --- delta: change only the .j2 → it updates, the others stay --- j2 = source_dir / "templates" / "deploy.j2" j2.write_text(j2.read_text(encoding="utf-8") + "\n# RESE-JINJA-DELTA-CHANGED\n") s2 = asyncio.run(import_sources([source_dir], llm, session=db)) assert (s2.added, s2.updated, s2.unchanged, s2.pruned) == (0, 1, 2, 0) changed = db.scalar( select(Document).where( Document.source == source_dir.name, Document.path == "templates/deploy.j2" ) ) assert changed is not None and "RESE-JINJA-DELTA-CHANGED" in changed.content # --- idempotent: a clean re-run re-embeds nothing --- calls_before = len(llm.calls) s3 = asyncio.run(import_sources([source_dir], llm, session=db)) assert (s3.added, s3.updated, s3.unchanged, s3.pruned) == (0, 0, 3, 0) assert len(llm.calls) == calls_before # unchanged → no embedding requests # --- prune: delete the .volume → its row + chunks cascade away --- (source_dir / "quadlet" / "cache.volume").unlink() s4 = asyncio.run(import_sources([source_dir], llm, session=db, prune=True)) assert (s4.unchanged, s4.pruned) == (2, 1) assert db.scalar( select(Document).where( Document.source == source_dir.name, Document.path == "quadlet/cache.volume" ) ) is None # Its chunks (content + summary) are gone — FK cascade. assert db.scalar( select(func.count()) .select_from(Chunk) .join(Document, Document.id == Chunk.document_id) .where( Document.source == source_dir.name, Document.path == "quadlet/cache.volume", ) ) == 0 # The other two docs survived the prune. assert db.scalar( select(func.count()).select_from(Document).where(Document.source == source_dir.name) ) == 2