brain-of-reese/tests/integration/test_import_quadlet_jinja.py

"""Integration test: the phase-47 quadlet + j2 formats ride the import
machinery unchanged (walk → delta → prune).

The temp source tree holds byte-identical copies of the real fixture
files (``tests/fixtures/docs/homelab/quadlet/*.container|.volume`` and
``templates/deploy.j2``), so this test tracks the fixtures even if their
bytes change. Runs against the local compose Postgres (the ``db`` fixture
from ``tests/conftest.py``) with the deterministic ``FakeEmbedder`` —
same harness as ``test_importer_e2e.py``. A separate file (not an
extension of that one) because the delta/prune lifecycle mutates the
tree between runs, while the fixture e2e stays a single import +
idempotent re-run over the shared fixture tree.

Runs (DB must be up: ``podman compose up -d db``):

    uv run pytest tests/integration/test_import_quadlet_jinja.py -v
"""
from __future__ import annotations

import asyncio
from collections.abc import Iterator
from pathlib import Path

import pytest
from sqlalchemy import func, select
from sqlalchemy.orm import Session

from app.models import Chunk, Document
from app.rag.importer import import_sources
from tests.fakes import FakeEmbedder

FIXTURES = Path(__file__).resolve().parents[1] / "fixtures" / "docs" / "homelab"

#: Fixture files copied into the temp source tree (a ``.container``, a
#: ``.volume``, and a ``.j2``).
QUADLET_RELS = ("quadlet/compose.container", "quadlet/cache.volume")
JINJA_RELS = ("templates/deploy.j2",)
ALL_RELS = QUADLET_RELS + JINJA_RELS


def _cleanup_source(db: Session, source: str) -> None:
    for doc in db.scalars(select(Document).where(Document.source == source)).all():
        db.delete(doc)
    db.commit()


@pytest.fixture()
def source_dir(db: Session, tmp_path: Path) -> Iterator[Path]:
    """A temp source tree seeded with copies of the real fixture files."""
    root = tmp_path / "quadsrc"
    for rel in ALL_RELS:
        target = root / rel
        target.parent.mkdir(parents=True, exist_ok=True)
        target.write_bytes((FIXTURES / rel).read_bytes())
    try:
        yield root
    finally:
        _cleanup_source(db, root.name)


def test_quadlet_and_jinja_import_delta_and_prune(source_dir: Path, db: Session) -> None:
    llm = FakeEmbedder()

    # --- fresh import: all three docs land with stem titles + chunks ---
    s1 = asyncio.run(import_sources([source_dir], llm, prune=False, session=db))
    assert (s1.files, s1.added, s1.unchanged, s1.updated, s1.pruned) == (3, 3, 0, 0, 0)
    assert s1.formats == {"container": 1, "volume": 1, "j2": 1}
    assert s1.errors == 0
    docs = {
        d.path: d
        for d in db.scalars(select(Document).where(Document.source == source_dir.name)).all()
    }
    assert set(docs) == {
        "quadlet/compose.container",
        "quadlet/cache.volume",
        "templates/deploy.j2",
    }
    # Non-markdown titles come from the file stem (a leading ``#`` is a
    # comment in these formats, not a heading).
    assert docs["quadlet/compose.container"].title == "compose"
    assert docs["quadlet/cache.volume"].title == "cache"
    assert docs["templates/deploy.j2"].title == "deploy"
    for path, doc in docs.items():
        content = [c for c in doc.chunks if not c.is_summary]
        assert content, f"{path} has no content chunks"
        assert all(
            c.embedding is not None and len(c.embedding) == 768 for c in content
        ), f"{path} content chunks not embedded"
        # Phase 30 parity: the new non-markdown formats also get their
        # lite summary + one ``is_summary`` chunk.
        assert doc.summary is not None, f"{path} should have a summary"
        schunks = [c for c in doc.chunks if c.is_summary]
        assert len(schunks) == 1 and schunks[0].position == -1

    # --- delta: change only the .j2 → it updates, the others stay ---
    j2 = source_dir / "templates" / "deploy.j2"
    j2.write_text(j2.read_text(encoding="utf-8") + "\n# RESE-JINJA-DELTA-CHANGED\n")
    s2 = asyncio.run(import_sources([source_dir], llm, session=db))
    assert (s2.added, s2.updated, s2.unchanged, s2.pruned) == (0, 1, 2, 0)
    changed = db.scalar(
        select(Document).where(
            Document.source == source_dir.name, Document.path == "templates/deploy.j2"
        )
    )
    assert changed is not None and "RESE-JINJA-DELTA-CHANGED" in changed.content

    # --- idempotent: a clean re-run re-embeds nothing ---
    calls_before = len(llm.calls)
    s3 = asyncio.run(import_sources([source_dir], llm, session=db))
    assert (s3.added, s3.updated, s3.unchanged, s3.pruned) == (0, 0, 3, 0)
    assert len(llm.calls) == calls_before  # unchanged → no embedding requests

    # --- prune: delete the .volume → its row + chunks cascade away ---
    (source_dir / "quadlet" / "cache.volume").unlink()
    s4 = asyncio.run(import_sources([source_dir], llm, session=db, prune=True))
    assert (s4.unchanged, s4.pruned) == (2, 1)
    assert db.scalar(
        select(Document).where(
            Document.source == source_dir.name, Document.path == "quadlet/cache.volume"
        )
    ) is None
    # Its chunks (content + summary) are gone — FK cascade.
    assert db.scalar(
        select(func.count())
        .select_from(Chunk)
        .join(Document, Document.id == Chunk.document_id)
        .where(
            Document.source == source_dir.name,
            Document.path == "quadlet/cache.volume",
        )
    ) == 0
    # The other two docs survived the prune.
    assert db.scalar(
        select(func.count()).select_from(Document).where(Document.source == source_dir.name)
    ) == 2