brain-of-reese/tests/integration/test_importer_e2e.py

"""Integration test: importer end-to-end against ``tests/fixtures/docs/``.

Runs the real import pipeline (walk → chunk → embed → upsert) into the
local compose Postgres, then checks the DB state *and* the API shape a
browser would consume. Embeddings come from a deterministic in-process
fake, so no network is needed.

Phase 187 (task 02) pins the PLAN §9 cron-safe summary line's ONE new
trailing field (``rss_mb=``) on both the initial import and the
idempotent re-run, with every earlier field byte-identical.
"""
from __future__ import annotations

import asyncio
import logging
import re
from pathlib import Path

import pytest
from sqlalchemy import func, select, text

from app.models import Chunk, Document
from app.rag.importer import import_sources
from tests.fakes import FakeEmbedder

FIXTURES = Path(__file__).resolve().parents[1] / "fixtures" / "docs"

EXPECTED_DOCS = {
    ("docs", "homelab/kubernetes.md"),
    ("docs", "homelab/backups.md"),
    ("docs", "deployments/new-service.md"),
    ("docs", "homelab/container_gitlab/gitlab.md"),
    ("docs", "homelab/container_gitlab/gitlab-compose.yaml"),
    ("docs", "homelab/networking/static-dns.json"),
    ("docs", "homelab/scripts/uptime_probe.py"),
    ("docs", "homelab/ssh/ssh_aliases.txt"),
    ("docs", "homelab/tables.md"),  # phase 44: the markdown-tables fixture
    # phase 47 (A9 revised 2026-08-27): quadlet family + jinja fixtures
    ("docs", "homelab/quadlet/compose.container"),
    ("docs", "homelab/quadlet/lan.network"),
    ("docs", "homelab/quadlet/cache.volume"),
    ("docs", "homelab/templates/deploy.j2"),
}


#: Phase 187 (task 02): the ONE rendering :func:`app.core.memory.fmt_mb`
#: allows for an MB field — ``?`` when the platform cannot know, else a
#: positive integer (never ``None``, never a float or a negative).
MB = r"(?:\?|[1-9][0-9]*)"


def test_import_fixtures_end_to_end(
    admin_client, db, caplog: pytest.LogCaptureFixture
) -> None:
    db.execute(text("TRUNCATE chunks, documents, query_log"))
    db.commit()
    # Phase 187 (task 02): the PLAN §9 cron-safe summary line gains a
    # trailing ``rss_mb=`` — captured for BOTH runs below (the initial
    # import and the idempotent re-run), the level set once so every
    # ``import_sources`` call of this test logs at INFO.
    caplog.set_level(logging.INFO, logger="app.importer")
    llm = FakeEmbedder()

    summary = asyncio.run(import_sources([FIXTURES], llm, session=db))
    # Thirteen A9-format files (phase 44 added homelab/tables.md, phase 47
    # added the quadlet + j2 fixtures); .hidden/junk.md is out of scope
    # (A9 revised).
    assert (summary.files, summary.added, summary.unchanged) == (13, 13, 0)
    assert summary.chunks >= 13
    assert summary.formats == {
        "md": 5, "yaml": 1, "json": 1, "py": 1, "txt": 1,
        "container": 1, "network": 1, "volume": 1, "j2": 1,  # phase 47
    }
    # PLAN §9 per-format summary line: highest count first, then alpha.
    assert summary.format_counts() == (
        "md:5,container:1,j2:1,json:1,network:1,py:1,txt:1,volume:1,yaml:1"
    )

    docs = db.scalars(select(Document)).all()
    assert {(d.source, d.path) for d in docs} == EXPECTED_DOCS
    titles = {d.path: d.title for d in docs}
    assert titles["homelab/kubernetes.md"] == "Kubernetes Homelab Cluster"
    assert titles["deployments/new-service.md"] == "Deploying a New Service"
    assert titles["homelab/container_gitlab/gitlab.md"] == "Gitlab"
    # Non-markdown titles come from the file stem (a leading ``#`` or docstring
    # line is a comment there, not a heading).
    assert titles["homelab/container_gitlab/gitlab-compose.yaml"] == "gitlab-compose"
    assert titles["homelab/scripts/uptime_probe.py"] == "uptime_probe"
    assert titles["homelab/networking/static-dns.json"] == "static-dns"
    assert titles["homelab/ssh/ssh_aliases.txt"] == "ssh_aliases"
    assert titles["homelab/quadlet/compose.container"] == "compose"
    assert titles["homelab/quadlet/lan.network"] == "lan"
    assert titles["homelab/quadlet/cache.volume"] == "cache"
    assert titles["homelab/templates/deploy.j2"] == "deploy"
    # Hidden junk was never imported.
    assert not any(".hidden" in d.path for d in docs)
    # Full content is stored — that is what the RAG context will be.
    k8s = next(d for d in docs if d.path == "homelab/kubernetes.md")
    assert "Talos Linux" in k8s.content and k8s.content_hash

    # Phase 30; phase 118 (A2): EVERY fixture gained one embedded
    # ``is_summary`` chunk (markdown included), so the DB holds content +
    # summary chunks.
    n_chunks = db.scalar(select(func.count()).select_from(Chunk))
    assert n_chunks == summary.chunks + summary.summaries
    for c in db.scalars(select(Chunk)).all():
        assert c.embedding is not None and len(c.embedding) == 768

    assert summary.summary_errors == 0
    for d in docs:
        # Phase 118 (A2): EVERY document — markdown included — carries a
        # stored summary + exactly one embedded summary chunk (−1).
        schunks = [c for c in d.chunks if c.is_summary]
        assert d.summary is not None, f"{d.path} should have a summary"
        assert len(schunks) == 1
        assert schunks[0].position == -1
        assert schunks[0].content == d.summary
        assert schunks[0].embedding is not None
    assert summary.summaries == len(docs)  # one per doc, no exceptions

    # The Sources page consumes exactly this shape.
    r = admin_client.get("/api/docs")  # phase 16: the catalog is admin-only
    assert r.status_code == 200
    body = r.json()
    assert len(body["documents"]) == 13
    assert all(d["chunks"] >= 1 for d in body["documents"])

    # Idempotent re-run: nothing re-embedded.
    calls_before = len(llm.calls)
    s2 = asyncio.run(import_sources([FIXTURES], llm, session=db))
    assert s2.unchanged == 13 and s2.added == 0
    assert len(llm.calls) == calls_before  # unchanged → no embedding requests

    # Phase 187 (task 02): every run's summary line ends with the ONE
    # appended memory field, and the pre-phase fields are byte-identical
    # (LOCKED A2 — append-only, so every existing grep keeps working).
    summary_lines = [
        r.getMessage()
        for r in caplog.records
        if r.getMessage().startswith("import: summary files=")
    ]
    assert len(summary_lines) == 2, summary_lines
    for line in summary_lines:
        assert re.search(rf" rss_mb={MB}$", line), line
    assert summary_lines[0].startswith(
        "import: summary files=13 added=13 updated=0 unchanged=0 pruned=0 errors=0 "
    )
    assert re.search(
        rf"formats=md:5,container:1,j2:1,json:1,network:1,py:1,txt:1,volume:1,yaml:1"
        rf" rss_mb={MB}$",
        summary_lines[0],
    ), summary_lines[0]
    assert summary_lines[1].startswith(
        "import: summary files=13 added=0 updated=0 unchanged=13 pruned=0 errors=0 "
    )

    db.execute(text("TRUNCATE chunks, documents, query_log"))
    db.commit()