"""Integration: the sync-time folder-summary wiring (phase 94, task 02). Extends the KB-overview sync pattern (``test_import_docs_overview.py`` is the template) to the phase-94 folder summaries — both sync paths regenerate ``folder_summaries`` change-gated (added + updated > 0) or on an empty table (the first full run after migration 0017 / a ``--limit`` first walk), per-folder fail-soft, best-effort, and in the run's own short-lived session (the phase-53 flush-then-caller-commits convention — the generator only flushes, the sync path commits). Script path (``scripts.import_docs.main`` end to end, fake LLM, real DB, explicit ``--source``): - a KB-changing import → one row per ≥ 2-doc subtree (the source root + the 2-doc folder; the 1-doc folder gets none), committed in the run's transaction, the summary line ending ``folder_summaries=//``; - an unchanged re-import → zero ``lite`` calls, ``folder_summaries=skipped``, rows untouched; - a subtree dropping below 2 docs after a changed re-walk → its row pruned; - one folder's ``lite`` failure → its previous row kept, the other lands, exit code 0, stats ``1/1/0``; - a ``--limit`` debug run → no generation, no rows, ``folder_summaries=skipped``; - a fresh (empty) table after a ``--limit`` first walk → an unchanged full walk generates (the table-empty first-run trigger). API path (``POST /api/sync`` end to end, real import over a host temp local dir, deterministic ``FakeEmbedder``): - a KB-changing sync → the rows land (visible via the test's own session) and the status detail keeps its exact pre-phase key set (no folder-summary surface — the stats are log-only); - an unchanged re-sync → zero ``FOLDER_SUMMARY_MODE`` calls; - a ``lite`` outage (one folder failing) → the failed folder's row is kept, the run reports ``success`` (never ``failed``), and the sources-version bump still lands (the bump is change-gated on the KB, not on the summaries); - an empty table after a populated sync (the migration-0017 scenario) → an unchanged walk regenerates (the overview's API gate, purely change-gated, does not). """ from __future__ import annotations import logging import time from collections.abc import Iterator from datetime import datetime from pathlib import Path import pytest from fastapi.testclient import TestClient from sqlalchemy import select, text from sqlalchemy.orm import Session from app.api import sync as sync_api from app.config import Settings from app.db import SessionLocal, db_available from app.main import app as fastapi_app from app.models import FolderSummary, GitSource from app.rag import git_sources as git_sources_resolver from app.rag.llm import LLMError from app.rag.sources_meta import current_sources_version from scripts import import_docs from tests.conftest import ADMIN_PASSWORD from tests.fakes import FakeEmbedder # --- shared helpers --------------------------------------------------------- def _folder_calls(llm: FakeEmbedder) -> list[list[dict[str, str]]]: """The ``FOLDER_SUMMARY_MODE`` chat calls on the client (the marker in the system prompt — the KB-overview marker never contains it).""" return [ msgs for msgs in llm.chat_calls if any( "FOLDER_SUMMARY_MODE" in m.get("content", "") for m in msgs if m.get("role") == "system" ) ] def _rows(db: Session) -> dict[tuple[str, str], str]: """The stored folder summaries as ``{(source, folder_path): summary}``.""" db.expire_all() return { (source, folder_path): summary for source, folder_path, summary in db.execute( select(FolderSummary.source, FolderSummary.folder_path, FolderSummary.summary) ).all() } def _updated_at(db: Session, source: str, folder_path: str) -> datetime | None: db.expire_all() row = db.get(FolderSummary, (source, folder_path)) return row.updated_at if row is not None else None class FolderFailingChatEmbedder(FakeEmbedder): """``lite`` stand-in that fails exactly one folder-summary call — the one whose user message contains *fail_label* (a ``Folder: …`` header) — and answers everything else (incl. the KB overview) normally. Drives the per-folder fail-soft path.""" def __init__(self, fail_label: str) -> None: super().__init__() self.fail_label = fail_label async def chat( self, messages: list[dict[str, str]], model: str | None = None ) -> str: user = next((m["content"] for m in messages if m.get("role") == "user"), "") if self.fail_label in user: self.chat_calls.append(list(messages)) raise LLMError("simulated folder-summary outage (test sentinel)") return await super().chat(messages, model) # --- fixtures ---------------------------------------------------------------- @pytest.fixture(autouse=True) def _clean_kb(db: Session) -> Iterator[None]: """Global KB + registry state, truncated around every test (the real import and the real generator write ``documents``/``chunks``/ ``folder_summaries``; the API tests seed ``git_sources``).""" db.execute( text("TRUNCATE chunks, documents, kb_overview, folder_summaries, git_sources") ) db.commit() yield db.execute( text("TRUNCATE chunks, documents, kb_overview, folder_summaries, git_sources") ) db.commit() @pytest.fixture(autouse=True) def _reset_sources_version() -> Iterator[None]: """The sources version counter is global mutable state — pin it to the migration-0010 seed (0) around every test (own session: both sync paths bump through their own short-lived ``SessionLocal``). Skips like the ``db`` fixture when Postgres is down.""" if not db_available(): pytest.skip("Postgres not reachable — run `podman compose up -d db` first") session = SessionLocal() try: session.execute(text("UPDATE sources_meta SET version = 0 WHERE id = 1")) session.commit() yield finally: session.execute(text("UPDATE sources_meta SET version = 0 WHERE id = 1")) session.commit() session.close() @pytest.fixture(autouse=True) def _fresh_sync_state() -> Iterator[None]: """The module-level status object + task are process-global: reset them around every test (harmless for the script-path tests).""" sync_api._status = sync_api.SyncStatus() sync_api._task = None yield sync_api._status = sync_api.SyncStatus() sync_api._task = None @pytest.fixture() def src(tmp_path: Path) -> Path: """MyDocs: a/ (2 docs) + b/ (1 doc) → subtree counts root 3, a 2, b 1. md → no document-summary chat calls, so the ``lite`` traffic is exactly the overview + the folder summaries.""" root = tmp_path / "MyDocs" (root / "a").mkdir(parents=True) (root / "b").mkdir() (root / "a" / "one.md").write_text("# A One\nFirst folder document.\n", encoding="utf-8") (root / "a" / "two.md").write_text("# A Two\nSecond folder document.\n", encoding="utf-8") (root / "b" / "three.md").write_text("# B Three\nLone folder document.\n", encoding="utf-8") return root def _run_main( monkeypatch: pytest.MonkeyPatch, llm: FakeEmbedder, argv: list[str], capsys: pytest.CaptureFixture[str], ) -> tuple[int, str]: """Run ``import_docs.main`` with fresh settings, a fake LLM, and a fail-loud git mock (``--source`` always wins, so git must stay idle) — the phase-31 overview test's runner.""" monkeypatch.setattr( import_docs, "get_settings", lambda: Settings(_env_file=None), # pyright: ignore[reportCallIssue] ) def _no_git(url: str, dest: Path | str) -> Path: raise AssertionError("git sync must not run with explicit --source") monkeypatch.setattr(import_docs, "clone_or_pull", _no_git) monkeypatch.setattr(import_docs, "LLMClient", lambda: llm) rc = import_docs.main(argv) return rc, capsys.readouterr().out # --- script path: scripts/import_docs.py ------------------------------------- def test_changed_import_generates_folder_rows( db: Session, src: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], ) -> None: """A KB-changing import upserts one row per ≥ 2-doc subtree in the run's own transaction — committed and visible afterwards — with the stats on the summary line (PLAN §9).""" llm = FakeEmbedder() records: list[logging.LogRecord] = [] class _Sink(logging.Handler): def emit(self, record: logging.LogRecord) -> None: records.append(record) fs_logger = logging.getLogger("app.rag.folder_summaries") sink = _Sink() fs_logger.addHandler(sink) fs_logger.setLevel(logging.INFO) try: rc, out = _run_main(monkeypatch, llm, ["--source", str(src)], capsys) finally: fs_logger.removeHandler(sink) assert rc == 0 assert "added=3" in out assert out.rstrip().endswith( "overview=updated sources_version=1 folder_summaries=2/0/0" ) # Two FOLDER_SUMMARY_MODE calls — the source root (3 docs) and the # 2-doc folder a; the 1-doc folder b gets no row (no lite burn). calls = _folder_calls(llm) assert len(calls) == 2 headers = [c[1]["content"].splitlines()[0] for c in calls] assert headers == ["Folder: MyDocs", "Folder: MyDocs/a"] for c in calls: assert "FOLDER_SUMMARY_MODE" in c[0]["content"] # The rows land committed (the run's own session committed them) — # one per ≥ 2-doc subtree, never empty, stamped. rows = _rows(db) assert set(rows) == {("MyDocs", ""), ("MyDocs", "a")} assert all(rows.values()) assert _updated_at(db, "MyDocs", "a") is not None # The stats log line (PLAN §9 ample logging). assert any( "folder_summaries: generated=2 failed=0 pruned=0" in r.getMessage() for r in records ) def test_unchanged_reimport_burns_zero_folder_calls( db: Session, src: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], ) -> None: """Same hashes → no KB change → the populated table stays untouched and zero ``lite`` calls burn (overview AND folder summaries).""" llm1 = FakeEmbedder() rc, out = _run_main(monkeypatch, llm1, ["--source", str(src)], capsys) assert rc == 0 assert out.rstrip().endswith( "overview=updated sources_version=1 folder_summaries=2/0/0" ) rows = _rows(db) assert set(rows) == {("MyDocs", ""), ("MyDocs", "a")} llm2 = FakeEmbedder() rc, out = _run_main(monkeypatch, llm2, ["--source", str(src)], capsys) assert rc == 0 assert "unchanged=3" in out assert out.rstrip().endswith( "overview=skipped sources_version=skipped folder_summaries=skipped" ) assert llm2.chat_calls == [] # zero lite calls, any mode assert _rows(db) == rows # rows byte-identical def test_subtree_dropping_below_two_docs_is_pruned( db: Session, src: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], ) -> None: """A subtree that drops below the ≥ 2 rule on a changed re-walk loses its (stale) row; the remaining qualifiers regenerate.""" llm1 = FakeEmbedder() rc, _ = _run_main(monkeypatch, llm1, ["--source", str(src)], capsys) assert rc == 0 assert set(_rows(db)) == {("MyDocs", ""), ("MyDocs", "a")} # a/ loses one of its two docs (below the ≥ 2 rule) while b's doc # changes → the re-walk is a KB change, so generation runs and the # stale a/ row is pruned; b (1 doc) still gets no row. (src / "a" / "two.md").unlink() (src / "b" / "three.md").write_text("# B Three\nChanged content.\n", encoding="utf-8") llm2 = FakeEmbedder() rc, out = _run_main(monkeypatch, llm2, ["--source", str(src), "--prune"], capsys) assert rc == 0 assert "pruned=1" in out assert "updated=1" in out assert out.rstrip().endswith( "overview=updated sources_version=2 folder_summaries=1/0/1" ) # Only the source root (the 2 remaining docs) still qualifies. assert set(_rows(db)) == {("MyDocs", "")} assert _updated_at(db, "MyDocs", "a") is None # the row is gone def test_folder_lite_failure_keeps_previous_row_and_stays_green( db: Session, src: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], ) -> None: """One folder's ``lite`` failure: its previous row is kept, the other folder lands, and the run's exit code stays 0 — the stats carry the failure (``1/1/0``).""" llm1 = FakeEmbedder() rc, _ = _run_main(monkeypatch, llm1, ["--source", str(src)], capsys) assert rc == 0 rows_before = _rows(db) a_stamp_before = _updated_at(db, "MyDocs", "a") root_stamp_before = _updated_at(db, "MyDocs", "") assert a_stamp_before is not None and root_stamp_before is not None # a/ changes (a KB change → generation runs); lite fails for a/ # only — its previous row must stay, the root row must land. (src / "a" / "one.md").write_text("# A One\nChanged content.\n", encoding="utf-8") llm2 = FolderFailingChatEmbedder(fail_label="Folder: MyDocs/a") rc, out = _run_main(monkeypatch, llm2, ["--source", str(src)], capsys) assert rc == 0 # a failed folder must not fail the import assert "updated=1" in out assert out.rstrip().endswith( "overview=updated sources_version=2 folder_summaries=1/1/0" ) rows_after = _rows(db) assert rows_after["MyDocs", "a"] == rows_before["MyDocs", "a"] # kept assert _updated_at(db, "MyDocs", "a") == a_stamp_before # untouched root_stamp_after = _updated_at(db, "MyDocs", "") assert root_stamp_after is not None and root_stamp_after > root_stamp_before def test_limit_run_skips_folder_generation( db: Session, src: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], ) -> None: """A ``--limit`` debug run (an incomplete walk) never generates — no ``lite`` call at all, no rows, the line says ``skipped``.""" llm = FakeEmbedder() rc, out = _run_main(monkeypatch, llm, ["--source", str(src), "--limit", "2"], capsys) assert rc == 0 assert "added=2" in out assert out.rstrip().endswith( "overview=skipped sources_version=skipped folder_summaries=skipped" ) assert llm.chat_calls == [] # no lite call, any mode assert _rows(db) == {} # an incomplete walk never writes rows def test_empty_table_generates_on_unchanged_walk( db: Session, src: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], ) -> None: """The table-empty first-run trigger: after a ``--limit`` first walk (populated KB, empty table), an unchanged full walk generates — for the folder summaries AND the missing outline, still never bumping the version.""" llm1 = FakeEmbedder() rc, out = _run_main(monkeypatch, llm1, ["--source", str(src), "--limit", "3"], capsys) assert rc == 0 assert "added=3" in out assert out.rstrip().endswith( "overview=skipped sources_version=skipped folder_summaries=skipped" ) assert _rows(db) == {} llm2 = FakeEmbedder() rc, out = _run_main(monkeypatch, llm2, ["--source", str(src)], capsys) assert rc == 0 assert "unchanged=3" in out assert out.rstrip().endswith( "overview=updated sources_version=skipped folder_summaries=2/0/0" ) assert set(_rows(db)) == {("MyDocs", ""), ("MyDocs", "a")} assert len(_folder_calls(llm2)) == 2 assert current_sources_version(db) == 0 # the unchanged walk never bumps # --- API path: POST /api/sync ------------------------------------------------- @pytest.fixture() def sync_client() -> Iterator[TestClient]: """Context-managed TestClient — one app event loop across requests (the background task must survive between the POST and the polls).""" with TestClient(fastapi_app) as client: yield client def _settings(sources_dir: str) -> Settings: return Settings(_env_file=None, sources_dir=sources_dir) # pyright: ignore[reportCallIssue] def _stub_env(monkeypatch: pytest.MonkeyPatch, git_sources: str = "") -> None: """The resolver's env fallback, driven by a fresh ``Settings`` (the dev ``.env`` never leaks in).""" monkeypatch.setattr( git_sources_resolver, "get_settings", lambda: Settings(_env_file=None, git_sources=git_sources), # pyright: ignore[reportCallIssue] ) def _seed_local(db: Session, path: Path) -> None: """A ``kind=local`` row as the phase-38 API stores it: the expanded absolute path in both ``path`` and the NOT-NULL ``url`` column.""" db.add(GitSource(url=str(path), kind="local", path=str(path))) db.commit() def _capture_llm(monkeypatch: pytest.MonkeyPatch) -> list[FakeEmbedder]: """The pipeline's ``LLMClient`` becomes a recorded ``FakeEmbedder`` (real import + real generator, no network).""" clients: list[FakeEmbedder] = [] def _factory() -> FakeEmbedder: client = FakeEmbedder() clients.append(client) return client monkeypatch.setattr(sync_api, "LLMClient", _factory) return clients def _login(client: TestClient) -> None: r = client.post("/api/login", json={"password": ADMIN_PASSWORD}) assert r.status_code == 204, f"admin login failed: {r.status_code} {r.text}" def _poll(client: TestClient, want: str, timeout: float = 10.0) -> dict: """Poll ``GET /api/sync/status`` until ``state == want`` (terminal). Any state other than ``running`` before the deadline fails loudly — an unexpected ``failed`` must never be masked by the wait.""" deadline = time.monotonic() + timeout while time.monotonic() < deadline: body = client.get("/api/sync/status").json() if body["state"] == want: return body assert body["state"] == "running", ( f"unexpected state {body['state']!r} while waiting for {want!r}: {body}" ) time.sleep(0.05) raise AssertionError(f"sync did not reach {want!r} within {timeout}s") @pytest.fixture() def local_dir(tmp_path: Path) -> Path: """LocalDocs: a/ (2 docs) → subtree counts root 2, a 2 (one qualifying source-root row + one folder row).""" d = tmp_path / "LocalDocs" (d / "a").mkdir(parents=True) (d / "a" / "one.md").write_text("# A One\nfirst local file\n", encoding="utf-8") (d / "a" / "two.md").write_text("# A Two\nsecond local file\n", encoding="utf-8") return d def test_api_changed_sync_generates_folder_rows( sync_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, local_dir: Path, ) -> None: """A KB-changing sync upserts the rows (visible via the test's own session) and keeps the status detail's exact pre-phase shape — the folder stats are log-only, not a new status surface.""" _seed_local(db, local_dir) _stub_env(monkeypatch) monkeypatch.setattr( sync_api, "get_settings", lambda: _settings(str(local_dir.parent / "bor")), ) clients = _capture_llm(monkeypatch) _login(sync_client) assert sync_client.post("/api/sync").status_code == 202 body = _poll(sync_client, "success") assert body["error"] is None # No new sync-status surface: the detail keeps its exact key set. assert set(body["detail"]) == { "files", "added", "updated", "unchanged", "pruned", "errors", "chunks", "summaries", "summary_errors", "overview", "sources_version", } assert body["detail"]["files"] == 2 assert body["detail"]["added"] == 2 assert body["detail"]["overview"] is True assert body["detail"]["sources_version"] == 1 # One LLM client for probe + import + overview + folder summaries; # exactly two FOLDER_SUMMARY_MODE calls (root + the 2-doc a/ folder). assert len(clients) == 1 calls = _folder_calls(clients[0]) assert len(calls) == 2 headers = [c[1]["content"].splitlines()[0] for c in calls] assert headers == ["Folder: LocalDocs", "Folder: LocalDocs/a"] # The rows land committed — visible from the test's own session. rows = _rows(db) assert set(rows) == {("LocalDocs", ""), ("LocalDocs", "a")} assert all(rows.values()) assert current_sources_version(db) == 1 def test_api_unchanged_resync_burns_zero_folder_calls( sync_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, local_dir: Path, ) -> None: """Same files → unchanged re-sync: no ``lite`` call of any kind (populated table), rows untouched, version unadvanced.""" _seed_local(db, local_dir) _stub_env(monkeypatch) monkeypatch.setattr( sync_api, "get_settings", lambda: _settings(str(local_dir.parent / "bor")), ) clients = _capture_llm(monkeypatch) _login(sync_client) assert sync_client.post("/api/sync").status_code == 202 _poll(sync_client, "success") rows = _rows(db) assert set(rows) == {("LocalDocs", ""), ("LocalDocs", "a")} assert sync_client.post("/api/sync").status_code == 202 body = _poll(sync_client, "success") assert body["detail"]["overview"] is False assert body["detail"]["sources_version"] == 1 # unchanged → no bump assert len(clients) == 2 # Zero GENERATION calls — no folder summary, no KB overview. The # only chat the re-sync's client makes is the phase-41 probe's ping. assert _folder_calls(clients[1]) == [] assert all( "KB_OVERVIEW_MODE" not in m.get("content", "") for msgs in clients[1].chat_calls for m in msgs if m.get("role") == "system" ) assert len(clients[1].chat_calls) == 1 assert clients[1].chat_calls[0][0]["content"] == "ping" assert _rows(db) == rows assert current_sources_version(db) == 1 def test_api_folder_lite_failure_keeps_rows_stays_green_and_bumps( sync_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, local_dir: Path, ) -> None: """A ``lite`` outage for one folder: the run stays ``success`` (never ``failed``), the failed folder's previous row is kept, the rest regenerates, and the sources-version bump still lands (the bump is change-gated on the KB, not on the summaries).""" _seed_local(db, local_dir) _stub_env(monkeypatch) monkeypatch.setattr( sync_api, "get_settings", lambda: _settings(str(local_dir.parent / "bor")), ) clients = _capture_llm(monkeypatch) _login(sync_client) assert sync_client.post("/api/sync").status_code == 202 _poll(sync_client, "success") rows_before = _rows(db) a_stamp_before = _updated_at(db, "LocalDocs", "a") root_stamp_before = _updated_at(db, "LocalDocs", "") assert a_stamp_before is not None and root_stamp_before is not None assert len(clients) == 1 # sync 1 used the recording factory # a/ changes (the KB changes) while lite fails for a/ only. (local_dir / "a" / "one.md").write_text( "# A One\nchanged local file\n", encoding="utf-8" ) monkeypatch.setattr( sync_api, "LLMClient", lambda: FolderFailingChatEmbedder(fail_label="Folder: LocalDocs/a"), ) assert sync_client.post("/api/sync").status_code == 202 body = _poll(sync_client, "success") # GREEN — a lite outage is fail-soft assert body["error"] is None assert body["detail"]["updated"] == 1 assert body["detail"]["sources_version"] == 2 # the bump was not blocked assert current_sources_version(db) == 2 rows_after = _rows(db) assert rows_after["LocalDocs", "a"] == rows_before["LocalDocs", "a"] assert _updated_at(db, "LocalDocs", "a") == a_stamp_before # kept as-is root_stamp_after = _updated_at(db, "LocalDocs", "") assert root_stamp_after is not None and root_stamp_after > root_stamp_before def test_api_empty_table_first_sync_regenerates( sync_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, local_dir: Path, ) -> None: """The migration-0017 scenario: the KB predates the table — wipe the rows and re-sync an unchanged KB: the empty-table trigger fires for the folder summaries (the overview's API gate, purely change-gated, does not).""" _seed_local(db, local_dir) _stub_env(monkeypatch) monkeypatch.setattr( sync_api, "get_settings", lambda: _settings(str(local_dir.parent / "bor")), ) clients = _capture_llm(monkeypatch) _login(sync_client) assert sync_client.post("/api/sync").status_code == 202 _poll(sync_client, "success") assert set(_rows(db)) == {("LocalDocs", ""), ("LocalDocs", "a")} db.execute(text("TRUNCATE folder_summaries")) db.commit() assert sync_client.post("/api/sync").status_code == 202 body = _poll(sync_client, "success") assert body["detail"]["overview"] is False # unchanged → no overview assert body["detail"]["sources_version"] == 1 # unchanged → no bump assert len(clients) == 2 assert len(_folder_calls(clients[1])) == 2 # the folders regenerated assert set(_rows(db)) == {("LocalDocs", ""), ("LocalDocs", "a")}