brain-of-reese/tests/integration/test_sync_folder_summaries.py

"""Integration: the sync-time folder-summary wiring (phase 94, task 02).

Extends the KB-overview sync pattern (``test_import_docs_overview.py``
is the template) to the phase-94 folder summaries — both sync paths
regenerate ``folder_summaries`` change-gated (added + updated > 0) or
on an empty table (the first full run after migration 0017 / a
``--limit`` first walk), per-folder fail-soft, best-effort, and in the
run's own short-lived session (the phase-53 flush-then-caller-commits
convention — the generator only flushes, the sync path commits).

Script path (``scripts.import_docs.main`` end to end, fake LLM, real
DB, explicit ``--source``):

- a KB-changing import → one row per existing subtree (the source
  root, the 2-doc folder a, AND the 1-doc folder b — the ≥ 1 rule),
  committed in the run's transaction, the summary line ending
  ``folder_summaries=<generated>/<failed>/<pruned>`` (unchanged by
  phase 96 — no gap-fill suffix on a full regeneration);
- an unchanged re-import with a COMPLETE table → zero ``lite`` calls,
  ``folder_summaries=skipped``, rows untouched (the phase-94
  zero-burn invariant);
- an unchanged re-import with a GAP (one stored row deleted) →
  exactly one ``FOLDER_SUMMARY_MODE`` call (the missing folder only),
  the row back with the deterministic fake text, every other row
  byte-identical (summary AND ``updated_at``), the line ending
  ``folder_summaries=1/0/0 (gap-fill)`` (phase 96, task 03 — the
  failed folder summary self-heals on the next sync);
- a KB change on a second run → still a FULL regeneration (call count
  == candidate count, every row re-stamped, no gap-fill suffix);
- a subtree losing its LAST doc after a changed re-walk → its row
  pruned;
- one folder's ``lite`` failure → its previous row kept, the other
  land, exit code 0, stats ``2/1/0``;
- a ``--limit`` debug run → no generation, no rows,
  ``folder_summaries=skipped``;
- a fresh (empty) table after a ``--limit`` first walk → an unchanged
  full walk generates via the gap-fill path (the subsumed table-empty
  first-run trigger — every candidate is missing), the line carrying
  `` (gap-fill)``.

API path (``POST /api/sync`` end to end, real import over a host temp
local dir, deterministic ``FakeEmbedder``):

- a KB-changing sync → the rows land (visible via the test's own
  session) and the status detail keeps its exact pre-phase key set
  (no folder-summary surface — the stats are log-only);
- an unchanged re-sync (complete table) → zero ``FOLDER_SUMMARY_MODE``
  calls (the phase-94 zero-burn invariant);
- an unchanged re-sync with a GAP (one stored row deleted) → targeted
  fill of exactly that row (one ``FOLDER_SUMMARY_MODE`` call, the
  ``gap-fill`` log line), every other row byte-identical, the status
  detail shape untouched (phase 96, task 03);
- a ``lite`` outage (one folder failing) → the failed folder's row is
  kept, the run reports ``success`` (never ``failed``), and the
  sources-version bump still lands (the bump is change-gated on the
  KB, not on the summaries);
- an empty table after a populated sync (the migration-0017 scenario)
  → an unchanged walk regenerates via the gap probe (the subsumed
  table-empty trigger; the overview's API gate, purely change-gated,
  does not fire).
"""
from __future__ import annotations

import logging
import time
from collections.abc import Iterator
from datetime import datetime
from pathlib import Path

import pytest
from fastapi.testclient import TestClient
from sqlalchemy import select, text
from sqlalchemy.orm import Session

from app.api import sync as sync_api
from app.config import Settings
from app.db import SessionLocal, db_available
from app.main import app as fastapi_app
from app.models import FolderSummary, GitSource
from app.rag import git_sources as git_sources_resolver
from app.rag.llm import LLMError
from app.rag.sources_meta import current_sources_version
from scripts import import_docs
from tests.conftest import ADMIN_PASSWORD
from tests.fakes import FakeEmbedder

# --- shared helpers ---------------------------------------------------------


def _folder_calls(llm: FakeEmbedder) -> list[list[dict[str, str]]]:
    """The ``FOLDER_SUMMARY_MODE`` chat calls on the client (the marker
    in the system prompt — the KB-overview marker never contains it)."""
    return [
        msgs
        for msgs in llm.chat_calls
        if any(
            "FOLDER_SUMMARY_MODE" in m.get("content", "")
            for m in msgs
            if m.get("role") == "system"
        )
    ]


def _rows(db: Session) -> dict[tuple[str, str], str]:
    """The stored folder summaries as ``{(source, folder_path): summary}``."""
    db.expire_all()
    return {
        (source, folder_path): summary
        for source, folder_path, summary in db.execute(
            select(FolderSummary.source, FolderSummary.folder_path, FolderSummary.summary)
        ).all()
    }


def _updated_at(db: Session, source: str, folder_path: str) -> datetime | None:
    db.expire_all()
    row = db.get(FolderSummary, (source, folder_path))
    return row.updated_at if row is not None else None


class FolderFailingChatEmbedder(FakeEmbedder):
    """``lite`` stand-in that fails exactly one folder-summary call —
    the one whose user message contains *fail_label* (a ``Folder: …``
    header) — and answers everything else (incl. the KB overview)
    normally. Drives the per-folder fail-soft path."""

    def __init__(self, fail_label: str) -> None:
        super().__init__()
        self.fail_label = fail_label

    async def chat(
        self, messages: list[dict[str, str]], model: str | None = None
    ) -> str:
        user = next((m["content"] for m in messages if m.get("role") == "user"), "")
        if self.fail_label in user:
            self.chat_calls.append(list(messages))
            raise LLMError("simulated folder-summary outage (test sentinel)")
        return await super().chat(messages, model)


# --- fixtures ----------------------------------------------------------------


@pytest.fixture(autouse=True)
def _clean_kb(db: Session) -> Iterator[None]:
    """Global KB + registry state, truncated around every test (the
    real import and the real generator write ``documents``/``chunks``/
    ``folder_summaries``; the API tests seed ``git_sources``)."""
    db.execute(
        text("TRUNCATE chunks, documents, kb_overview, folder_summaries, git_sources")
    )
    db.commit()
    yield
    db.execute(
        text("TRUNCATE chunks, documents, kb_overview, folder_summaries, git_sources")
    )
    db.commit()


@pytest.fixture(autouse=True)
def _reset_sources_version() -> Iterator[None]:
    """The sources version counter is global mutable state — pin it to
    the migration-0010 seed (0) around every test (own session: both
    sync paths bump through their own short-lived ``SessionLocal``).
    Skips like the ``db`` fixture when Postgres is down."""
    if not db_available():
        pytest.skip("Postgres not reachable — run `podman compose up -d db` first")
    session = SessionLocal()
    try:
        session.execute(text("UPDATE sources_meta SET version = 0 WHERE id = 1"))
        session.commit()
        yield
    finally:
        session.execute(text("UPDATE sources_meta SET version = 0 WHERE id = 1"))
        session.commit()
        session.close()


@pytest.fixture(autouse=True)
def _fresh_sync_state() -> Iterator[None]:
    """The module-level status object + task are process-global: reset
    them around every test (harmless for the script-path tests)."""
    sync_api._status = sync_api.SyncStatus()
    sync_api._task = None
    yield
    sync_api._status = sync_api.SyncStatus()
    sync_api._task = None


@pytest.fixture()
def src(tmp_path: Path) -> Path:
    """MyDocs: a/ (2 docs) + b/ (1 doc) → subtree counts root 3, a 2, b 1
    (b/ is a candidate too — the ≥ 1 rule).

    md → no document-summary chat calls, so the ``lite`` traffic is
    exactly the overview + the folder summaries."""
    root = tmp_path / "MyDocs"
    (root / "a").mkdir(parents=True)
    (root / "b").mkdir()
    (root / "a" / "one.md").write_text("# A One\nFirst folder document.\n", encoding="utf-8")
    (root / "a" / "two.md").write_text("# A Two\nSecond folder document.\n", encoding="utf-8")
    (root / "b" / "three.md").write_text("# B Three\nLone folder document.\n", encoding="utf-8")
    return root


def _run_main(
    monkeypatch: pytest.MonkeyPatch,
    llm: FakeEmbedder,
    argv: list[str],
    capsys: pytest.CaptureFixture[str],
) -> tuple[int, str]:
    """Run ``import_docs.main`` with fresh settings, a fake LLM, and a
    fail-loud git mock (``--source`` always wins, so git must stay
    idle) — the phase-31 overview test's runner."""
    monkeypatch.setattr(
        import_docs,
        "get_settings",
        lambda: Settings(_env_file=None),  # pyright: ignore[reportCallIssue]
    )

    def _no_git(url: str, dest: Path | str) -> Path:
        raise AssertionError("git sync must not run with explicit --source")

    monkeypatch.setattr(import_docs, "clone_or_pull", _no_git)
    monkeypatch.setattr(import_docs, "LLMClient", lambda: llm)
    rc = import_docs.main(argv)
    return rc, capsys.readouterr().out


# --- script path: scripts/import_docs.py -------------------------------------


def test_changed_import_generates_folder_rows(
    db: Session,
    src: Path,
    monkeypatch: pytest.MonkeyPatch,
    capsys: pytest.CaptureFixture[str],
) -> None:
    """A KB-changing import upserts one row per existing subtree in
    the run's own transaction — committed and visible afterwards —
    with the stats on the summary line (PLAN §9)."""
    llm = FakeEmbedder()
    records: list[logging.LogRecord] = []

    class _Sink(logging.Handler):
        def emit(self, record: logging.LogRecord) -> None:
            records.append(record)

    fs_logger = logging.getLogger("app.rag.folder_summaries")
    sink = _Sink()
    fs_logger.addHandler(sink)
    fs_logger.setLevel(logging.INFO)
    try:
        rc, out = _run_main(monkeypatch, llm, ["--source", str(src)], capsys)
    finally:
        fs_logger.removeHandler(sink)

    assert rc == 0
    assert "added=3" in out
    assert out.rstrip().endswith(
        "overview=updated sources_version=1 folder_summaries=3/0/0"
    )
    # Three FOLDER_SUMMARY_MODE calls — the source root (3 docs), the
    # 2-doc folder a, and the 1-doc folder b (the ≥ 1 rule: every
    # existing folder is a candidate).
    calls = _folder_calls(llm)
    assert len(calls) == 3
    headers = [c[1]["content"].splitlines()[0] for c in calls]
    assert headers == ["Folder: MyDocs", "Folder: MyDocs/a", "Folder: MyDocs/b"]
    for c in calls:
        assert "FOLDER_SUMMARY_MODE" in c[0]["content"]
    # The rows land committed (the run's own session committed them) —
    # one per existing subtree, never empty, stamped.
    rows = _rows(db)
    assert set(rows) == {("MyDocs", ""), ("MyDocs", "a"), ("MyDocs", "b")}
    assert all(rows.values())
    assert _updated_at(db, "MyDocs", "a") is not None
    # The stats log line (PLAN §9 ample logging).
    assert any(
        "folder_summaries: generated=3 failed=0 pruned=0 kept_manual=0"
        in r.getMessage()
        for r in records
    )


def test_unchanged_reimport_burns_zero_folder_calls(
    db: Session,
    src: Path,
    monkeypatch: pytest.MonkeyPatch,
    capsys: pytest.CaptureFixture[str],
) -> None:
    """Same hashes → no KB change → the populated table stays untouched
    and zero ``lite`` calls burn (overview AND folder summaries)."""
    llm1 = FakeEmbedder()
    rc, out = _run_main(monkeypatch, llm1, ["--source", str(src)], capsys)
    assert rc == 0
    assert out.rstrip().endswith(
        "overview=updated sources_version=1 folder_summaries=3/0/0"
    )
    rows = _rows(db)
    assert set(rows) == {("MyDocs", ""), ("MyDocs", "a"), ("MyDocs", "b")}

    llm2 = FakeEmbedder()
    rc, out = _run_main(monkeypatch, llm2, ["--source", str(src)], capsys)
    assert rc == 0
    assert "unchanged=3" in out
    assert out.rstrip().endswith(
        "overview=skipped sources_version=skipped folder_summaries=skipped"
    )
    assert llm2.chat_calls == []  # zero lite calls, any mode
    assert _rows(db) == rows  # rows byte-identical


def test_unchanged_reimport_with_gap_fills_only_the_missing_row(
    db: Session,
    src: Path,
    monkeypatch: pytest.MonkeyPatch,
    capsys: pytest.CaptureFixture[str],
) -> None:
    """Phase 96 (task 03): an unchanged walk with ONE deleted stored
    row → exactly one ``FOLDER_SUMMARY_MODE`` call (the deleted
    folder only), the row back with the deterministic fake text, every
    OTHER row byte-identical (summary AND ``updated_at``), the summary
    line ending ``folder_summaries=1/0/0 (gap-fill)`` — the failed
    folder summary self-heals on the next sync instead of persisting
    until a KB change."""
    llm1 = FakeEmbedder()
    rc, out = _run_main(monkeypatch, llm1, ["--source", str(src)], capsys)
    assert rc == 0
    rows_before = _rows(db)
    assert set(rows_before) == {("MyDocs", ""), ("MyDocs", "a"), ("MyDocs", "b")}
    root_stamp_before = _updated_at(db, "MyDocs", "")
    assert root_stamp_before is not None

    # Simulate the phase-96 incident: a lost row (an exhausted
    # one-shot retry leaves a candidate without its row).
    db.execute(
        text(
            "DELETE FROM folder_summaries "
            "WHERE source = 'MyDocs' AND folder_path = 'a'"
        )
    )
    db.commit()

    llm2 = FakeEmbedder()
    rc, out = _run_main(monkeypatch, llm2, ["--source", str(src)], capsys)
    assert rc == 0
    assert "unchanged=3" in out
    assert out.rstrip().endswith(
        "overview=skipped sources_version=skipped "
        "folder_summaries=1/0/0 (gap-fill)"
    )
    # Exactly ONE new folder call — the deleted row's folder only.
    calls = _folder_calls(llm2)
    assert len(calls) == 1
    assert calls[0][1]["content"].splitlines()[0] == "Folder: MyDocs/a"
    assert len(llm2.chat_calls) == 1  # no other lite traffic at all
    # The row is back with the deterministic fake text ...
    rows_after = _rows(db)
    assert rows_after == rows_before
    # ... and every OTHER row byte-identical (the root was never
    # re-stamped by the targeted fill).
    assert _updated_at(db, "MyDocs", "") == root_stamp_before


def test_changed_reimport_is_a_full_regeneration(
    db: Session,
    src: Path,
    monkeypatch: pytest.MonkeyPatch,
    capsys: pytest.CaptureFixture[str],
) -> None:
    """Phase 96 (task 03): a KB change is STILL a full regeneration —
    call count == candidate count, every row re-stamped, and NO
    `` (gap-fill)`` suffix (byte-identical to today's behavior)."""
    llm1 = FakeEmbedder()
    rc, _ = _run_main(monkeypatch, llm1, ["--source", str(src)], capsys)
    assert rc == 0
    rows_before = _rows(db)
    root_stamp_before = _updated_at(db, "MyDocs", "")
    a_stamp_before = _updated_at(db, "MyDocs", "a")
    assert root_stamp_before is not None and a_stamp_before is not None

    # A KB change (one doc edited) — the gate is the change, not the
    # gap.
    (src / "a" / "one.md").write_text("# A One\nChanged content.\n", encoding="utf-8")
    llm2 = FakeEmbedder()
    rc, out = _run_main(monkeypatch, llm2, ["--source", str(src)], capsys)
    assert rc == 0
    assert "updated=1" in out
    # Full-regeneration token — the stats without the gap-fill suffix.
    assert out.rstrip().endswith(
        "overview=updated sources_version=2 folder_summaries=3/0/0"
    )
    # Call count == candidate count — ALL existing folders, not a
    # targeted fill.
    calls = _folder_calls(llm2)
    assert [c[1]["content"].splitlines()[0] for c in calls] == [
        "Folder: MyDocs",
        "Folder: MyDocs/a",
        "Folder: MyDocs/b",
    ]
    # All rows re-stamped (the full regeneration re-writes every
    # candidate, even the unchanging one).
    root_stamp_after = _updated_at(db, "MyDocs", "")
    a_stamp_after = _updated_at(db, "MyDocs", "a")
    assert root_stamp_after is not None and root_stamp_after > root_stamp_before
    assert a_stamp_after is not None and a_stamp_after > a_stamp_before
    assert _rows(db) == rows_before  # deterministic fake → same texts


def test_subtree_losing_its_last_doc_is_pruned(
    db: Session,
    src: Path,
    monkeypatch: pytest.MonkeyPatch,
    capsys: pytest.CaptureFixture[str],
) -> None:
    """A subtree that loses its LAST doc (0 recursive docs — below the
    ≥ 1 minimum) on a changed re-walk loses its (stale) row; the
    remaining qualifiers (≥ 1 doc — the 1-doc b/ included) regenerate."""
    llm1 = FakeEmbedder()
    rc, _ = _run_main(monkeypatch, llm1, ["--source", str(src)], capsys)
    assert rc == 0
    assert set(_rows(db)) == {("MyDocs", ""), ("MyDocs", "a"), ("MyDocs", "b")}

    # a/ loses BOTH its docs (the folder vanishes — 0 docs) while b's
    # doc changes → the re-walk is a KB change, so generation runs and
    # the stale a/ row is pruned; b (1 doc) keeps qualifying.
    (src / "a" / "one.md").unlink()
    (src / "a" / "two.md").unlink()
    (src / "b" / "three.md").write_text("# B Three\nChanged content.\n", encoding="utf-8")
    llm2 = FakeEmbedder()
    rc, out = _run_main(monkeypatch, llm2, ["--source", str(src), "--prune"], capsys)

    assert rc == 0
    assert "pruned=2" in out  # the two a/ documents left the index
    assert "updated=1" in out
    assert out.rstrip().endswith(
        "overview=updated sources_version=2 folder_summaries=2/0/1"
    )
    # The source root (1 remaining doc) and b/ (1 doc) still qualify.
    assert set(_rows(db)) == {("MyDocs", ""), ("MyDocs", "b")}
    assert _updated_at(db, "MyDocs", "a") is None  # the row is gone


def test_folder_lite_failure_keeps_previous_row_and_stays_green(
    db: Session,
    src: Path,
    monkeypatch: pytest.MonkeyPatch,
    capsys: pytest.CaptureFixture[str],
) -> None:
    """One folder's ``lite`` failure: its previous row is kept, the
    other folders land, and the run's exit code stays 0 — the stats
    carry the failure (``2/1/0``)."""
    llm1 = FakeEmbedder()
    rc, _ = _run_main(monkeypatch, llm1, ["--source", str(src)], capsys)
    assert rc == 0
    rows_before = _rows(db)
    a_stamp_before = _updated_at(db, "MyDocs", "a")
    root_stamp_before = _updated_at(db, "MyDocs", "")
    assert a_stamp_before is not None and root_stamp_before is not None

    # a/ changes (a KB change → generation runs); lite fails for a/
    # only — its previous row must stay, the root row must land.
    (src / "a" / "one.md").write_text("# A One\nChanged content.\n", encoding="utf-8")
    llm2 = FolderFailingChatEmbedder(fail_label="Folder: MyDocs/a")
    rc, out = _run_main(monkeypatch, llm2, ["--source", str(src)], capsys)

    assert rc == 0  # a failed folder must not fail the import
    assert "updated=1" in out
    assert out.rstrip().endswith(
        "overview=updated sources_version=2 folder_summaries=2/1/0"
    )
    rows_after = _rows(db)
    assert rows_after["MyDocs", "a"] == rows_before["MyDocs", "a"]  # kept
    assert _updated_at(db, "MyDocs", "a") == a_stamp_before  # untouched
    root_stamp_after = _updated_at(db, "MyDocs", "")
    assert root_stamp_after is not None and root_stamp_after > root_stamp_before


def test_changed_import_never_overwrites_a_manual_row(
    db: Session,
    src: Path,
    monkeypatch: pytest.MonkeyPatch,
    capsys: pytest.CaptureFixture[str],
) -> None:
    """Phase 97 (task 01): a manually-edited folder description survives
    a KB-changing sync — the generator SKIPS it (zero
    ``FOLDER_SUMMARY_MODE`` calls for it), the summary-line token
    STAYS 3 fields (``folder_summaries=<generated>/<failed>/<pruned>``
    — ``kept_manual`` is a stat, not a token), the manual row's text,
    stamp, and flag are untouched, and the other folders regenerate
    (the ``kept_manual`` stat lands on the generator's log line)."""
    # First sync: full generation — root + a/ + b/ (the ≥ 1 rule: b/
    # holds 1 doc — still a candidate).
    llm1 = FakeEmbedder()
    rc, out = _run_main(monkeypatch, llm1, ["--source", str(src)], capsys)
    assert rc == 0
    assert out.rstrip().endswith(
        "overview=updated sources_version=1 folder_summaries=3/0/0"
    )
    assert set(_rows(db)) == {("MyDocs", ""), ("MyDocs", "a"), ("MyDocs", "b")}

    # The owner edits the source-root description (task 03's PATCH is
    # the writer; task 01 pins the generator's behavior, so the row is
    # inserted directly — the ``test_import_docs_overview.py`` pattern).
    manual_text = "Owner's own words about MyDocs."
    db.execute(
        text(
            "UPDATE folder_summaries SET summary = :s, manually_edited = true"
            " WHERE source = 'MyDocs' AND folder_path = ''"
        ),
        {"s": manual_text},
    )
    db.commit()
    root_stamp_before = _updated_at(db, "MyDocs", "")
    assert root_stamp_before is not None

    records: list[logging.LogRecord] = []

    class _Sink(logging.Handler):
        def emit(self, record: logging.LogRecord) -> None:
            records.append(record)

    fs_logger = logging.getLogger("app.rag.folder_summaries")
    sink = _Sink()
    fs_logger.addHandler(sink)
    fs_logger.setLevel(logging.INFO)
    try:
        # A KB-changing re-sync (a new doc under a/) — the gate fires a
        # full regeneration ...
        (src / "a" / "three.md").write_text(
            "# A Three\nAnother folder document.\n", encoding="utf-8"
        )
        llm2 = FakeEmbedder()
        rc, out = _run_main(monkeypatch, llm2, ["--source", str(src)], capsys)
    finally:
        fs_logger.removeHandler(sink)

    assert rc == 0
    assert "added=1" in out
    # The token STAYS 3 fields — kept_manual is a stat, not a token.
    assert out.rstrip().endswith(
        "overview=updated sources_version=2 folder_summaries=2/0/0"
    )
    # Zero folder calls for the owner's folder — a/ (now 3 docs) and
    # b/ regenerate.
    calls = _folder_calls(llm2)
    assert [c[1]["content"].splitlines()[0] for c in calls] == [
        "Folder: MyDocs/a",
        "Folder: MyDocs/b",
    ]
    # The owner's text, stamp, and flag are untouched ...
    rows_after = _rows(db)
    assert rows_after[("MyDocs", "")] == manual_text
    assert _updated_at(db, "MyDocs", "") == root_stamp_before, (
        "the manual row is never re-stamped"
    )
    flag = db.execute(
        text(
            "SELECT manually_edited FROM folder_summaries"
            " WHERE source = 'MyDocs' AND folder_path = ''"
        )
    ).scalar_one()
    assert flag is True, "the generator never clears the flag"
    # ... while the other folder regenerates.
    assert rows_after[("MyDocs", "a")] is not None
    # The 4-field stats line carries the skip (PLAN §9 ample logging).
    assert any(
        "folder_summaries: generated=2 failed=0 pruned=0 kept_manual=1"
        in r.getMessage()
        for r in records
    )


def test_limit_run_skips_folder_generation(
    db: Session,
    src: Path,
    monkeypatch: pytest.MonkeyPatch,
    capsys: pytest.CaptureFixture[str],
) -> None:
    """A ``--limit`` debug run (an incomplete walk) never generates —
    no overview/folder ``lite`` call, no rows, the line says
    ``skipped``."""
    llm = FakeEmbedder()
    rc, out = _run_main(monkeypatch, llm, ["--source", str(src), "--limit", "2"], capsys)
    assert rc == 0
    assert "added=2" in out
    assert out.rstrip().endswith(
        "overview=skipped sources_version=skipped folder_summaries=skipped"
    )
    # Phase 118 (A2): the two walked docs still burn their document
    # summaries — but the folder generation never fires (and neither
    # does the overview).
    assert len(llm.chat_calls) == 2  # doc summaries only, any other mode absent
    assert _rows(db) == {}  # an incomplete walk never writes rows


def test_empty_table_generates_on_unchanged_walk(
    db: Session,
    src: Path,
    monkeypatch: pytest.MonkeyPatch,
    capsys: pytest.CaptureFixture[str],
) -> None:
    """The subsumed table-empty first-run trigger (phase 96, task
    03): after a ``--limit`` first walk (populated KB, empty table),
    an unchanged full walk generates — for the folder summaries (now
    via the gap-fill path — every candidate is missing) AND the
    missing outline, still never bumping the version."""
    llm1 = FakeEmbedder()
    rc, out = _run_main(monkeypatch, llm1, ["--source", str(src), "--limit", "3"], capsys)
    assert rc == 0
    assert "added=3" in out
    assert out.rstrip().endswith(
        "overview=skipped sources_version=skipped folder_summaries=skipped"
    )
    assert _rows(db) == {}

    llm2 = FakeEmbedder()
    rc, out = _run_main(monkeypatch, llm2, ["--source", str(src)], capsys)
    assert rc == 0
    assert "unchanged=3" in out
    # Phase 96 (task 03): the old table-empty trigger is now the
    # subsumed gap case — every candidate is missing, so the unchanged
    # walk takes the targeted-fill path and the token carries
    # `` (gap-fill)`` (the generated set is the full candidate set).
    assert out.rstrip().endswith(
        "overview=updated sources_version=skipped "
        "folder_summaries=3/0/0 (gap-fill)"
    )
    assert set(_rows(db)) == {("MyDocs", ""), ("MyDocs", "a"), ("MyDocs", "b")}
    assert len(_folder_calls(llm2)) == 3
    assert current_sources_version(db) == 0  # the unchanged walk never bumps


# --- API path: POST /api/sync -------------------------------------------------


@pytest.fixture()
def sync_client() -> Iterator[TestClient]:
    """Context-managed TestClient — one app event loop across requests
    (the background task must survive between the POST and the polls)."""
    with TestClient(fastapi_app) as client:
        yield client


def _settings(sources_dir: str) -> Settings:
    return Settings(_env_file=None, sources_dir=sources_dir)  # pyright: ignore[reportCallIssue]


def _stub_env(monkeypatch: pytest.MonkeyPatch, git_sources: str = "") -> None:
    """The resolver's env fallback, driven by a fresh ``Settings`` (the
    dev ``.env`` never leaks in)."""
    monkeypatch.setattr(
        git_sources_resolver,
        "get_settings",
        lambda: Settings(_env_file=None, git_sources=git_sources),  # pyright: ignore[reportCallIssue]
    )


def _seed_local(db: Session, path: Path) -> None:
    """A ``kind=local`` row as the phase-38 API stores it: the expanded
    absolute path in both ``path`` and the NOT-NULL ``url`` column."""
    db.add(GitSource(url=str(path), kind="local", path=str(path)))
    db.commit()


def _capture_llm(monkeypatch: pytest.MonkeyPatch) -> list[FakeEmbedder]:
    """The pipeline's ``LLMClient`` becomes a recorded
    ``FakeEmbedder`` (real import + real generator, no network)."""
    clients: list[FakeEmbedder] = []

    def _factory() -> FakeEmbedder:
        client = FakeEmbedder()
        clients.append(client)
        return client

    monkeypatch.setattr(sync_api, "LLMClient", _factory)
    return clients


def _login(client: TestClient) -> None:
    r = client.post("/api/login", json={"password": ADMIN_PASSWORD})
    assert r.status_code == 204, f"admin login failed: {r.status_code} {r.text}"


def _poll(client: TestClient, want: str, timeout: float = 10.0) -> dict:
    """Poll ``GET /api/sync/status`` until ``state == want`` (terminal).

    Any state other than ``running`` before the deadline fails loudly —
    an unexpected ``failed`` must never be masked by the wait."""
    deadline = time.monotonic() + timeout
    while time.monotonic() < deadline:
        body = client.get("/api/sync/status").json()
        if body["state"] == want:
            return body
        assert body["state"] == "running", (
            f"unexpected state {body['state']!r} while waiting for {want!r}: {body}"
        )
        time.sleep(0.05)
    raise AssertionError(f"sync did not reach {want!r} within {timeout}s")


@pytest.fixture()
def local_dir(tmp_path: Path) -> Path:
    """LocalDocs: a/ (2 docs) → subtree counts root 2, a 2 (one
    qualifying source-root row + one folder row)."""
    d = tmp_path / "LocalDocs"
    (d / "a").mkdir(parents=True)
    (d / "a" / "one.md").write_text("# A One\nfirst local file\n", encoding="utf-8")
    (d / "a" / "two.md").write_text("# A Two\nsecond local file\n", encoding="utf-8")
    return d


def test_api_changed_sync_generates_folder_rows(
    sync_client: TestClient,
    monkeypatch: pytest.MonkeyPatch,
    db: Session,
    local_dir: Path,
) -> None:
    """A KB-changing sync upserts the rows (visible via the test's own
    session) and keeps the status detail's exact pre-phase shape — the
    folder stats are log-only, not a new status surface."""
    _seed_local(db, local_dir)
    _stub_env(monkeypatch)
    monkeypatch.setattr(
        sync_api,
        "get_settings",
        lambda: _settings(str(local_dir.parent / "bor")),
    )
    clients = _capture_llm(monkeypatch)

    _login(sync_client)
    assert sync_client.post("/api/sync").status_code == 202
    body = _poll(sync_client, "success")

    assert body["error"] is None
    # No new sync-status surface: the detail keeps its exact key set.
    assert set(body["detail"]) == {
        "files", "added", "updated", "unchanged", "pruned", "errors",
        "chunks", "summaries", "summary_errors", "dates_updated", "overview",
        "sources_version",
    }
    assert body["detail"]["files"] == 2
    assert body["detail"]["added"] == 2
    assert body["detail"]["overview"] is True
    assert body["detail"]["sources_version"] == 1
    # Phase 98 (task 01): the progress hook fired on the CHANGED-KB
    # regeneration branch — the terminal clears phase + current_summary
    # and keeps the hook's final counts (2 candidates: root + a/ — the
    # only writer of those counters is the hook itself).
    assert body["phase"] is None
    assert body["current_summary"] is None
    assert body["summaries_done"] == 2 and body["summaries_total"] == 2
    # One LLM client for probe + import + overview + folder summaries;
    # exactly two FOLDER_SUMMARY_MODE calls (root + the 2-doc a/ folder).
    assert len(clients) == 1
    calls = _folder_calls(clients[0])
    assert len(calls) == 2
    headers = [c[1]["content"].splitlines()[0] for c in calls]
    assert headers == ["Folder: LocalDocs", "Folder: LocalDocs/a"]
    # The rows land committed — visible from the test's own session.
    rows = _rows(db)
    assert set(rows) == {("LocalDocs", ""), ("LocalDocs", "a")}
    assert all(rows.values())
    assert current_sources_version(db) == 1


def test_api_unchanged_resync_burns_zero_folder_calls(
    sync_client: TestClient,
    monkeypatch: pytest.MonkeyPatch,
    db: Session,
    local_dir: Path,
) -> None:
    """Same files → unchanged re-sync: no ``lite`` call of any kind
    (populated table), rows untouched, version unadvanced."""
    _seed_local(db, local_dir)
    _stub_env(monkeypatch)
    monkeypatch.setattr(
        sync_api,
        "get_settings",
        lambda: _settings(str(local_dir.parent / "bor")),
    )
    clients = _capture_llm(monkeypatch)

    _login(sync_client)
    assert sync_client.post("/api/sync").status_code == 202
    _poll(sync_client, "success")
    rows = _rows(db)
    assert set(rows) == {("LocalDocs", ""), ("LocalDocs", "a")}

    assert sync_client.post("/api/sync").status_code == 202
    body = _poll(sync_client, "success")
    assert body["detail"]["overview"] is False
    assert body["detail"]["sources_version"] == 1  # unchanged → no bump
    # Phase 98 (task 01): the no-gap skip branch never CALLS the
    # generator — the hook never fires, so the terminal's summary
    # counters stay 0/0 (the run stayed in the import phase through to
    # the terminal, which then cleared it). The counter keep at 0/0 is
    # the deterministic pin of "never fired": a fired hook's counts
    # would survive to the terminal (the keep-final-counts rule).
    assert body["phase"] is None
    assert body["current_summary"] is None
    assert body["summaries_done"] == 0 and body["summaries_total"] == 0
    assert len(clients) == 2
    # Zero GENERATION calls — no folder summary, no KB overview. The
    # only chat the re-sync's client makes is the phase-41 probe's ping.
    assert _folder_calls(clients[1]) == []
    assert all(
        "KB_OVERVIEW_MODE" not in m.get("content", "")
        for msgs in clients[1].chat_calls
        for m in msgs
        if m.get("role") == "system"
    )
    assert len(clients[1].chat_calls) == 1
    assert clients[1].chat_calls[0][0]["content"] == "ping"
    assert _rows(db) == rows
    assert current_sources_version(db) == 1


def test_api_unchanged_resync_with_gap_fills_only_the_missing_row(
    sync_client: TestClient,
    monkeypatch: pytest.MonkeyPatch,
    db: Session,
    local_dir: Path,
) -> None:
    """Phase 96 (task 03), API path: an unchanged re-sync with ONE
    deleted stored row → targeted fill of exactly that row (one
    ``FOLDER_SUMMARY_MODE`` call), the ``gap-fill`` log line, every
    other row byte-identical (summary AND ``updated_at``), status
    ``success``, and the status detail shape untouched (the stats stay
    log-only — the phase-94 contract)."""
    _seed_local(db, local_dir)
    _stub_env(monkeypatch)
    monkeypatch.setattr(
        sync_api,
        "get_settings",
        lambda: _settings(str(local_dir.parent / "bor")),
    )
    clients = _capture_llm(monkeypatch)

    _login(sync_client)
    assert sync_client.post("/api/sync").status_code == 202
    _poll(sync_client, "success")
    rows_before = _rows(db)
    assert set(rows_before) == {("LocalDocs", ""), ("LocalDocs", "a")}
    root_stamp_before = _updated_at(db, "LocalDocs", "")
    assert root_stamp_before is not None

    # The phase-96 incident shape: a lost row, deleted directly.
    db.execute(
        text(
            "DELETE FROM folder_summaries "
            "WHERE source = 'LocalDocs' AND folder_path = 'a'"
        )
    )
    db.commit()

    records: list[logging.LogRecord] = []

    class _Sink(logging.Handler):
        def emit(self, record: logging.LogRecord) -> None:
            records.append(record)

    sync_logger = logging.getLogger("app.api.sync")
    sink = _Sink()
    sync_logger.addHandler(sink)
    sync_logger.setLevel(logging.INFO)
    try:
        assert sync_client.post("/api/sync").status_code == 202
        body = _poll(sync_client, "success")
    finally:
        sync_logger.removeHandler(sink)

    assert body["error"] is None
    assert body["detail"]["overview"] is False  # unchanged → no overview
    assert body["detail"]["sources_version"] == 1  # unchanged → no bump
    # No new sync-status surface: the detail keeps its exact key set.
    assert set(body["detail"]) == {
        "files", "added", "updated", "unchanged", "pruned", "errors",
        "chunks", "summaries", "summary_errors", "dates_updated", "overview",
        "sources_version",
    }
    # Phase 98 (task 01): the progress hook fired on the UNCHANGED-KB
    # GAP-FILL branch — total is the missing count (the one deleted
    # row), kept at the terminal next to the cleared phase keys.
    assert body["phase"] is None
    assert body["current_summary"] is None
    assert body["summaries_done"] == 1 and body["summaries_total"] == 1
    assert len(clients) == 2
    # Targeted fill — exactly ONE folder call, the missing folder only
    # (plus the phase-41 probe's ping on the same client).
    calls = _folder_calls(clients[1])
    assert len(calls) == 1
    assert calls[0][1]["content"].splitlines()[0] == "Folder: LocalDocs/a"
    assert len(clients[1].chat_calls) == 2  # ping + the one fill
    # The row is back with the deterministic fake text ...
    assert _rows(db) == rows_before
    # ... every other row byte-identical (the root was never re-stamped
    # by the targeted fill).
    assert _updated_at(db, "LocalDocs", "") == root_stamp_before
    # The gap-fill log line (PLAN §9 ample logging).
    assert any(
        "sync: folder_summaries gap-fill" in r.getMessage() for r in records
    )


def test_api_folder_lite_failure_keeps_rows_stays_green_and_bumps(
    sync_client: TestClient,
    monkeypatch: pytest.MonkeyPatch,
    db: Session,
    local_dir: Path,
) -> None:
    """A ``lite`` outage for one folder: the run stays ``success``
    (never ``failed``), the failed folder's previous row is kept, the
    rest regenerates, and the sources-version bump still lands (the
    bump is change-gated on the KB, not on the summaries)."""
    _seed_local(db, local_dir)
    _stub_env(monkeypatch)
    monkeypatch.setattr(
        sync_api,
        "get_settings",
        lambda: _settings(str(local_dir.parent / "bor")),
    )
    clients = _capture_llm(monkeypatch)

    _login(sync_client)
    assert sync_client.post("/api/sync").status_code == 202
    _poll(sync_client, "success")
    rows_before = _rows(db)
    a_stamp_before = _updated_at(db, "LocalDocs", "a")
    root_stamp_before = _updated_at(db, "LocalDocs", "")
    assert a_stamp_before is not None and root_stamp_before is not None
    assert len(clients) == 1  # sync 1 used the recording factory

    # a/ changes (the KB changes) while lite fails for a/ only.
    (local_dir / "a" / "one.md").write_text(
        "# A One\nchanged local file\n", encoding="utf-8"
    )
    monkeypatch.setattr(
        sync_api,
        "LLMClient",
        lambda: FolderFailingChatEmbedder(fail_label="Folder: LocalDocs/a"),
    )
    assert sync_client.post("/api/sync").status_code == 202
    body = _poll(sync_client, "success")  # GREEN — a lite outage is fail-soft

    assert body["error"] is None
    assert body["detail"]["updated"] == 1
    assert body["detail"]["sources_version"] == 2  # the bump was not blocked
    # Phase 98 (task 01): the failed (fail-soft) folder still advanced
    # the hook's counter — the hook fires BEFORE the attempt, so the
    # terminal keeps 2/2, not 1/2.
    assert body["phase"] is None and body["current_summary"] is None
    assert body["summaries_done"] == 2 and body["summaries_total"] == 2
    assert current_sources_version(db) == 2
    rows_after = _rows(db)
    assert rows_after["LocalDocs", "a"] == rows_before["LocalDocs", "a"]
    assert _updated_at(db, "LocalDocs", "a") == a_stamp_before  # kept as-is
    root_stamp_after = _updated_at(db, "LocalDocs", "")
    assert root_stamp_after is not None and root_stamp_after > root_stamp_before


def test_api_empty_table_first_sync_regenerates(
    sync_client: TestClient,
    monkeypatch: pytest.MonkeyPatch,
    db: Session,
    local_dir: Path,
) -> None:
    """The migration-0017 scenario (the subsumed table-empty trigger —
    phase 96, task 03): the KB predates the table — wipe the rows and
    re-sync an unchanged KB: the gap probe fires (every candidate is
    missing) and the targeted fill regenerates the full candidate set
    for the folder summaries (the overview's API gate, purely
    change-gated, does not fire)."""
    _seed_local(db, local_dir)
    _stub_env(monkeypatch)
    monkeypatch.setattr(
        sync_api,
        "get_settings",
        lambda: _settings(str(local_dir.parent / "bor")),
    )
    clients = _capture_llm(monkeypatch)

    _login(sync_client)
    assert sync_client.post("/api/sync").status_code == 202
    _poll(sync_client, "success")
    assert set(_rows(db)) == {("LocalDocs", ""), ("LocalDocs", "a")}

    db.execute(text("TRUNCATE folder_summaries"))
    db.commit()
    assert sync_client.post("/api/sync").status_code == 202
    body = _poll(sync_client, "success")

    assert body["detail"]["overview"] is False  # unchanged → no overview
    assert body["detail"]["sources_version"] == 1  # unchanged → no bump
    # Phase 98 (task 01): the gap-fill branch over the emptied table —
    # every candidate missing (2), the hook's final counts kept at the
    # terminal next to the cleared phase keys.
    assert body["phase"] is None and body["current_summary"] is None
    assert body["summaries_done"] == 2 and body["summaries_total"] == 2
    assert len(clients) == 2
    assert len(_folder_calls(clients[1])) == 2  # the folders regenerated
    assert set(_rows(db)) == {("LocalDocs", ""), ("LocalDocs", "a")}