brain-of-reese/tests/integration/test_source_removal_api.py

"""Integration: total source removal (phase 69, task 01).

Real Postgres (``podman compose up -d db``); the settings the router
reads are monkeypatched at fresh ``tmp_path`` dirs per test (the
``test_git_sources_upload.py`` ``_point_at`` pattern — the dev ``.env``
never leaks in, and no rmtree can ever aim at a real checkout or
upload), and the LLM is never hit: the endpoint's
``regenerate_overview`` becomes a counting spy (faked/spied per the
house pattern — the best-effort branch gets a spy that raises
``LLMError``).

Contract under test — ``DELETE /api/git-sources/{id}`` is a total
removal (owner request 2026-09-02), in the locked order:

* row + the source's documents (chunks + embeddings via the
  ``all, delete-orphan`` cascade) commit **first**; the app-managed
  on-disk dir (git checkout / unpacked upload folder) is deleted
  **after** the commit — absent dir is a no-op, foreign local dirs
  (the owner's own) are never touched on disk;
* sibling guard — another stored row resolving to the same source name
  (``https://e.com/r`` vs ``https://e.com/r.git``) keeps the shared
  documents + files: only the row goes;
* ``sources_version`` bumps exactly once when documents were pruned
  (the phase-53 saved-chat invalidation, same gate as sync) — no docs,
  no bump;
* the overview refresh runs when pruned > 0 and is best-effort: a
  failing spy still lands the 204 (and the bump — overview first, bump
  second, mirroring ``_run_sync``);
* the 204 no-body contract and the 404/422 pins stay unchanged (the
  404/422 pins live in ``test_git_sources_api.py``);
* one per-operation INFO line (PLAN §9) —
  ``source removed: kind=… name=… docs_pruned=… files_removed=yes|no|
  skipped overview=… total_ms=…`` (``skipped`` for the sibling guard
  and for foreign local dirs).

``git_sources`` / ``documents`` / ``chunks`` / ``kb_overview`` are
global state: reset around every test; the single-row ``sources_meta``
counter resets to 0 (the ``test_sync_api.py`` pattern).
"""
from __future__ import annotations

import logging
import re
import uuid
from collections.abc import Iterator
from pathlib import Path

import pytest
from fastapi.testclient import TestClient
from sqlalchemy import text
from sqlalchemy.orm import Session

from app.api import git_sources as git_sources_api
from app.config import Settings
from app.models import Chunk, Document, GitSource
from app.rag.llm import LLMError

# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------


@pytest.fixture(autouse=True)
def clean_state(db: Session) -> Iterator[None]:
    """The registry + KB tables are global state: reset around every
    test; the seeded ``sources_meta`` row resets to version 0."""
    db.execute(text("TRUNCATE chunks, documents, kb_overview, git_sources"))
    db.execute(text("UPDATE sources_meta SET version = 0 WHERE id = 1"))
    db.commit()
    yield
    db.execute(text("TRUNCATE chunks, documents, kb_overview, git_sources"))
    db.execute(text("UPDATE sources_meta SET version = 0 WHERE id = 1"))
    db.commit()


class _OverviewSpy:
    """The endpoint's ``regenerate_overview`` seam: counts calls and
    (optionally) raises — the LLM outage the best-effort branch must
    swallow."""

    def __init__(self, fail: BaseException | None = None) -> None:
        self.calls = 0
        self.fail = fail

    async def __call__(self, llm: object, session: object = None) -> bool:  # noqa: ARG002
        self.calls += 1
        if self.fail is not None:
            raise self.fail
        return True


@pytest.fixture(autouse=True)
def overview_spy(monkeypatch: pytest.MonkeyPatch) -> _OverviewSpy:
    """The LLM is never hit: by default the spy succeeds (the log line
    pins ``overview=True``); the best-effort test swaps in a spy that
    raises ``LLMError``."""
    spy = _OverviewSpy()
    monkeypatch.setattr(git_sources_api, "regenerate_overview", spy)
    return spy


def _point_at(monkeypatch: pytest.MonkeyPatch, sources_dir: Path, upload_dir: Path) -> None:
    """Fresh settings on the router's module: the tmp sources dir and
    upload dir — the dev ``.env`` never leaks in, and a rmtree can
    never aim at a real checkout/upload."""
    monkeypatch.setattr(
        git_sources_api,
        "get_settings",
        lambda: Settings(
            _env_file=None,  # pyright: ignore[reportCallIssue]
            sources_dir=str(sources_dir),
            upload_dir=str(upload_dir),
        ),
    )


def _seed_doc(db: Session, source: str, path: str = "alpha.md") -> None:
    """One indexed document (+ two chunks) under ``source`` — the prune
    + cascade target. Seed directly (the task allows it): the import
    pipeline is out of scope here."""
    doc = Document(
        source=source,
        path=path,
        full_path=f"/srv/brain/{source}/{path}",
        title="Alpha",
        content="# Alpha\nsentinel content\n",
        content_hash=uuid.uuid4().hex,
    )
    db.add(doc)
    db.flush()
    db.add_all(
        [
            Chunk(document_id=doc.id, position=0, content="chunk zero"),
            Chunk(document_id=doc.id, position=1, content="chunk one"),
        ]
    )
    db.commit()


def _seed_git_row(db: Session, url: str) -> GitSource:
    row = GitSource(url=url, kind="git")
    db.add(row)
    db.commit()
    return row


def _seed_local_row(db: Session, path: Path) -> GitSource:
    """A ``kind='local'`` row exactly as the create endpoint stores it
    (phase 38: the expanded path mirrored in the NOT-NULL ``url``)."""
    row = GitSource(url=str(path), kind="local", path=str(path))
    db.add(row)
    db.commit()
    return row


def _version(db: Session) -> int:
    """The raw counter — a text query dodges the session identity map
    (the bump commits in the endpoint's own short-lived session)."""
    row = db.execute(text("SELECT version FROM sources_meta WHERE id = 1")).first()
    return row[0] if row is not None else 0


def _docs(client: TestClient) -> list[tuple[str, str]]:
    body = client.get("/api/docs").json()
    return [(d["source"], d["path"]) for d in body["documents"]]


def _rows(client: TestClient) -> list[dict[str, object]]:
    """The stored rows (the settings carry no env URLs, so the
    empty-table fallback also reports ``[]`` — the assertions only ever
    rely on the DB rows' ids/urls)."""
    return client.get("/api/git-sources").json()["sources"]


def _removal_lines(caplog: pytest.LogCaptureFixture) -> list[str]:
    return [
        rec.getMessage()
        for rec in caplog.records
        if rec.getMessage().startswith("source removed: ")
    ]


# ---------------------------------------------------------------------------
# git row: row + docs + checkout dir, one 204
# ---------------------------------------------------------------------------


def test_git_removal_deletes_row_docs_and_checkout_dir(
    admin_client: TestClient,
    db: Session,
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
    caplog: pytest.LogCaptureFixture,
    overview_spy: _OverviewSpy,
) -> None:
    sources = tmp_path / "sources"
    _point_at(monkeypatch, sources, tmp_path / "uploads")
    url = "https://gitlab.example.com/reese/homelab.git"
    row = _seed_git_row(db, url)
    checkout = sources / "homelab"
    checkout.mkdir(parents=True)
    marker = checkout / "marker.md"
    marker.write_text("sentinel\n", encoding="utf-8")
    _seed_doc(db, "homelab")
    _seed_doc(db, "homelab", path="bravo.md")
    version_before = _version(db)

    with caplog.at_level(logging.INFO, logger="app.api.git_sources"):
        r = admin_client.delete(f"/api/git-sources/{row.id}")
    assert r.status_code == 204
    assert r.content == b""  # the 204 no-body contract, unchanged

    # The row is gone (and the table is empty → the env fallback shape,
    # no env URLs set here).
    body = admin_client.get("/api/git-sources").json()
    assert body["from_env"] is True
    assert body["sources"] == []

    # The app-managed checkout dir is gone from disk — marker included.
    assert not checkout.exists()
    assert not marker.exists()

    # The source's documents are gone from the index (admin /api/docs)…
    assert _docs(admin_client) == []
    # …and every chunk row too (the ``all, delete-orphan`` cascade).
    assert db.execute(text("SELECT count(*) FROM chunks")).scalar_one() == 0
    assert db.execute(text("SELECT count(*) FROM documents")).scalar_one() == 0

    # The KB changed → exactly one version bump (phase 53) and exactly
    # one (spied) overview call.
    assert _version(db) == version_before + 1
    assert overview_spy.calls == 1

    # The per-operation INFO line (PLAN §9) with the counts.
    lines = _removal_lines(caplog)
    assert len(lines) == 1, lines
    match = re.match(
        r"^source removed: kind=git name=homelab docs_pruned=2 files_removed=yes "
        r"overview=True total_ms=\d+$",
        lines[0],
    )
    assert match, lines[0]


def test_git_removal_without_checkout_dir_is_a_noop_on_disk(
    admin_client: TestClient,
    db: Session,
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
    caplog: pytest.LogCaptureFixture,
    overview_spy: _OverviewSpy,
) -> None:
    """No checkout yet (never cloned) → the disk step is a silent no-op;
    the row + index removal still lands, 204."""
    sources = tmp_path / "sources"
    _point_at(monkeypatch, sources, tmp_path / "uploads")
    row = _seed_git_row(db, "https://gitlab.example.com/reese/ops.git")
    _seed_doc(db, "ops")
    version_before = _version(db)

    with caplog.at_level(logging.INFO, logger="app.api.git_sources"):
        assert admin_client.delete(f"/api/git-sources/{row.id}").status_code == 204

    assert not (sources / "ops").exists()
    assert _docs(admin_client) == []
    assert db.execute(text("SELECT count(*) FROM git_sources")).scalar_one() == 0
    # Docs were pruned → the bump + the overview still run.
    assert _version(db) == version_before + 1
    assert overview_spy.calls == 1
    line = _removal_lines(caplog)[0]
    assert "files_removed=no" in line  # the absent dir is the "no" case


# ---------------------------------------------------------------------------
# local rows: upload folder removed, foreign dir never touched
# ---------------------------------------------------------------------------


def test_local_upload_removal_deletes_row_docs_and_folder(
    admin_client: TestClient,
    db: Session,
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
    caplog: pytest.LogCaptureFixture,
) -> None:
    upload = tmp_path / "uploads"
    _point_at(monkeypatch, tmp_path / "sources", upload)
    folder = upload / "my-notes"
    folder.mkdir(parents=True)
    marker = folder / "readme.md"
    marker.write_text("sentinel\n", encoding="utf-8")
    row = _seed_local_row(db, folder)
    _seed_doc(db, "my-notes")
    version_before = _version(db)

    with caplog.at_level(logging.INFO, logger="app.api.git_sources"):
        assert admin_client.delete(f"/api/git-sources/{row.id}").status_code == 204

    # Row + index entries gone…
    assert _rows(admin_client) == []
    assert _docs(admin_client) == []
    # …and the app-managed upload folder is gone from disk.
    assert not folder.exists()
    assert not marker.exists()
    assert _version(db) == version_before + 1
    line = _removal_lines(caplog)[0]
    assert re.match(
        r"^source removed: kind=local name=my-notes docs_pruned=1 files_removed=yes "
        r"overview=True total_ms=\d+$",
        line,
    ), line


def test_local_foreign_dir_removal_never_touches_disk(
    admin_client: TestClient,
    db: Session,
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
    caplog: pytest.LogCaptureFixture,
) -> None:
    """A ``kind='local'`` row pointing at the owner's own directory:
    row + index entries go, the directory NEVER does (the locked
    decision) — and the KB change still bumps the version."""
    upload = tmp_path / "uploads"
    _point_at(monkeypatch, tmp_path / "sources", upload)
    foreign = tmp_path / "own" / "docs"
    foreign.mkdir(parents=True)
    marker = foreign / "precious.md"
    marker.write_text("keep me\n", encoding="utf-8")
    row = _seed_local_row(db, foreign)
    _seed_doc(db, "docs")  # the local source label = the dir's name
    version_before = _version(db)

    with caplog.at_level(logging.INFO, logger="app.api.git_sources"):
        assert admin_client.delete(f"/api/git-sources/{row.id}").status_code == 204

    assert _rows(admin_client) == []
    assert _docs(admin_client) == []
    # The foreign directory — and its files — are exactly as they were.
    assert foreign.is_dir()
    assert marker.read_text(encoding="utf-8") == "keep me\n"
    # Index change → the bump still lands; the disk step was skipped.
    assert _version(db) == version_before + 1
    line = _removal_lines(caplog)[0]
    assert "files_removed=skipped" in line, line


def test_local_prefix_sibling_dir_is_not_contained(
    admin_client: TestClient,
    db: Session,
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
    caplog: pytest.LogCaptureFixture,
) -> None:
    """The containment edge end-to-end: ``upload_dir = tmp/'u'`` and a
    local row at ``tmp/'u-evil'`` — the prefix-sharing sibling is NOT
    under the upload dir, so its files are never deleted (the unit test
    pins ``managed_dir_for``; this pins the endpoint honoring it)."""
    upload = tmp_path / "u"
    upload.mkdir()
    _point_at(monkeypatch, tmp_path / "sources", upload)
    evil = tmp_path / "u-evil"
    evil.mkdir()
    marker = evil / "marker.md"
    marker.write_text("not an upload\n", encoding="utf-8")
    row = _seed_local_row(db, evil)
    _seed_doc(db, "u-evil")

    with caplog.at_level(logging.INFO, logger="app.api.git_sources"):
        assert admin_client.delete(f"/api/git-sources/{row.id}").status_code == 204

    assert _rows(admin_client) == []
    assert _docs(admin_client) == []
    assert evil.is_dir()
    assert marker.read_text(encoding="utf-8") == "not an upload\n"
    assert "files_removed=skipped" in _removal_lines(caplog)[0]


# ---------------------------------------------------------------------------
# sibling guard
# ---------------------------------------------------------------------------


def test_sibling_guard_keeps_shared_docs_and_files_until_last_row(
    admin_client: TestClient,
    db: Session,
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
    caplog: pytest.LogCaptureFixture,
    overview_spy: _OverviewSpy,
) -> None:
    """``https://example.com/reese/r`` and ``https://example.com/reese/
    r.git`` both index under ``r`` — the first removal deletes ONLY the
    row (docs + shared checkout stay, loudly logged, no bump/overview);
    the second is the total removal."""
    sources = tmp_path / "sources"
    _point_at(monkeypatch, sources, tmp_path / "uploads")
    row_a = _seed_git_row(db, "https://example.com/reese/r")
    row_b = _seed_git_row(db, "https://example.com/reese/r.git")
    checkout = sources / "r"
    checkout.mkdir(parents=True)
    marker = checkout / "shared.md"
    marker.write_text("belongs to both rows\n", encoding="utf-8")
    _seed_doc(db, "r")
    version_before = _version(db)

    # --- first row: sibling guard — only the row goes -----------------
    # INFO level so both the loudly-logged warning and the per-operation
    # INFO line of this removal are captured.
    with caplog.at_level(logging.INFO, logger="app.api.git_sources"):
        assert admin_client.delete(f"/api/git-sources/{row_a.id}").status_code == 204

    rows = _rows(admin_client)
    assert [s["id"] for s in rows] == [str(row_b.id)]
    assert rows[0]["url"] == "https://example.com/reese/r.git"
    assert rows[0]["kind"] == "git" and rows[0]["path"] is None
    # The shared documents + files still belong to the sibling…
    assert _docs(admin_client) == [("r", "alpha.md")]
    assert checkout.is_dir()
    assert marker.read_text(encoding="utf-8") == "belongs to both rows\n"
    # …and nothing KB-side happened: no prune, no bump, no overview.
    assert _version(db) == version_before
    assert overview_spy.calls == 0
    line = _removal_lines(caplog)[0]
    assert re.match(
        r"^source removed: kind=git name=r docs_pruned=0 files_removed=skipped "
        r"overview=False total_ms=\d+$",
        line,
    ), line
    # …and the guard is logged loudly (naming the row + the shared name).
    warnings = [
        rec.getMessage() for rec in caplog.records if rec.levelno == logging.WARNING
    ]
    assert any(
        "shares source name r" in w and str(row_a.id) in w for w in warnings
    ), warnings

    # --- second row: the total removal, at last ------------------------
    with caplog.at_level(logging.INFO, logger="app.api.git_sources"):
        assert admin_client.delete(f"/api/git-sources/{row_b.id}").status_code == 204

    assert _rows(admin_client) == []
    assert _docs(admin_client) == []
    assert not checkout.exists()
    assert _version(db) == version_before + 1
    assert overview_spy.calls == 1
    assert "files_removed=yes" in _removal_lines(caplog)[1]


# ---------------------------------------------------------------------------
# version bump + best-effort overview gates
# ---------------------------------------------------------------------------


def test_overview_failure_is_best_effort_still_204(
    admin_client: TestClient,
    db: Session,
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
    caplog: pytest.LogCaptureFixture,
    overview_spy: _OverviewSpy,
) -> None:
    """A failing overview (LLM outage) never fails the delete: the 204
    lands, the docs stay pruned, and the bump still lands — overview
    first, bump second (mirroring ``_run_sync``)."""
    sources = tmp_path / "sources"
    _point_at(monkeypatch, sources, tmp_path / "uploads")
    row = _seed_git_row(db, "https://gitlab.example.com/reese/notes.git")
    checkout = sources / "notes"
    checkout.mkdir(parents=True)
    (checkout / "n.md").write_text("x\n", encoding="utf-8")
    _seed_doc(db, "notes")
    version_before = _version(db)

    failing = _OverviewSpy(fail=LLMError("simulated lite-model outage"))
    monkeypatch.setattr(git_sources_api, "regenerate_overview", failing)

    # INFO level — the per-operation line (INFO) and the best-effort
    # failure (ERROR) are both captured.
    with caplog.at_level(logging.INFO, logger="app.api.git_sources"):
        assert admin_client.delete(f"/api/git-sources/{row.id}").status_code == 204

    assert failing.calls == 1
    assert _docs(admin_client) == []
    assert not checkout.exists()
    # The bump lands even though the best-effort overview failed…
    assert _version(db) == version_before + 1
    # …and the failure is logged, never fatal.
    line = _removal_lines(caplog)[0]
    assert "overview=False" in line, line
    errors = [
        rec.getMessage() for rec in caplog.records if rec.levelno >= logging.ERROR
    ]
    assert any("overview regeneration failed" in e for e in errors), errors


def test_no_docs_pruned_means_no_overview_and_no_bump(
    admin_client: TestClient,
    db: Session,
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
    caplog: pytest.LogCaptureFixture,
    overview_spy: _OverviewSpy,
) -> None:
    """A row with no indexed documents: the checkout dir is still
    removed (files_removed=yes) but the KB is unchanged — no overview
    call, no version bump (the gate is docs pruned, not files)."""
    sources = tmp_path / "sources"
    _point_at(monkeypatch, sources, tmp_path / "uploads")
    row = _seed_git_row(db, "https://gitlab.example.com/reese/bare.git")
    checkout = sources / "bare"
    checkout.mkdir(parents=True)
    (checkout / "b.md").write_text("never imported\n", encoding="utf-8")
    version_before = _version(db)

    with caplog.at_level(logging.INFO, logger="app.api.git_sources"):
        assert admin_client.delete(f"/api/git-sources/{row.id}").status_code == 204

    assert not checkout.exists()
    assert _rows(admin_client) == []
    assert _version(db) == version_before
    assert overview_spy.calls == 0
    line = _removal_lines(caplog)[0]
    assert re.match(
        r"^source removed: kind=git name=bare docs_pruned=0 files_removed=yes "
        r"overview=False total_ms=\d+$",
        line,
    ), line