feat(sources): removing a source deletes its files and index entries behind a confirmation modal

tests/integration/test_source_removal_api.py

@@ -0,0 +1,528 @@
+"""Integration: total source removal (phase 69, task 01).
+
+Real Postgres (``podman compose up -d db``); the settings the router
+reads are monkeypatched at fresh ``tmp_path`` dirs per test (the
+``test_git_sources_upload.py`` ``_point_at`` pattern — the dev ``.env``
+never leaks in, and no rmtree can ever aim at a real checkout or
+upload), and the LLM is never hit: the endpoint's
+``regenerate_overview`` becomes a counting spy (faked/spied per the
+house pattern — the best-effort branch gets a spy that raises
+``LLMError``).
+
+Contract under test — ``DELETE /api/git-sources/{id}`` is a total
+removal (owner request 2026-09-02), in the locked order:
+
+* row + the source's documents (chunks + embeddings via the
+  ``all, delete-orphan`` cascade) commit **first**; the app-managed
+  on-disk dir (git checkout / unpacked upload folder) is deleted
+  **after** the commit — absent dir is a no-op, foreign local dirs
+  (the owner's own) are never touched on disk;
+* sibling guard — another stored row resolving to the same source name
+  (``https://e.com/r`` vs ``https://e.com/r.git``) keeps the shared
+  documents + files: only the row goes;
+* ``sources_version`` bumps exactly once when documents were pruned
+  (the phase-53 saved-chat invalidation, same gate as sync) — no docs,
+  no bump;
+* the overview refresh runs when pruned > 0 and is best-effort: a
+  failing spy still lands the 204 (and the bump — overview first, bump
+  second, mirroring ``_run_sync``);
+* the 204 no-body contract and the 404/422 pins stay unchanged (the
+  404/422 pins live in ``test_git_sources_api.py``);
+* one per-operation INFO line (PLAN §9) —
+  ``source removed: kind=… name=… docs_pruned=… files_removed=yes|no|
+  skipped overview=… total_ms=…`` (``skipped`` for the sibling guard
+  and for foreign local dirs).
+
+``git_sources`` / ``documents`` / ``chunks`` / ``kb_overview`` are
+global state: reset around every test; the single-row ``sources_meta``
+counter resets to 0 (the ``test_sync_api.py`` pattern).
+"""
+from __future__ import annotations
+
+import logging
+import re
+import uuid
+from collections.abc import Iterator
+from pathlib import Path
+
+import pytest
+from fastapi.testclient import TestClient
+from sqlalchemy import text
+from sqlalchemy.orm import Session
+
+from app.api import git_sources as git_sources_api
+from app.config import Settings
+from app.models import Chunk, Document, GitSource
+from app.rag.llm import LLMError
+
+# ---------------------------------------------------------------------------
+# Fixtures
+# ---------------------------------------------------------------------------
+
+
+@pytest.fixture(autouse=True)
+def clean_state(db: Session) -> Iterator[None]:
+    """The registry + KB tables are global state: reset around every
+    test; the seeded ``sources_meta`` row resets to version 0."""
+    db.execute(text("TRUNCATE chunks, documents, kb_overview, git_sources"))
+    db.execute(text("UPDATE sources_meta SET version = 0 WHERE id = 1"))
+    db.commit()
+    yield
+    db.execute(text("TRUNCATE chunks, documents, kb_overview, git_sources"))
+    db.execute(text("UPDATE sources_meta SET version = 0 WHERE id = 1"))
+    db.commit()
+
+
+class _OverviewSpy:
+    """The endpoint's ``regenerate_overview`` seam: counts calls and
+    (optionally) raises — the LLM outage the best-effort branch must
+    swallow."""
+
+    def __init__(self, fail: BaseException | None = None) -> None:
+        self.calls = 0
+        self.fail = fail
+
+    async def __call__(self, llm: object, session: object = None) -> bool:  # noqa: ARG002
+        self.calls += 1
+        if self.fail is not None:
+            raise self.fail
+        return True
+
+
+@pytest.fixture(autouse=True)
+def overview_spy(monkeypatch: pytest.MonkeyPatch) -> _OverviewSpy:
+    """The LLM is never hit: by default the spy succeeds (the log line
+    pins ``overview=True``); the best-effort test swaps in a spy that
+    raises ``LLMError``."""
+    spy = _OverviewSpy()
+    monkeypatch.setattr(git_sources_api, "regenerate_overview", spy)
+    return spy
+
+
+def _point_at(monkeypatch: pytest.MonkeyPatch, sources_dir: Path, upload_dir: Path) -> None:
+    """Fresh settings on the router's module: the tmp sources dir and
+    upload dir — the dev ``.env`` never leaks in, and a rmtree can
+    never aim at a real checkout/upload."""
+    monkeypatch.setattr(
+        git_sources_api,
+        "get_settings",
+        lambda: Settings(
+            _env_file=None,  # pyright: ignore[reportCallIssue]
+            sources_dir=str(sources_dir),
+            upload_dir=str(upload_dir),
+        ),
+    )
+
+
+def _seed_doc(db: Session, source: str, path: str = "alpha.md") -> None:
+    """One indexed document (+ two chunks) under ``source`` — the prune
+    + cascade target. Seed directly (the task allows it): the import
+    pipeline is out of scope here."""
+    doc = Document(
+        source=source,
+        path=path,
+        full_path=f"/srv/brain/{source}/{path}",
+        title="Alpha",
+        content="# Alpha\nsentinel content\n",
+        content_hash=uuid.uuid4().hex,
+    )
+    db.add(doc)
+    db.flush()
+    db.add_all(
+        [
+            Chunk(document_id=doc.id, position=0, content="chunk zero"),
+            Chunk(document_id=doc.id, position=1, content="chunk one"),
+        ]
+    )
+    db.commit()
+
+
+def _seed_git_row(db: Session, url: str) -> GitSource:
+    row = GitSource(url=url, kind="git")
+    db.add(row)
+    db.commit()
+    return row
+
+
+def _seed_local_row(db: Session, path: Path) -> GitSource:
+    """A ``kind='local'`` row exactly as the create endpoint stores it
+    (phase 38: the expanded path mirrored in the NOT-NULL ``url``)."""
+    row = GitSource(url=str(path), kind="local", path=str(path))
+    db.add(row)
+    db.commit()
+    return row
+
+
+def _version(db: Session) -> int:
+    """The raw counter — a text query dodges the session identity map
+    (the bump commits in the endpoint's own short-lived session)."""
+    row = db.execute(text("SELECT version FROM sources_meta WHERE id = 1")).first()
+    return row[0] if row is not None else 0
+
+
+def _docs(client: TestClient) -> list[tuple[str, str]]:
+    body = client.get("/api/docs").json()
+    return [(d["source"], d["path"]) for d in body["documents"]]
+
+
+def _rows(client: TestClient) -> list[dict[str, object]]:
+    """The stored rows (the settings carry no env URLs, so the
+    empty-table fallback also reports ``[]`` — the assertions only ever
+    rely on the DB rows' ids/urls)."""
+    return client.get("/api/git-sources").json()["sources"]
+
+
+def _removal_lines(caplog: pytest.LogCaptureFixture) -> list[str]:
+    return [
+        rec.getMessage()
+        for rec in caplog.records
+        if rec.getMessage().startswith("source removed: ")
+    ]
+
+
+# ---------------------------------------------------------------------------
+# git row: row + docs + checkout dir, one 204
+# ---------------------------------------------------------------------------
+
+
+def test_git_removal_deletes_row_docs_and_checkout_dir(
+    admin_client: TestClient,
+    db: Session,
+    monkeypatch: pytest.MonkeyPatch,
+    tmp_path: Path,
+    caplog: pytest.LogCaptureFixture,
+    overview_spy: _OverviewSpy,
+) -> None:
+    sources = tmp_path / "sources"
+    _point_at(monkeypatch, sources, tmp_path / "uploads")
+    url = "https://gitlab.example.com/reese/homelab.git"
+    row = _seed_git_row(db, url)
+    checkout = sources / "homelab"
+    checkout.mkdir(parents=True)
+    marker = checkout / "marker.md"
+    marker.write_text("sentinel\n", encoding="utf-8")
+    _seed_doc(db, "homelab")
+    _seed_doc(db, "homelab", path="bravo.md")
+    version_before = _version(db)
+
+    with caplog.at_level(logging.INFO, logger="app.api.git_sources"):
+        r = admin_client.delete(f"/api/git-sources/{row.id}")
+    assert r.status_code == 204
+    assert r.content == b""  # the 204 no-body contract, unchanged
+
+    # The row is gone (and the table is empty → the env fallback shape,
+    # no env URLs set here).
+    body = admin_client.get("/api/git-sources").json()
+    assert body["from_env"] is True
+    assert body["sources"] == []
+
+    # The app-managed checkout dir is gone from disk — marker included.
+    assert not checkout.exists()
+    assert not marker.exists()
+
+    # The source's documents are gone from the index (admin /api/docs)…
+    assert _docs(admin_client) == []
+    # …and every chunk row too (the ``all, delete-orphan`` cascade).
+    assert db.execute(text("SELECT count(*) FROM chunks")).scalar_one() == 0
+    assert db.execute(text("SELECT count(*) FROM documents")).scalar_one() == 0
+
+    # The KB changed → exactly one version bump (phase 53) and exactly
+    # one (spied) overview call.
+    assert _version(db) == version_before + 1
+    assert overview_spy.calls == 1
+
+    # The per-operation INFO line (PLAN §9) with the counts.
+    lines = _removal_lines(caplog)
+    assert len(lines) == 1, lines
+    match = re.match(
+        r"^source removed: kind=git name=homelab docs_pruned=2 files_removed=yes "
+        r"overview=True total_ms=\d+$",
+        lines[0],
+    )
+    assert match, lines[0]
+
+
+def test_git_removal_without_checkout_dir_is_a_noop_on_disk(
+    admin_client: TestClient,
+    db: Session,
+    monkeypatch: pytest.MonkeyPatch,
+    tmp_path: Path,
+    caplog: pytest.LogCaptureFixture,
+    overview_spy: _OverviewSpy,
+) -> None:
+    """No checkout yet (never cloned) → the disk step is a silent no-op;
+    the row + index removal still lands, 204."""
+    sources = tmp_path / "sources"
+    _point_at(monkeypatch, sources, tmp_path / "uploads")
+    row = _seed_git_row(db, "https://gitlab.example.com/reese/ops.git")
+    _seed_doc(db, "ops")
+    version_before = _version(db)
+
+    with caplog.at_level(logging.INFO, logger="app.api.git_sources"):
+        assert admin_client.delete(f"/api/git-sources/{row.id}").status_code == 204
+
+    assert not (sources / "ops").exists()
+    assert _docs(admin_client) == []
+    assert db.execute(text("SELECT count(*) FROM git_sources")).scalar_one() == 0
+    # Docs were pruned → the bump + the overview still run.
+    assert _version(db) == version_before + 1
+    assert overview_spy.calls == 1
+    line = _removal_lines(caplog)[0]
+    assert "files_removed=no" in line  # the absent dir is the "no" case
+
+
+# ---------------------------------------------------------------------------
+# local rows: upload folder removed, foreign dir never touched
+# ---------------------------------------------------------------------------
+
+
+def test_local_upload_removal_deletes_row_docs_and_folder(
+    admin_client: TestClient,
+    db: Session,
+    monkeypatch: pytest.MonkeyPatch,
+    tmp_path: Path,
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    upload = tmp_path / "uploads"
+    _point_at(monkeypatch, tmp_path / "sources", upload)
+    folder = upload / "my-notes"
+    folder.mkdir(parents=True)
+    marker = folder / "readme.md"
+    marker.write_text("sentinel\n", encoding="utf-8")
+    row = _seed_local_row(db, folder)
+    _seed_doc(db, "my-notes")
+    version_before = _version(db)
+
+    with caplog.at_level(logging.INFO, logger="app.api.git_sources"):
+        assert admin_client.delete(f"/api/git-sources/{row.id}").status_code == 204
+
+    # Row + index entries gone…
+    assert _rows(admin_client) == []
+    assert _docs(admin_client) == []
+    # …and the app-managed upload folder is gone from disk.
+    assert not folder.exists()
+    assert not marker.exists()
+    assert _version(db) == version_before + 1
+    line = _removal_lines(caplog)[0]
+    assert re.match(
+        r"^source removed: kind=local name=my-notes docs_pruned=1 files_removed=yes "
+        r"overview=True total_ms=\d+$",
+        line,
+    ), line
+
+
+def test_local_foreign_dir_removal_never_touches_disk(
+    admin_client: TestClient,
+    db: Session,
+    monkeypatch: pytest.MonkeyPatch,
+    tmp_path: Path,
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    """A ``kind='local'`` row pointing at the owner's own directory:
+    row + index entries go, the directory NEVER does (the locked
+    decision) — and the KB change still bumps the version."""
+    upload = tmp_path / "uploads"
+    _point_at(monkeypatch, tmp_path / "sources", upload)
+    foreign = tmp_path / "own" / "docs"
+    foreign.mkdir(parents=True)
+    marker = foreign / "precious.md"
+    marker.write_text("keep me\n", encoding="utf-8")
+    row = _seed_local_row(db, foreign)
+    _seed_doc(db, "docs")  # the local source label = the dir's name
+    version_before = _version(db)
+
+    with caplog.at_level(logging.INFO, logger="app.api.git_sources"):
+        assert admin_client.delete(f"/api/git-sources/{row.id}").status_code == 204
+
+    assert _rows(admin_client) == []
+    assert _docs(admin_client) == []
+    # The foreign directory — and its files — are exactly as they were.
+    assert foreign.is_dir()
+    assert marker.read_text(encoding="utf-8") == "keep me\n"
+    # Index change → the bump still lands; the disk step was skipped.
+    assert _version(db) == version_before + 1
+    line = _removal_lines(caplog)[0]
+    assert "files_removed=skipped" in line, line
+
+
+def test_local_prefix_sibling_dir_is_not_contained(
+    admin_client: TestClient,
+    db: Session,
+    monkeypatch: pytest.MonkeyPatch,
+    tmp_path: Path,
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    """The containment edge end-to-end: ``upload_dir = tmp/'u'`` and a
+    local row at ``tmp/'u-evil'`` — the prefix-sharing sibling is NOT
+    under the upload dir, so its files are never deleted (the unit test
+    pins ``managed_dir_for``; this pins the endpoint honoring it)."""
+    upload = tmp_path / "u"
+    upload.mkdir()
+    _point_at(monkeypatch, tmp_path / "sources", upload)
+    evil = tmp_path / "u-evil"
+    evil.mkdir()
+    marker = evil / "marker.md"
+    marker.write_text("not an upload\n", encoding="utf-8")
+    row = _seed_local_row(db, evil)
+    _seed_doc(db, "u-evil")
+
+    with caplog.at_level(logging.INFO, logger="app.api.git_sources"):
+        assert admin_client.delete(f"/api/git-sources/{row.id}").status_code == 204
+
+    assert _rows(admin_client) == []
+    assert _docs(admin_client) == []
+    assert evil.is_dir()
+    assert marker.read_text(encoding="utf-8") == "not an upload\n"
+    assert "files_removed=skipped" in _removal_lines(caplog)[0]
+
+
+# ---------------------------------------------------------------------------
+# sibling guard
+# ---------------------------------------------------------------------------
+
+
+def test_sibling_guard_keeps_shared_docs_and_files_until_last_row(
+    admin_client: TestClient,
+    db: Session,
+    monkeypatch: pytest.MonkeyPatch,
+    tmp_path: Path,
+    caplog: pytest.LogCaptureFixture,
+    overview_spy: _OverviewSpy,
+) -> None:
+    """``https://example.com/reese/r`` and ``https://example.com/reese/
+    r.git`` both index under ``r`` — the first removal deletes ONLY the
+    row (docs + shared checkout stay, loudly logged, no bump/overview);
+    the second is the total removal."""
+    sources = tmp_path / "sources"
+    _point_at(monkeypatch, sources, tmp_path / "uploads")
+    row_a = _seed_git_row(db, "https://example.com/reese/r")
+    row_b = _seed_git_row(db, "https://example.com/reese/r.git")
+    checkout = sources / "r"
+    checkout.mkdir(parents=True)
+    marker = checkout / "shared.md"
+    marker.write_text("belongs to both rows\n", encoding="utf-8")
+    _seed_doc(db, "r")
+    version_before = _version(db)
+
+    # --- first row: sibling guard — only the row goes -----------------
+    # INFO level so both the loudly-logged warning and the per-operation
+    # INFO line of this removal are captured.
+    with caplog.at_level(logging.INFO, logger="app.api.git_sources"):
+        assert admin_client.delete(f"/api/git-sources/{row_a.id}").status_code == 204
+
+    rows = _rows(admin_client)
+    assert [s["id"] for s in rows] == [str(row_b.id)]
+    assert rows[0]["url"] == "https://example.com/reese/r.git"
+    assert rows[0]["kind"] == "git" and rows[0]["path"] is None
+    # The shared documents + files still belong to the sibling…
+    assert _docs(admin_client) == [("r", "alpha.md")]
+    assert checkout.is_dir()
+    assert marker.read_text(encoding="utf-8") == "belongs to both rows\n"
+    # …and nothing KB-side happened: no prune, no bump, no overview.
+    assert _version(db) == version_before
+    assert overview_spy.calls == 0
+    line = _removal_lines(caplog)[0]
+    assert re.match(
+        r"^source removed: kind=git name=r docs_pruned=0 files_removed=skipped "
+        r"overview=False total_ms=\d+$",
+        line,
+    ), line
+    # …and the guard is logged loudly (naming the row + the shared name).
+    warnings = [
+        rec.getMessage() for rec in caplog.records if rec.levelno == logging.WARNING
+    ]
+    assert any(
+        "shares source name r" in w and str(row_a.id) in w for w in warnings
+    ), warnings
+
+    # --- second row: the total removal, at last ------------------------
+    with caplog.at_level(logging.INFO, logger="app.api.git_sources"):
+        assert admin_client.delete(f"/api/git-sources/{row_b.id}").status_code == 204
+
+    assert _rows(admin_client) == []
+    assert _docs(admin_client) == []
+    assert not checkout.exists()
+    assert _version(db) == version_before + 1
+    assert overview_spy.calls == 1
+    assert "files_removed=yes" in _removal_lines(caplog)[1]
+
+
+# ---------------------------------------------------------------------------
+# version bump + best-effort overview gates
+# ---------------------------------------------------------------------------
+
+
+def test_overview_failure_is_best_effort_still_204(
+    admin_client: TestClient,
+    db: Session,
+    monkeypatch: pytest.MonkeyPatch,
+    tmp_path: Path,
+    caplog: pytest.LogCaptureFixture,
+    overview_spy: _OverviewSpy,
+) -> None:
+    """A failing overview (LLM outage) never fails the delete: the 204
+    lands, the docs stay pruned, and the bump still lands — overview
+    first, bump second (mirroring ``_run_sync``)."""
+    sources = tmp_path / "sources"
+    _point_at(monkeypatch, sources, tmp_path / "uploads")
+    row = _seed_git_row(db, "https://gitlab.example.com/reese/notes.git")
+    checkout = sources / "notes"
+    checkout.mkdir(parents=True)
+    (checkout / "n.md").write_text("x\n", encoding="utf-8")
+    _seed_doc(db, "notes")
+    version_before = _version(db)
+
+    failing = _OverviewSpy(fail=LLMError("simulated lite-model outage"))
+    monkeypatch.setattr(git_sources_api, "regenerate_overview", failing)
+
+    # INFO level — the per-operation line (INFO) and the best-effort
+    # failure (ERROR) are both captured.
+    with caplog.at_level(logging.INFO, logger="app.api.git_sources"):
+        assert admin_client.delete(f"/api/git-sources/{row.id}").status_code == 204
+
+    assert failing.calls == 1
+    assert _docs(admin_client) == []
+    assert not checkout.exists()
+    # The bump lands even though the best-effort overview failed…
+    assert _version(db) == version_before + 1
+    # …and the failure is logged, never fatal.
+    line = _removal_lines(caplog)[0]
+    assert "overview=False" in line, line
+    errors = [
+        rec.getMessage() for rec in caplog.records if rec.levelno >= logging.ERROR
+    ]
+    assert any("overview regeneration failed" in e for e in errors), errors
+
+
+def test_no_docs_pruned_means_no_overview_and_no_bump(
+    admin_client: TestClient,
+    db: Session,
+    monkeypatch: pytest.MonkeyPatch,
+    tmp_path: Path,
+    caplog: pytest.LogCaptureFixture,
+    overview_spy: _OverviewSpy,
+) -> None:
+    """A row with no indexed documents: the checkout dir is still
+    removed (files_removed=yes) but the KB is unchanged — no overview
+    call, no version bump (the gate is docs pruned, not files)."""
+    sources = tmp_path / "sources"
+    _point_at(monkeypatch, sources, tmp_path / "uploads")
+    row = _seed_git_row(db, "https://gitlab.example.com/reese/bare.git")
+    checkout = sources / "bare"
+    checkout.mkdir(parents=True)
+    (checkout / "b.md").write_text("never imported\n", encoding="utf-8")
+    version_before = _version(db)
+
+    with caplog.at_level(logging.INFO, logger="app.api.git_sources"):
+        assert admin_client.delete(f"/api/git-sources/{row.id}").status_code == 204
+
+    assert not checkout.exists()
+    assert _rows(admin_client) == []
+    assert _version(db) == version_before
+    assert overview_spy.calls == 0
+    line = _removal_lines(caplog)[0]
+    assert re.match(
+        r"^source removed: kind=git name=bare docs_pruned=0 files_removed=yes "
+        r"overview=False total_ms=\d+$",
+        line,
+    ), line