feat(sources): removing a source deletes its files and index entries behind a confirmation modal

app/api/git_sources.py

@@ -28,8 +28,10 @@ task** — see :func:`upload_archive` and :func:`_run_upload`),
 state of that run — incl. the phase-64 ``current_file`` /
 ``files_done`` / ``files_total`` progress fields; navigating away from
 the page mid-scan no longer aborts anything), ``DELETE /{source_id}``
-(204). The whole router sits behind :func:`app.core.auth.require_admin`
-— anonymous callers get 403 on every route.
+(204 — total removal, phase 69: row + the source's documents (chunks +
+embeddings) committed first, then the app-managed on-disk dir). The
+whole router sits behind :func:`app.core.auth.require_admin` —
+anonymous callers get 403 on every route.
 
 No credential-echo path: git URLs may embed ``user:pass@`` (phase 32's
 masking discipline), so every git 409/422 detail is a fixed generic
@@ -38,9 +40,20 @@ secrets — the local 422/409 details name the (expanded) path so the
 owner sees exactly which directory failed.
 
 Scope boundary (phase locked decisions): the CRUD routes do NOT
-clone, import, or prune anything — the existing Sync button performs
-that (a removal prunes on the next sync, ``prune=True``). The upload
-route is the exception (phase 64, task 03): after the 202 receive
+clone or import anything — the existing Sync button performs that, and
+its ``prune=True`` stays for **upstream file churn** (files deleted
+upstream or dropped from a local dir), not for row removal: ``DELETE``
+is a total removal in itself (phase 69) — the row + the source's
+documents (chunks + embeddings via the ``all, delete-orphan`` cascade)
+commit atomically **first** (the RAG is always consistent with the
+registry), then the app-managed on-disk dir (git checkout / unpacked
+upload folder) is deleted; an ``OSError`` there is logged, never fatal.
+Foreign local dirs (the owner's own) are never touched on disk, a
+sibling row sharing the source name keeps the shared documents +
+files (only the row goes), and a pruned KB bumps ``sources_version``
+exactly once (the phase-53 saved-chat invalidation) with a
+best-effort overview refresh. The upload route is the other exception
+(phase 64, task 03): after the 202 receive
 answer, its background task unpacks the archive, swaps it in, upserts
 the row, probes the models, scans the single source
 (``import_sources`` with ``prune=True`` + the change-gated overview
@@ -69,7 +82,7 @@ from app.api.sync import _sanitize_error
 from app.config import get_settings
 from app.core.auth import require_admin
 from app.db import SessionLocal, get_db
-from app.models import GitSource
+from app.models import Document, GitSource
 from app.rag.archive_upload import (
     ARCHIVE_SUFFIXES,
     ArchiveUploadError,
@@ -80,6 +93,13 @@ from app.rag.archive_upload import (
 from app.rag.importer import import_sources
 from app.rag.llm import LLMClient, check_models
 from app.rag.overview import regenerate_overview
+from app.rag.source_removal import (
+    has_sibling,
+    managed_dir_for,
+    remove_managed_dir,
+    resolve_source_name,
+)
+from app.rag.sources_meta import bump_sources_version
 from app.schemas import (
     GitSourceIn,
     GitSourceList,
@@ -564,18 +584,113 @@ async def _run_upload(
 
 
 @router.delete("/{source_id}", status_code=204)
-def delete_git_source(
+async def delete_git_source(
     source_id: uuid.UUID,
     db: Session = Depends(get_db),  # noqa: B008
 ) -> Response:
-    """Remove a stored row; 404 when the id is unknown.
+    """Remove a stored source — **totally** (phase 69); 404 unknown id.
 
-    Removing does not touch the clones or the index — the next Sync
-    (``prune=True``) prunes the dropped repo (phase scope boundary).
+    Total removal, one action: the row, every indexed document of the
+    source (chunks + embeddings via the ``all, delete-orphan`` cascade
+    — ``app.models``), and, for app-managed sources, the on-disk dir
+    (the git checkout or the unpacked upload folder). Foreign local
+    directories (the owner's own) are never touched on disk — their row
+    + index entries still go.
+
+    Locked order (phase 69):
+
+    1. **Sibling guard** — another stored row resolves to the same
+       source name (e.g. ``…/r`` and ``…/r.git``): it still owns the
+       shared documents + files, so only this row is deleted (logged
+       loudly); no index/disk work.
+    2. **DB first** — in the request transaction: the source's
+       documents are deleted, then the row, one ``commit``. A DB
+       failure propagates as 500 **before any disk work** — the 204
+       contract below never sees a half-removal.
+    3. **Disk second** — the app-managed dir is removed after the
+       commit; an ``OSError`` is logged, never fatal (a leftover dir
+       is inert and self-heals on re-add; the reverse order is
+       forbidden — a disk failure must never leave a row pointing at
+       deleted files). Foreign local dir → skipped.
+    4. **When documents were pruned** — the best-effort overview
+       refresh (an LLM failure logs, never fails the 204; the next
+       added/updated change refreshes it, as today) and then exactly
+       one ``sources_version`` bump (phase 53 — the bump lands even if
+       the overview failed, mirroring ``_run_sync``'s order). No docs
+       pruned → no overview, no bump.
+
+    The route contract is unchanged: 204 with no body; the per-
+    operation INFO line (PLAN §9) carries the counts —
+    ``files_removed`` is ``yes|no|skipped`` (``skipped`` for the
+    sibling guard and for foreign local dirs).
     """
+    started = time.monotonic()
     row = db.get(GitSource, source_id)
     if row is None:
         raise HTTPException(status_code=404, detail="git source not found")
-    db.delete(row)
-    db.commit()
+
+    name = resolve_source_name(row)
+    settings = get_settings()
+    managed = managed_dir_for(row, Path(settings.sources_dir), Path(settings.upload_dir))
+
+    if has_sibling(db, row):
+        # Sibling guard: the shared documents + files still belong to
+        # the sibling row — only this row goes (phase 69 locked
+        # decision), so no index prune and no disk work.
+        logger.warning(
+            "source removal: row %s (url=%s) shares source name %s with "
+            "another stored row — deleting only the row; the shared "
+            "documents and files stay",
+            row.id,
+            row.url,
+            name,
+        )
+        db.delete(row)
+        db.commit()
+        docs_pruned, files_removed, overview = 0, "skipped", False
+    else:
+        # DB first: prune every document of the source (the cascade
+        # drops all chunks incl. embeddings) and the row itself in one
+        # commit — the RAG is always consistent with the registry.
+        docs = db.scalars(select(Document).where(Document.source == name)).all()
+        for doc in docs:
+            db.delete(doc)
+        db.delete(row)
+        db.commit()
+        docs_pruned = len(docs)
+        # Disk second: only the app-managed dir (git checkout / upload
+        # folder) — ``managed is None`` is the foreign local dir (the
+        # owner's own), never touched on disk.
+        if managed is None:
+            files_removed = "skipped"
+        else:
+            files_removed = "yes" if remove_managed_dir(managed) else "no"
+        # KB changed (documents pruned) → best-effort overview refresh,
+        # then exactly one sources_version bump — the bump in its own
+        # short-lived session (the ``_run_sync`` pattern) so it lands
+        # even if the best-effort overview failed.
+        overview = False
+        if docs_pruned > 0:
+            try:
+                overview = await regenerate_overview(LLMClient())
+            except Exception:  # noqa: BLE001 — best-effort: never fail the 204
+                logger.exception("source removal: overview regeneration failed (best-effort)")
+                overview = False
+            bump_db = SessionLocal()
+            try:
+                bump_sources_version(bump_db)
+                bump_db.commit()
+            finally:
+                bump_db.close()
+
+    logger.info(
+        "source removed: kind=%s name=%s docs_pruned=%d files_removed=%s "
+        "overview=%s total_ms=%d",
+        row.kind,
+        name,
+        docs_pruned,
+        files_removed,
+        overview,
+        round((time.monotonic() - started) * 1000),
+    )
     return Response(status_code=204)

app/rag/source_removal.py

@@ -0,0 +1,138 @@
+"""Total source removal helpers (phase 69, task 01).
+
+Removing a source is a **total removal** (owner request 2026-09-02):
+the stored row, every indexed document of that source (chunks +
+embeddings), and — for app-managed sources — the files on disk (the git
+checkout or the unpacked upload folder), all in one action. These
+helpers carry the row-agnostic pieces (the source-name resolution, the
+managed-dir mapping, the disk removal, the sibling guard) so the
+``DELETE /api/git-sources/{id}`` endpoint stays thin.
+
+Locked order (phase 69, ``00_phase.md``):
+
+* **DB first, disk second.** The row + document prune commit atomically
+  first (the RAG is always consistent with the registry — the owner's
+  core ask); the disk removal runs **after** the commit, and an
+  ``OSError`` is logged but never fails the 204 (a leftover dir is
+  inert — no row → never imported — and self-heals on re-add). The
+  reverse order is forbidden: a disk failure must never leave a row
+  pointing at deleted files.
+* **App-managed files only.** Git checkouts live under
+  ``sources_dir/<repo>/`` and unpacked uploads under
+  ``upload_dir/<name>/`` — both app-managed. A ``kind='local'`` row
+  pointing at any other directory (the owner's own) is never touched
+  on disk — only its row + index entries are removed.
+* **Sibling guard.** Two rows that resolve to the same source name
+  (e.g. ``https://e.com/r`` and ``https://e.com/r.git`` → both ``r``)
+  share documents and files — removing one of them deletes only the
+  row; the shared documents and files stay.
+
+The module is stdlib + SQLAlchemy + ``app``/``scripts`` imports only
+(**no FastAPI**), so every helper unit-tests with plain objects and
+``tmp_path``.
+"""
+from __future__ import annotations
+
+import logging
+import shutil
+from pathlib import Path
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from app.models import GitSource
+from scripts.import_docs import repo_name
+
+logger = logging.getLogger("app.rag.source_removal")
+
+
+def resolve_source_name(row: GitSource) -> str:
+    """The label under which the sync/importer index this row's documents.
+
+    Exactly the expressions the import pipeline walks — reuse them here
+    or removal would prune the wrong documents: ``kind='git'`` →
+    :func:`scripts.import_docs.repo_name` (the phase-28 helper — strips
+    the trailing ``.git`` and handles scp-style ``git@host:repo``);
+    ``kind='local'`` → ``Path(row.path or row.url).expanduser().name``
+    (the same expression ``app.api.sync._run_sync`` walks; phase 38
+    mirrors the expanded path in the NOT-NULL ``url`` column, the
+    ``or`` keeps the type checker honest).
+    """
+    if row.kind == "git":
+        return repo_name(row.url)
+    return Path(row.path or row.url).expanduser().name
+
+
+def managed_dir_for(row: GitSource, sources_dir: Path, upload_dir: Path) -> Path | None:
+    """The app-managed on-disk directory whose files belong to ``row``.
+
+    * ``kind='git'`` → ``sources_dir/<repo-name>/`` — the checkout
+      location (the ``clone_or_pull`` target of the sync/import).
+    * ``kind='local'`` → the stored directory **only when** it equals
+      ``upload_dir`` or is nested under it (the unpacked uploads,
+      phase 49). Containment is checked on **resolved** paths with
+      ``parents``, so a sibling directory whose name merely shares a
+      prefix (``…/uploads-foo`` next to ``…/uploads``) never counts —
+      and a symlink that escapes the upload dir never counts either
+      (``resolve()`` follows it to its target).
+    * any other ``kind='local'`` path → ``None``: the owner's own
+      directory, **never** touched on disk (row + index entries only).
+
+    ``None`` as well when containment cannot be established (an
+    unresolvable path) — never delete when in doubt.
+    """
+    if row.kind == "git":
+        return Path(sources_dir).expanduser() / repo_name(row.url)
+    stored = Path(row.path or row.url).expanduser()
+    upload = Path(upload_dir).expanduser()
+    try:
+        stored_resolved = stored.resolve()
+        upload_resolved = upload.resolve()
+    except OSError:
+        return None
+    if stored_resolved == upload_resolved or upload_resolved in stored_resolved.parents:
+        return stored
+    return None
+
+
+def remove_managed_dir(directory: Path | None) -> bool:
+    """Delete the app-managed directory (recursively), best-effort.
+
+    ``None`` or an absent directory → ``False`` (a no-op, no
+    filesystem write — the row may simply have no checkout yet).
+    Present → ``shutil.rmtree`` → ``True`` (only a dir that was
+    actually removed). An ``OSError`` (permissions, a busy file, a
+    symlink-to-dir refusal, …) is logged with ``logger.exception`` and
+    returns ``False`` — this function **never raises**: the row + index
+    are already committed at call time, and a leftover dir is inert
+    (no row → never imported) and self-heals on re-add (git re-clones,
+    a re-upload recreates the folder).
+    """
+    if directory is None or not directory.exists():
+        return False
+    try:
+        shutil.rmtree(directory)
+    except OSError:
+        logger.exception(
+            "source removal: could not remove the on-disk directory %s", directory
+        )
+        return False
+    return True
+
+
+def has_sibling(db: Session, row: GitSource) -> bool:
+    """Whether another stored row resolves to the same source name.
+
+    The registry is tiny — every row is resolved in Python (no SQL
+    trickery): ``True`` when a different-id row's
+    :func:`resolve_source_name` equals this row's (e.g.
+    ``https://e.com/r`` and ``https://e.com/r.git`` → both ``r``). Such
+    a sibling still owns the shared documents and files, so the caller
+    must delete **only the row** and skip the index/disk work (the
+    phase-69 sibling guard).
+    """
+    name = resolve_source_name(row)
+    for other in db.scalars(select(GitSource)).all():
+        if other.id != row.id and resolve_source_name(other) == name:
+            return True
+    return False