feat(admin): one-click sources sync — admin-only button triggers git clone/pull + re-import + KB overview refresh with polled live status

app/api/sync.py

@@ -0,0 +1,177 @@
+"""Sources sync API — one-click KB mirror (phase 32, task 01).
+
+Admin-only ``POST /api/sync`` + ``GET /api/sync/status`` behind the
+existing :func:`app.core.auth.require_admin` (A10 extended, phase 16
+pattern — the public API surface stays stateless, the signed cookie
+remains the only session state, same as ``/api/steering``).
+
+The button's backend runs the full document sync **in-process** (A12
+untouched — no queue, no new services): one ``asyncio`` background task
+plus a module-level :class:`SyncStatus` that the UI polls every 2 s
+(task 02). One sync at a time — ``POST`` while a run is in flight is
+409; the status object is authoritative, so the UI can never sit on a
+stale button state (§7.4 adaptation, phase locked decisions).
+
+Pipeline (the canonical "mirror the repos" action — phase locked
+decisions):
+
+1. resolve the ``BOR_GIT_SOURCES`` URLs — empty/missing fails loudly
+   (``no git sources configured``) instead of silently importing the
+   legacy local directories;
+2. :func:`scripts.git_sync.clone_or_pull` each repo into
+   ``BOR_SOURCES_DIR/<repo-name>/`` (phase 28 — reused, not
+   re-implemented; a failing repo aborts before any import);
+3. ``import_sources(..., prune=True)`` — prune so files deleted
+   upstream leave the index (the CLI's no-prune default is unchanged);
+4. when the import changed the KB (added + updated > 0),
+   ``regenerate_overview`` refreshes the single ``kb_overview`` row
+   (phase 31 trigger, best-effort inside).
+
+Status is in memory: a restart mid-sync loses the running state
+(accepted — the next click re-syncs idempotently).
+"""
+from __future__ import annotations
+
+import asyncio
+import logging
+import re
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any, Literal
+
+from fastapi import APIRouter, Depends, HTTPException
+
+from app.config import get_settings
+from app.core.auth import require_admin
+from app.rag.importer import ImportSummary, import_sources
+from app.rag.llm import LLMClient
+from app.rag.overview import regenerate_overview
+from scripts.git_sync import GitSyncError, clone_or_pull
+from scripts.import_docs import repo_name
+
+logger = logging.getLogger("app.api.sync")
+
+router = APIRouter(
+    prefix="/sync",
+    tags=["sync"],
+    dependencies=[Depends(require_admin)],  # phase 16 pattern: admin-only surface
+)
+
+#: ``user:pass@`` inside any error text (git stderr, endpoint URLs) —
+#: masked so a sync failure can never leak credentials into the UI.
+_CREDS_RE = re.compile(r"[A-Za-z0-9._~%*-]+:[A-Za-z0-9._~%*-]+@")
+
+
+def _sanitize_error(message: str) -> str:
+    """Mask credentials embedded in an error string (no secrets in the UI).
+
+    Git's stderr is otherwise surfaced verbatim (phase locked decisions) —
+    it names the failing repo and git's reason, which is what the admin
+    needs to fix things.
+    """
+    return _CREDS_RE.sub("*****@", message)
+
+
+@dataclass
+class SyncStatus:
+    """In-memory state of the (at most one) in-flight sync run.
+
+    ``state`` is a four-state machine: ``idle`` (never run / reset),
+    ``running``, ``success``, ``failed``. Terminal states carry the run's
+    ``detail`` (success) or ``error`` (failure) so the UI can render the
+    last result after a page reload (task 02's re-attach behavior).
+    """
+
+    state: Literal["idle", "running", "success", "failed"] = "idle"
+    started_at: datetime | None = None
+    finished_at: datetime | None = None
+    detail: dict[str, Any] = field(default_factory=dict)
+    error: str | None = None
+
+
+_status = SyncStatus()
+_task: asyncio.Task[None] | None = None
+
+
+@router.get("/status")
+def sync_status() -> dict[str, Any]:
+    """Current sync state (the UI polls this every 2 s — task 02).
+
+    ``started_at`` / ``finished_at`` are ISO-8601 strings or null.
+    """
+    return {
+        "state": _status.state,
+        "started_at": _status.started_at.isoformat() if _status.started_at else None,
+        "finished_at": _status.finished_at.isoformat() if _status.finished_at else None,
+        "detail": _status.detail,
+        "error": _status.error,
+    }
+
+
+@router.post("", status_code=202)
+async def start_sync() -> dict[str, str]:
+    """Start the clone → import → overview sync as a background task.
+
+    202 + ``sync started`` kicks off :func:`_run_sync` on the app's event
+    loop. 409 when a run is already in flight (one sync at a time — the
+    status endpoint is the single source of truth for the run, and the
+    UI re-attaches to it rather than starting a second one).
+    """
+    global _task
+    if _task is not None and not _task.done():
+        raise HTTPException(status_code=409, detail="a sync is already running")
+    _task = asyncio.create_task(_run_sync())
+    return {"detail": "sync started"}
+
+
+async def _run_sync() -> None:
+    """The full sync pipeline, one in-process background task.
+
+    Every failure mode (git, embeddings, anything else) lands in the
+    ``failed`` state with a sanitized ``error`` string — a background
+    task must die in state, never as an unobserved exception.
+    ``CancelledError`` is deliberately *not* caught: app shutdown
+    cancels the task, and swallowing that would mask a real stop.
+    """
+    _status.state = "running"
+    _status.started_at = datetime.now(UTC)
+    _status.finished_at = None
+    _status.detail = {}
+    _status.error = None
+    try:
+        settings = get_settings()
+        git_urls = settings.git_source_list
+        if not git_urls:
+            # The button targets BOR_GIT_SOURCES only (manual --source
+            # dirs have no repo to clone) — an empty config fails loudly
+            # instead of silently importing the legacy directories.
+            raise GitSyncError("no git sources configured (BOR_GIT_SOURCES)")
+        logger.info("sync: started repos=%d", len(git_urls))
+        sources_root = Path(settings.sources_dir).expanduser()
+        sources = [clone_or_pull(url, sources_root / repo_name(url)) for url in git_urls]
+        llm = LLMClient()
+        summary: ImportSummary = await import_sources(sources, llm, prune=True)
+        overview = False
+        if summary.added + summary.updated > 0:
+            overview = await regenerate_overview(llm)
+        _status.state = "success"
+        _status.finished_at = datetime.now(UTC)
+        _status.detail = {
+            "files": summary.files,
+            "added": summary.added,
+            "updated": summary.updated,
+            "unchanged": summary.unchanged,
+            "pruned": summary.pruned,
+            "errors": summary.errors,
+            "chunks": summary.chunks,
+            "summaries": summary.summaries,
+            "summary_errors": summary.summary_errors,
+            "overview": overview,
+        }
+        logger.info("sync: done detail=%s", _status.detail)
+    except Exception as e:  # noqa: BLE001 — a background task dies in state, see above
+        logger.exception("sync: failed")
+        _status.state = "failed"
+        _status.finished_at = datetime.now(UTC)
+        _status.error = _sanitize_error(str(e))

app/main.py

@@ -25,6 +25,7 @@ from app.api.docs import router as docs_router
 from app.api.health import router as health_router
 from app.api.steering import router as steering_router
 from app.api.suggestions import router as suggestions_router
+from app.api.sync import router as sync_router
 from app.config import get_settings
 from app.core.auth import ensure_admin_configured
 from app.core.debugging import configure_debugging
@@ -64,6 +65,7 @@ def create_app() -> FastAPI:
     app.include_router(docs_router, prefix="/api")
     app.include_router(chat_router, prefix="/api")
     app.include_router(steering_router, prefix="/api")
+    app.include_router(sync_router, prefix="/api")
 
     static_dir = Path(settings.static_dir).resolve()
     if static_dir.is_dir():