feat(admin): one-click sources sync — admin-only button triggers git clone/pull + re-import + KB overview refresh with polled live status
This commit is contained in:
+177
@@ -0,0 +1,177 @@
|
||||
"""Sources sync API — one-click KB mirror (phase 32, task 01).
|
||||
|
||||
Admin-only ``POST /api/sync`` + ``GET /api/sync/status`` behind the
|
||||
existing :func:`app.core.auth.require_admin` (A10 extended, phase 16
|
||||
pattern — the public API surface stays stateless, the signed cookie
|
||||
remains the only session state, same as ``/api/steering``).
|
||||
|
||||
The button's backend runs the full document sync **in-process** (A12
|
||||
untouched — no queue, no new services): one ``asyncio`` background task
|
||||
plus a module-level :class:`SyncStatus` that the UI polls every 2 s
|
||||
(task 02). One sync at a time — ``POST`` while a run is in flight is
|
||||
409; the status object is authoritative, so the UI can never sit on a
|
||||
stale button state (§7.4 adaptation, phase locked decisions).
|
||||
|
||||
Pipeline (the canonical "mirror the repos" action — phase locked
|
||||
decisions):
|
||||
|
||||
1. resolve the ``BOR_GIT_SOURCES`` URLs — empty/missing fails loudly
|
||||
(``no git sources configured``) instead of silently importing the
|
||||
legacy local directories;
|
||||
2. :func:`scripts.git_sync.clone_or_pull` each repo into
|
||||
``BOR_SOURCES_DIR/<repo-name>/`` (phase 28 — reused, not
|
||||
re-implemented; a failing repo aborts before any import);
|
||||
3. ``import_sources(..., prune=True)`` — prune so files deleted
|
||||
upstream leave the index (the CLI's no-prune default is unchanged);
|
||||
4. when the import changed the KB (added + updated > 0),
|
||||
``regenerate_overview`` refreshes the single ``kb_overview`` row
|
||||
(phase 31 trigger, best-effort inside).
|
||||
|
||||
Status is in memory: a restart mid-sync loses the running state
|
||||
(accepted — the next click re-syncs idempotently).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
|
||||
from app.config import get_settings
|
||||
from app.core.auth import require_admin
|
||||
from app.rag.importer import ImportSummary, import_sources
|
||||
from app.rag.llm import LLMClient
|
||||
from app.rag.overview import regenerate_overview
|
||||
from scripts.git_sync import GitSyncError, clone_or_pull
|
||||
from scripts.import_docs import repo_name
|
||||
|
||||
logger = logging.getLogger("app.api.sync")
|
||||
|
||||
router = APIRouter(
|
||||
prefix="/sync",
|
||||
tags=["sync"],
|
||||
dependencies=[Depends(require_admin)], # phase 16 pattern: admin-only surface
|
||||
)
|
||||
|
||||
#: ``user:pass@`` inside any error text (git stderr, endpoint URLs) —
|
||||
#: masked so a sync failure can never leak credentials into the UI.
|
||||
_CREDS_RE = re.compile(r"[A-Za-z0-9._~%*-]+:[A-Za-z0-9._~%*-]+@")
|
||||
|
||||
|
||||
def _sanitize_error(message: str) -> str:
|
||||
"""Mask credentials embedded in an error string (no secrets in the UI).
|
||||
|
||||
Git's stderr is otherwise surfaced verbatim (phase locked decisions) —
|
||||
it names the failing repo and git's reason, which is what the admin
|
||||
needs to fix things.
|
||||
"""
|
||||
return _CREDS_RE.sub("*****@", message)
|
||||
|
||||
|
||||
@dataclass
|
||||
class SyncStatus:
|
||||
"""In-memory state of the (at most one) in-flight sync run.
|
||||
|
||||
``state`` is a four-state machine: ``idle`` (never run / reset),
|
||||
``running``, ``success``, ``failed``. Terminal states carry the run's
|
||||
``detail`` (success) or ``error`` (failure) so the UI can render the
|
||||
last result after a page reload (task 02's re-attach behavior).
|
||||
"""
|
||||
|
||||
state: Literal["idle", "running", "success", "failed"] = "idle"
|
||||
started_at: datetime | None = None
|
||||
finished_at: datetime | None = None
|
||||
detail: dict[str, Any] = field(default_factory=dict)
|
||||
error: str | None = None
|
||||
|
||||
|
||||
_status = SyncStatus()
|
||||
_task: asyncio.Task[None] | None = None
|
||||
|
||||
|
||||
@router.get("/status")
|
||||
def sync_status() -> dict[str, Any]:
|
||||
"""Current sync state (the UI polls this every 2 s — task 02).
|
||||
|
||||
``started_at`` / ``finished_at`` are ISO-8601 strings or null.
|
||||
"""
|
||||
return {
|
||||
"state": _status.state,
|
||||
"started_at": _status.started_at.isoformat() if _status.started_at else None,
|
||||
"finished_at": _status.finished_at.isoformat() if _status.finished_at else None,
|
||||
"detail": _status.detail,
|
||||
"error": _status.error,
|
||||
}
|
||||
|
||||
|
||||
@router.post("", status_code=202)
|
||||
async def start_sync() -> dict[str, str]:
|
||||
"""Start the clone → import → overview sync as a background task.
|
||||
|
||||
202 + ``sync started`` kicks off :func:`_run_sync` on the app's event
|
||||
loop. 409 when a run is already in flight (one sync at a time — the
|
||||
status endpoint is the single source of truth for the run, and the
|
||||
UI re-attaches to it rather than starting a second one).
|
||||
"""
|
||||
global _task
|
||||
if _task is not None and not _task.done():
|
||||
raise HTTPException(status_code=409, detail="a sync is already running")
|
||||
_task = asyncio.create_task(_run_sync())
|
||||
return {"detail": "sync started"}
|
||||
|
||||
|
||||
async def _run_sync() -> None:
|
||||
"""The full sync pipeline, one in-process background task.
|
||||
|
||||
Every failure mode (git, embeddings, anything else) lands in the
|
||||
``failed`` state with a sanitized ``error`` string — a background
|
||||
task must die in state, never as an unobserved exception.
|
||||
``CancelledError`` is deliberately *not* caught: app shutdown
|
||||
cancels the task, and swallowing that would mask a real stop.
|
||||
"""
|
||||
_status.state = "running"
|
||||
_status.started_at = datetime.now(UTC)
|
||||
_status.finished_at = None
|
||||
_status.detail = {}
|
||||
_status.error = None
|
||||
try:
|
||||
settings = get_settings()
|
||||
git_urls = settings.git_source_list
|
||||
if not git_urls:
|
||||
# The button targets BOR_GIT_SOURCES only (manual --source
|
||||
# dirs have no repo to clone) — an empty config fails loudly
|
||||
# instead of silently importing the legacy directories.
|
||||
raise GitSyncError("no git sources configured (BOR_GIT_SOURCES)")
|
||||
logger.info("sync: started repos=%d", len(git_urls))
|
||||
sources_root = Path(settings.sources_dir).expanduser()
|
||||
sources = [clone_or_pull(url, sources_root / repo_name(url)) for url in git_urls]
|
||||
llm = LLMClient()
|
||||
summary: ImportSummary = await import_sources(sources, llm, prune=True)
|
||||
overview = False
|
||||
if summary.added + summary.updated > 0:
|
||||
overview = await regenerate_overview(llm)
|
||||
_status.state = "success"
|
||||
_status.finished_at = datetime.now(UTC)
|
||||
_status.detail = {
|
||||
"files": summary.files,
|
||||
"added": summary.added,
|
||||
"updated": summary.updated,
|
||||
"unchanged": summary.unchanged,
|
||||
"pruned": summary.pruned,
|
||||
"errors": summary.errors,
|
||||
"chunks": summary.chunks,
|
||||
"summaries": summary.summaries,
|
||||
"summary_errors": summary.summary_errors,
|
||||
"overview": overview,
|
||||
}
|
||||
logger.info("sync: done detail=%s", _status.detail)
|
||||
except Exception as e: # noqa: BLE001 — a background task dies in state, see above
|
||||
logger.exception("sync: failed")
|
||||
_status.state = "failed"
|
||||
_status.finished_at = datetime.now(UTC)
|
||||
_status.error = _sanitize_error(str(e))
|
||||
Reference in New Issue
Block a user