brain-of-reese/app/api/sync.py

"""Sources sync API — one-click KB mirror (phase 32, task 01).

Admin-only ``POST /api/sync`` + ``GET /api/sync/status`` behind the
existing :func:`app.core.auth.require_admin` (A10 extended, phase 16
pattern — the public API surface stays stateless, the signed cookie
remains the only session state, same as ``/api/steering``).

The button's backend runs the full document sync **in-process** (A12
untouched — no queue, no new services): one ``asyncio`` background task
plus a module-level :class:`SyncStatus` that the UI polls every 2 s
(task 02). One sync at a time — ``POST`` while a run is in flight is
409; the status object is authoritative, so the UI can never sit on a
stale button state (§7.4 adaptation, phase locked decisions).

Pipeline (the canonical "mirror the repos" action — phase locked
decisions):

1. resolve the effective git sources — the ``git_sources`` DB rows,
   else the ``BOR_GIT_SOURCES`` fallback
   (:func:`app.rag.git_sources.effective_git_sources`, shared with the
   CLI) — empty on both origins fails loudly (``no git sources
   configured``) instead of silently importing the legacy local
   directories;
2. :func:`scripts.git_sync.clone_or_pull` each repo into
   ``BOR_SOURCES_DIR/<repo-name>/`` (phase 28 — reused, not
   re-implemented; a failing repo aborts before any import);
3. ``import_sources(..., prune=True)`` — prune so files deleted
   upstream leave the index (the CLI's no-prune default is unchanged);
4. when the import changed the KB (added + updated > 0),
   ``regenerate_overview`` refreshes the single ``kb_overview`` row
   (phase 31 trigger, best-effort inside).

Status is in memory: a restart mid-sync loses the running state
(accepted — the next click re-syncs idempotently).
"""
from __future__ import annotations

import asyncio
import logging
import re
from dataclasses import dataclass, field
from datetime import UTC, datetime
from pathlib import Path
from typing import Any, Literal

from fastapi import APIRouter, Depends, HTTPException

from app.config import get_settings
from app.core.auth import require_admin
from app.db import SessionLocal
from app.rag.git_sources import effective_git_sources
from app.rag.importer import ImportSummary, import_sources
from app.rag.llm import LLMClient
from app.rag.overview import regenerate_overview
from scripts.git_sync import GitSyncError, clone_or_pull
from scripts.import_docs import repo_name

logger = logging.getLogger("app.api.sync")

router = APIRouter(
    prefix="/sync",
    tags=["sync"],
    dependencies=[Depends(require_admin)],  # phase 16 pattern: admin-only surface
)

#: ``user:pass@`` inside any error text (git stderr, endpoint URLs) —
#: masked so a sync failure can never leak credentials into the UI.
_CREDS_RE = re.compile(r"[A-Za-z0-9._~%*-]+:[A-Za-z0-9._~%*-]+@")


def _sanitize_error(message: str) -> str:
    """Mask credentials embedded in an error string (no secrets in the UI).

    Git's stderr is otherwise surfaced verbatim (phase locked decisions) —
    it names the failing repo and git's reason, which is what the admin
    needs to fix things.
    """
    return _CREDS_RE.sub("*****@", message)


@dataclass
class SyncStatus:
    """In-memory state of the (at most one) in-flight sync run.

    ``state`` is a four-state machine: ``idle`` (never run / reset),
    ``running``, ``success``, ``failed``. Terminal states carry the run's
    ``detail`` (success) or ``error`` (failure) so the UI can render the
    last result after a page reload (task 02's re-attach behavior).
    """

    state: Literal["idle", "running", "success", "failed"] = "idle"
    started_at: datetime | None = None
    finished_at: datetime | None = None
    detail: dict[str, Any] = field(default_factory=dict)
    error: str | None = None


_status = SyncStatus()
_task: asyncio.Task[None] | None = None


@router.get("/status")
def sync_status() -> dict[str, Any]:
    """Current sync state (the UI polls this every 2 s — task 02).

    ``started_at`` / ``finished_at`` are ISO-8601 strings or null.
    """
    return {
        "state": _status.state,
        "started_at": _status.started_at.isoformat() if _status.started_at else None,
        "finished_at": _status.finished_at.isoformat() if _status.finished_at else None,
        "detail": _status.detail,
        "error": _status.error,
    }


@router.post("", status_code=202)
async def start_sync() -> dict[str, str]:
    """Start the clone → import → overview sync as a background task.

    202 + ``sync started`` kicks off :func:`_run_sync` on the app's event
    loop. 409 when a run is already in flight (one sync at a time — the
    status endpoint is the single source of truth for the run, and the
    UI re-attaches to it rather than starting a second one).
    """
    global _task
    if _task is not None and not _task.done():
        raise HTTPException(status_code=409, detail="a sync is already running")
    _task = asyncio.create_task(_run_sync())
    return {"detail": "sync started"}


async def _run_sync() -> None:
    """The full sync pipeline, one in-process background task.

    Every failure mode (git, embeddings, anything else) lands in the
    ``failed`` state with a sanitized ``error`` string — a background
    task must die in state, never as an unobserved exception.
    ``CancelledError`` is deliberately *not* caught: app shutdown
    cancels the task, and swallowing that would mask a real stop.
    """
    _status.state = "running"
    _status.started_at = datetime.now(UTC)
    _status.finished_at = None
    _status.detail = {}
    _status.error = None
    try:
        settings = get_settings()
        # The background task has no request session: open a short-lived
        # one around the shared phase-35 resolver (DB rows win, the
        # BOR_GIT_SOURCES list is a fallback while the table is empty).
        db = SessionLocal()
        try:
            git_urls, origin = effective_git_sources(db)
        finally:
            db.close()
        if not git_urls:
            # The button targets git sources only (manual --source dirs
            # have no repo to clone) — an empty config on *both* origins
            # fails loudly instead of silently importing the legacy
            # directories.
            raise GitSyncError(
                "no git sources configured (git_sources table empty and BOR_GIT_SOURCES unset)"
            )
        logger.info("sync: started repos=%d origin=%s", len(git_urls), origin)
        sources_root = Path(settings.sources_dir).expanduser()
        sources = [clone_or_pull(url, sources_root / repo_name(url)) for url in git_urls]
        llm = LLMClient()
        summary: ImportSummary = await import_sources(sources, llm, prune=True)
        overview = False
        if summary.added + summary.updated > 0:
            overview = await regenerate_overview(llm)
        _status.state = "success"
        _status.finished_at = datetime.now(UTC)
        _status.detail = {
            "files": summary.files,
            "added": summary.added,
            "updated": summary.updated,
            "unchanged": summary.unchanged,
            "pruned": summary.pruned,
            "errors": summary.errors,
            "chunks": summary.chunks,
            "summaries": summary.summaries,
            "summary_errors": summary.summary_errors,
            "overview": overview,
        }
        logger.info("sync: done detail=%s", _status.detail)
    except Exception as e:  # noqa: BLE001 — a background task dies in state, see above
        logger.exception("sync: failed")
        _status.state = "failed"
        _status.finished_at = datetime.now(UTC)
        _status.error = _sanitize_error(str(e))