feat(rag): git-based import sources — BOR_GIT_SOURCES repos cloned (first run, --depth 1) or pulled (--ff-only) into BOR_SOURCES_DIR/<repo>/ then indexed; --source still wins; a failed sync aborts before importing anything

scripts/git_sync.py

@@ -0,0 +1,60 @@
+"""Git source sync for import_docs (phase 28).
+
+clone_or_pull(url, dest) clones ``url`` into ``dest`` (shallow, depth 1)
+the first time, or fast-forwards an existing checkout with ``git pull
+--ff-only`` on subsequent runs.
+
+Auth: nothing special — an ``https://…`` URL uses the OS credential
+helper / prompts; a ``git@host:repo.git`` URL uses the machine's SSH key.
+No credentials are stored here; whatever the URL/SSH config supplies is
+used.
+
+This module is the only place the ``git`` CLI is invoked (A11: stdlib
+``subprocess`` only, no new packages).
+"""
+from __future__ import annotations
+
+import subprocess
+from pathlib import Path
+
+__all__ = ["GitSyncError", "clone_or_pull"]
+
+
+class GitSyncError(RuntimeError):
+    """A git clone/pull failed (or git is missing); carries git's stderr."""
+
+
+def clone_or_pull(url: str, dest: Path | str) -> Path:
+    """Clone ``url`` into ``dest`` (shallow, first run) or fast-forward it.
+
+    - dest without a ``.git`` (or absent) → ``git clone --depth 1 url dest``
+      (shallow: the KB is re-imported incrementally anyway).
+    - dest with a ``.git`` → ``git pull --ff-only`` (refuses to merge
+      unrelated histories — a broken checkout fails loudly rather than
+      producing a dirty index).
+
+    Returns the destination path. Raises :class:`GitSyncError` when git is
+    missing or a git invocation exits non-zero (with git's stderr in the
+    message, so the caller can name the failing repo + reason).
+    """
+    dest = Path(dest)
+    if not dest.exists() or not (dest / ".git").exists():
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        _run(["git", "clone", "--depth", "1", url, str(dest)], cwd=dest.parent)
+    else:
+        _run(["git", "pull", "--ff-only"], cwd=dest)
+    return dest
+
+
+def _run(argv: list[str], cwd: Path) -> str:
+    """Run a git command, capturing output; raise GitSyncError on failure."""
+    try:
+        proc = subprocess.run(argv, cwd=cwd, capture_output=True, text=True)
+    except FileNotFoundError:
+        raise GitSyncError("git was not found on PATH — install git and retry") from None
+    if proc.returncode != 0:
+        raise GitSyncError(
+            f"git {' '.join(argv[1:])} failed (exit {proc.returncode}): "
+            f"{proc.stderr.strip()}"
+        )
+    return proc.stdout

scripts/import_docs.py

@@ -2,16 +2,28 @@
 
 Examples::
 
-    uv run python -m scripts.import_docs                      # ~/Homelab + ~/Deployments
-    uv run python -m scripts.import_docs --source ~/OtherDocs # extra dir (repeatable)
+    uv run python -m scripts.import_docs      # BOR_GIT_SOURCES, else ~/Homelab + ~/Deployments
+    uv run python -m scripts.import_docs --source ~/OtherDocs # explicit dir(s); always wins
     uv run python -m scripts.import_docs --prune              # also drop deleted/out-of-scope files
     uv run python -m scripts.import_docs --limit 5            # debug: first 5 files only
 
+Source resolution (phase 28), in precedence order:
+
+1. ``--source PATH`` — explicit manual directories (repeatable) always win;
+   ``BOR_GIT_SOURCES`` is ignored when this flag is used.
+2. ``BOR_GIT_SOURCES`` (comma-separated git URLs) — each repo is cloned
+   (first run, shallow ``--depth 1``) or fast-forwarded (``git pull
+   --ff-only``) into ``BOR_SOURCES_DIR/<repo-name>/`` (default
+   ``~/bor-sources``) and the resulting checkouts are imported. A failing
+   clone/pull aborts the whole run *before* anything is imported.
+3. Fallback — the legacy ``DEFAULT_SOURCES`` (``~/Homelab`` +
+   ``~/Deployments``), kept for backwards compatibility.
+
 Imported formats (PLAN anchor A9, revised): ``md, markdown, txt, yaml,
 yml, json, py`` (case-insensitive; narrow with ``BOR_IMPORT_EXTENSIONS``).
 Any path with a dot-prefixed component (hidden files/dirs — vendored
 caches) is skipped, along with non-content dirs (``.venv``,
-``node_modules``, ``.git``, ``__pycache__``, ``.pytest_cache``, ``dist``,
+``node_modules``, ``.git``, ``__pycache``, ``.pytest_cache``, ``dist``,
 ``build``). Re-runs are cheap: files are diffed by sha256 and unchanged
 ones are not re-embedded; ``--prune`` also drops documents whose files no
 longer match the format filter.
@@ -20,14 +32,19 @@ from __future__ import annotations
 
 import argparse
 import asyncio
+import logging
+import re
 import sys
 from pathlib import Path
 
-from app.config import get_settings
+from app.config import Settings, get_settings
 from app.core.debugging import configure_debugging
 from app.core.logging import configure_logging
 from app.rag.importer import import_sources
 from app.rag.llm import LLMClient
+from scripts.git_sync import GitSyncError, clone_or_pull
+
+logger = logging.getLogger("scripts.import_docs")
 
 DEFAULT_SOURCES: list[Path] = [Path("~/Homelab"), Path("~/Deployments")]
 
@@ -42,7 +59,10 @@ def build_parser() -> argparse.ArgumentParser:
         action="append",
         type=Path,
         metavar="PATH",
-        help="directory to import (repeatable; default: ~/Homelab ~/Deployments)",
+        help=(
+            "directory to import (repeatable; always wins over BOR_GIT_SOURCES; "
+            "default when neither is given: ~/Homelab ~/Deployments)"
+        ),
     )
     p.add_argument(
         "--prune",
@@ -59,12 +79,63 @@ def build_parser() -> argparse.ArgumentParser:
     return p
 
 
+def repo_name(url: str) -> str:
+    """Local directory name for a git URL (phase 28).
+
+    Strips a trailing ``.git`` and takes the basename after the last ``/``
+    (``:`` for scp-style ``git@host:repo.git`` URLs); falls back to a slug
+    of the whole URL when no usable basename remains.
+    """
+    name = url.strip()
+    if name.endswith(".git"):
+        name = name[: -len(".git")]
+    base = name.rsplit("/", 1)[-1].rsplit(":", 1)[-1].strip()
+    if base:
+        return base
+    slug = re.sub(r"[^A-Za-z0-9]+", "-", name).strip("-")
+    return slug or "repo"
+
+
+def _resolve_sources(cli_sources: list[Path] | None, settings: Settings) -> list[Path]:
+    """Resolve the directories to import (phase 28).
+
+    Precedence: ``--source`` (explicit manual paths — always wins) >
+    ``BOR_GIT_SOURCES`` (each URL cloned/pulled via
+    :func:`scripts.git_sync.clone_or_pull` into
+    ``BOR_SOURCES_DIR/<repo-name>/``) > the legacy ``DEFAULT_SOURCES``.
+
+    A :class:`GitSyncError` from a failing clone/pull propagates to
+    :func:`main`, which aborts the run before importing anything.
+    """
+    if cli_sources:
+        return [path.expanduser() for path in cli_sources]
+    git_urls = settings.git_source_list
+    if git_urls:
+        sources_root = Path(settings.sources_dir).expanduser()
+        return [clone_or_pull(url, sources_root / repo_name(url)) for url in git_urls]
+    return [path.expanduser() for path in DEFAULT_SOURCES]
+
+
 def main(argv: list[str] | None = None) -> int:
     args = build_parser().parse_args(argv)
-    configure_logging(get_settings().log_level)
+    settings = get_settings()
+    configure_logging(settings.log_level)
     configure_debugging()
 
-    sources = [path.expanduser() for path in (args.source or DEFAULT_SOURCES)]
+    # Git sources resolve (and clone/pull) *before* any import: a failing
+    # repo aborts the run with a non-zero exit, naming the failure — a bad
+    # URL must never silently import partial junk.
+    try:
+        sources = _resolve_sources(args.source, settings)
+    except GitSyncError as e:
+        print(f"import_docs: git sync failed: {e}", file=sys.stderr)
+        return 1
+
+    logger.info(
+        "import_docs: importing %d source dir(s): %s",
+        len(sources),
+        ", ".join(str(s) for s in sources),
+    )
     missing = [s for s in sources if not s.is_dir()]
     for s in missing:
         print(f"import_docs: source dir not found: {s}", file=sys.stderr)