brain-of-reese/scripts/git_sync.py

"""Git source sync for import_docs (phase 28).

clone_or_pull(url, dest, branch=None) clones ``url`` into ``dest`` with
FULL history the first time — ``git clone --branch <b>`` when a branch
is pinned (phase 135: the source's branch is checked out from the
first commit), plain ``git clone`` when not — or, for an existing
checkout, runs the branch guard first when a branch is pinned (a
checkout on any other branch fails loudly — the source must be removed
and re-added to switch, phase 135), unshallows a shallow one first
(``git fetch --unshallow``, the one-time self-heal), and then
fast-forwards it — ``git pull --ff-only`` with no branch, or the
single-fetch ``git fetch origin <b>`` + ``git merge --ff-only
FETCH_HEAD`` of the pinned ref with one (``branch=None`` keeps the
pre-phase argv byte-identical — the unit argv pins are the proof).

Auth: nothing special — an ``https://…`` URL uses the OS credential
helper / prompts; a ``git@host:repo.git`` URL uses the machine's SSH key.
No credentials are stored here; whatever the URL/SSH config supplies is
used.

This module is the only place the ``git`` CLI is invoked (A11: stdlib
``subprocess`` only, no new packages) — every git command goes through
:func:`run_git`: the clone (with ``--branch`` when a branch is
pinned, phase 135), the branch guard's ``git rev-parse --abbrev-ref
HEAD`` probe, the shallow probe, the one-time ``git fetch
--unshallow``, the pinned branch's ``git fetch origin <b>`` +
``git merge --ff-only FETCH_HEAD`` pair, and the ``git pull
--ff-only`` in :func:`clone_or_pull`; the per-file last-commit-date
walk in :func:`file_commit_dates` (phase 106 — it walks the
checkout's own log, so a branch checkout yields the BRANCH's history:
the head-of-branch semantics phase 135 pins, untouched); and the
docs-push sequence in :mod:`app.core.docs_push` (phase 59).

Per-file last-commit dates (phase 106 D2; phase 107 D11 supersedes
phase 106 D10) — behavior verified against scratch and live repos
2026-09-13 / 2026-09-16:

* every :func:`clone_or_pull` checkout is FULL history for EVERY
  transport (https/ssh/``file://``/local-path): a fresh clone carries
  no ``--depth`` (D11 — D10's ``--depth 1`` shallow URL clones are
gone; ``--branch`` clones, phase 135, included), and an existing
  shallow checkout (made pre-phase 107 — i.e. every deployed one,
  live + dev included) is probed with ``git
  rev-parse --is-shallow-repository`` and, while shallow, self-healed
  with the ONE-TIME ``git fetch --unshallow`` before the usual ``git
  pull --ff-only`` (no branch) / ``git merge --ff-only FETCH_HEAD``
  (pinned branch) — no re-clone;
* on a full-history checkout :func:`file_commit_dates` yields the
  TRUE per-file last-commit date for ALL git sources. The old
  shallow-clone behavior was the bug the owner reported 2026-09-16:
  in a shallow clone git reports the TIP commit as every existing
  file's last commit (the shallow boundary is each file's history
  root) → a uniform per-repo tip date (``container_bifrost`` shown as
  created on the repo tip, months off); after ``git fetch
  --unshallow`` on the live homelab checkout (438 commits visible)
  the walk returned the true dates — e.g. ``bifrost.md`` 2026-05-05,
  not the 2026-09-07 tip.
"""
from __future__ import annotations

import logging
import subprocess
from datetime import datetime
from pathlib import Path

logger = logging.getLogger(__name__)

__all__ = ["GitSyncError", "clone_or_pull", "file_commit_dates", "run_git"]


class GitSyncError(RuntimeError):
    """A git clone/pull failed (or git is missing); carries git's stderr."""


def clone_or_pull(url: str, dest: Path | str, branch: str | None = None) -> Path:
    """Clone ``url`` into ``dest`` (full history, first run) or fast-forward it.

    - dest without a ``.git`` (or absent) → ``git clone url dest`` (full
      history for every transport — phase 107 D11: no ``--depth``, so a
      URL-transport checkout carries the whole commit log, not just the
      tip); with a pinned ``branch`` → ``git clone --branch <b> url
      dest`` (phase 135: the source's branch is checked out from the
      first commit — no post-clone checkout; ``branch=None`` keeps the
      two-argument argv byte-identical).
    - dest with a ``.git`` and no ``branch`` → probe ``git rev-parse
      --is-shallow-repository``; while shallow, ``git fetch
      --unshallow`` (the ONE-TIME self-heal for checkouts made shallow
      pre-phase 107 — the next sync of a deployed checkout becomes
      full-history without a re-clone), then ``git pull --ff-only``
      (refuses to merge unrelated histories — a broken checkout fails
      loudly rather than producing a dirty index).
    - dest with a ``.git`` and a pinned ``branch`` (phase 135), in
      order: the HEAD GUARD first — ``git rev-parse --abbrev-ref
      HEAD``; a checkout sitting on any other branch raises the
      remove-and-re-add :class:`GitSyncError` (naming the checkout and
      BOTH branch names — never the URL, so no credential exposure)
      BEFORE any fetch or merge (no silent branch switch: the checkout
      may serve other rows' documents, and a switch would silently
      re-map the source's file set); a matching checkout then runs the
      SAME shallow probe + one-time unshallow self-heal, and pulls the
      pinned ref with the single-fetch sequence ``git fetch origin
      <b>`` + ``git merge --ff-only FETCH_HEAD`` (a non-ff merge still
      fails loudly HERE — the force-push reset fallback is phase 136's
      extension of this sequence, not this phase's).

    Returns the destination path. Raises :class:`GitSyncError` when git is
    missing or a git invocation exits non-zero (with git's stderr in the
    message, so the caller can name the failing repo + reason). D12
    fail-loud: a failed probe/unshallow/pull/merge propagates exactly
    like a clone failure — never a silent fallback to tip dates or
    mtimes (a continued shallow checkout would silently re-serve the
    uniform tip date, i.e. the bug D11 fixes) and never a silent branch
    switch (phase 135 — the fail-loud guard, above).
    """
    dest = Path(dest)
    if not dest.exists() or not (dest / ".git").exists():
        dest.parent.mkdir(parents=True, exist_ok=True)
        argv = ["git", "clone"]
        if branch is not None:
            argv += ["--branch", branch]
        argv += [url, str(dest)]
        run_git(argv, cwd=dest.parent)
    elif branch is None:
        _unshallow_if_shallow(dest)
        run_git(["git", "pull", "--ff-only"], cwd=dest)
    else:
        # Phase 135 HEAD guard (first — before the probe, any fetch,
        # or any merge): a mismatched checkout fails loudly. The
        # message names the checkout and both branch names; it never
        # carries the URL (the only place a token could ride).
        current = run_git(["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=dest).strip()
        if current != branch:
            raise GitSyncError(
                f"checkout {dest} is on branch '{current}' but the source "
                f"specifies branch '{branch}' — remove and re-add the "
                "source to switch branches"
            )
        _unshallow_if_shallow(dest)
        # The single-fetch ff pull of the pinned ref (phase 136
        # extends this sequence with its reset fallback).
        run_git(["git", "fetch", "origin", branch], cwd=dest)
        run_git(["git", "merge", "--ff-only", "FETCH_HEAD"], cwd=dest)
    return dest


def _unshallow_if_shallow(dest: Path) -> None:
    """The one-time shallow self-heal (phase 107 D11).

    Probe ``git rev-parse --is-shallow-repository``; while shallow,
    ``git fetch --unshallow`` (a pre-phase-107 deployed checkout
    becomes full-history on its next sync without a re-clone). BOTH
    pull paths of :func:`clone_or_pull` (no branch / pinned branch)
    run this before their pull — the argv sequence is the pre-phase
    one in both (the unit argv pins are the proof).
    """
    shallow = (
        run_git(["git", "rev-parse", "--is-shallow-repository"], cwd=dest)
        .strip()
        == "true"
    )
    if shallow:
        run_git(["git", "fetch", "--unshallow"], cwd=dest)


def _parse_commit_dates(output: str) -> dict[str, datetime]:
    """Parse ``git log --name-only --format=@@%cI`` output (newest first).

    A ``@@<ISO-8601>`` line starts a commit (``%cI`` is ISO-strict, so
    the date is always aware — parsed with ``datetime.fromisoformat``);
    the following non-empty, non-``@@`` lines are repo-relative paths.
    The FIRST sighting of a path wins (the walk is newest-first) — that
    is the file's last-commit date. Paths are split on whitespace (like
    name-only output), ``\\``-normalized to ``/``, and a leading ``/``
    is stripped. Raises ``ValueError`` on a malformed commit date or a
    path line before any commit header (the caller fails soft).
    """
    dates: dict[str, datetime] = {}
    commit: datetime | None = None
    for line in output.splitlines():
        line = line.strip()
        if not line:
            continue
        if line.startswith("@@"):
            commit = datetime.fromisoformat(line[2:])
            continue
        if commit is None:
            raise ValueError(f"path line before any commit header: {line!r}")
        for raw_path in line.split():
            path = raw_path.replace("\\", "/").lstrip("/")
            if path:
                dates.setdefault(path, commit)
    return dates


def file_commit_dates(dest: Path | str) -> dict[str, datetime]:
    """Per-file last-commit dates for one checkout (phase 106, D2).

    ONE ``git log --name-only --format=@@%cI`` walk through
    :func:`run_git` (the A11 single-invocation site, one git call per
    source per sync) → ``{repo-relative POSIX path: last-commit
    datetime}``, newest-first so the first sighting of a path wins.

    Every ``clone_or_pull`` checkout is FULL history (fresh: no
    ``--depth``; an existing shallow checkout is unshallowed on its
    next sync — phase 107 D11 supersedes phase 106 D10's shallow
    tip-date behavior) → TRUE per-file last-commit dates for ALL git
    sources, local AND URL (verified 2026-09-16 — see the module
    docstring).

    Fail-soft (pinned): a missing/non-directory checkout, a git failure
    (:class:`GitSyncError`), or ANY parse anomaly logs a warning and
    returns ``{}`` — the importer falls back to file mtimes; a date
    walk must never break a sync.
    """
    dest = Path(dest)
    if not dest.is_dir():
        logger.warning(
            "file_commit_dates: %s is not a directory — no git dates "
            "(the importer will fall back to file mtimes)",
            dest,
        )
        return {}
    try:
        output = run_git(["git", "log", "--name-only", "--format=@@%cI"], cwd=dest)
    except GitSyncError as exc:
        logger.warning(
            "file_commit_dates: git log failed for %s: %s — the importer "
            "will fall back to file mtimes",
            dest,
            exc,
        )
        return {}
    try:
        return _parse_commit_dates(output)
    except ValueError as exc:
        logger.warning(
            "file_commit_dates: unparseable git log output for %s (%s) — the "
            "importer will fall back to file mtimes",
            dest,
            exc,
        )
        return {}


def run_git(argv: list[str], cwd: Path) -> str:
    """Run one git command, capturing output; raise GitSyncError on failure.

    The single ``git`` invocation point for the whole app (A11). Every
    step of :func:`clone_or_pull` and of the docs-push sequence
    (:mod:`app.core.docs_push`, phase 59) goes through here, so error
    handling stays uniform: captured stdout on success, and
    :class:`GitSyncError` carrying git's stderr on a non-zero exit (or
    when the git binary is missing from PATH).
    """
    try:
        proc = subprocess.run(argv, cwd=cwd, capture_output=True, text=True)
    except FileNotFoundError:
        raise GitSyncError("git was not found on PATH — install git and retry") from None
    if proc.returncode != 0:
        raise GitSyncError(
            f"git {' '.join(argv[1:])} failed (exit {proc.returncode}): "
            f"{proc.stderr.strip()}"
        )
    return proc.stdout