"""Unit tests: git clone/pull utility (phase 28; phase 107 D11/D12; phase 135 adds the branch pin; phase 136 replaces the existing- checkout update with the UNIFIED ff-or-reset sequence — the argv matrix). ``clone_or_pull`` dispatches to ``git clone`` (full history, fresh dest) or the UNIFIED existing-checkout update (phase 136 — one sequence for both branch shapes) with ``subprocess`` fully mocked — no real git CLI is needed, so the tests run anywhere: ref resolution (``rev-parse --abbrev-ref HEAD`` — the phase-135 HEAD guard for a pinned ``branch``, the checkout's own branch + the detached-HEAD fail-loud for ``branch=None``) → the shallow probe → optional ``git fetch --unshallow`` (the one-time D11 self-heal) → ``git fetch origin `` (a fetch failure STILL raises) → ``git merge --ff-only FETCH_HEAD`` → on a non-ff merge: the ``rev-parse HEAD`` / ``rev-parse FETCH_HEAD`` capture + ``git reset --hard FETCH_HEAD`` + one WARNING line, then RETURN (a force push never fails a sync — phase 136's fix). The fresh-clone argv (with/without ``--branch``) is the pre-phase one — the argv pins below are the regression gate for every existing deployment. Phase 186 adds the commit-date PARSER pins and the STREAMING git site. ``_parse_commit_dates`` accepts ``str | Iterable[str]`` so the walk can stream off the subprocess pipe instead of buffering the whole ``git log`` in RAM, and ``stream_git_lines`` is the A11 sibling of ``run_git`` — its success / error-shape / huge-stderr cases run the REAL git CLI through the house ``_git`` helper (skipped, not failed, without it, like ``tests/integration/test_git_file_dates.py``), while the abandon-the-stream case uses a recorded ``Popen`` stand-in so the terminate-and-reap arm is deterministic instead of a race. The ``test_only_two_git_spawn_sites`` pin is the AST version of the old A11 grep: exactly one ``subprocess`` spawn per shape, both inside ``git_sync.py``. The memory pin's synthetic corpus is ~300,000 lines because that is the size of the real thing on the incident box (2026-10-15, measured read-only inside the app container): ``litellm`` 54,502 commits / 346,829 lines / 15,077,351 bytes; ``llama.cpp`` 11,659 commits / 69,922 lines / 1,867,046 bytes; ``gufo`` 9,155 lines; ``vllm-radiance`` 4,855 lines — i.e. the walk scales with a repository's ENTIRE history, not its working tree, which is why the corpus sits at 306,000 lines (3,000 commits × 100 path lines + the header/blank pair per commit) rather than at a toy size. """ from __future__ import annotations import ast import io import logging import os import shutil import subprocess import time import tracemalloc from collections.abc import Callable, Generator, Iterable, Iterator from datetime import UTC, datetime from pathlib import Path from typing import cast import pytest import scripts.git_sync as git_sync from app.core import docs_push from scripts.git_sync import ( GitSyncError, _parse_commit_dates, # pyright: ignore[reportPrivateUsage] clone_or_pull, stream_git_lines, ) class _FakeProc: def __init__(self, returncode: int = 0, stdout: str = "", stderr: str = "") -> None: self.returncode = returncode self.stdout = stdout self.stderr = stderr def _fake_run( monkeypatch: pytest.MonkeyPatch, returncode: int = 0, stdout: str = "", stderr: str = "", missing_git: bool = False, responses: dict[tuple[str, ...], tuple[int, str, str]] | None = None, ) -> list[dict[str, object]]: """Monkeypatch scripts.git_sync.subprocess.run; record each call. ``responses`` dispatches by the full argv (clone / shallow-probe / ``fetch --unshallow`` / pull) — a per-command ``(returncode, stdout, stderr)``; any argv not in the map gets the default ``returncode``/``stdout``/``stderr``. """ calls: list[dict[str, object]] = [] def fake_run(argv: list[str], cwd: Path | None = None, **kwargs: object) -> _FakeProc: calls.append({"argv": list(argv), "cwd": cwd, **kwargs}) if missing_git: raise FileNotFoundError("git") if responses is not None: rc, out, err = responses.get(tuple(argv), (returncode, stdout, stderr)) return _FakeProc(rc, out, err) return _FakeProc(returncode, stdout, stderr) monkeypatch.setattr(subprocess, "run", fake_run) return calls def test_clone_or_pull_clones_when_dest_has_no_git_dir( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """First run: dest absent → ``git clone`` (full history — NO ``--depth``, phase 107 D11) from the parent dir.""" url = "https://github.com/user/homelab.git" dest = tmp_path / "homelab" calls = _fake_run(monkeypatch) result = clone_or_pull(url, dest) assert result == dest assert len(calls) == 1 call = calls[0] assert call["argv"] == ["git", "clone", url, str(dest)] assert call["cwd"] == dest.parent assert call["capture_output"] is True assert call["text"] is True def test_clone_or_pull_existing_checkout_ff_updates_the_tracked_ref( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """Phase 136 steady state (``branch=None``): dest/.git present → the UNIFIED sequence — the checkout's OWN branch is resolved (``rev-parse --abbrev-ref HEAD`` → ``main``), the shallow probe says NOT shallow (``false``), and ``fetch origin main`` + ``merge --ff-only FETCH_HEAD`` ff-update the ref (the legacy ``git pull --ff-only`` is deliberately gone — every row gets the ff-or-reset update). A SUCCESSFUL merge leaves NO ``reset`` argv and NO ``rev-parse HEAD`` / ``rev-parse FETCH_HEAD`` capture pair (the capture is only on failure).""" dest = tmp_path / "homelab" (dest / ".git").mkdir(parents=True) calls = _fake_run( monkeypatch, responses={ ("git", "rev-parse", "--abbrev-ref", "HEAD"): (0, "main\n", ""), ("git", "rev-parse", "--is-shallow-repository"): (0, "false\n", ""), ("git", "fetch", "origin", "main"): (0, "", ""), ("git", "merge", "--ff-only", "FETCH_HEAD"): (0, "Already up to date.\n", ""), }, ) result = clone_or_pull("https://github.com/user/homelab.git", dest) assert result == dest assert [call["argv"] for call in calls] == [ ["git", "rev-parse", "--abbrev-ref", "HEAD"], ["git", "rev-parse", "--is-shallow-repository"], ["git", "fetch", "origin", "main"], ["git", "merge", "--ff-only", "FETCH_HEAD"], ] for call in calls: assert call["cwd"] == dest # ff succeeded — the exact 4-step sequence above already excludes # the reset tail (and the rev-parse HEAD/FETCH_HEAD capture pair, # which is only on failure). assert ["git", "reset", "--hard", "FETCH_HEAD"] not in [c["argv"] for c in calls] def test_clone_or_pull_unshallows_existing_shallow_checkout( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """Existing shallow checkout (probe ``true`` — the one-time D11 self-heal of deployed checkouts) → ``git fetch --unshallow`` BETWEEN the ref-resolution probe and the unified update's ``fetch origin `` + ``merge --ff-only FETCH_HEAD`` (order pinned — the self-heal still fires in front of the fetch), all in dest.""" dest = tmp_path / "homelab" (dest / ".git").mkdir(parents=True) calls = _fake_run( monkeypatch, responses={ ("git", "rev-parse", "--abbrev-ref", "HEAD"): (0, "main\n", ""), ("git", "rev-parse", "--is-shallow-repository"): (0, "true\n", ""), ("git", "fetch", "--unshallow"): (0, "Fetching full history\n", ""), ("git", "fetch", "origin", "main"): (0, "", ""), ("git", "merge", "--ff-only", "FETCH_HEAD"): (0, "", ""), }, ) clone_or_pull("https://github.com/user/homelab.git", dest) assert len(calls) == 5 assert [call["argv"] for call in calls] == [ ["git", "rev-parse", "--abbrev-ref", "HEAD"], ["git", "rev-parse", "--is-shallow-repository"], ["git", "fetch", "--unshallow"], ["git", "fetch", "origin", "main"], ["git", "merge", "--ff-only", "FETCH_HEAD"], ] for call in calls: assert call["cwd"] == dest def test_clone_or_pull_unshallow_failure_propagates( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """D12 fail-loud: a failed ``git fetch --unshallow`` raises :class:`GitSyncError` (with git's stderr) and the update NEVER runs — continuing a failed self-heal would silently re-serve the uniform tip date (the bug D11 fixes).""" dest = tmp_path / "homelab" (dest / ".git").mkdir(parents=True) calls = _fake_run( monkeypatch, responses={ ("git", "rev-parse", "--abbrev-ref", "HEAD"): (0, "main\n", ""), ("git", "rev-parse", "--is-shallow-repository"): (0, "true\n", ""), ("git", "fetch", "--unshallow"): (128, "", "fatal: could not fetch\n"), }, ) with pytest.raises( GitSyncError, match=r"git fetch --unshallow failed \(exit 128\): fatal: could not fetch" ): clone_or_pull("https://github.com/user/homelab.git", dest) assert [call["argv"] for call in calls] == [ ["git", "rev-parse", "--abbrev-ref", "HEAD"], ["git", "rev-parse", "--is-shallow-repository"], ["git", "fetch", "--unshallow"], ] # the fetch/merge/reset never ran def test_clone_or_pull_creates_missing_parent_before_clone( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """The (nested) parent of the destination is mkdir'd before git runs.""" dest = tmp_path / "nested" / "deeper" / "homelab" assert not dest.parent.exists() calls = _fake_run(monkeypatch) clone_or_pull("https://github.com/user/homelab.git", dest) assert dest.parent.is_dir() assert calls[0]["argv"] == [ "git", "clone", "https://github.com/user/homelab.git", str(dest), ] def test_existing_dir_without_git_dir_is_treated_as_fresh( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """A leftover non-repo dir at the dest still takes the clone path — git fails loudly (non-zero exit) if the dir is not empty.""" dest = tmp_path / "homelab" dest.mkdir() calls = _fake_run(monkeypatch) clone_or_pull("https://github.com/user/homelab.git", dest) assert calls[0]["argv"] == [ "git", "clone", "https://github.com/user/homelab.git", str(dest), ] def test_failing_git_raises_error_carrying_stderr( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """Non-zero exit → GitSyncError naming the subcommand, exit code, stderr.""" dest = tmp_path / "homelab" _fake_run(monkeypatch, returncode=128, stderr="fatal: repository not found\n") with pytest.raises( GitSyncError, match=r"git clone .* failed \(exit 128\): fatal: repository not found", ): clone_or_pull("https://example.com/nope.git", dest) # Same for the unified update path — but the reset fallback is # scoped to the MERGE step: a genuinely broken FETCH (dead # network, the branch deleted upstream, git missing) STILL fails # loud, with no merge and no reset argv. (dest / ".git").mkdir(parents=True) calls = _fake_run( monkeypatch, returncode=1, stderr="fatal: couldn't find remote ref main", responses={ ("git", "rev-parse", "--abbrev-ref", "HEAD"): (0, "main\n", ""), ("git", "rev-parse", "--is-shallow-repository"): (0, "false\n", ""), ("git", "fetch", "origin", "main"): ( 128, "", "fatal: couldn't find remote ref main\n" ), }, ) with pytest.raises( GitSyncError, match=r"git fetch origin main failed \(exit 128\): fatal: couldn't find remote ref main", ): clone_or_pull("https://example.com/homelab.git", dest) assert [call["argv"] for call in calls] == [ ["git", "rev-parse", "--abbrev-ref", "HEAD"], ["git", "rev-parse", "--is-shallow-repository"], ["git", "fetch", "origin", "main"], ] # the merge (and therefore any reset) never ran def test_missing_git_raises_named_error( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """No git binary on PATH → GitSyncError telling the user to install it.""" _fake_run(monkeypatch, missing_git=True) with pytest.raises(GitSyncError, match="git was not found on PATH"): clone_or_pull("https://example.com/homelab.git", tmp_path / "homelab") def test_run_git_captures_and_returns_stdout( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """run_git returns the captured stdout on success (git output is not lost).""" calls = _fake_run(monkeypatch, stdout="From example.com\n + abc..def main") assert git_sync.run_git(["git", "status"], cwd=tmp_path) == "From example.com\n + abc..def main" assert len(calls) == 1 def test_run_git_default_env_is_none_and_reaches_subprocess( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """Phase 173 (C5 byte-identity pin): the default ``env=None`` is passed through verbatim to ``subprocess.run`` — the pre-phase call shape, so every sync-path invocation is unchanged.""" calls = _fake_run(monkeypatch, stdout="ok") assert git_sync.run_git(["git", "version"], cwd=tmp_path) == "ok" assert len(calls) == 1 assert "env" in calls[0] assert calls[0]["env"] is None def test_run_git_explicit_env_reaches_subprocess_verbatim( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """Phase 173 (C2): an explicit env dict reaches ``subprocess.run`` verbatim — ``run_git`` neither merges over nor mutates the parent environment (the env-replacement trap is the caller's contract; the docs-push sequence does its own ``os.environ`` merge).""" calls = _fake_run(monkeypatch, stdout="ok") env = {"A": "b"} assert git_sync.run_git(["git", "version"], cwd=tmp_path, env=env) == "ok" assert len(calls) == 1 assert calls[0]["env"] is env # the SAME object — passed through verbatim assert calls[0]["env"] == {"A": "b"} # --- phase 135: the branch pin (the argv matrix) ---------------------------- def test_branch_none_argv_pins_the_unified_phase_136_sequence( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """The ``branch=None`` argv pins (explicit ``None`` is the call sites' form for every row without a branch pin). FRESH: one ``git clone`` without ``--branch`` — the pre-phase argv, UNCHANGED (the regression gate for every existing deployment). EXISTING: the legacy ``git pull --ff-only`` is deliberately GONE — that is the fix (phase 136): every row, branched or not, gets the ff-or-reset update. The pinned sequence: ref resolution (``rev- parse --abbrev-ref HEAD``) → the shallow probe → (``fetch --unshallow`` while shallow) → ``fetch origin `` → ``merge --ff-only FETCH_HEAD``. A future refactor that re-introduces ``git pull`` (or re-splits the two branch shapes) breaks this pin.""" url = "https://github.com/user/homelab.git" dest = tmp_path / "homelab" # fresh (dest absent) — the pre-phase argv, byte-identical calls = _fake_run(monkeypatch) clone_or_pull(url, dest, None) assert [call["argv"] for call in calls] == [["git", "clone", url, str(dest)]] # existing, not shallow — the unified ff-or-reset sequence (dest / ".git").mkdir(parents=True) calls = _fake_run( monkeypatch, responses={ ("git", "rev-parse", "--abbrev-ref", "HEAD"): (0, "main\n", ""), ("git", "rev-parse", "--is-shallow-repository"): (0, "false\n", ""), }, ) clone_or_pull(url, dest, None) assert [call["argv"] for call in calls] == [ ["git", "rev-parse", "--abbrev-ref", "HEAD"], ["git", "rev-parse", "--is-shallow-repository"], ["git", "fetch", "origin", "main"], ["git", "merge", "--ff-only", "FETCH_HEAD"], ] # existing, shallow (the one-time D11 self-heal between probe # and fetch — still in front of the fetch) calls = _fake_run( monkeypatch, responses={ ("git", "rev-parse", "--abbrev-ref", "HEAD"): (0, "main\n", ""), ("git", "rev-parse", "--is-shallow-repository"): (0, "true\n", ""), ("git", "fetch", "--unshallow"): (0, "", ""), }, ) clone_or_pull(url, dest, None) assert [call["argv"] for call in calls] == [ ["git", "rev-parse", "--abbrev-ref", "HEAD"], ["git", "rev-parse", "--is-shallow-repository"], ["git", "fetch", "--unshallow"], ["git", "fetch", "origin", "main"], ["git", "merge", "--ff-only", "FETCH_HEAD"], ] def test_fresh_clone_with_branch_uses_clone_branch( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """Phase 135: fresh dest + pinned branch → ``git clone --branch url dest`` (the branch is checked out from the first commit — no post-clone checkout; still full history, no ``--depth``).""" url = "https://github.com/user/homelab.git" dest = tmp_path / "homelab" calls = _fake_run(monkeypatch) result = clone_or_pull(url, dest, "release") assert result == dest assert [call["argv"] for call in calls] == [ ["git", "clone", "--branch", "release", url, str(dest)] ] assert calls[0]["cwd"] == dest.parent def test_existing_checkout_matching_branch_fetches_and_ff_merges_the_ref( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """Phase 135: existing checkout ALREADY on the pinned branch → the full sequence in order: the HEAD guard (``rev-parse --abbrev-ref HEAD``) → the shallow probe → the single-fetch ``fetch origin `` → ``merge --ff-only FETCH_HEAD`` (not ``git pull`` — the default path's argv is reserved for ``branch=None``).""" dest = tmp_path / "homelab" (dest / ".git").mkdir(parents=True) calls = _fake_run( monkeypatch, responses={ ("git", "rev-parse", "--abbrev-ref", "HEAD"): (0, "release\n", ""), ("git", "rev-parse", "--is-shallow-repository"): (0, "false\n", ""), ("git", "fetch", "origin", "release"): (0, " * branch release -> FETCH_HEAD", ""), ("git", "merge", "--ff-only", "FETCH_HEAD"): (0, "Already up to date.\n", ""), }, ) result = clone_or_pull("https://github.com/user/homelab.git", dest, "release") assert result == dest assert [call["argv"] for call in calls] == [ ["git", "rev-parse", "--abbrev-ref", "HEAD"], ["git", "rev-parse", "--is-shallow-repository"], ["git", "fetch", "origin", "release"], ["git", "merge", "--ff-only", "FETCH_HEAD"], ] for call in calls: assert call["cwd"] == dest def test_existing_shallow_checkout_matching_branch_unshallows_before_fetch( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """Phase 135 + phase 107 D11: the branch path runs the SAME one-time self-heal — a shallow matching checkout is unshallowed BETWEEN the guard/probe and the branch fetch (order pinned).""" dest = tmp_path / "homelab" (dest / ".git").mkdir(parents=True) calls = _fake_run( monkeypatch, responses={ ("git", "rev-parse", "--abbrev-ref", "HEAD"): (0, "release\n", ""), ("git", "rev-parse", "--is-shallow-repository"): (0, "true\n", ""), ("git", "fetch", "--unshallow"): (0, "", ""), ("git", "fetch", "origin", "release"): (0, "", ""), ("git", "merge", "--ff-only", "FETCH_HEAD"): (0, "", ""), }, ) clone_or_pull("https://github.com/user/homelab.git", dest, "release") assert [call["argv"] for call in calls] == [ ["git", "rev-parse", "--abbrev-ref", "HEAD"], ["git", "rev-parse", "--is-shallow-repository"], ["git", "fetch", "--unshallow"], ["git", "fetch", "origin", "release"], ["git", "merge", "--ff-only", "FETCH_HEAD"], ] def test_existing_checkout_branch_mismatch_fails_loud_before_any_fetch( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """Phase 135 (ASSUMPTION B5, D12): a checkout on a DIFFERENT branch raises the remove-and-re-add ``GitSyncError`` — naming the checkout and BOTH branch names — and the guard is FIRST: no shallow probe, no fetch, no merge argv was invoked (no silent branch switch).""" url = "https://github.com/user/homelab.git" dest = tmp_path / "homelab" (dest / ".git").mkdir(parents=True) calls = _fake_run( monkeypatch, responses={("git", "rev-parse", "--abbrev-ref", "HEAD"): (0, "main\n", "")}, ) with pytest.raises(GitSyncError) as excinfo: clone_or_pull(url, dest, "release") msg = str(excinfo.value) assert f"checkout {dest}" in msg # the checkout is named assert "'main'" in msg # the checkout's branch assert "'release'" in msg # the source's pinned branch assert "remove and re-add the source to switch branches" in msg # credential safety: the message carries NO URL (the only place a # token could ride) — even for a plain URL. assert url not in msg assert "github" not in msg # the guard is first: exactly ONE argv (the guard probe) ran. assert [call["argv"] for call in calls] == [ ["git", "rev-parse", "--abbrev-ref", "HEAD"] ] def test_branch_mismatch_message_never_contains_url_even_with_token( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """Phase 135 credential-safety: even when the CLONE URL carries the injected token (the phase-121 ``clone_url_for`` form), the guard message is URL-free — the checkout path and the two branch names only.""" url = "https://x-access-token:ghp_supersecret@git.example.com/user/repo.git" dest = tmp_path / "repo" (dest / ".git").mkdir(parents=True) calls = _fake_run( monkeypatch, responses={("git", "rev-parse", "--abbrev-ref", "HEAD"): (0, "main\n", "")}, ) with pytest.raises(GitSyncError) as excinfo: clone_or_pull(url, dest, "release") msg = str(excinfo.value) assert url not in msg assert "ghp_supersecret" not in msg assert "git.example.com" not in msg assert f"checkout {dest} is on branch 'main' but the source " \ "specifies branch 'release'" in msg assert [call["argv"] for call in calls] == [ ["git", "rev-parse", "--abbrev-ref", "HEAD"] ] def test_existing_checkout_branch_fetch_failure_propagates_before_merge( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """Phase 135 D12 fail-loud: a failed ``fetch origin `` raises (with git's stderr) and the merge NEVER runs.""" dest = tmp_path / "homelab" (dest / ".git").mkdir(parents=True) calls = _fake_run( monkeypatch, responses={ ("git", "rev-parse", "--abbrev-ref", "HEAD"): (0, "release\n", ""), ("git", "rev-parse", "--is-shallow-repository"): (0, "false\n", ""), ("git", "fetch", "origin", "release"): (128, "", "fatal: couldn't find remote ref\n"), }, ) with pytest.raises( GitSyncError, match=r"git fetch origin release failed \(exit 128\): fatal: couldn't find remote ref", ): clone_or_pull("https://github.com/user/homelab.git", dest, "release") assert [call["argv"] for call in calls] == [ ["git", "rev-parse", "--abbrev-ref", "HEAD"], ["git", "rev-parse", "--is-shallow-repository"], ["git", "fetch", "origin", "release"], ] # the merge never ran def test_existing_checkout_branch_non_ff_merge_resets_to_the_ref_head( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, ) -> None: """Phase 136: a non-ff ``merge --ff-only FETCH_HEAD`` (diverged / force-pushed origin) NO LONGER fails the sync — the sequence captures the old/new SHAs, resets the checkout hard to ``FETCH_HEAD`` (the branch's ref — the owner's verbatim "reset origin/"), logs ONE WARNING line (PLAN §9), and RETURNS the dest (no raise — the run continues). The WARNING is the only observable side effect besides the checkout state: no user-facing error, no status change.""" url = "https://github.com/user/homelab.git" dest = tmp_path / "homelab" (dest / ".git").mkdir(parents=True) old, new = "a" * 40, "b" * 40 calls = _fake_run( monkeypatch, responses={ ("git", "rev-parse", "--abbrev-ref", "HEAD"): (0, "release\n", ""), ("git", "rev-parse", "--is-shallow-repository"): (0, "false\n", ""), ("git", "fetch", "origin", "release"): (0, "", ""), ("git", "merge", "--ff-only", "FETCH_HEAD"): ( 1, "", "fatal: Not possible to fast-forward, aborting.\n" ), ("git", "rev-parse", "HEAD"): (0, old + "\n", ""), ("git", "rev-parse", "FETCH_HEAD"): (0, new + "\n", ""), ("git", "reset", "--hard", "FETCH_HEAD"): (0, "", ""), }, ) with caplog.at_level(logging.WARNING, logger="scripts.git_sync"): result = clone_or_pull(url, dest, "release") # NO raise — the run continues assert result == dest assert [call["argv"] for call in calls] == [ ["git", "rev-parse", "--abbrev-ref", "HEAD"], ["git", "rev-parse", "--is-shallow-repository"], ["git", "fetch", "origin", "release"], ["git", "merge", "--ff-only", "FETCH_HEAD"], ["git", "rev-parse", "HEAD"], ["git", "rev-parse", "FETCH_HEAD"], ["git", "reset", "--hard", "FETCH_HEAD"], ] for call in calls: assert call["cwd"] == dest # ONE WARNING line (PLAN §9): the checkout PATH (never the URL — # credential-safety), both short SHAs, the tracked ref, and the # merge's stderr (diagnosability, log-only — never user-facing). warnings = [r for r in caplog.records if r.levelno == logging.WARNING] assert len(warnings) == 1 msg = warnings[0].getMessage() assert msg == ( f"git sync: divergent remote head on {dest} (force push) — reset " f"{old[:7]} → {new[:7]} to origin/release " "(git merge --ff-only FETCH_HEAD failed (exit 1): " "fatal: Not possible to fast-forward, aborting.)" ) assert url not in msg # credential-safety: never the URL assert "github" not in msg def test_null_branch_non_ff_merge_resets_and_returns( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """Phase 136 (``branch=None`` — the legacy ``git pull --ff-only`` path, the owner's reported break): a non-ff merge of the checkout's OWN branch takes the SAME reset tail — ``rev-parse HEAD`` → ``rev-parse FETCH_HEAD`` → ``reset --hard FETCH_HEAD`` — and RETURNS the dest (no raise; pre-phase, this exact shape was the ``fatal: Not possible to fast-forward`` that failed the whole sync). The reset targets the checkout's branch ref (``fetch origin main``).""" dest = tmp_path / "homelab" (dest / ".git").mkdir(parents=True) old, new = "c" * 40, "d" * 40 calls = _fake_run( monkeypatch, responses={ ("git", "rev-parse", "--abbrev-ref", "HEAD"): (0, "main\n", ""), ("git", "rev-parse", "--is-shallow-repository"): (0, "false\n", ""), ("git", "fetch", "origin", "main"): (0, "", ""), ("git", "merge", "--ff-only", "FETCH_HEAD"): ( 1, "", "fatal: Not possible to fast-forward, aborting.\n" ), ("git", "rev-parse", "HEAD"): (0, old + "\n", ""), ("git", "rev-parse", "FETCH_HEAD"): (0, new + "\n", ""), ("git", "reset", "--hard", "FETCH_HEAD"): (0, "", ""), }, ) result = clone_or_pull("https://github.com/user/homelab.git", dest, None) assert result == dest # the function RETURNS the dest (no raise) assert [call["argv"] for call in calls] == [ ["git", "rev-parse", "--abbrev-ref", "HEAD"], ["git", "rev-parse", "--is-shallow-repository"], ["git", "fetch", "origin", "main"], ["git", "merge", "--ff-only", "FETCH_HEAD"], ["git", "rev-parse", "HEAD"], ["git", "rev-parse", "FETCH_HEAD"], ["git", "reset", "--hard", "FETCH_HEAD"], ] def test_null_branch_detached_head_fails_loud( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """Phase 136 D12 fail-loud: ``branch=None`` on a DETACHED checkout (the ref resolution returns ``"HEAD"``) raises the remove-and- re-add ``GitSyncError`` naming the checkout — the app never CREATES a detached checkout (a fresh clone always lands on a branch), so this is a genuinely broken checkout, not a force-push case: no shallow probe, no fetch, no merge, NO reset argv.""" dest = tmp_path / "homelab" (dest / ".git").mkdir(parents=True) calls = _fake_run( monkeypatch, responses={("git", "rev-parse", "--abbrev-ref", "HEAD"): (0, "HEAD\n", "")}, ) with pytest.raises(GitSyncError) as excinfo: clone_or_pull("https://github.com/user/homelab.git", dest, None) msg = str(excinfo.value) assert f"checkout {dest} is in a detached-HEAD state" in msg assert "remove and re-add the source to rebuild it" in msg assert [call["argv"] for call in calls] == [ ["git", "rev-parse", "--abbrev-ref", "HEAD"] ] # the guard is first: exactly ONE argv ran (no fetch, no reset) # --- phase 186: the parser takes a stream (semantics untouched) ------------ #: One commit header of the synthetic corpus, in the exact byte shape #: ``git log --name-only --format=@@%cI`` emits (``%cI`` is ISO-strict). _CORPUS_HEADER = "@@2026-01-01T00:00:00+00:00" #: The synthetic corpus's size — see the module docstring for the #: measured real-world line counts this is modelled on. _CORPUS_COMMITS = 3_000 _CORPUS_PATHS_PER_COMMIT = 100 def _corpus_path(commit: int, slot: int) -> str: """One path line of the synthetic corpus. The pool cycles (``(commit * 100 + slot) % 10_000``), so the walk touches 10,000 distinct paths across 300,000 path lines — the same shape as a real repo, where a handful of hot files reappear in commit after commit while history's total path count stays a fraction of the line count. """ return f"path_{(commit * _CORPUS_PATHS_PER_COMMIT + slot) % 10_000:05d}/file_{slot:03d}.md" def _corpus_stream() -> Iterator[str]: """The corpus as a LAZY stream: nothing is materialised, each line is built when pulled (the ``stream_git_lines`` shape of task 02).""" for commit in range(_CORPUS_COMMITS): yield _CORPUS_HEADER yield "" for slot in range(_CORPUS_PATHS_PER_COMMIT): yield _corpus_path(commit, slot) def _corpus_string() -> str: """The SAME corpus as one big ``str`` (what ``run_git`` returns today) — built OUTSIDE any measured region.""" return "\n".join(_corpus_stream()) + "\n" class _OneShot: """An iterable that refuses to be iterated twice. ``__iter__`` raises on the SECOND call, so a parser that materialises its input and rescans it (or loops over ``lines`` twice) fails loudly instead of silently passing. """ def __init__(self, lines: Iterable[str]) -> None: self._lines = lines self.iter_calls = 0 def __iter__(self) -> Iterator[str]: self.iter_calls += 1 if self.iter_calls > 1: raise AssertionError("re-iterated") return iter(self._lines) class _CountingStream: """A stream that records how far the consumer pulled.""" def __init__(self, lines: list[str]) -> None: self._lines = lines self.pulled = 0 def __iter__(self) -> Iterator[str]: for pulled, line in enumerate(self._lines, start=1): self.pulled = pulled yield line def _peak_bytes[T](fn: Callable[[], T]) -> tuple[float, T]: """Run ``fn`` and return its tracemalloc PEAK in KiB (+ result). ``reset_peak`` runs after ``start`` so corpus construction (and any earlier test's residue) is excluded — only what ``fn`` itself allocates counts. """ import gc gc.collect() tracemalloc.start() try: tracemalloc.reset_peak() result = fn() peak = tracemalloc.get_traced_memory()[1] finally: tracemalloc.stop() return peak / 1024, result def test_parser_accepts_a_line_iterable() -> None: """Phase 186 D2: the SAME corpus fed as ``iter([...])`` yields the identical dict as the string form — every parsing rule (the ``@@%cI`` header, the blank-line skip, the whitespace path split, the ``\\``→``/`` normalisation, the leading-``/`` strip, first-sighting-wins) is untouched.""" lines = [ "@@2024-06-15T10:00:00+00:00", "", "b.md", "docs\\deep.md /rooted.md", "@@2020-01-02T03:04:06+00:00", "", "a.md", "b.md", ] output = "\n".join(lines) + "\n" expected = { "b.md": datetime.fromisoformat("2024-06-15T10:00:00+00:00"), "docs/deep.md": datetime.fromisoformat("2024-06-15T10:00:00+00:00"), "rooted.md": datetime.fromisoformat("2024-06-15T10:00:00+00:00"), "a.md": datetime.fromisoformat("2020-01-02T03:04:06+00:00"), } assert _parse_commit_dates(iter(lines)) == expected assert _parse_commit_dates(output) == expected # the string form, unchanged assert _parse_commit_dates(_OneShot(lines)) == expected def test_parser_rejects_the_malformed_streams_too() -> None: """Both ``ValueError`` arms fire through the iterable path exactly as through the string path (the caller's fail-soft is unchanged).""" with pytest.raises(ValueError, match="before any commit header"): _parse_commit_dates(iter(["stray.md", "@@2024-06-15T10:00:00+00:00"])) with pytest.raises(ValueError): _parse_commit_dates(iter(["@@yesterday", "b.md"])) def test_parser_consumes_exactly_once() -> None: """Phase 186 D4: the parser walks its input in ONE pass. The one-shot wrapper raises ``AssertionError("re-iterated")`` on a second ``iter()``; the parse succeeds and the wrapper reports exactly one ``__iter__`` call, so no materialise-then-rescan (and no accidental ``list(lines)`` re-entering the loop) exists. The second half is the stronger laziness pin: a corpus whose offending line sits at index 3 raises there, having pulled EXACTLY 4 lines — a parser that pre-materialised its input would have pulled the whole tail first. """ corpus = [ "@@2024-06-15T10:00:00+00:00", "", "b.md", "@@2020-01-02T03:04:06+00:00", "a.md", ] one_shot = _OneShot(corpus) assert _parse_commit_dates(one_shot) == { "b.md": datetime.fromisoformat("2024-06-15T10:00:00+00:00"), "a.md": datetime.fromisoformat("2020-01-02T03:04:06+00:00"), } assert one_shot.iter_calls == 1 counting = _CountingStream( [ "@@2024-06-15T10:00:00+00:00", "", "b.md", "@@not-a-date", "b.md", *(["a.md"] * 1_000), # a long tail the parser must never reach ] ) with pytest.raises(ValueError): _parse_commit_dates(counting) assert counting.pulled == 4 # stopped AT the bad line, no full read def test_streamed_parse_alloculates_less_than_buffered() -> None: """Phase 186 D4: streaming the same corpus allocates materially less than the buffered path, measured comparatively on THIS machine. Both arms parse the identical 306,000-line corpus (3,000 commits × 100 paths — see the module docstring for the measured real repos it is modelled on). (a) is the buffered path exactly as ``run_git`` serves it today: one big ``str`` that the parser splits (``splitlines()`` materialises a 306,001-entry list of fresh ``str`` objects). (b) is a generator building each line lazily. The assertion is ``streamed < buffered * 0.6`` — a wide but not tautological margin (``<=`` would prove nothing); the numbers below are measured on this box (Linux 7.2.9-200.fc44.x86_64, CPython 3.13.13, glibc 2.43): buffered **21,504 KiB** peak (the 306,001-entry ``splitlines()`` list over a ~7 MB corpus) vs streamed **823 KiB** — a ratio of **0.038**, i.e. the streamed arm allocates ~26× less. The 823 KiB is the retained 10,000-entry result dict, which BOTH arms pay, so the streaming arm is effectively allocation-free. """ corpus = _corpus_string() buffered_kb, buffered = _peak_bytes(lambda: _parse_commit_dates(corpus)) streamed_kb, streamed = _peak_bytes(lambda: _parse_commit_dates(_corpus_stream())) assert buffered_kb > 1_024, f"buffered peak {buffered_kb:,.0f} KiB is not a real corpus" assert streamed_kb < buffered_kb * 0.6, ( f"streamed peak {streamed_kb:,.0f} KiB is not materially below the " f"buffered {buffered_kb:,.0f} KiB (ratio " f"{streamed_kb / buffered_kb:.3f}, budget 0.600)" ) # memory is not bought by losing data: identical results, ~300k lines assert streamed == buffered assert len(buffered) == 10_000 # --- phase 186: the streaming git site (stream_git_lines) ------------------- #: The real-git cases below build a throwaway repo (the house ``_git`` #: helper) — skipped, not failed, without the git CLI, exactly like #: ``tests/integration/test_git_file_dates.py``. GIT = shutil.which("git") is not None requires_git = pytest.mark.skipif(not GIT, reason="git CLI not available") #: Lines spewed to stderr by the deadlock pin — ≈340 KB, i.e. five #: times the ~64 KB pipe buffer, so a non-drained stderr blocks the #: child long before the 10 s wall-clock guard trips. _SPEW_LINES = 60_000 def _git(cwd: Path, *argv: str, when: str | None = None) -> None: """Run one git command for the test harness (fixture setup); a non-zero exit fails the fixture, not the test under test.""" env = os.environ.copy() if when is not None: env["GIT_AUTHOR_DATE"] = when env["GIT_COMMITTER_DATE"] = when env["GIT_AUTHOR_NAME"] = "T" env["GIT_AUTHOR_EMAIL"] = "t@example.com" env["GIT_COMMITTER_NAME"] = "T" env["GIT_COMMITTER_EMAIL"] = "t@example.com" proc = subprocess.run( ["git", *argv], cwd=cwd, env=env, capture_output=True, text=True, check=False ) assert proc.returncode == 0, f"git {' '.join(argv)} failed: {proc.stderr}" def _scratch_repo(tmp_path: Path) -> Path: """A throwaway repo with ONE commit (``a.md``) dated 2020-01-02.""" repo = tmp_path / "repo" repo.mkdir() _git(repo, "init", "-q") _git(repo, "config", "user.email", "t@example.com") _git(repo, "config", "user.name", "T") _git(repo, "config", "commit.gpgsign", "false") (repo / "a.md").write_text("# A\n", encoding="utf-8") _git(repo, "add", "-A", when="2020-01-02T03:04:06+00:00") _git(repo, "commit", "-qm", "one", when="2020-01-02T03:04:06+00:00") return repo class _StreamProc: """A ``Popen`` stand-in for the abandon-the-stream pin. Real pipes would make the case a race (the child has to still be alive at ``close()``); this one holds an unbounded ``StringIO`` so the generator is guaranteed to be parked at its ``yield`` — which is exactly the state whose cleanup the pin checks: terminate, close both pipes, join the stderr thread, reap. """ def __init__(self, out: str, err: str) -> None: self.stdout = io.StringIO(out) self.stderr = io.StringIO(err) self.returncode: int | None = None self.terminated = False self.waits = 0 def terminate(self) -> None: self.terminated = True self.returncode = -15 def wait(self) -> int: self.waits += 1 if self.returncode is None: self.returncode = 0 return self.returncode @requires_git def test_stream_git_lines_yields_lines_and_raises_run_git_shaped_error( tmp_path: Path, ) -> None: """The streaming site's two shapes against REAL git: success yields the ``git log`` lines straight off the pipe (each one still newline-terminated — nothing is buffered or re-split), and a failure raises the ``run_git``-shaped ``GitSyncError`` (``git failed (exit N): ``), so every caller's fail-soft text is unchanged.""" repo = _scratch_repo(tmp_path) argv = ["git", "log", "--name-only", "--format=@@%cI"] lines = list(stream_git_lines(argv, cwd=repo)) stripped = [line.rstrip("\n") for line in lines] assert stripped[1:] == ["", "a.md"] assert stripped[0].startswith("@@") # the %cI commit header, unsplit # (%cI prints UTC as "Z"; fromisoformat reads both forms) assert datetime.fromisoformat(stripped[0][2:]) == datetime(2020, 1, 2, 3, 4, 6, tzinfo=UTC) assert all(line.endswith("\n") for line in lines) plain = tmp_path / "not-a-repo" plain.mkdir() with pytest.raises( GitSyncError, match=r"^git log --name-only --format=@@%cI failed \(exit \d+\): " ) as excinfo: list(stream_git_lines(argv, cwd=plain)) assert "not a git repository" in str(excinfo.value) @requires_git def test_stream_git_lines_reports_stderr_without_deadlocking(tmp_path: Path) -> None: """A2: a child spilling ≈340 KB on stderr while stdout is being consumed neither deadlocks nor truncates. The command is a git alias running a shell that writes ``seq 1 60_000`` (≈340 KB — five pipe buffers) to stderr and exits 3, so git's own stderr pipe is what must be drained. Without the drain thread the child blocks on a full stderr pipe while the parent blocks on stdout and neither ever finishes (the classic two-pipe ``Popen`` trap); here the call returns in well under the 10 s guard and the ``GitSyncError`` carries git's FULL stderr, which is why ``stderr=DEVNULL`` is not the answer (and ``stderr=STDOUT`` would corrupt the date parse). """ repo = _scratch_repo(tmp_path) argv = ["git", "-c", f'alias.spew=!sh -c "seq 1 {_SPEW_LINES} >&2; exit 3"', "spew"] started = time.monotonic() with pytest.raises(GitSyncError) as excinfo: list(stream_git_lines(argv, cwd=repo)) elapsed = time.monotonic() - started msg = str(excinfo.value) assert elapsed < 10, f"streaming a chatty stderr took {elapsed:.1f}s — deadlock?" assert "(exit 3)" in msg assert len(msg.encode()) > 100_000, "git's stderr was truncated, not drained" assert msg.rstrip().endswith(str(_SPEW_LINES)) # the LAST line is there def test_stream_git_lines_terminates_an_abandoned_stream( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """Closing the generator early (``GeneratorExit``) — and an exception thrown into it — never leaks a ``git`` child: the process is terminated, both pipes closed, the drain thread joined, the child reaped.""" proc = _StreamProc("".join(f"line {i}\n" for i in range(5_000)), "warning: noisy stderr\n") monkeypatch.setattr(subprocess, "Popen", lambda *a, **k: proc) # The public shape is ``Iterator[str]`` (the parser's contract); this # pin exercises the generator protocol behind it — close()/throw(). gen = cast( "Generator[str, None, None]", stream_git_lines(["git", "log", "--name-only", "--format=@@%cI"], cwd=tmp_path), ) assert next(gen) == "line 0\n" # parked at the yield, 4,999 lines unread gen.close() assert proc.terminated is True assert proc.returncode == -15 # wait() reaped it — no zombie, no orphan assert proc.waits >= 1 assert proc.stdout.closed and proc.stderr.closed # the same arm fires for a consumer that dies on its own exception proc2 = _StreamProc("line 0\nline 1\n", "") monkeypatch.setattr(subprocess, "Popen", lambda *a, **k: proc2) gen2 = cast("Generator[str, None, None]", stream_git_lines(["git", "log"], cwd=tmp_path)) assert next(gen2) == "line 0\n" with pytest.raises(RuntimeError, match="the parser blew up"): gen2.throw(RuntimeError("the parser blew up")) assert proc2.terminated is True def test_stream_git_lines_missing_git_raises_named_error( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """A missing git binary is the same ``GitSyncError`` as through :func:`run_git` — the walk's fail-soft never sees a bare ``FileNotFoundError``.""" def boom(*args: object, **kwargs: object) -> None: raise FileNotFoundError("git") monkeypatch.setattr(subprocess, "Popen", boom) with pytest.raises(GitSyncError, match="git was not found on PATH"): list(stream_git_lines(["git", "log"], cwd=tmp_path)) # …and the walk's fail-soft keeps working on it: {} + the mtime # fallback, never a bare FileNotFoundError out of the sync. assert git_sync.file_commit_dates(tmp_path) == {} def test_file_commit_dates_fails_soft_on_a_mid_stream_git_failure( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture ) -> None: """Phase 186 D3: a failure AFTER some lines were parsed still yields ``{}`` + the same warning (the D2 contract is about the RESULT). Pre-streaming, ``run_git`` could only fail before a single line was seen; the stream can fail half-way through a huge history. The importer must still fall back to mtimes — a date walk must never break a sync — and the partial map must be DROPPED, not returned. """ def half_then_boom(argv: list[str], cwd: Path) -> Iterator[str]: yield "@@2024-06-15T10:00:00+00:00" yield "a.md" raise GitSyncError("git log --name-only --format=@@%cI failed (exit 128): fatal: boom") monkeypatch.setattr(git_sync, "stream_git_lines", half_then_boom) with caplog.at_level(logging.WARNING, logger="scripts.git_sync"): assert git_sync.file_commit_dates(tmp_path) == {} warnings = [r.getMessage() for r in caplog.records if r.levelno == logging.WARNING] assert len(warnings) == 1 assert warnings[0].startswith(f"file_commit_dates: git log failed for {tmp_path}: ") assert "fall back to file mtimes" in warnings[0] #: Child-process spawns by name (``subprocess.run``/``Popen``/… and the #: ``os`` equivalents) — what an A11 audit has to catch, not just the #: literal string "subprocess". _SUBPROCESS_SPAWNS = frozenset( {"Popen", "run", "call", "check_call", "check_output", "getoutput", "getstatusoutput"} ) _OS_SPAWNS = frozenset( {"system", "popen", "execl", "execle", "execlp", "execv", "execve", "execvp", "posix_spawn"} ) class _SpawnVisitor(ast.NodeVisitor): """Collects ``(enclosing function, spawn call)`` for every child process.""" def __init__(self) -> None: self.stack: list[str] = [] self.sites: list[tuple[str, str]] = [] @property def _scope(self) -> str: return ".".join(self.stack) if self.stack else "" def _visit_func(self, node: ast.FunctionDef | ast.AsyncFunctionDef) -> None: self.stack.append(node.name) self.generic_visit(node) self.stack.pop() visit_FunctionDef = _visit_func visit_AsyncFunctionDef = _visit_func def visit_Call(self, node: ast.Call) -> None: fn = node.func if isinstance(fn, ast.Attribute) and isinstance(fn.value, ast.Name): if fn.value.id == "subprocess" and fn.attr in _SUBPROCESS_SPAWNS: self.sites.append((self._scope, f"subprocess.{fn.attr}")) elif fn.value.id == "os" and fn.attr in _OS_SPAWNS: self.sites.append((self._scope, f"os.{fn.attr}")) self.generic_visit(node) def _spawn_sites(path: Path) -> list[tuple[str, str]]: """Every real spawn in one module, tagged with its function. Parsing the AST (not grepping text) keeps docstrings that MENTION ``subprocess.run`` out of the result — this module has several. """ visitor = _SpawnVisitor() visitor.visit(ast.parse(path.read_text(encoding="utf-8"))) return visitor.sites def test_only_two_git_spawn_sites() -> None: """A11, phase-186 shape: exactly TWO spawn sites, one per shape, both inside ``scripts/git_sync.py`` — the buffered ``run_git`` (``subprocess.run``) and the streaming ``stream_git_lines`` (``subprocess.Popen``). The repo-wide half asserts no NEW spawn appears in ``app/`` or ``scripts/``. It enumerates the ONE pre-existing spawn outside the module: ``app/core/caching.py``'s boot-time asset-version probe (a 5-second-timeouted ``git rev-parse --short HEAD`` that never raises and is not part of any sync path) — the honest state of A11, which the module docstring now states too, instead of a pin that quietly ignores it. ``app/core/docs_push.py`` passes by construction: it calls ``run_git``, so it never appears here. """ module = Path(git_sync.__file__) assert _spawn_sites(module) == [ ("run_git", "subprocess.run"), ("stream_git_lines", "subprocess.Popen"), ] repo_root = module.parents[1] offenders: dict[str, list[tuple[str, str]]] = {} for package in ("app", "scripts"): for py in sorted((repo_root / package).rglob("*.py")): if py == module: continue found = _spawn_sites(py) if found: offenders[str(py.relative_to(repo_root))] = found assert offenders == {"app/core/caching.py": [("_git_short_sha", "subprocess.run")]} # docs-push rides the buffered site rather than spawning its own — # the reason it is absent from `offenders`. assert docs_push.run_git is git_sync.run_git