All green — verification complete. Final report: **Phase 107 — final verification pass (all tasks already in `complete/`)** - **Verified:** `clone_or_pull` full-history fix (no `--depth` clone; `--is-shallow-repository` probe → one-time `git fetch --unshallow` self-heal → `git pull --ff-only`; D12 fail-loud) in `scripts/git_sync.py`; E2E suite `tests/e2e/test_git_source_dates.py` present; all five stale shallow/tip-date narratives corrected (no residual claims repo-wide; `docs_push.py` `--depth` fetches untouched as designed). - **Unit+integration:** `uv run pytest tests/unit/test_git_sync.py tests/integration/test_git_file_dates.py tests/integration/test_import_docs_git.py tests/integration/test_sync_api.py -v --no-cov` → **66 passed** (D10 tip-date test replaced by `test_url_clone_yields_true_per_file_dates` + `test_existing_shallow_checkout_self_heals`; unit pins: clone argv w/o `--depth`, probe→unshallow→pull order, unshallow-failure → `GitSyncError`). - **New E2E in isolation:** `uv run pytest tests/e2e/test_git_source_dates.py -v --no-cov` → **4 passed** (old file 2020-06-15 ≠ tip file 2024-06-15 in `GET /api/docs`, Sources `Created`/`Updated` columns, viewer `Created` badge ISO title; folder `updated_at` subtree maxes). - **Phase-106 regressions in isolation:** `test_document_dates.py` → 6 passed; `test_sync_button.py` → 3 passed; `test_git_sources_admin.py` → 6 passed. - **Full gates:** `uv run pytest` → **2302 passed**; `uv run pytest --cov=app --cov-report=term-missing` → **TOTAL 99%** (>90%); `uv run ruff check . && uv run pyright` → **clean, 0 errors**. - **Completion criteria:** 1 ✅ non-shallow URL clone + true per-file dates · 2 ✅ self-heal / plain-pull / fail-loud · 3 ✅ both suites green, bug-pin replaced · 4 ✅ E2E green in isolation · 5 ✅ all regressions + full suite + coverage + lint · 6 ✅ narratives corrected · 7 — no commit made (harness override: changes left in working tree; task files already in `complete/`). - **No defects found; no deviations.** Next pending phase: **108_history_wire_check**.
295 lines
11 KiB
Python
295 lines
11 KiB
Python
"""Integration: phase 106 task 03 — ``file_commit_dates`` against real
|
|
git scratch repos (D2; phase 107 D11 supersedes phase 106 D10).
|
|
|
|
Builds scratch repositories with controlled ``GIT_COMMITTER_DATE``s
|
|
(the 2026-09-13 verification recipe: file ``a.md`` committed once in
|
|
2020, file ``b.md`` committed in 2020 and touched again in 2024, a
|
|
``docs/deep.md`` subdirectory file committed once in 2020) and pins
|
|
the VERIFIED checkout behavior (phase 107):
|
|
|
|
* a LOCAL-PATH ``clone_or_pull`` checkout is FULL history → TRUE
|
|
per-file last-commit dates (first-sighting wins: ``a.md`` 2020,
|
|
``b.md`` 2024, ``docs/deep.md`` 2020);
|
|
* a URL-TRANSPORT (``file://``) ``clone_or_pull`` checkout is FULL
|
|
history TOO (D11: no ``--depth`` on the clone) → the SAME true
|
|
per-file dates — the regression pin for the 2026-09-16 bug
|
|
(pre-fix, D10's shallow ``--depth 1`` clone returned the TIP date
|
|
for EVERY file, uniform per repo);
|
|
* an EXISTING shallow checkout (made ``--depth 1`` directly by the
|
|
test harness — simulating every pre-phase-107 deployed checkout)
|
|
self-heals on the next ``clone_or_pull``: the
|
|
``--is-shallow-repository`` probe finds it shallow → ``git fetch
|
|
--unshallow`` → ``git pull --ff-only`` → no longer shallow, true
|
|
per-file dates;
|
|
* fail-soft: a directory without ``.git``, an empty repo (no
|
|
commits), a git failure, and a malformed log output all yield
|
|
``{}`` — a date walk must never break a sync (the importer, task
|
|
04, falls back to file mtimes).
|
|
|
|
DB-free by design: ``file_commit_dates`` takes a path, no session.
|
|
Skipped (not failed) on a machine without the git CLI (the
|
|
``test_doc_drafts_api.py`` guard).
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import subprocess
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from scripts.git_sync import (
|
|
GitSyncError,
|
|
_parse_commit_dates, # pyright: ignore[reportPrivateUsage]
|
|
clone_or_pull,
|
|
file_commit_dates,
|
|
run_git,
|
|
)
|
|
|
|
|
|
def _git_available() -> bool:
|
|
try:
|
|
proc = subprocess.run(["git", "--version"], capture_output=True, check=False)
|
|
return proc.returncode == 0
|
|
except (FileNotFoundError, OSError):
|
|
return False
|
|
|
|
|
|
#: Real ``git`` in the test environment — skipped cleanly without it
|
|
#: (the ``test_doc_drafts_api.py`` house pattern).
|
|
GIT = _git_available()
|
|
|
|
pytestmark = pytest.mark.skipif(not GIT, reason="git CLI not available")
|
|
|
|
#: The two controlled commit dates (the 2026-09-13 verification recipe).
|
|
DATE_A = datetime(2020, 1, 2, 3, 4, 6, tzinfo=UTC) # commit one (2020)
|
|
DATE_B = datetime(2024, 6, 15, 10, 0, 0, tzinfo=UTC) # commit two = the tip (2024)
|
|
|
|
|
|
def _git(cwd: Path, *argv: str, when: datetime | None = None) -> None:
|
|
"""Run one git command for the test harness (fixture setup); a
|
|
non-zero exit fails the fixture, not the test under test."""
|
|
env = os.environ.copy()
|
|
if when is not None:
|
|
iso = when.isoformat()
|
|
env["GIT_AUTHOR_DATE"] = iso
|
|
env["GIT_COMMITTER_DATE"] = iso
|
|
env["GIT_AUTHOR_NAME"] = "T"
|
|
env["GIT_AUTHOR_EMAIL"] = "t@example.com"
|
|
env["GIT_COMMITTER_NAME"] = "T"
|
|
env["GIT_COMMITTER_EMAIL"] = "t@example.com"
|
|
proc = subprocess.run(
|
|
["git", *argv], cwd=cwd, env=env, capture_output=True, text=True, check=False
|
|
)
|
|
assert proc.returncode == 0, f"git {' '.join(argv)} failed: {proc.stderr}"
|
|
|
|
|
|
@pytest.fixture()
|
|
def scratch_repo(tmp_path: Path) -> Path:
|
|
"""The 2026-09-13 recipe: commit one (2020-01-02) adds ``a.md``,
|
|
``b.md``, ``docs/deep.md``; commit two (2024-06-15, the tip)
|
|
touches ONLY ``b.md``."""
|
|
repo = tmp_path / "repo"
|
|
repo.mkdir()
|
|
_git(repo, "init", "-q")
|
|
_git(repo, "config", "user.email", "t@example.com")
|
|
_git(repo, "config", "user.name", "T")
|
|
_git(repo, "config", "commit.gpgsign", "false")
|
|
(repo / "docs").mkdir()
|
|
(repo / "a.md").write_text("# A\nstable since 2020\n", encoding="utf-8")
|
|
(repo / "b.md").write_text("# B\nfirst version\n", encoding="utf-8")
|
|
(repo / "docs" / "deep.md").write_text("# Deep\nalso 2020\n", encoding="utf-8")
|
|
_git(repo, "add", "-A", when=DATE_A)
|
|
_git(repo, "commit", "-qm", "one", when=DATE_A)
|
|
(repo / "b.md").write_text("# B\nupdated 2024\n", encoding="utf-8")
|
|
_git(repo, "add", "-A", when=DATE_B)
|
|
_git(repo, "commit", "-qm", "two", when=DATE_B)
|
|
return repo
|
|
|
|
|
|
def test_local_clone_yields_true_per_file_dates(scratch_repo: Path, tmp_path: Path) -> None:
|
|
"""(a) LOCAL-PATH ``clone_or_pull`` → full history (no ``--depth``
|
|
at all, phase 107 D11) → TRUE per-file last-commit dates: the
|
|
first (newest) sighting of each path wins — ``b.md`` the 2024
|
|
touch, the rest the 2020 commit."""
|
|
dest = tmp_path / "local"
|
|
clone_or_pull(str(scratch_repo), dest)
|
|
assert (dest / ".git").exists() # a real checkout
|
|
assert file_commit_dates(dest) == {
|
|
"a.md": DATE_A,
|
|
"b.md": DATE_B, # touched again by the tip commit
|
|
"docs/deep.md": DATE_A,
|
|
}
|
|
|
|
|
|
def test_url_clone_yields_true_per_file_dates(scratch_repo: Path, tmp_path: Path) -> None:
|
|
"""(b) URL-TRANSPORT ``clone_or_pull`` (``file://`` through the REAL
|
|
function — the 2026-09-16 regression pin): the checkout is FULL
|
|
history (no ``--depth`` — phase 107 D11 supersedes phase 106 D10)
|
|
→ NOT shallow, and TRUE per-file last-commit dates: the 2020 files
|
|
stay 2020 and ``b.md`` (touched again at the tip) gets the 2024
|
|
tip date. PRE-FIX this returned DATE_B (the uniform tip date) for
|
|
all three files — the exact bug the owner reported.
|
|
"""
|
|
dest = tmp_path / "url-clone"
|
|
clone_or_pull(f"file://{scratch_repo}", dest)
|
|
assert (dest / ".git").exists() # a real checkout
|
|
assert (
|
|
run_git(["git", "rev-parse", "--is-shallow-repository"], cwd=dest).strip()
|
|
== "false"
|
|
) # a full-history checkout, not shallow
|
|
assert file_commit_dates(dest) == {
|
|
"a.md": DATE_A,
|
|
"b.md": DATE_B, # touched again by the tip commit
|
|
"docs/deep.md": DATE_A,
|
|
}
|
|
|
|
|
|
def test_existing_shallow_checkout_self_heals(scratch_repo: Path, tmp_path: Path) -> None:
|
|
"""(c) an EXISTING shallow checkout — built ``--depth 1`` directly
|
|
by the harness, simulating every pre-phase-107 deployed checkout
|
|
(live + dev homelab included) — self-heals on the next
|
|
``clone_or_pull`` (D11): the ``--is-shallow-repository`` probe
|
|
finds it shallow → the ONE-TIME ``git fetch --unshallow`` restores
|
|
the full history (no re-clone) → the usual ``--ff-only`` pull. Pre-
|
|
heal: shallow + the uniform tip date for all files; post-heal:
|
|
full history + true per-file dates."""
|
|
dest = tmp_path / "shallow"
|
|
_git(tmp_path, "clone", "-q", "--depth", "1", f"file://{scratch_repo}", str(dest))
|
|
# Pre-heal: the deployed state the fix must cure.
|
|
assert (
|
|
run_git(["git", "rev-parse", "--is-shallow-repository"], cwd=dest).strip()
|
|
== "true"
|
|
)
|
|
assert file_commit_dates(dest) == {
|
|
"a.md": DATE_B,
|
|
"b.md": DATE_B,
|
|
"docs/deep.md": DATE_B,
|
|
}
|
|
clone_or_pull(f"file://{scratch_repo}", dest)
|
|
# Post-heal: full history and true per-file dates, same as (b).
|
|
assert (
|
|
run_git(["git", "rev-parse", "--is-shallow-repository"], cwd=dest).strip()
|
|
== "false"
|
|
)
|
|
assert file_commit_dates(dest) == {
|
|
"a.md": DATE_A,
|
|
"b.md": DATE_B,
|
|
"docs/deep.md": DATE_A,
|
|
}
|
|
|
|
|
|
def test_directory_without_dotgit_fails_soft(tmp_path: Path) -> None:
|
|
"""(c) a plain directory (no ``.git``) → ``git log`` exits
|
|
non-zero → ``{}`` (fail-soft, no raise) — the importer falls back
|
|
to file mtimes."""
|
|
plain = tmp_path / "notarepo"
|
|
plain.mkdir()
|
|
(plain / "a.md").write_text("# A\nnot a git repo\n", encoding="utf-8")
|
|
assert file_commit_dates(plain) == {}
|
|
|
|
|
|
def test_nonexistent_directory_fails_soft(tmp_path: Path) -> None:
|
|
"""(c) a missing checkout directory → ``{}`` without even
|
|
invoking git (no raise)."""
|
|
assert file_commit_dates(tmp_path / "gone") == {}
|
|
|
|
|
|
def test_empty_repo_fails_soft(tmp_path: Path) -> None:
|
|
"""(c) an initialized repo with NO commits → ``git log`` fails
|
|
(nothing to log) → ``{}`` (a cloned-but-empty source must not
|
|
break the sync)."""
|
|
empty = tmp_path / "emptyrepo"
|
|
empty.mkdir()
|
|
_git(empty, "init", "-q")
|
|
_git(empty, "config", "commit.gpgsign", "false")
|
|
assert file_commit_dates(empty) == {}
|
|
|
|
|
|
def test_git_error_fails_soft(
|
|
scratch_repo: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
|
|
) -> None:
|
|
"""(c) a ``GitSyncError`` from the walk (git missing/failed) →
|
|
``{}`` + a logged warning naming the fallback — the fail-soft
|
|
contract (pinned)."""
|
|
|
|
def boom(argv: list[str], cwd: Path) -> str:
|
|
raise GitSyncError("git log failed (exit 128): fatal: bad object")
|
|
|
|
monkeypatch.setattr("scripts.git_sync.run_git", boom)
|
|
with caplog.at_level("WARNING"):
|
|
assert file_commit_dates(scratch_repo) == {}
|
|
assert any("file_commit_dates" in record.message for record in caplog.records)
|
|
|
|
|
|
def test_malformed_log_output_fails_soft(
|
|
scratch_repo: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
"""(c) ANY parse anomaly (a commit date ``fromisoformat`` cannot
|
|
read) → ``{}`` (fail-soft) — the walk is all-or-nothing: a
|
|
partially parsed date map would be worse than none."""
|
|
monkeypatch.setattr(
|
|
"scripts.git_sync.run_git",
|
|
lambda argv, cwd: "@@not-a-date\nb.md\n",
|
|
)
|
|
assert file_commit_dates(scratch_repo) == {}
|
|
|
|
|
|
# --- the pure parser (canned git output — no git, no DB) -------------------
|
|
|
|
|
|
def test_parser_first_sighting_wins_newest_first() -> None:
|
|
"""The walk is newest-first, so the FIRST sighting of a path is
|
|
its last-commit date: ``b.md`` appears under both commits and keeps
|
|
the 2024 (newest) date; the 2020 ``a.md`` keeps 2020. Blank lines
|
|
(git's commit separators) are skipped."""
|
|
output = "\n".join(
|
|
[
|
|
"@@2024-06-15T10:00:00+00:00",
|
|
"",
|
|
"b.md",
|
|
"@@2020-01-02T03:04:06+00:00",
|
|
"",
|
|
"a.md",
|
|
"b.md",
|
|
]
|
|
)
|
|
assert _parse_commit_dates(output) == {"a.md": DATE_A, "b.md": DATE_B}
|
|
|
|
|
|
def test_parser_normalizes_paths() -> None:
|
|
"""Path lines are whitespace-split (defensively - git's name-only
|
|
output is one path per line), backslash-normalized to ``/``, and a
|
|
leading ``/`` is stripped (repo-relative POSIX keys); the line is
|
|
stripped first."""
|
|
output = "\n".join(
|
|
[
|
|
"@@2024-06-15T10:00:00+00:00",
|
|
"",
|
|
"docs\\deep.md",
|
|
"/rooted.md",
|
|
" padded.md ",
|
|
]
|
|
)
|
|
assert _parse_commit_dates(output) == {
|
|
"docs/deep.md": DATE_B,
|
|
"rooted.md": DATE_B,
|
|
"padded.md": DATE_B,
|
|
}
|
|
|
|
|
|
def test_parser_rejects_path_before_header() -> None:
|
|
"""A path line before ANY commit header is a malformed walk →
|
|
``ValueError`` (the caller's fail-soft path turns it into
|
|
``{}``)."""
|
|
with pytest.raises(ValueError, match="before any commit header"):
|
|
_parse_commit_dates("stray.md\n@@2024-06-15T10:00:00+00:00\n")
|
|
|
|
|
|
def test_parser_rejects_bad_date() -> None:
|
|
"""A commit date ``fromisoformat`` cannot read → ``ValueError``
|
|
(ISO-strict ``%cI`` always parses — this is the anomaly guard)."""
|
|
with pytest.raises(ValueError):
|
|
_parse_commit_dates("@@yesterday\nb.md\n")
|