From 6bcee831ec220f2d541ef1fb8533a4094facc4d7 Mon Sep 17 00:00:00 2001 From: ducoterra Date: Sun, 13 Sep 2026 23:21:39 -0400 Subject: [PATCH] phase: 107_git_full_history_dates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All green — verification complete. Final report: **Phase 107 — final verification pass (all tasks already in `complete/`)** - **Verified:** `clone_or_pull` full-history fix (no `--depth` clone; `--is-shallow-repository` probe → one-time `git fetch --unshallow` self-heal → `git pull --ff-only`; D12 fail-loud) in `scripts/git_sync.py`; E2E suite `tests/e2e/test_git_source_dates.py` present; all five stale shallow/tip-date narratives corrected (no residual claims repo-wide; `docs_push.py` `--depth` fetches untouched as designed). - **Unit+integration:** `uv run pytest tests/unit/test_git_sync.py tests/integration/test_git_file_dates.py tests/integration/test_import_docs_git.py tests/integration/test_sync_api.py -v --no-cov` → **66 passed** (D10 tip-date test replaced by `test_url_clone_yields_true_per_file_dates` + `test_existing_shallow_checkout_self_heals`; unit pins: clone argv w/o `--depth`, probe→unshallow→pull order, unshallow-failure → `GitSyncError`). - **New E2E in isolation:** `uv run pytest tests/e2e/test_git_source_dates.py -v --no-cov` → **4 passed** (old file 2020-06-15 ≠ tip file 2024-06-15 in `GET /api/docs`, Sources `Created`/`Updated` columns, viewer `Created` badge ISO title; folder `updated_at` subtree maxes). - **Phase-106 regressions in isolation:** `test_document_dates.py` → 6 passed; `test_sync_button.py` → 3 passed; `test_git_sources_admin.py` → 6 passed. - **Full gates:** `uv run pytest` → **2302 passed**; `uv run pytest --cov=app --cov-report=term-missing` → **TOTAL 99%** (>90%); `uv run ruff check . && uv run pyright` → **clean, 0 errors**. - **Completion criteria:** 1 ✅ non-shallow URL clone + true per-file dates · 2 ✅ self-heal / plain-pull / fail-loud · 3 ✅ both suites green, bug-pin replaced · 4 ✅ E2E green in isolation · 5 ✅ all regressions + full suite + coverage + lint · 6 ✅ narratives corrected · 7 — no commit made (harness override: changes left in working tree; task files already in `complete/`). - **No defects found; no deviations.** Next pending phase: **108_history_wire_check**. --- .../107_git_full_history_dates/00_phase.md | 0 .../01_full_history_checkouts.md | 0 .../02_e2e_git_source_dates.md | 0 .../03_gates_and_commit.md | 0 ...07_git_full_history_dates__00_phase.a1.err | 0 ...107_git_full_history_dates__00_phase.a1.md | 11 + ...t_full_history_dates__00_phase.a1.validate | 100 +++ ...ry_dates__01_full_history_checkouts.a1.err | 0 ...ory_dates__01_full_history_checkouts.a1.md | 19 + ...tes__01_full_history_checkouts.a1.validate | 100 +++ ...tory_dates__02_e2e_git_source_dates.a1.err | 0 ...story_dates__02_e2e_git_source_dates.a1.md | 18 + ...dates__02_e2e_git_source_dates.a1.validate | 100 +++ ..._history_dates__03_gates_and_commit.a1.err | 0 ...l_history_dates__03_gates_and_commit.a1.md | 14 + ...ory_dates__03_gates_and_commit.a1.validate | 100 +++ app/api/sync.py | 9 +- scripts/git_sync.py | 91 ++- scripts/import_docs.py | 3 +- tests/e2e/test_git_source_dates.py | 637 ++++++++++++++++++ tests/integration/test_git_file_dates.py | 92 ++- tests/integration/test_import_docs_git.py | 2 +- tests/integration/test_sync_api.py | 2 +- tests/unit/test_git_sync.py | 113 +++- 24 files changed, 1337 insertions(+), 74 deletions(-) rename .agents/phases/{todo => complete}/107_git_full_history_dates/00_phase.md (100%) rename .agents/phases/{todo => complete}/107_git_full_history_dates/01_full_history_checkouts.md (100%) rename .agents/phases/{todo => complete}/107_git_full_history_dates/02_e2e_git_source_dates.md (100%) rename .agents/phases/{todo => complete}/107_git_full_history_dates/03_gates_and_commit.md (100%) create mode 100644 .agents/reports/107_git_full_history_dates/107_git_full_history_dates__00_phase.a1.err create mode 100644 .agents/reports/107_git_full_history_dates/107_git_full_history_dates__00_phase.a1.md create mode 100644 .agents/reports/107_git_full_history_dates/107_git_full_history_dates__00_phase.a1.validate create mode 100644 .agents/reports/107_git_full_history_dates/107_git_full_history_dates__01_full_history_checkouts.a1.err create mode 100644 .agents/reports/107_git_full_history_dates/107_git_full_history_dates__01_full_history_checkouts.a1.md create mode 100644 .agents/reports/107_git_full_history_dates/107_git_full_history_dates__01_full_history_checkouts.a1.validate create mode 100644 .agents/reports/107_git_full_history_dates/107_git_full_history_dates__02_e2e_git_source_dates.a1.err create mode 100644 .agents/reports/107_git_full_history_dates/107_git_full_history_dates__02_e2e_git_source_dates.a1.md create mode 100644 .agents/reports/107_git_full_history_dates/107_git_full_history_dates__02_e2e_git_source_dates.a1.validate create mode 100644 .agents/reports/107_git_full_history_dates/107_git_full_history_dates__03_gates_and_commit.a1.err create mode 100644 .agents/reports/107_git_full_history_dates/107_git_full_history_dates__03_gates_and_commit.a1.md create mode 100644 .agents/reports/107_git_full_history_dates/107_git_full_history_dates__03_gates_and_commit.a1.validate create mode 100644 tests/e2e/test_git_source_dates.py diff --git a/.agents/phases/todo/107_git_full_history_dates/00_phase.md b/.agents/phases/complete/107_git_full_history_dates/00_phase.md similarity index 100% rename from .agents/phases/todo/107_git_full_history_dates/00_phase.md rename to .agents/phases/complete/107_git_full_history_dates/00_phase.md diff --git a/.agents/phases/todo/107_git_full_history_dates/01_full_history_checkouts.md b/.agents/phases/complete/107_git_full_history_dates/01_full_history_checkouts.md similarity index 100% rename from .agents/phases/todo/107_git_full_history_dates/01_full_history_checkouts.md rename to .agents/phases/complete/107_git_full_history_dates/01_full_history_checkouts.md diff --git a/.agents/phases/todo/107_git_full_history_dates/02_e2e_git_source_dates.md b/.agents/phases/complete/107_git_full_history_dates/02_e2e_git_source_dates.md similarity index 100% rename from .agents/phases/todo/107_git_full_history_dates/02_e2e_git_source_dates.md rename to .agents/phases/complete/107_git_full_history_dates/02_e2e_git_source_dates.md diff --git a/.agents/phases/todo/107_git_full_history_dates/03_gates_and_commit.md b/.agents/phases/complete/107_git_full_history_dates/03_gates_and_commit.md similarity index 100% rename from .agents/phases/todo/107_git_full_history_dates/03_gates_and_commit.md rename to .agents/phases/complete/107_git_full_history_dates/03_gates_and_commit.md diff --git a/.agents/reports/107_git_full_history_dates/107_git_full_history_dates__00_phase.a1.err b/.agents/reports/107_git_full_history_dates/107_git_full_history_dates__00_phase.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/107_git_full_history_dates/107_git_full_history_dates__00_phase.a1.md b/.agents/reports/107_git_full_history_dates/107_git_full_history_dates__00_phase.a1.md new file mode 100644 index 0000000..36ec6a4 --- /dev/null +++ b/.agents/reports/107_git_full_history_dates/107_git_full_history_dates__00_phase.a1.md @@ -0,0 +1,11 @@ +All green — verification complete. Final report: + +**Phase 107 — final verification pass (all tasks already in `complete/`)** + +- **Verified:** `clone_or_pull` full-history fix (no `--depth` clone; `--is-shallow-repository` probe → one-time `git fetch --unshallow` self-heal → `git pull --ff-only`; D12 fail-loud) in `scripts/git_sync.py`; E2E suite `tests/e2e/test_git_source_dates.py` present; all five stale shallow/tip-date narratives corrected (no residual claims repo-wide; `docs_push.py` `--depth` fetches untouched as designed). +- **Unit+integration:** `uv run pytest tests/unit/test_git_sync.py tests/integration/test_git_file_dates.py tests/integration/test_import_docs_git.py tests/integration/test_sync_api.py -v --no-cov` → **66 passed** (D10 tip-date test replaced by `test_url_clone_yields_true_per_file_dates` + `test_existing_shallow_checkout_self_heals`; unit pins: clone argv w/o `--depth`, probe→unshallow→pull order, unshallow-failure → `GitSyncError`). +- **New E2E in isolation:** `uv run pytest tests/e2e/test_git_source_dates.py -v --no-cov` → **4 passed** (old file 2020-06-15 ≠ tip file 2024-06-15 in `GET /api/docs`, Sources `Created`/`Updated` columns, viewer `Created` badge ISO title; folder `updated_at` subtree maxes). +- **Phase-106 regressions in isolation:** `test_document_dates.py` → 6 passed; `test_sync_button.py` → 3 passed; `test_git_sources_admin.py` → 6 passed. +- **Full gates:** `uv run pytest` → **2302 passed**; `uv run pytest --cov=app --cov-report=term-missing` → **TOTAL 99%** (>90%); `uv run ruff check . && uv run pyright` → **clean, 0 errors**. +- **Completion criteria:** 1 ✅ non-shallow URL clone + true per-file dates · 2 ✅ self-heal / plain-pull / fail-loud · 3 ✅ both suites green, bug-pin replaced · 4 ✅ E2E green in isolation · 5 ✅ all regressions + full suite + coverage + lint · 6 ✅ narratives corrected · 7 — no commit made (harness override: changes left in working tree; task files already in `complete/`). +- **No defects found; no deviations.** Next pending phase: **108_history_wire_check**. diff --git a/.agents/reports/107_git_full_history_dates/107_git_full_history_dates__00_phase.a1.validate b/.agents/reports/107_git_full_history_dates/107_git_full_history_dates__00_phase.a1.validate new file mode 100644 index 0000000..e9a1c53 --- /dev/null +++ b/.agents/reports/107_git_full_history_dates/107_git_full_history_dates__00_phase.a1.validate @@ -0,0 +1,100 @@ +........................................................................ [ 3%] +........................................................................ [ 6%] +........................................................................ [ 9%] +........................................................................ [ 12%] +........................................................................ [ 15%] +........................................................................ [ 18%] +........................................................................ [ 21%] +........................................................................ [ 25%] +........................................................................ [ 28%] +........................................................................ [ 31%] +........................................................................ [ 34%] +........................................................................ [ 37%] +........................................................................ [ 40%] +........................................................................ [ 43%] +........................................................................ [ 46%] +........................................................................ [ 50%] +........................................................................ [ 53%] +........................................................................ [ 56%] +........................................................................ [ 59%] +........................................................................ [ 62%] +........................................................................ [ 65%] +........................................................................ [ 68%] +........................................................................ [ 71%] +........................................................................ [ 75%] +........................................................................ [ 78%] +........................................................................ [ 81%] +........................................................................ [ 84%] +........................................................................ [ 87%] +........................................................................ [ 90%] +........................................................................ [ 93%] +........................................................................ [ 96%] +...................................................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 181 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 94 0 100% +app/api/docs.py 156 1 99% +app/api/git_sources.py 232 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 139 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 154 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 20 0 100% +app/core/theming.py 38 0 100% +app/core/tokens.py 44 0 100% +app/db.py 21 0 100% +app/main.py 66 0 100% +app/models.py 128 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 316 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 206 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 123 0 100% +app/rag/git_sources.py 14 0 100% +app/rag/importer.py 215 3 99% +app/rag/llm.py 243 1 99% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 88 0 100% +app/rag/retriever.py 172 3 98% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 24 0 100% +app/schemas.py 327 0 100% +-------------------------------------------------- +TOTAL 4018 15 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/107_git_full_history_dates/107_git_full_history_dates__01_full_history_checkouts.a1.err b/.agents/reports/107_git_full_history_dates/107_git_full_history_dates__01_full_history_checkouts.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/107_git_full_history_dates/107_git_full_history_dates__01_full_history_checkouts.a1.md b/.agents/reports/107_git_full_history_dates/107_git_full_history_dates__01_full_history_checkouts.a1.md new file mode 100644 index 0000000..349d2f7 --- /dev/null +++ b/.agents/reports/107_git_full_history_dates/107_git_full_history_dates__01_full_history_checkouts.a1.md @@ -0,0 +1,19 @@ +All work for task 01 is complete and verified. Final state: all edits made, then all gates run green. + +## Report — Phase 107, Task 01 (`clone_or_pull` full-history checkouts, D11/D12) + +**Implemented:** +- `scripts/git_sync.py::clone_or_pull`: fresh clone now `git clone url dest` (no `--depth`); existing checkouts probe `git rev-parse --is-shallow-repository` → one-time `git fetch --unshallow` while shallow → `git pull --ff-only`; all failures propagate `GitSyncError` (D12 fail-loud, no silent fallback) +- `file_commit_dates` unchanged in code (unchanged per design); all five stale shallow/tip-date narratives corrected (git_sync module + both function docstrings, `import_docs.py` docstring, `sync.py` comment); also refreshed two fabricated error strings in `test_sync_api.py`/`test_import_docs_git.py` +- Unit: argv re-pins (no `--depth`), probe-then-pull, new `test_clone_or_pull_unshallows_existing_shallow_checkout` (order pinned), new `test_clone_or_pull_unshallow_failure_propagates` (pull never runs); `_fake_run` extended with argv-dispatched responses +- Integration: replaced the D10 tip-date test with `test_url_clone_yields_true_per_file_dates` (regression pin over real `file://` — 2020 files stay 2020, tip-touched file gets 2024, checkout non-shallow) + new `test_existing_shallow_checkout_self_heals` + +**Results:** +- `uv run pytest tests/unit/test_git_sync.py tests/integration/test_git_file_dates.py -v` → 21 passed +- `uv run pytest` (full) → 2302 passed; `--cov=app` TOTAL 99% (>90%) +- `uv run ruff check . && uv run pyright` → clean (0 errors) +- `uv run pytest tests/e2e/test_sync_button.py -v --no-cov` → 3 passed (real `file://` sync regression) + +**Decisions:** none — followed the phase design exactly (no env knob, fail-loud per D12). + +**Next pending task:** `02_e2e_git_source_dates.md` (dedicated Playwright suite). diff --git a/.agents/reports/107_git_full_history_dates/107_git_full_history_dates__01_full_history_checkouts.a1.validate b/.agents/reports/107_git_full_history_dates/107_git_full_history_dates__01_full_history_checkouts.a1.validate new file mode 100644 index 0000000..e9a1c53 --- /dev/null +++ b/.agents/reports/107_git_full_history_dates/107_git_full_history_dates__01_full_history_checkouts.a1.validate @@ -0,0 +1,100 @@ +........................................................................ [ 3%] +........................................................................ [ 6%] +........................................................................ [ 9%] +........................................................................ [ 12%] +........................................................................ [ 15%] +........................................................................ [ 18%] +........................................................................ [ 21%] +........................................................................ [ 25%] +........................................................................ [ 28%] +........................................................................ [ 31%] +........................................................................ [ 34%] +........................................................................ [ 37%] +........................................................................ [ 40%] +........................................................................ [ 43%] +........................................................................ [ 46%] +........................................................................ [ 50%] +........................................................................ [ 53%] +........................................................................ [ 56%] +........................................................................ [ 59%] +........................................................................ [ 62%] +........................................................................ [ 65%] +........................................................................ [ 68%] +........................................................................ [ 71%] +........................................................................ [ 75%] +........................................................................ [ 78%] +........................................................................ [ 81%] +........................................................................ [ 84%] +........................................................................ [ 87%] +........................................................................ [ 90%] +........................................................................ [ 93%] +........................................................................ [ 96%] +...................................................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 181 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 94 0 100% +app/api/docs.py 156 1 99% +app/api/git_sources.py 232 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 139 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 154 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 20 0 100% +app/core/theming.py 38 0 100% +app/core/tokens.py 44 0 100% +app/db.py 21 0 100% +app/main.py 66 0 100% +app/models.py 128 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 316 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 206 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 123 0 100% +app/rag/git_sources.py 14 0 100% +app/rag/importer.py 215 3 99% +app/rag/llm.py 243 1 99% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 88 0 100% +app/rag/retriever.py 172 3 98% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 24 0 100% +app/schemas.py 327 0 100% +-------------------------------------------------- +TOTAL 4018 15 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/107_git_full_history_dates/107_git_full_history_dates__02_e2e_git_source_dates.a1.err b/.agents/reports/107_git_full_history_dates/107_git_full_history_dates__02_e2e_git_source_dates.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/107_git_full_history_dates/107_git_full_history_dates__02_e2e_git_source_dates.a1.md b/.agents/reports/107_git_full_history_dates/107_git_full_history_dates__02_e2e_git_source_dates.a1.md new file mode 100644 index 0000000..f0f01da --- /dev/null +++ b/.agents/reports/107_git_full_history_dates/107_git_full_history_dates__02_e2e_git_source_dates.a1.md @@ -0,0 +1,18 @@ +Task complete. Final report: + +**Implemented** +- `tests/e2e/test_git_source_dates.py` (new, 4 tests) — phase 107's dedicated Playwright suite: two-commit `file://` fixture repo with controlled `GIT_COMMITTER_DATE`s (2020-06-15 / 2024-06-15, mid-year for TZ-stable years), module-scoped app with `BOR_GIT_SOURCES=file://` + own `BOR_SOURCES_DIR` (port 8130, test_sync_button idiom), per-test KB truncate, per-test real in-app admin sync. +- Test 1 = regression pin: `GET /api/docs` `created_at[:10]` 2020-06-15 vs 2024-06-15, values DIFFER + verbatim ISOs; tree file nodes verbatim, folder/source `updated_at` subtree maxes (D9 over true dates). +- Test 2: file-table `Created` (col 5) + folder-table `Updated` (col 3) render 2020/2024 (year regexes, full ISO on cell `title`), two cells differ. +- Test 3: viewer `.doc-created` badge `title == "2020-06-15T12:00:00+00:00"`, text `Created …`, precedes Indexed badge, Escape closes cleanly (focus restored). +- Test 4: landmarks, both tables' `Created`/`Updated` ``s, 3px `:focus-visible` on a row link via real Tab walk, same-origin assets only. + +**Gates (exact)** +- `uv run pytest tests/e2e/test_git_source_dates.py -v --no-cov` → 4 passed (ran twice, stable) +- `uv run pytest --cov=app --cov-report=term-missing` → 2302 passed, TOTAL 99% (>90%) +- `uv run pytest tests/e2e/test_document_dates.py / test_sync_button.py / test_git_sources_admin.py -v --no-cov` (each in isolation) → 6/3/6 passed +- `uv run ruff check . && uv run pyright` → clean (0 errors) + +**Notable decisions**: pre-click sync-label assertion accepts `"Sync sources"` or an adopted `"Synced HH:MM"` (the module app's server-side status survives per-test pages — the boot re-attach adopts the previous terminal state; the run is still proven by observing `Syncing…` after the click). No app code touched; no git commit (harness owns it). + +**Next pending task**: `03_gates_and_commit.md` (full gate + atomic commit + phase dir move). diff --git a/.agents/reports/107_git_full_history_dates/107_git_full_history_dates__02_e2e_git_source_dates.a1.validate b/.agents/reports/107_git_full_history_dates/107_git_full_history_dates__02_e2e_git_source_dates.a1.validate new file mode 100644 index 0000000..e9a1c53 --- /dev/null +++ b/.agents/reports/107_git_full_history_dates/107_git_full_history_dates__02_e2e_git_source_dates.a1.validate @@ -0,0 +1,100 @@ +........................................................................ [ 3%] +........................................................................ [ 6%] +........................................................................ [ 9%] +........................................................................ [ 12%] +........................................................................ [ 15%] +........................................................................ [ 18%] +........................................................................ [ 21%] +........................................................................ [ 25%] +........................................................................ [ 28%] +........................................................................ [ 31%] +........................................................................ [ 34%] +........................................................................ [ 37%] +........................................................................ [ 40%] +........................................................................ [ 43%] +........................................................................ [ 46%] +........................................................................ [ 50%] +........................................................................ [ 53%] +........................................................................ [ 56%] +........................................................................ [ 59%] +........................................................................ [ 62%] +........................................................................ [ 65%] +........................................................................ [ 68%] +........................................................................ [ 71%] +........................................................................ [ 75%] +........................................................................ [ 78%] +........................................................................ [ 81%] +........................................................................ [ 84%] +........................................................................ [ 87%] +........................................................................ [ 90%] +........................................................................ [ 93%] +........................................................................ [ 96%] +...................................................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 181 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 94 0 100% +app/api/docs.py 156 1 99% +app/api/git_sources.py 232 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 139 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 154 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 20 0 100% +app/core/theming.py 38 0 100% +app/core/tokens.py 44 0 100% +app/db.py 21 0 100% +app/main.py 66 0 100% +app/models.py 128 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 316 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 206 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 123 0 100% +app/rag/git_sources.py 14 0 100% +app/rag/importer.py 215 3 99% +app/rag/llm.py 243 1 99% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 88 0 100% +app/rag/retriever.py 172 3 98% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 24 0 100% +app/schemas.py 327 0 100% +-------------------------------------------------- +TOTAL 4018 15 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/107_git_full_history_dates/107_git_full_history_dates__03_gates_and_commit.a1.err b/.agents/reports/107_git_full_history_dates/107_git_full_history_dates__03_gates_and_commit.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/107_git_full_history_dates/107_git_full_history_dates__03_gates_and_commit.a1.md b/.agents/reports/107_git_full_history_dates/107_git_full_history_dates__03_gates_and_commit.a1.md new file mode 100644 index 0000000..c1619d7 --- /dev/null +++ b/.agents/reports/107_git_full_history_dates/107_git_full_history_dates__03_gates_and_commit.a1.md @@ -0,0 +1,14 @@ +**Phase 107, task 03 — full gate: VERIFIED GREEN.** + +**Verified (all run by me, not taken on faith from task reports):** +- `uv run pytest` → 2302 passed, 1 warning, exit 0 +- `uv run pytest --cov=app --cov-report=term-missing` → **TOTAL 99%** (>90%), exit 0 +- New E2E `uv run pytest tests/e2e/test_git_source_dates.py -v --no-cov` → 4 passed (13.2s) +- Regressions in isolation: `test_document_dates.py` → 6 passed; `test_sync_button.py` → 3 passed; `test_git_sources_admin.py` → 6 passed +- `uv run ruff check .` → All checks passed; `uv run pyright` → 0 errors, 0 warnings + +**Review:** code diffs (full-history clone, shallow-probe → `fetch --unshallow` self-heal, fail-loud, argv/test pins, 5 stale-narrative sites) all match D11/D12; no `frontend/` changes → UI structure check + byte-identical contracts untouched (UI asserted by E2E is phase-106's, re-verified only). + +**Deviations:** task steps 7–8 (commit, phase-dir move) intentionally NOT executed — my executor protocol explicitly overrides: no `git add`/`git commit`, harness owns the one atomic `--no-gpg-sign` commit and the `todo/ → complete/` move. All changes left in the working tree (git status shows exactly the phase's code/tests/phase-files/reports). + +**Next pending task:** phase 108 `108_history_wire_check` (phase 107 is now complete). diff --git a/.agents/reports/107_git_full_history_dates/107_git_full_history_dates__03_gates_and_commit.a1.validate b/.agents/reports/107_git_full_history_dates/107_git_full_history_dates__03_gates_and_commit.a1.validate new file mode 100644 index 0000000..e9a1c53 --- /dev/null +++ b/.agents/reports/107_git_full_history_dates/107_git_full_history_dates__03_gates_and_commit.a1.validate @@ -0,0 +1,100 @@ +........................................................................ [ 3%] +........................................................................ [ 6%] +........................................................................ [ 9%] +........................................................................ [ 12%] +........................................................................ [ 15%] +........................................................................ [ 18%] +........................................................................ [ 21%] +........................................................................ [ 25%] +........................................................................ [ 28%] +........................................................................ [ 31%] +........................................................................ [ 34%] +........................................................................ [ 37%] +........................................................................ [ 40%] +........................................................................ [ 43%] +........................................................................ [ 46%] +........................................................................ [ 50%] +........................................................................ [ 53%] +........................................................................ [ 56%] +........................................................................ [ 59%] +........................................................................ [ 62%] +........................................................................ [ 65%] +........................................................................ [ 68%] +........................................................................ [ 71%] +........................................................................ [ 75%] +........................................................................ [ 78%] +........................................................................ [ 81%] +........................................................................ [ 84%] +........................................................................ [ 87%] +........................................................................ [ 90%] +........................................................................ [ 93%] +........................................................................ [ 96%] +...................................................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 181 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 94 0 100% +app/api/docs.py 156 1 99% +app/api/git_sources.py 232 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 139 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 154 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 20 0 100% +app/core/theming.py 38 0 100% +app/core/tokens.py 44 0 100% +app/db.py 21 0 100% +app/main.py 66 0 100% +app/models.py 128 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 316 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 206 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 123 0 100% +app/rag/git_sources.py 14 0 100% +app/rag/importer.py 215 3 99% +app/rag/llm.py 243 1 99% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 88 0 100% +app/rag/retriever.py 172 3 98% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 24 0 100% +app/schemas.py 327 0 100% +-------------------------------------------------- +TOTAL 4018 15 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/app/api/sync.py b/app/api/sync.py index 117791d..47e4063 100644 --- a/app/api/sync.py +++ b/app/api/sync.py @@ -296,10 +296,11 @@ async def _run_sync() -> None: root = clone_or_pull(row.url, sources_root / repo_name(row.url)) # Phase 106 (D2): the checkout's per-file last-commit # dates, keyed by the SAME root string the importer - # sees (local-path checkouts → true per-file dates, - # shallow URL checkouts → the tip date, D10). Local - # rows contribute nothing — the importer's mtime - # fallback applies to them. + # sees (full-history checkouts → true per-file + # last-commit dates for every git source — phase 107 + # D11 supersedes phase 106 D10's shallow tip-date + # behavior). Local rows contribute nothing — the + # importer's mtime fallback applies to them. doc_dates_by_root[str(root)] = file_commit_dates(root) else: # kind=local — the stored expanded path (phase 38 also diff --git a/scripts/git_sync.py b/scripts/git_sync.py index bee008a..a5475a9 100644 --- a/scripts/git_sync.py +++ b/scripts/git_sync.py @@ -1,8 +1,9 @@ """Git source sync for import_docs (phase 28). -clone_or_pull(url, dest) clones ``url`` into ``dest`` (shallow, depth 1) -the first time, or fast-forwards an existing checkout with ``git pull ---ff-only`` on subsequent runs. +clone_or_pull(url, dest) clones ``url`` into ``dest`` with FULL history +the first time, or — for an existing checkout — unshallows a shallow +one first (``git fetch --unshallow``, the one-time self-heal) and then +fast-forwards it with ``git pull --ff-only``. Auth: nothing special — an ``https://…`` URL uses the OS credential helper / prompts; a ``git@host:repo.git`` URL uses the machine's SSH key. @@ -11,22 +12,34 @@ used. This module is the only place the ``git`` CLI is invoked (A11: stdlib ``subprocess`` only, no new packages) — every git command goes through -:func:`run_git`: the clone/pull in :func:`clone_or_pull`, the per-file -last-commit-date walk in :func:`file_commit_dates` (phase 106), and the -docs-push sequence in :mod:`app.core.docs_push` (phase 59). +:func:`run_git`: the clone, the shallow probe, the one-time +``git fetch --unshallow``, and the ``git pull --ff-only`` in +:func:`clone_or_pull`, the per-file last-commit-date walk in +:func:`file_commit_dates` (phase 106), and the docs-push sequence in +:mod:`app.core.docs_push` (phase 59). -Per-file last-commit dates (phase 106, D2/D10) — behavior verified -against scratch repos 2026-09-13: +Per-file last-commit dates (phase 106 D2; phase 107 D11 supersedes +phase 106 D10) — behavior verified against scratch and live repos +2026-09-13 / 2026-09-16: -* a LOCAL-PATH checkout made by :func:`clone_or_pull` keeps FULL - history (``git clone --depth 1 /local/path`` prints "--depth is - ignored in local clones" and does not shallow) → - :func:`file_commit_dates` yields TRUE per-file last-commit dates; -* a URL-TRANSPORT checkout (https/ssh/``file://``) is shallow, and in a - shallow clone git reports the TIP commit as every existing file's - last commit (the shallow boundary is each file's history root) → a - uniform per-repo tip date: no intra-repo distortion, a real - cross-source signal, refreshed on every pull. +* every :func:`clone_or_pull` checkout is FULL history for EVERY + transport (https/ssh/``file://``/local-path): a fresh clone carries + no ``--depth`` (D11 — D10's ``--depth 1`` shallow URL clones are +gone), and an existing shallow checkout (made pre-phase 107 — i.e. + every deployed one, live + dev included) is probed with ``git + rev-parse --is-shallow-repository`` and, while shallow, self-healed + with the ONE-TIME ``git fetch --unshallow`` before the usual ``git + pull --ff-only`` — no re-clone; +* on a full-history checkout :func:`file_commit_dates` yields the + TRUE per-file last-commit date for ALL git sources. The old + shallow-clone behavior was the bug the owner reported 2026-09-16: + in a shallow clone git reports the TIP commit as every existing + file's last commit (the shallow boundary is each file's history + root) → a uniform per-repo tip date (``container_bifrost`` shown as + created on the repo tip, months off); after ``git fetch + --unshallow`` on the live homelab checkout (438 commits visible) + the walk returned the true dates — e.g. ``bifrost.md`` 2026-05-05, + not the 2026-09-07 tip. """ from __future__ import annotations @@ -45,23 +58,40 @@ class GitSyncError(RuntimeError): def clone_or_pull(url: str, dest: Path | str) -> Path: - """Clone ``url`` into ``dest`` (shallow, first run) or fast-forward it. + """Clone ``url`` into ``dest`` (full history, first run) or fast-forward it. - - dest without a ``.git`` (or absent) → ``git clone --depth 1 url dest`` - (shallow: the KB is re-imported incrementally anyway). - - dest with a ``.git`` → ``git pull --ff-only`` (refuses to merge - unrelated histories — a broken checkout fails loudly rather than - producing a dirty index). + - dest without a ``.git`` (or absent) → ``git clone url dest`` (full + history for every transport — phase 107 D11: no ``--depth``, so a + URL-transport checkout carries the whole commit log, not just the + tip). + - dest with a ``.git`` → probe ``git rev-parse + --is-shallow-repository``; while shallow, ``git fetch + --unshallow`` (the ONE-TIME self-heal for checkouts made shallow + pre-phase 107 — the next sync of a deployed checkout becomes + full-history without a re-clone), then ``git pull --ff-only`` + (refuses to merge unrelated histories — a broken checkout fails + loudly rather than producing a dirty index). Returns the destination path. Raises :class:`GitSyncError` when git is missing or a git invocation exits non-zero (with git's stderr in the - message, so the caller can name the failing repo + reason). + message, so the caller can name the failing repo + reason). D12 + fail-loud: a failed probe/unshallow/pull propagates exactly like a + clone failure — never a silent fallback to tip dates or mtimes (a + continued shallow checkout would silently re-serve the uniform tip + date, i.e. the bug D11 fixes). """ dest = Path(dest) if not dest.exists() or not (dest / ".git").exists(): dest.parent.mkdir(parents=True, exist_ok=True) - run_git(["git", "clone", "--depth", "1", url, str(dest)], cwd=dest.parent) + run_git(["git", "clone", url, str(dest)], cwd=dest.parent) else: + shallow = ( + run_git(["git", "rev-parse", "--is-shallow-repository"], cwd=dest) + .strip() + == "true" + ) + if shallow: + run_git(["git", "fetch", "--unshallow"], cwd=dest) run_git(["git", "pull", "--ff-only"], cwd=dest) return dest @@ -104,11 +134,12 @@ def file_commit_dates(dest: Path | str) -> dict[str, datetime]: source per sync) → ``{repo-relative POSIX path: last-commit datetime}``, newest-first so the first sighting of a path wins. - The checkout behavior is pinned (verified 2026-09-13 — see the - module docstring): a local-path ``clone_or_pull`` checkout keeps - FULL history → TRUE per-file dates; a URL-transport checkout is - shallow → the repo's TIP-commit date for every working-tree file - (D10: uniform within the repo, real across sources). + Every ``clone_or_pull`` checkout is FULL history (fresh: no + ``--depth``; an existing shallow checkout is unshallowed on its + next sync — phase 107 D11 supersedes phase 106 D10's shallow + tip-date behavior) → TRUE per-file last-commit dates for ALL git + sources, local AND URL (verified 2026-09-16 — see the module + docstring). Fail-soft (pinned): a missing/non-directory checkout, a git failure (:class:`GitSyncError`), or ANY parse anomaly logs a warning and diff --git a/scripts/import_docs.py b/scripts/import_docs.py index 34f9bf3..8dd05c1 100644 --- a/scripts/import_docs.py +++ b/scripts/import_docs.py @@ -16,7 +16,8 @@ precedence order: (both kinds), else the ``BOR_GIT_SOURCES`` (comma-separated) git-only fallback (:func:`app.rag.git_sources.effective_sources`, the same shared resolver the in-app Sync button uses). Git rows are cloned - (first run, shallow ``--depth 1``) or fast-forwarded + (first run, full history — no ``--depth``; an existing shallow + checkout is unshallowed first, phase 107) or fast-forwarded (``git pull --ff-only``) into ``BOR_SOURCES_DIR//`` (default ``~/bor-sources``); local rows are the existing directories themselves, walked directly. A failing clone/pull — or a local diff --git a/tests/e2e/test_git_source_dates.py b/tests/e2e/test_git_source_dates.py new file mode 100644 index 0000000..1a43004 --- /dev/null +++ b/tests/e2e/test_git_source_dates.py @@ -0,0 +1,637 @@ +"""Phase 107 (task 02) E2E (Playwright): TRUE per-file git dates end to +end for a URL-transport git source. + +Story: n/a — owner bug report 2026-09-16 (phase 106 follow-up): the +live ``https://gitea…/homelab.git`` URL source displayed the repo's +TIP-commit date for every document (``container_bifrost`` "created +8/16/2026", months off) because phase 106 D10 locked ``--depth 1`` +shallow URL clones. Phase 107 D11 (task 01) removed the shallow clone +and added the one-time ``git fetch --unshallow`` self-heal — this +suite proves the FIX END TO END over the ``file://`` transport (the +transport-true stand-in for the live https source: pre-fix it rendered +the uniform tip date too). + +Run in isolation (DB must be up: ``podman compose up -d db``; git on +PATH — a documented environment prerequisite, phase 28): + + uv run pytest tests/e2e/test_git_source_dates.py -v --no-cov + +The whole owner scenario through the REAL pipeline: a real +``file://`` git fixture with TWO commits of controlled +``GIT_COMMITTER_DATE``s (``old/old-note.md`` @ 2020-06-15, +``recent/recent-note.md`` @ the 2024-06-15 tip — MID-YEAR dates so +the browser's locale/TZ rendering of the YEAR is stable in any +timezone, the test_document_dates.py L455 lesson), a real in-app +admin Sync (``clone_or_pull`` task-01 full-history path → real +importer → real KB overview, mock LLM), and the assertions that the +OLD file shows its OLD commit date and the NEW file shows the TIP +date — in ``GET /api/docs`` / ``GET /api/docs/tree`` (deterministic +ISO) and in the Sources tables + document viewer (locale-tolerant +year regexes / the badge's full-ISO ``title``). + +Per-module app env (the E2E conftest pattern, module-scoped — copied +verbatim in shape from ``test_sync_button.py``): this suite's app +boots with ``BOR_GIT_SOURCES=file://`` and its own +``BOR_SOURCES_DIR`` (fresh dir — the first sync takes the NEW-CLONE +path, the later per-test syncs the existing-checkout probe→pull +path); the session app (no git sources) is never started in this +isolated run, so no port clash. + +Test → contract mapping (four tests): + +1. ``test_api_created_dates_are_true_per_file`` — THE REGRESSION PIN: + after a real sync, ``GET /api/docs`` carries the OLD commit date + (2020-06-15) for the old file and the TIP date (2024-06-15) for + the new file — the two DIFFER (the phase-106 bug made both carry + the tip date). ``GET /api/docs/tree``: the file nodes carry the + same ISO dates verbatim; the folder ``updated_at`` values are the + subtree MAXES (D9 — now over TRUE dates): ``old`` = 2020, + ``recent`` = 2024, the source node = the 2024 max. +2. ``test_sources_tables_render_distinct_created_dates`` — the RAG + view's FILE table ``Created`` column (phase 106 D8 — between + ``Chunks`` and ``Indexed``) renders 2020 for the old file and + 2024 for the new one (year regexes — ``toLocaleString`` is + locale/TZ-dependent, the year is stable for mid-year dates; the + full ISO rides the cell's ``title``, asserted verbatim); the + FOLDER table ``Updated`` column (between ``Documents`` and + ``Description``) renders 2020 / 2024 per folder, the source row + the 2024 max. The two file cells' visible text differs. +3. ``test_viewer_created_badge_is_the_true_git_date`` — opening the + old document (the phase-26 same-page modal) shows the + ``.doc-created`` badge (phase 106, before the ``Indexed`` badge) + with the RAW ISO on its ``title`` — ``2020-06-15T12:00:00+00:00`` + (``metaBadge``'s title, deterministic under any locale/TZ — the + house solution to the L455 rendering trap) and ``Created …`` in + the visible text; Escape closes the modal cleanly (focus + restored to the row link). +4. ``test_page_a11y_and_no_cdn_basics`` — the standard light pass + (AGENTS.md rules 5/6, the test_git_sources_admin.py a11y test + shape): landmarks on the Sources view, the ``Created`` / + ``Updated`` ```` cells present in both tables (this phase + adds no color — no new contrast surface), the 3px + ``:focus-visible`` outline on a table row link (a real keyboard + Tab walk), and same-origin assets only. + +Isolation: per-test TRUNCATE of the KB tables (the +``test_sync_button.py`` ``_truncate_kb`` shape, extended with +``sources_meta``) so each test's sync counts are its own; own +``BOR_SOURCES_DIR``; the suite touches no other suite's fixtures. +""" +from __future__ import annotations + +import os +import re +import subprocess +import sys +from collections.abc import Iterator +from pathlib import Path +from typing import Any + +import pytest +from playwright.sync_api import Page, expect +from sqlalchemy import text + +from app.db import SessionLocal +from e2e.auth_helpers import login +from e2e.conftest import ( + ADMIN_PASSWORD, + SESSION_SECRET, + USE_REAL_LLM, + _wait_http, +) + +REPO = Path(__file__).resolve().parents[2] + +# Phase 79 (task 04, full inventory): the conftest session app owns its +# port in a combined run — this module app binds its own port instead +# (a same-port second uvicorn dies on bind and would drive the wrong +# server). Env-overridable. +APP_PORT = int(os.environ.get("E2E_APP_PORT_GITDATES", "8130")) +APP_URL = f"http://127.0.0.1:{APP_PORT}" + +# -------------------------------------------------------------------------- +# Fixture constants (deterministic — see the module docstring) +# -------------------------------------------------------------------------- + +#: The fixture repo's directory name — the source name the import +#: records for it (the repo_name basename rule, phase 28). +REPO_NAME = "git-dates" + +OLD_PATH = "old/old-note.md" +TIP_PATH = "recent/recent-note.md" + +#: The controlled commit dates (MID-YEAR, 12:00Z — the local year of +#: the instant is the committed year in ANY timezone, from UTC-12 to +#: UTC+14, so the browser's locale/TZ rendering of the year is stable, +#: the test_document_dates.py L455 lesson). +OLD_COMMIT = "2020-06-15T12:00:00Z" +TIP_COMMIT = "2024-06-15T12:00:00Z" + +#: The same instants as stored (``%cI`` → ``datetime.fromisoformat`` → +#: ``normalize_doc_date`` → Postgres → ``isoformat()``): the exact ISO +#: the API, the cell ``title``s, and the viewer badge ``title`` carry. +OLD_ISO = "2020-06-15T12:00:00+00:00" +TIP_ISO = "2024-06-15T12:00:00+00:00" + +OLD_TEXT = """\ +# Old stable note + +An old, stable note that has not changed since it was first written. +""" + +TIP_TEXT = """\ +# Recent note + +A recent note added at the tip of the repository. +""" + +#: "Synced HH:MM" — the local-time last-result label (sources.js's +#: fmtSyncTime), any hour/minute. +SYNCED_LABEL = re.compile(r"Synced \d{1,2}:\d{2}") + +#: Real git clone + embed against the mock LLM — generous budget +#: (the sync can legitimately take a while, no client timeout). +SYNC_TIMEOUT_MS = 60_000 + + +# -------------------------------------------------------------------------- +# The fixture repo (module-scoped — real git, two controlled commits) +# -------------------------------------------------------------------------- + + +def _git(cwd: Path, *args: str, env: dict[str, str] | None = None) -> None: + """Run git in *cwd*; a non-zero exit fails the fixture loudly. + + *env* overrides (the ``GIT_AUTHOR_DATE`` / ``GIT_COMMITTER_DATE`` + commit-date controls — the phase's whole point). + """ + full_env = dict(os.environ) + if env: + full_env.update(env) + proc = subprocess.run(["git", *args], cwd=cwd, capture_output=True, text=True, + env=full_env) + if proc.returncode != 0: + raise AssertionError(f"git {' '.join(args)} failed: {proc.stderr.strip()}") + + +def _git_out(cwd: Path, *args: str) -> str: + proc = subprocess.run(["git", *args], cwd=cwd, capture_output=True, text=True) + if proc.returncode != 0: + raise AssertionError(f"git {' '.join(args)} failed: {proc.stderr.strip()}") + return proc.stdout + + +def _commit(cwd: Path, message: str, when: str) -> None: + """A ``git add -A`` + commit with BOTH the author and the committer + dates pinned to *when* (``file_commit_dates`` reads ``%cI`` — the + committer date; the author date is pinned too so the fixture has + one unambiguous date per commit).""" + _git(cwd, "add", "-A") + _git( + cwd, + "-c", "user.email=e@x", "-c", "user.name=t", + "-c", "commit.gpgsign=false", # the fixture commits never sign + "commit", "-qm", message, + env={"GIT_AUTHOR_DATE": when, "GIT_COMMITTER_DATE": when}, + ) + + +@pytest.fixture(scope="module") +def git_dates_repo(tmp_path_factory: pytest.TempPathFactory) -> Path: + """A real two-commit git repo the sync must clone over ``file://`` + (task 02 step 1): commit one @ 2020-06-15 adds ``old/old-note.md``; + commit two (the tip) @ 2024-06-15 adds ``recent/recent-note.md``. + + Built under ``tmp_path_factory`` (``tmp_path`` is function-scoped + while the module-scoped app fixture needs the repo for the module's + lifetime) via real ``git`` subprocesses — the + ``test_sync_button.py`` idiom. + """ + root = tmp_path_factory.mktemp("bor_git_dates") + repo = root / REPO_NAME + (repo / "old").mkdir(parents=True) + (repo / "old" / "old-note.md").write_text(OLD_TEXT, encoding="utf-8") + _git(repo, "init", "-q") + _commit(repo, "old note", OLD_COMMIT) + (repo / "recent").mkdir() + (repo / "recent" / "recent-note.md").write_text(TIP_TEXT, encoding="utf-8") + _commit(repo, "recent note", TIP_COMMIT) + assert (repo / ".git").is_dir() + assert _git_out(repo, "rev-list", "--count", "HEAD").strip() == "2" + return repo + + +# -------------------------------------------------------------------------- +# The app under test (module-scoped env, the test_sync_button.py shape) +# -------------------------------------------------------------------------- + + +@pytest.fixture(scope="module") +def app_server(mock_llm: int, git_dates_repo: Path) -> Iterator[str]: + """The real app under test — per-module env: the sync's subject is + a real ``file://`` URL-transport git source with its own checkout + dir (the conftest session app boots without ``BOR_GIT_SOURCES`` + and is never started in this isolated run).""" + env = dict(os.environ) + env.pop("DEBUGPY", None) + env["BOR_ENVIRONMENT"] = "e2e" + env["BOR_STATIC_DIR"] = str(REPO / "frontend") + env["BOR_LLM_BASE_URL"] = ( + "https://aipi.reeseapps.com/v1" + if USE_REAL_LLM + else f"http://127.0.0.1:{mock_llm}/v1" + ) + # Mock-calibrated threshold (conftest pattern) — no chat turn is + # sent in this suite, but the app boots with the same env shape. + env["BOR_RELEVANCE_THRESHOLD"] = "0.30" + env.setdefault( + "BOR_DATABASE_URL", + "postgresql+psycopg://reese:reese@localhost:5432/brain_of_reese", + ) + # Phase 16: admin auth must be set or create_app() refuses to boot. + env["BOR_ADMIN_PASSWORD"] = ADMIN_PASSWORD + env["BOR_SESSION_SECRET"] = SESSION_SECRET + # Phase 107: the subject — one real local repo over the URL + # transport, checked out under its own fresh dir (first sync = + # the task-01 NEW-CLONE full-history path; later per-test syncs = + # the existing-checkout probe → pull path). + env["BOR_GIT_SOURCES"] = f"file://{git_dates_repo}" + env["BOR_SOURCES_DIR"] = str(git_dates_repo.parent / "checkouts") + proc = subprocess.Popen( + [sys.executable, "-m", "uvicorn", "app.main:app", + "--host", "127.0.0.1", "--port", str(APP_PORT), "--log-level", "warning"], + cwd=REPO, + env=env, + ) + try: + _wait_http(f"{APP_URL}/api/health") + yield APP_URL + finally: + proc.terminate() + try: + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + proc.kill() + + +@pytest.fixture(scope="module") +def app_url(app_server: str) -> str: + return app_server + + +def _truncate_kb() -> None: + """Fresh KB per test (the E2E isolation pattern): each test's sync + counts are its own. ``git_sources`` too: the shared Postgres may + carry leftover registry rows from other suites, and + effective_sources prefers the DB list over this app's + ``BOR_GIT_SOURCES`` env fallback. ``sources_meta`` (the KB + generation counter) resets with the KB — the bump is an upsert, so + the sync re-creates the row.""" + with SessionLocal() as db: + db.execute(text( + "TRUNCATE chunks, documents, query_log, kb_overview, " + "git_sources, sources_meta" + )) + db.commit() + + +@pytest.fixture(autouse=True) +def _clean_kb(db_ready: None) -> Iterator[None]: + _truncate_kb() + yield + + +# -------------------------------------------------------------------------- +# Helpers +# -------------------------------------------------------------------------- + + +def _sync_in_app(page: Page, app_url: str) -> None: + """Admin login → the admin Sources page → click "Sync sources" → + the terminal success label + the fresh-import counts (the + ``test_sync_button.py`` lifecycle wait, generous timeout — a real + clone/pull + the real import + the KB overview, mock LLM).""" + login(page, app_url) # lands on /sources.html (the button's home) + btn = page.locator("#sync-btn") + expect(btn).to_be_visible() + # Fresh app state: "Sync sources" — or, when this app process has + # ALREADY run a previous test's sync (the per-module server-side + # status survives the per-test page), the boot re-attach adopted + # that terminal state ("Synced HH:MM") — either way the button is + # clickable (never stale). + expect(page.locator("#sync-label")).to_have_text( + re.compile(r"^Sync sources$|^Synced \d{1,2}:\d{2}$") + ) + + btn.click() + expect(btn).to_be_disabled() + expect(page.locator("#sync-label")).to_have_text("Syncing…") + expect(page.locator("#sync-label")).to_have_text(SYNCED_LABEL, timeout=SYNC_TIMEOUT_MS) + expect(btn).to_be_enabled() # never stale — re-enabled at the terminal state + # Both fixture docs are fresh after the per-test truncate. + expect(page.locator("#sync-result")).to_have_text("2 added") + + +def _source_node(tree_json: dict[str, Any]) -> dict[str, Any]: + for s in tree_json["sources"]: + if s["name"] == REPO_NAME: + return s + raise AssertionError(f"source {REPO_NAME!r} not in the tree") + + +def _folder_node(node: dict[str, Any], path: str) -> dict[str, Any]: + for child in node.get("children", []): + if child.get("kind") == "folder" and child["path"] == path: + return child + raise AssertionError(f"folder {path!r} not under the node") + + +def _find_file(node: dict[str, Any], path: str) -> dict[str, Any]: + """The file node with *path* anywhere under *node* (recursing into + the folder children).""" + for child in node.get("children", []): + if child.get("kind") == "file" and child["path"] == path: + return child + if child.get("kind") == "folder": + try: + return _find_file(child, path) + except AssertionError: + continue + raise AssertionError(f"file {path!r} not under the node") + + +def _drill_to_folder(page: Page, folder: str) -> None: + """Top level → the source → *folder* (the two real clicks).""" + page.locator("#folders-tbody .folder-link").first.wait_for(state="visible") + page.click(f'#folders-tbody a.folder-link:text-is("{REPO_NAME}")') + page.click(f'#folders-tbody a.folder-link:text-is("{folder}")') + + +def _back_to_source_level(page: Page) -> None: + """From a folder level, the breadcrumb's source segment (the + "Knowledge base" segment goes all the way to the top).""" + page.click(f'#kb-crumb a.kb-crumb-link:text-is("{REPO_NAME}")') + expect(page.locator("#folders-tbody .folder-link")).to_have_count(2) + + +# --------------------------------------------------------------------------- +# 1. THE REGRESSION PIN: the API dates are the TRUE per-file commit +# dates — 2020 for the old file, the tip date for the new file +# --------------------------------------------------------------------------- + + +def test_api_created_dates_are_true_per_file( + page: Page, app_url: str, db_ready: None +) -> None: + """After a real in-app sync of the ``file://`` fixture: the old + file's ``created_at`` is its OLD commit date (2020-06-15) and the + new file's is the TIP date (2024-06-15) — the two DIFFER (the + phase-106 bug made every URL-source file carry the tip date, i.e. + both ``2024-06-15``). The tree carries the same dates verbatim and + the folder/source ``updated_at`` values are the subtree MAXES (D9). + """ + _sync_in_app(page, app_url) + + # GET /api/docs (the page context's request client — the login's + # cookie jar rides along). + r = page.request.get(f"{app_url}/api/docs") + assert r.status == 200, r.text + docs = {d["path"]: d for d in r.json()["documents"]} + old_created = docs[OLD_PATH]["created_at"] + tip_created = docs[TIP_PATH]["created_at"] + assert old_created[:10] == "2020-06-15" + assert tip_created[:10] == "2024-06-15" + assert old_created != tip_created, ( + "the two dates must DIFFER — the phase-106 bug made a URL " + f"source carry the uniform tip date for both ({old_created!r}, {tip_created!r})" + ) + # The TRUE commit dates, verbatim (the %cI instants through D3 — + # past dates pass normalize_doc_date unchanged). + assert old_created == OLD_ISO + assert tip_created == TIP_ISO + + # GET /api/docs/tree: the file nodes carry the same ISO dates + # verbatim; the folder updated_at values are the subtree MAXES + # (old folder = the 2020 date alone, recent folder = the 2024 + # date alone, source = the whole subtree's 2024 max — D9, now over + # TRUE dates). + t = page.request.get(f"{app_url}/api/docs/tree") + assert t.status == 200, t.text + source = _source_node(t.json()) + assert source["documents"] == 2 + assert _find_file(source, OLD_PATH)["created_at"] == OLD_ISO + assert _find_file(source, TIP_PATH)["created_at"] == TIP_ISO + old_folder = _folder_node(source, "old") + recent_folder = _folder_node(source, "recent") + assert old_folder["documents"] == 1 + assert old_folder["updated_at"] is not None + assert old_folder["updated_at"][:10] == "2020-06-15" + assert recent_folder["documents"] == 1 + assert recent_folder["updated_at"] is not None + assert recent_folder["updated_at"][:10] == "2024-06-15" + assert source["updated_at"] is not None + assert source["updated_at"][:10] == "2024-06-15" # max(2020, 2024) + + +# --------------------------------------------------------------------------- +# 2. The Sources tables render the DISTINCT created dates +# --------------------------------------------------------------------------- + + +def test_sources_tables_render_distinct_created_dates( + page: Page, app_url: str, db_ready: None +) -> None: + """The RAG view's FILE table ``Created`` column (phase 106 D8 — + between ``Chunks`` and ``Indexed``) renders 2020 for the old file + and 2024 for the new one, the FOLDER table ``Updated`` column + (between ``Documents`` and ``Description``) renders 2020 / 2024 + per folder, and the two file cells' visible text differs. Year + regexes — ``toLocaleString`` is locale/TZ-dependent, the year is + stable for the mid-year fixture dates; the full ISO rides the + cells' ``title`` (asserted verbatim — the locale-stable idiom). + textContent reads only — never set innerHTML.""" + page.set_default_timeout(30_000) + _sync_in_app(page, app_url) + + # Top level: one source row; its Updated cell carries the 2024 + # subtree max. + page.locator("#folders-tbody .folder-link").first.wait_for(state="visible") + source_row = page.locator( + f'#folders-tbody tr:has(a.folder-link:text-is("{REPO_NAME}"))' + ) + expect(source_row).to_have_count(1) + expect(source_row.locator("td:nth-child(3)")).to_have_text(re.compile("2024")) + + # Drill into the source: the two folders, each Updated = its own + # year (D9 through the UI). + page.click(f'#folders-tbody a.folder-link:text-is("{REPO_NAME}")') + expect(page.locator("#folders-tbody .folder-link")).to_have_count(2) + old_row = page.locator('#folders-tbody tr:has(a.folder-link:text-is("old"))') + recent_row = page.locator('#folders-tbody tr:has(a.folder-link:text-is("recent"))') + expect(old_row).to_have_count(1) + expect(recent_row).to_have_count(1) + expect(old_row.locator("td:nth-child(3)")).to_have_text(re.compile("2020")) + expect(recent_row.locator("td:nth-child(3)")).to_have_text(re.compile("2024")) + + # The FILE table: the old file's Created cell (the column between + # Chunks and Indexed) carries the 2020 year in text + the FULL ISO + # on the cell's title. + page.click('#folders-tbody a.folder-link:text-is("old")') + old_file_row = page.locator("#docs-tbody tr", has_text=OLD_PATH) + expect(old_file_row).to_have_count(1) + old_created_cell = old_file_row.locator("td:nth-child(5)") + expect(old_created_cell).to_have_text(re.compile("2020"), timeout=15_000) + assert old_created_cell.get_attribute("title", timeout=15_000) == OLD_ISO + old_created_text = (old_created_cell.text_content() or "").strip() + + # Back to the source level, then the new file's Created cell: the + # 2024 year + its full ISO. + _back_to_source_level(page) + page.click('#folders-tbody a.folder-link:text-is("recent")') + recent_file_row = page.locator("#docs-tbody tr", has_text=TIP_PATH) + expect(recent_file_row).to_have_count(1) + recent_created_cell = recent_file_row.locator("td:nth-child(5)") + expect(recent_created_cell).to_have_text(re.compile("2024"), timeout=15_000) + assert recent_created_cell.get_attribute("title", timeout=15_000) == TIP_ISO + recent_created_text = (recent_created_cell.text_content() or "").strip() + + # The two cells' visible text differs (the bug made them identical + # — both the tip date). + assert old_created_text and recent_created_text + assert old_created_text != recent_created_text, ( + f"the two Created cells must differ, both show {old_created_text!r}" + ) + + +# --------------------------------------------------------------------------- +# 3. The viewer's Created badge carries the TRUE git date (raw ISO) +# --------------------------------------------------------------------------- + + +def test_viewer_created_badge_is_the_true_git_date( + page: Page, app_url: str, db_ready: None +) -> None: + """Opening the OLD document (the phase-26 same-page modal) shows + the ``.doc-created`` badge (phase 106, before the ``Indexed`` + badge) with the RAW ISO on its ``title`` — + ``2020-06-15T12:00:00+00:00`` (``metaBadge``'s title, + deterministic under any locale/TZ — the house solution to the L455 + rendering trap) and ``Created …`` in the visible text; Escape + closes the modal cleanly (focus restored to the row link).""" + page.set_default_timeout(30_000) + _sync_in_app(page, app_url) + + _drill_to_folder(page, "old") + row = page.locator("#docs-tbody tr", has_text=OLD_PATH) + expect(row).to_have_count(1) + before_tabs = len(page.context.pages) + row.locator("td:nth-child(2) a.doc-link").click() + assert len(page.context.pages) == before_tabs, "row link must not open a new tab" + + expect(page.locator(".doc-modal")).to_be_visible() + expect(page.locator("#doc-modal-title")).to_have_text("Old stable note") + + created = page.locator("#doc-modal-meta .doc-created") + expect(created).to_have_count(1) + # The badge text: "Created " (text + format pairing — + # the date is carried by text, never color alone, B5). + expect(created).to_have_text(re.compile(r"^Created \S")) + # THE assertion: the TRUE old commit date, the full raw ISO on the + # badge's title (hover precision, locale-independent). + assert created.get_attribute("title", timeout=15_000) == OLD_ISO + # Phase 106 D8 in the DOM: the Created badge PRECEDES the Indexed + # badge (unchanged by this phase — it now simply carries the true + # date). + assert page.evaluate( + """() => { + const a = document.querySelector('#doc-modal-meta .doc-created'); + const b = document.querySelector('#doc-modal-meta .doc-indexed'); + return a !== null && b !== null && + !!(a.compareDocumentPosition(b) & Node.DOCUMENT_POSITION_FOLLOWING); + }""" + ) + # No regression: the source/format/chunks badges are all still there. + meta = page.locator("#doc-modal-meta") + expect(meta.locator(".doc-source-badge")).to_have_text(REPO_NAME) + expect(meta.locator(".format-badge")).to_have_text("md") + expect(meta.locator(".doc-chunks")).to_have_text(re.compile(r"^\d+ chunk")) + + # Close: Escape — the viewer reverts cleanly (the modal hides, + # focus returns to the row link that opened it). + page.keyboard.press("Escape") + expect(page.locator(".doc-modal")).to_be_hidden() + assert page.evaluate( + "() => String(document.activeElement.className)" + ) == "doc-link" + + +# --------------------------------------------------------------------------- +# 4. UI Structure Check (AGENTS.md rule 5) + no CDN (rule 6) — light pass +# --------------------------------------------------------------------------- + + +def test_page_a11y_and_no_cdn_basics( + page: Page, app_url: str, db_ready: None +) -> None: + """The standard light pass (the test_git_sources_admin.py a11y test + shape): landmarks on the Sources view, the ``Created`` / + ``Updated`` ```` cells present in both tables (this phase adds + NO color — the phase-106 columns only, already ≥4.5:1 — so there + is no new contrast surface to pin), the 3px ``:focus-visible`` + outline on a table row link (a real keyboard Tab walk), and + same-origin assets only (rule 6).""" + page.set_default_timeout(30_000) + _sync_in_app(page, app_url) + + # Standard app frame: landmarks + skip link (PLAN §7.2). + expect(page.locator("header.app-header")).to_have_count(1) + expect(page.locator('nav[aria-label="Primary"]')).to_have_count(1) + expect(page.locator("main#main")).to_have_count(1) + expect(page.locator("footer.app-footer")).to_have_count(1) + expect(page.locator(".skip-link")).to_have_count(1) + + # The date columns' cells (phase 106 D8, verbatim — text_content, + # not inner_text: the rendered s are CSS-uppercased). + file_headers = [ + th.text_content() for th in page.locator("#docs-table thead th").all() + ] + assert file_headers == ["Source", "Path", "Title", "Chunks", "Created", "Indexed"] + folder_headers = [ + th.text_content() for th in page.locator("#folders-table thead th").all() + ] + assert folder_headers == ["Folder", "Documents", "Updated", "Description"] + + # :focus-visible draws the 3px outline on a table row link — a real + # keyboard Tab walk (the test_responsive_polish.py idiom): reset + # focus to the top of the document, then Tab until the old file's + # row link is focused (the page state is deterministic, so the + # walk order is — the bound is generous, never the contract). + _drill_to_folder(page, "old") + expect(page.locator("#docs-tbody tr", has_text=OLD_PATH)).to_have_count(1) + page.evaluate( + "() => { if (document.activeElement instanceof HTMLElement)" + " document.activeElement.blur(); }" + ) + reached = False + for _ in range(150): + page.keyboard.press("Tab") + if page.evaluate("() => String(document.activeElement.className)") == "doc-link": + reached = True + break + assert reached, "keyboard Tab never reached the table row link" + outline = page.evaluate("() => getComputedStyle(document.activeElement).outlineWidth") + assert outline == "3px", f"focus-visible outline missing on the row link: {outline!r}" + + # No CDN (rule 6): no https:// asset tags; every script/link ref is + # same-origin or a data: URI. + html = page.content() + assert 'src="https://' not in html and 'href="https://' not in html + refs = page.evaluate( + """() => [...document.querySelectorAll("script[src], link[href]")] + .map((el) => el.src || el.href)""" + ) + assert refs, "expected local asset references" + for ref in refs: + assert ref.startswith(app_url) or ref.startswith("data:"), ( + f"non-local asset reference: {ref}" + ) diff --git a/tests/integration/test_git_file_dates.py b/tests/integration/test_git_file_dates.py index d96cd03..9032d5e 100644 --- a/tests/integration/test_git_file_dates.py +++ b/tests/integration/test_git_file_dates.py @@ -1,21 +1,26 @@ """Integration: phase 106 task 03 — ``file_commit_dates`` against real -git scratch repos (D2/D10). +git scratch repos (D2; phase 107 D11 supersedes phase 106 D10). Builds scratch repositories with controlled ``GIT_COMMITTER_DATE``s (the 2026-09-13 verification recipe: file ``a.md`` committed once in 2020, file ``b.md`` committed in 2020 and touched again in 2024, a ``docs/deep.md`` subdirectory file committed once in 2020) and pins -the VERIFIED checkout behavior: +the VERIFIED checkout behavior (phase 107): -* a LOCAL-PATH ``clone_or_pull`` keeps FULL history (git's own - "--depth is ignored in local clones" warning — the ``--depth 1`` - flag stays, D10) → TRUE per-file last-commit dates (first-sighting - wins: ``a.md`` 2020, ``b.md`` 2024, ``docs/deep.md`` 2020); -* a shallow URL-transport clone (``file://``, made directly in this - test — the test harness, not ``clone_or_pull``, makes this one) → - the TIP commit's date for EVERY working-tree file (the - shallow-boundary property, D10: uniform per repo, real across - repos); +* a LOCAL-PATH ``clone_or_pull`` checkout is FULL history → TRUE + per-file last-commit dates (first-sighting wins: ``a.md`` 2020, + ``b.md`` 2024, ``docs/deep.md`` 2020); +* a URL-TRANSPORT (``file://``) ``clone_or_pull`` checkout is FULL + history TOO (D11: no ``--depth`` on the clone) → the SAME true + per-file dates — the regression pin for the 2026-09-16 bug + (pre-fix, D10's shallow ``--depth 1`` clone returned the TIP date + for EVERY file, uniform per repo); +* an EXISTING shallow checkout (made ``--depth 1`` directly by the + test harness — simulating every pre-phase-107 deployed checkout) + self-heals on the next ``clone_or_pull``: the + ``--is-shallow-repository`` probe finds it shallow → ``git fetch + --unshallow`` → ``git pull --ff-only`` → no longer shallow, true + per-file dates; * fail-soft: a directory without ``.git``, an empty repo (no commits), a git failure, and a malformed log output all yield ``{}`` — a date walk must never break a sync (the importer, task @@ -39,6 +44,7 @@ from scripts.git_sync import ( _parse_commit_dates, # pyright: ignore[reportPrivateUsage] clone_or_pull, file_commit_dates, + run_git, ) @@ -103,10 +109,10 @@ def scratch_repo(tmp_path: Path) -> Path: def test_local_clone_yields_true_per_file_dates(scratch_repo: Path, tmp_path: Path) -> None: - """(a) LOCAL-PATH ``clone_or_pull`` → full history (git warns - "--depth is ignored in local clones" and does not shallow) → TRUE - per-file last-commit dates: the first (newest) sighting of each - path wins — ``b.md`` the 2024 touch, the rest the 2020 commit.""" + """(a) LOCAL-PATH ``clone_or_pull`` → full history (no ``--depth`` + at all, phase 107 D11) → TRUE per-file last-commit dates: the + first (newest) sighting of each path wins — ``b.md`` the 2024 + touch, the rest the 2020 commit.""" dest = tmp_path / "local" clone_or_pull(str(scratch_repo), dest) assert (dest / ".git").exists() # a real checkout @@ -117,21 +123,61 @@ def test_local_clone_yields_true_per_file_dates(scratch_repo: Path, tmp_path: Pa } -def test_shallow_file_clone_yields_tip_date_for_every_file( - scratch_repo: Path, tmp_path: Path -) -> None: - """(b) SHALLOW URL-TRANSPORT clone (``file://``, made directly in - the test — D10): in a shallow clone git reports the TIP commit as - every existing file's last commit (the shallow boundary is each - file's history root) → EVERY working-tree file carries the tip - date, uniform within the repo.""" +def test_url_clone_yields_true_per_file_dates(scratch_repo: Path, tmp_path: Path) -> None: + """(b) URL-TRANSPORT ``clone_or_pull`` (``file://`` through the REAL + function — the 2026-09-16 regression pin): the checkout is FULL + history (no ``--depth`` — phase 107 D11 supersedes phase 106 D10) + → NOT shallow, and TRUE per-file last-commit dates: the 2020 files + stay 2020 and ``b.md`` (touched again at the tip) gets the 2024 + tip date. PRE-FIX this returned DATE_B (the uniform tip date) for + all three files — the exact bug the owner reported. + """ + dest = tmp_path / "url-clone" + clone_or_pull(f"file://{scratch_repo}", dest) + assert (dest / ".git").exists() # a real checkout + assert ( + run_git(["git", "rev-parse", "--is-shallow-repository"], cwd=dest).strip() + == "false" + ) # a full-history checkout, not shallow + assert file_commit_dates(dest) == { + "a.md": DATE_A, + "b.md": DATE_B, # touched again by the tip commit + "docs/deep.md": DATE_A, + } + + +def test_existing_shallow_checkout_self_heals(scratch_repo: Path, tmp_path: Path) -> None: + """(c) an EXISTING shallow checkout — built ``--depth 1`` directly + by the harness, simulating every pre-phase-107 deployed checkout + (live + dev homelab included) — self-heals on the next + ``clone_or_pull`` (D11): the ``--is-shallow-repository`` probe + finds it shallow → the ONE-TIME ``git fetch --unshallow`` restores + the full history (no re-clone) → the usual ``--ff-only`` pull. Pre- + heal: shallow + the uniform tip date for all files; post-heal: + full history + true per-file dates.""" dest = tmp_path / "shallow" _git(tmp_path, "clone", "-q", "--depth", "1", f"file://{scratch_repo}", str(dest)) + # Pre-heal: the deployed state the fix must cure. + assert ( + run_git(["git", "rev-parse", "--is-shallow-repository"], cwd=dest).strip() + == "true" + ) assert file_commit_dates(dest) == { "a.md": DATE_B, "b.md": DATE_B, "docs/deep.md": DATE_B, } + clone_or_pull(f"file://{scratch_repo}", dest) + # Post-heal: full history and true per-file dates, same as (b). + assert ( + run_git(["git", "rev-parse", "--is-shallow-repository"], cwd=dest).strip() + == "false" + ) + assert file_commit_dates(dest) == { + "a.md": DATE_A, + "b.md": DATE_B, + "docs/deep.md": DATE_A, + } def test_directory_without_dotgit_fails_soft(tmp_path: Path) -> None: diff --git a/tests/integration/test_import_docs_git.py b/tests/integration/test_import_docs_git.py index a85b722..e3d75bf 100644 --- a/tests/integration/test_import_docs_git.py +++ b/tests/integration/test_import_docs_git.py @@ -689,7 +689,7 @@ def test_main_git_failure_aborts_before_import( def failing_clone(url: str, dest: Path | str) -> Path: raise GitSyncError( - f"git clone --depth 1 {url} failed (exit 128): " + f"git clone {url} failed (exit 128): " "fatal: repository not found" ) diff --git a/tests/integration/test_sync_api.py b/tests/integration/test_sync_api.py index 34ab8c2..cd449c6 100644 --- a/tests/integration/test_sync_api.py +++ b/tests/integration/test_sync_api.py @@ -862,7 +862,7 @@ def test_git_failure_marks_failed_and_skips_import( def failing_clone(url: str, dest: Path | str) -> Path: raise GitSyncError( - f"git clone --depth 1 {url} failed (exit 128): " + f"git clone {url} failed (exit 128): " "fatal: repository not found" ) diff --git a/tests/unit/test_git_sync.py b/tests/unit/test_git_sync.py index f7a2e9a..d2ea65c 100644 --- a/tests/unit/test_git_sync.py +++ b/tests/unit/test_git_sync.py @@ -1,6 +1,7 @@ -"""Unit tests: git clone/pull utility (phase 28). +"""Unit tests: git clone/pull utility (phase 28; phase 107 D11/D12). -``clone_or_pull`` dispatches to ``git clone --depth 1`` (fresh dest) or +``clone_or_pull`` dispatches to ``git clone`` (full history, fresh dest) +or a shallow-probe + optional ``git fetch --unshallow`` + ``git pull --ff-only`` (existing checkout) with ``subprocess`` fully mocked — the real git CLI is never invoked, so the tests run anywhere. """ @@ -28,14 +29,24 @@ def _fake_run( stdout: str = "", stderr: str = "", missing_git: bool = False, + responses: dict[tuple[str, ...], tuple[int, str, str]] | None = None, ) -> list[dict[str, object]]: - """Monkeypatch scripts.git_sync.subprocess.run; record each call.""" + """Monkeypatch scripts.git_sync.subprocess.run; record each call. + + ``responses`` dispatches by the full argv (clone / shallow-probe / + ``fetch --unshallow`` / pull) — a per-command + ``(returncode, stdout, stderr)``; any argv not in the map gets the + default ``returncode``/``stdout``/``stderr``. + """ calls: list[dict[str, object]] = [] def fake_run(argv: list[str], cwd: Path | None = None, **kwargs: object) -> _FakeProc: calls.append({"argv": list(argv), "cwd": cwd, **kwargs}) if missing_git: raise FileNotFoundError("git") + if responses is not None: + rc, out, err = responses.get(tuple(argv), (returncode, stdout, stderr)) + return _FakeProc(rc, out, err) return _FakeProc(returncode, stdout, stderr) monkeypatch.setattr(subprocess, "run", fake_run) @@ -45,7 +56,8 @@ def _fake_run( def test_clone_or_pull_clones_when_dest_has_no_git_dir( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - """First run: dest absent → ``git clone --depth 1`` from the parent dir.""" + """First run: dest absent → ``git clone`` (full history — NO + ``--depth``, phase 107 D11) from the parent dir.""" url = "https://github.com/user/homelab.git" dest = tmp_path / "homelab" calls = _fake_run(monkeypatch) @@ -55,7 +67,7 @@ def test_clone_or_pull_clones_when_dest_has_no_git_dir( assert result == dest assert len(calls) == 1 call = calls[0] - assert call["argv"] == ["git", "clone", "--depth", "1", url, str(dest)] + assert call["argv"] == ["git", "clone", url, str(dest)] assert call["cwd"] == dest.parent assert call["capture_output"] is True assert call["text"] is True @@ -64,16 +76,79 @@ def test_clone_or_pull_clones_when_dest_has_no_git_dir( def test_clone_or_pull_pulls_when_git_dir_exists( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - """Subsequent run: dest/.git present → ``git pull --ff-only`` in-place.""" + """Steady state: dest/.git present and the probe says NOT shallow + (``false``) → straight to ``git pull --ff-only`` in-place (one cheap + local probe, no network fetch).""" dest = tmp_path / "homelab" (dest / ".git").mkdir(parents=True) - calls = _fake_run(monkeypatch) + calls = _fake_run( + monkeypatch, + responses={("git", "rev-parse", "--is-shallow-repository"): (0, "false\n", "")}, + ) result = clone_or_pull("https://github.com/user/homelab.git", dest) assert result == dest - assert calls[0]["argv"] == ["git", "pull", "--ff-only"] + assert len(calls) == 2 + assert calls[0]["argv"] == ["git", "rev-parse", "--is-shallow-repository"] assert calls[0]["cwd"] == dest + assert calls[1]["argv"] == ["git", "pull", "--ff-only"] + assert calls[1]["cwd"] == dest + + +def test_clone_or_pull_unshallows_existing_shallow_checkout( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Existing shallow checkout (probe ``true`` — the one-time D11 + self-heal of deployed checkouts) → ``git fetch --unshallow`` + BEFORE the usual ``git pull --ff-only`` (order pinned), all in + dest.""" + dest = tmp_path / "homelab" + (dest / ".git").mkdir(parents=True) + calls = _fake_run( + monkeypatch, + responses={ + ("git", "rev-parse", "--is-shallow-repository"): (0, "true\n", ""), + ("git", "fetch", "--unshallow"): (0, "Fetching full history\n", ""), + }, + ) + + clone_or_pull("https://github.com/user/homelab.git", dest) + + assert len(calls) == 3 + assert calls[0]["argv"] == ["git", "rev-parse", "--is-shallow-repository"] + assert calls[1]["argv"] == ["git", "fetch", "--unshallow"] + assert calls[2]["argv"] == ["git", "pull", "--ff-only"] + for call in calls: + assert call["cwd"] == dest + + +def test_clone_or_pull_unshallow_failure_propagates( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """D12 fail-loud: a failed ``git fetch --unshallow`` raises + :class:`GitSyncError` (with git's stderr) and the pull NEVER runs — + continuing a failed self-heal would silently re-serve the uniform + tip date (the bug D11 fixes).""" + dest = tmp_path / "homelab" + (dest / ".git").mkdir(parents=True) + calls = _fake_run( + monkeypatch, + responses={ + ("git", "rev-parse", "--is-shallow-repository"): (0, "true\n", ""), + ("git", "fetch", "--unshallow"): (128, "", "fatal: could not fetch\n"), + }, + ) + + with pytest.raises( + GitSyncError, match=r"git fetch --unshallow failed \(exit 128\): fatal: could not fetch" + ): + clone_or_pull("https://github.com/user/homelab.git", dest) + + assert [call["argv"] for call in calls] == [ + ["git", "rev-parse", "--is-shallow-repository"], + ["git", "fetch", "--unshallow"], + ] # the pull never ran def test_clone_or_pull_creates_missing_parent_before_clone( @@ -82,11 +157,17 @@ def test_clone_or_pull_creates_missing_parent_before_clone( """The (nested) parent of the destination is mkdir'd before git runs.""" dest = tmp_path / "nested" / "deeper" / "homelab" assert not dest.parent.exists() - _fake_run(monkeypatch) + calls = _fake_run(monkeypatch) clone_or_pull("https://github.com/user/homelab.git", dest) assert dest.parent.is_dir() + assert calls[0]["argv"] == [ + "git", + "clone", + "https://github.com/user/homelab.git", + str(dest), + ] def test_existing_dir_without_git_dir_is_treated_as_fresh( @@ -103,8 +184,6 @@ def test_existing_dir_without_git_dir_is_treated_as_fresh( assert calls[0]["argv"] == [ "git", "clone", - "--depth", - "1", "https://github.com/user/homelab.git", str(dest), ] @@ -119,13 +198,19 @@ def test_failing_git_raises_error_carrying_stderr( with pytest.raises( GitSyncError, - match=r"git clone --depth 1 .* failed \(exit 128\): fatal: repository not found", + match=r"git clone .* failed \(exit 128\): fatal: repository not found", ): clone_or_pull("https://example.com/nope.git", dest) - # Same for the pull path (broken checkout, e.g. diverged history). + # Same for the pull path (broken checkout, e.g. diverged history) — + # the probe answers "false" (not shallow) so the pull runs. (dest / ".git").mkdir(parents=True) - _fake_run(monkeypatch, returncode=1, stderr="error: cannot pull with rebase") + _fake_run( + monkeypatch, + returncode=1, + stderr="error: cannot pull with rebase", + responses={("git", "rev-parse", "--is-shallow-repository"): (0, "false\n", "")}, + ) with pytest.raises( GitSyncError, match=r"git pull --ff-only failed \(exit 1\): error: cannot pull with rebase"