phase: 89_source_ignore_paths
All verification complete — TODO.md was already cleared in the roadmap commit; the two extra unit-test diffs are necessary fake-signature adaptations for the new keywords. Everything is green, no fixes were needed. ## Phase 89 — final verification pass: ALL GREEN **Verified (all 6 task files present in `complete/`):** - `git_sources.ignore_paths` JSONB column + migration 0013; `alembic downgrade -1 && upgrade head` round-trips (head `0013`) - Importer: `normalize_ignore_path`/`is_ignored`/`_ignore_for_root`, `ignore` in walk + progress pre-walk, `ignore_by_root` in `import_sources` - API: GET/POST carry list; admin-only `PATCH` (replace, 404/422 fixed details, anonymous 403) - Pipelines wired: `_run_sync`, `_run_upload` re-upload, `scripts/import_docs.py` - Sources-page box: dialog, §7.4 save lifecycle, `N ignored` tag, a11y; env rows get no box **Test/lint results:** - `uv run pytest` → 1808 passed - `uv run pytest --cov=app --cov-report=term-missing` → TOTAL **99%** (>90%) - `uv run pytest tests/e2e/test_source_ignore_paths.py -v --no-cov` → 6 passed (isolated, DB up) - Regressions in isolation: `test_git_sources_admin` 6, `test_archive_upload_sources` 5, `test_sync_button` 3, `test_smoke` 3 — all passed - `uv run ruff check . && uv run pyright` → clean (0 errors) **Completion criteria:** box→PATCH 200→count+GET round-trip ✅ · sync excludes `ignore/` (no docs/chunks/embeddings/summaries) + prunes newly-ignored (pruned==2) ✅ · no-mid-path rule E2E ✅ · PATCH 404/422/replace/clear/403 ✅ · full gate green ✅ · commit + phase move left to harness per rules. **Deviations:** none blocking — E2E pins `files == 4` (overview's "5" was an off-by-one vs its own 6-file tree, documented in-test); `tests/unit/test_importer.py` + `test_sync_button.py` test-double fakes extended for the new keywords (needed for the suite to stay green). **Next pending phase:** none — `todo/` holds only this phase.
This commit is contained in:
@@ -291,6 +291,7 @@ class _GatedImport:
|
||||
limit: int | None = None,
|
||||
session: object = None,
|
||||
progress: Callable[[str, str, int, int], None] | None = None,
|
||||
ignore_by_root: dict[str, list[str]] | None = None, # phase 89
|
||||
) -> ImportSummary:
|
||||
self.prune_flags.append(prune)
|
||||
if progress is not None:
|
||||
@@ -1352,3 +1353,92 @@ def test_models_down_fails_the_run_and_leaves_folder_and_row(
|
||||
# The scan never ran: no docs, no stray temps.
|
||||
assert _docs(upload_client) == []
|
||||
assert [p.name for p in uploads.iterdir()] == ["homelab"]
|
||||
|
||||
|
||||
# --- phase 89: re-uploads honor the row's saved ignore list -------------------
|
||||
|
||||
|
||||
def test_reupload_honors_saved_ignore_list(
|
||||
upload_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, tmp_path: Path
|
||||
) -> None:
|
||||
"""Phase 89: a re-upload of an EXISTING source honors the ignore
|
||||
list saved on its row. First scan (fresh row, no list) indexes the
|
||||
ignored file too; once the list is saved via the API, re-uploading
|
||||
the same archive again scans only the kept file — the previously
|
||||
indexed ignored file leaves the KB (prune), and the run lands
|
||||
``success`` with counts that exclude it. The upload itself keeps
|
||||
every file on disk: the ignore is about the index, not the folder."""
|
||||
uploads = tmp_path / "uploads"
|
||||
_point_at(monkeypatch, uploads)
|
||||
_real_llm(monkeypatch)
|
||||
files = {
|
||||
"keep.md": "# Keep\nin scope\n",
|
||||
"ignore/secret.md": "# Secret\nignored\n",
|
||||
}
|
||||
r = _post(upload_client, "docs.tar.gz", _targz_bytes(files))
|
||||
assert r.status_code == 202, r.text
|
||||
status = _wait_status(upload_client)
|
||||
assert status["state"] == "success", status
|
||||
# First scan: the fresh row has no list → both files are indexed.
|
||||
assert status["detail"]["files"] == 2
|
||||
assert _docs(upload_client) == [("docs", "ignore/secret.md"), ("docs", "keep.md")]
|
||||
|
||||
# Save the ignore list on the row via the API (phase 89, task 03).
|
||||
folder = uploads / "docs"
|
||||
row = _row(db, str(folder))
|
||||
assert row is not None
|
||||
r = upload_client.patch(
|
||||
f"/api/git-sources/{row.id}", json={"ignore_paths": ["ignore/"]}
|
||||
)
|
||||
assert r.status_code == 200, r.text
|
||||
assert r.json()["ignore_paths"] == ["ignore"] # stored normalized
|
||||
|
||||
# Re-upload the SAME archive (same name → in-place replace).
|
||||
r = _post(upload_client, "docs.tar.gz", _targz_bytes(files))
|
||||
assert r.status_code == 202, r.text
|
||||
status = _wait_status(upload_client)
|
||||
assert status["state"] == "success", status
|
||||
detail = status["detail"]
|
||||
# The scan walked only keep.md (unchanged); the ignored file never
|
||||
# entered the walk, so prune dropped the first scan's row for it.
|
||||
assert detail["files"] == 1
|
||||
assert detail["added"] == 0
|
||||
assert detail["updated"] == 0
|
||||
assert detail["unchanged"] == 1
|
||||
assert detail["pruned"] == 1
|
||||
assert _docs(upload_client) == [("docs", "keep.md")]
|
||||
|
||||
# The re-upload left the row's list in place (the existing row is
|
||||
# untouched by the upsert), and the folder keeps every file.
|
||||
row_after = _row(db, str(folder))
|
||||
assert row_after is not None
|
||||
assert row_after.ignore_paths == ["ignore"]
|
||||
assert {p.name for p in folder.iterdir()} == {"keep.md", "ignore"}
|
||||
|
||||
|
||||
def test_upload_new_source_without_list_imports_everything(
|
||||
upload_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, tmp_path: Path
|
||||
) -> None:
|
||||
"""Phase 89 regression: an upload of a NEW source name (no row yet,
|
||||
hence no ignore list) imports everything in the archive — including
|
||||
nested files — exactly as before phase 89."""
|
||||
uploads = tmp_path / "uploads"
|
||||
_point_at(monkeypatch, uploads)
|
||||
_real_llm(monkeypatch)
|
||||
files = {
|
||||
"alpha.md": "# Alpha\nroot file\n",
|
||||
"sub/deep.md": "# Deep\nnested file\n",
|
||||
}
|
||||
r = _post(upload_client, "fresh.tar.gz", _targz_bytes(files))
|
||||
assert r.status_code == 202, r.text
|
||||
status = _wait_status(upload_client)
|
||||
assert status["state"] == "success", status
|
||||
detail = status["detail"]
|
||||
assert detail["files"] == 2
|
||||
assert detail["added"] == 2
|
||||
# The fresh row carries the server-default empty list: nothing was
|
||||
# ignored.
|
||||
row = _row(db, str(uploads / "fresh"))
|
||||
assert row is not None
|
||||
assert (row.ignore_paths or []) == []
|
||||
assert _docs(upload_client) == [("fresh", "alpha.md"), ("fresh", "sub/deep.md")]
|
||||
|
||||
Reference in New Issue
Block a user