phase: 89_source_ignore_paths
All verification complete — TODO.md was already cleared in the roadmap commit; the two extra unit-test diffs are necessary fake-signature adaptations for the new keywords. Everything is green, no fixes were needed. ## Phase 89 — final verification pass: ALL GREEN **Verified (all 6 task files present in `complete/`):** - `git_sources.ignore_paths` JSONB column + migration 0013; `alembic downgrade -1 && upgrade head` round-trips (head `0013`) - Importer: `normalize_ignore_path`/`is_ignored`/`_ignore_for_root`, `ignore` in walk + progress pre-walk, `ignore_by_root` in `import_sources` - API: GET/POST carry list; admin-only `PATCH` (replace, 404/422 fixed details, anonymous 403) - Pipelines wired: `_run_sync`, `_run_upload` re-upload, `scripts/import_docs.py` - Sources-page box: dialog, §7.4 save lifecycle, `N ignored` tag, a11y; env rows get no box **Test/lint results:** - `uv run pytest` → 1808 passed - `uv run pytest --cov=app --cov-report=term-missing` → TOTAL **99%** (>90%) - `uv run pytest tests/e2e/test_source_ignore_paths.py -v --no-cov` → 6 passed (isolated, DB up) - Regressions in isolation: `test_git_sources_admin` 6, `test_archive_upload_sources` 5, `test_sync_button` 3, `test_smoke` 3 — all passed - `uv run ruff check . && uv run pyright` → clean (0 errors) **Completion criteria:** box→PATCH 200→count+GET round-trip ✅ · sync excludes `ignore/` (no docs/chunks/embeddings/summaries) + prunes newly-ignored (pruned==2) ✅ · no-mid-path rule E2E ✅ · PATCH 404/422/replace/clear/403 ✅ · full gate green ✅ · commit + phase move left to harness per rules. **Deviations:** none blocking — E2E pins `files == 4` (overview's "5" was an off-by-one vs its own 6-file tree, documented in-test); `tests/unit/test_importer.py` + `test_sync_button.py` test-double fakes extended for the new keywords (needed for the suite to stay green). **Next pending phase:** none — `todo/` holds only this phase.
This commit is contained in:
@@ -238,6 +238,10 @@ class FakeImportSources:
|
||||
# Phase 64 (task 02): the progress hook the runner passes (a live
|
||||
# closure while wired, None if the wiring regresses).
|
||||
self.progress_hooks: list[object] = []
|
||||
# Phase 89: the per-root ignore map the runner builds from the
|
||||
# rows' ``ignore_paths`` (keyed by the root string the importer
|
||||
# sees; two rows sharing a root string get the union).
|
||||
self.ignore_maps: list[dict[str, list[str]]] = []
|
||||
|
||||
async def __call__(
|
||||
self,
|
||||
@@ -248,11 +252,13 @@ class FakeImportSources:
|
||||
limit: int | None = None,
|
||||
session: Session | None = None,
|
||||
progress: Callable[[str, str, int, int], None] | None = None,
|
||||
ignore_by_root: dict[str, list[str]] | None = None,
|
||||
) -> ImportSummary:
|
||||
self.sources.append(list(sources))
|
||||
self.llms.append(llm)
|
||||
self.prune_flags.append(prune)
|
||||
self.progress_hooks.append(progress)
|
||||
self.ignore_maps.append(ignore_by_root or {})
|
||||
if self.delay:
|
||||
await asyncio.sleep(self.delay)
|
||||
return self.summary
|
||||
@@ -747,6 +753,7 @@ def test_import_error_is_reported_with_credentials_masked(
|
||||
limit: int | None = None,
|
||||
session: Session | None = None,
|
||||
progress: Callable[[str, str, int, int], None] | None = None,
|
||||
ignore_by_root: dict[str, list[str]] | None = None,
|
||||
) -> ImportSummary:
|
||||
raise EmbeddingError(
|
||||
"embeddings request to https://user:secret@aipi.reeseapps.com/v1 "
|
||||
@@ -897,3 +904,128 @@ def test_probe_runs_before_source_resolution(
|
||||
assert order == ["probe", "effective_sources"]
|
||||
assert clone_calls == []
|
||||
assert "short-circuit" in body["error"] # the spy aborted the run
|
||||
|
||||
|
||||
# --- phase 89: per-row ignore lists ------------------------------------------
|
||||
|
||||
|
||||
def test_local_row_ignore_paths_excluded_from_sync(
|
||||
sync_client: TestClient,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
db: Session,
|
||||
tmp_path: Path,
|
||||
clean_documents: None,
|
||||
) -> None:
|
||||
"""Phase 89: a local row carrying ``ignore_paths`` — the button
|
||||
sync skips every matching file: the ignored file never lands in the
|
||||
KB (no document row — hence no embedding, no summary), and the
|
||||
success detail's ``files``/``added`` counts exclude it, while the
|
||||
kept file imports as usual."""
|
||||
local_dir = tmp_path / "LocalDocs"
|
||||
local_dir.mkdir()
|
||||
(local_dir / "keep.md").write_text("# Keep\nin scope\n", encoding="utf-8")
|
||||
(local_dir / "ignore").mkdir()
|
||||
(local_dir / "ignore" / "secret.md").write_text("# Secret\nignored\n",
|
||||
encoding="utf-8")
|
||||
db.add(
|
||||
GitSource(
|
||||
url=str(local_dir), kind="local", path=str(local_dir),
|
||||
ignore_paths=["ignore/"], # any spelling — the importer normalizes
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
_stub_env(monkeypatch)
|
||||
monkeypatch.setattr(
|
||||
sync_api,
|
||||
"get_settings",
|
||||
lambda: _settings(sources_dir=str(tmp_path / "bor")),
|
||||
)
|
||||
_real_llm(monkeypatch) # real import_sources, deterministic embeddings
|
||||
|
||||
_login(sync_client)
|
||||
assert sync_client.post("/api/sync").status_code == 202
|
||||
body = _poll(sync_client, "success")
|
||||
|
||||
# The ignored file is not counted: only keep.md was walked.
|
||||
assert body["detail"]["files"] == 1
|
||||
assert body["detail"]["added"] == 1
|
||||
assert body["detail"]["errors"] == 0
|
||||
# The KB holds exactly the kept file — ignore/secret.md is absent.
|
||||
docs = [(d["source"], d["path"]) for d in sync_client.get("/api/docs").json()["documents"]]
|
||||
assert docs == [("LocalDocs", "keep.md")]
|
||||
|
||||
|
||||
def test_sync_builds_ignore_map_by_root_string_with_union(
|
||||
sync_client: TestClient,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
db: Session,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""Phase 89 wiring (fake import): the runner keys the map by the
|
||||
SAME root string the importer sees, and two rows sharing that root
|
||||
string (the sibling/repo-name edge — ``…/shared`` and
|
||||
``…/shared.git`` clone into the same checkout dir) get the UNION
|
||||
of their lists, in row order; a row with an empty list contributes
|
||||
nothing."""
|
||||
url_a = f"file://{tmp_path / 'shared'}"
|
||||
url_b = f"{url_a}.git" # same repo name → same checkout dir
|
||||
# Distinct added_at: the resolver orders by (added_at, id) — a
|
||||
# same-timestamp pair would tie-break on the random uuid.
|
||||
db.add(GitSource(url=url_a, kind="git", ignore_paths=["a/"],
|
||||
added_at=datetime(2026, 1, 1, tzinfo=UTC)))
|
||||
db.add(GitSource(url=url_b, kind="git", ignore_paths=["b"],
|
||||
added_at=datetime(2026, 1, 2, tzinfo=UTC)))
|
||||
db.commit()
|
||||
_stub_env(monkeypatch)
|
||||
monkeypatch.setattr(
|
||||
sync_api,
|
||||
"get_settings",
|
||||
lambda: _settings(sources_dir=str(tmp_path / "bor")),
|
||||
)
|
||||
_, fake_clone = _fake_clone()
|
||||
monkeypatch.setattr(sync_api, "clone_or_pull", fake_clone)
|
||||
_stub_probe(monkeypatch)
|
||||
fake_import = FakeImportSources(ImportSummary(files=1, added=1))
|
||||
monkeypatch.setattr(sync_api, "import_sources", fake_import)
|
||||
monkeypatch.setattr(sync_api, "regenerate_overview", FakeOverview(ok=True))
|
||||
|
||||
_login(sync_client)
|
||||
assert sync_client.post("/api/sync").status_code == 202
|
||||
_poll(sync_client, "success")
|
||||
|
||||
shared = str(tmp_path / "bor" / "shared")
|
||||
# Both rows resolve to the SAME checkout (the collision itself) and
|
||||
# the map holds their union, keyed by that one root string.
|
||||
assert fake_import.sources == [[tmp_path / "bor" / "shared", tmp_path / "bor" / "shared"]]
|
||||
assert fake_import.ignore_maps == [{shared: ["a/", "b"]}]
|
||||
|
||||
|
||||
def test_sync_without_ignore_lists_passes_empty_map(
|
||||
sync_client: TestClient,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
db: Session,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""Phase 89 regression: rows without a list → the runner passes an
|
||||
EMPTY map (the importer's byte-identical pre-phase-89 behavior)."""
|
||||
local_dir = tmp_path / "LocalDocs"
|
||||
local_dir.mkdir()
|
||||
(local_dir / "notes.md").write_text("# Notes\nplain row\n", encoding="utf-8")
|
||||
_seed_local(db, local_dir) # no ignore_paths → []
|
||||
_stub_env(monkeypatch)
|
||||
monkeypatch.setattr(
|
||||
sync_api,
|
||||
"get_settings",
|
||||
lambda: _settings(sources_dir=str(tmp_path / "bor")),
|
||||
)
|
||||
_stub_probe(monkeypatch)
|
||||
fake_import = FakeImportSources(ImportSummary(files=1, added=1))
|
||||
monkeypatch.setattr(sync_api, "import_sources", fake_import)
|
||||
monkeypatch.setattr(sync_api, "regenerate_overview", FakeOverview(ok=True))
|
||||
|
||||
_login(sync_client)
|
||||
assert sync_client.post("/api/sync").status_code == 202
|
||||
_poll(sync_client, "success")
|
||||
|
||||
assert fake_import.sources == [[local_dir]]
|
||||
assert fake_import.ignore_maps == [{}] # no row carried a list
|
||||
|
||||
Reference in New Issue
Block a user