phase: 89_source_ignore_paths
All verification complete — TODO.md was already cleared in the roadmap commit; the two extra unit-test diffs are necessary fake-signature adaptations for the new keywords. Everything is green, no fixes were needed. ## Phase 89 — final verification pass: ALL GREEN **Verified (all 6 task files present in `complete/`):** - `git_sources.ignore_paths` JSONB column + migration 0013; `alembic downgrade -1 && upgrade head` round-trips (head `0013`) - Importer: `normalize_ignore_path`/`is_ignored`/`_ignore_for_root`, `ignore` in walk + progress pre-walk, `ignore_by_root` in `import_sources` - API: GET/POST carry list; admin-only `PATCH` (replace, 404/422 fixed details, anonymous 403) - Pipelines wired: `_run_sync`, `_run_upload` re-upload, `scripts/import_docs.py` - Sources-page box: dialog, §7.4 save lifecycle, `N ignored` tag, a11y; env rows get no box **Test/lint results:** - `uv run pytest` → 1808 passed - `uv run pytest --cov=app --cov-report=term-missing` → TOTAL **99%** (>90%) - `uv run pytest tests/e2e/test_source_ignore_paths.py -v --no-cov` → 6 passed (isolated, DB up) - Regressions in isolation: `test_git_sources_admin` 6, `test_archive_upload_sources` 5, `test_sync_button` 3, `test_smoke` 3 — all passed - `uv run ruff check . && uv run pyright` → clean (0 errors) **Completion criteria:** box→PATCH 200→count+GET round-trip ✅ · sync excludes `ignore/` (no docs/chunks/embeddings/summaries) + prunes newly-ignored (pruned==2) ✅ · no-mid-path rule E2E ✅ · PATCH 404/422/replace/clear/403 ✅ · full gate green ✅ · commit + phase move left to harness per rules. **Deviations:** none blocking — E2E pins `files == 4` (overview's "5" was an off-by-one vs its own 6-file tree, documented in-test); `tests/unit/test_importer.py` + `test_sync_button.py` test-double fakes extended for the new keywords (needed for the suite to stay green). **Next pending phase:** none — `todo/` holds only this phase.
This commit is contained in:
@@ -43,14 +43,15 @@ def _settings(git_sources: str = "", sources_dir: str = "~/bor-sources") -> Sett
|
||||
return Settings(_env_file=None, git_sources=git_sources, sources_dir=sources_dir) # pyright: ignore[reportCallIssue]
|
||||
|
||||
|
||||
def _git_row(url: str) -> GitSource:
|
||||
return GitSource(url=url, kind="git")
|
||||
def _git_row(url: str, ignore_paths: list[str] | None = None) -> GitSource:
|
||||
return GitSource(url=url, kind="git", ignore_paths=ignore_paths or [])
|
||||
|
||||
|
||||
def _local_row(path: str) -> GitSource:
|
||||
def _local_row(path: str, ignore_paths: list[str] | None = None) -> GitSource:
|
||||
"""A local row as the phase-38 API stores it: the expanded path in
|
||||
both ``path`` and the NOT-NULL ``url`` location column."""
|
||||
return GitSource(url=path, kind="local", path=path)
|
||||
both ``path`` and the NOT-NULL ``url`` location column (plus the
|
||||
phase-89 ignore list, empty by default)."""
|
||||
return GitSource(url=path, kind="local", path=path, ignore_paths=ignore_paths or [])
|
||||
|
||||
|
||||
class FakeImportSources:
|
||||
@@ -66,8 +67,12 @@ class FakeImportSources:
|
||||
*,
|
||||
prune: bool = False,
|
||||
limit: int | None = None,
|
||||
ignore_by_root: dict[str, list[str]] | None = None, # phase 89
|
||||
) -> ImportSummary:
|
||||
self.calls.append({"sources": list(sources), "prune": prune, "limit": limit})
|
||||
self.calls.append(
|
||||
{"sources": list(sources), "prune": prune, "limit": limit,
|
||||
"ignore_by_root": ignore_by_root}
|
||||
)
|
||||
return ImportSummary(files=1, added=1)
|
||||
|
||||
|
||||
@@ -146,9 +151,10 @@ def test_resolve_sources_git_urls_cloned_into_sources_dir(
|
||||
sources_dir=str(tmp_path / "bor"),
|
||||
)
|
||||
|
||||
sources = import_docs._resolve_sources(None, settings)
|
||||
sources, ignore_map = import_docs._resolve_sources(None, settings)
|
||||
|
||||
assert sources == [tmp_path / "bor" / "homelab", tmp_path / "bor" / "deploy"]
|
||||
assert ignore_map == {} # phase 89: no row carries a list → empty map
|
||||
assert calls == [
|
||||
("https://host/a/homelab.git", tmp_path / "bor" / "homelab"),
|
||||
("git@host:user/deploy.git", tmp_path / "bor" / "deploy"),
|
||||
@@ -163,9 +169,10 @@ def test_resolve_sources_cli_source_wins(
|
||||
settings = _settings(git_sources="https://host/a/repo.git")
|
||||
manual = tmp_path / "Manual"
|
||||
|
||||
sources = import_docs._resolve_sources([manual], settings)
|
||||
sources, ignore_map = import_docs._resolve_sources([manual], settings)
|
||||
|
||||
assert sources == [manual]
|
||||
assert ignore_map == {} # phase 89: manual dirs have no rows → no ignore
|
||||
assert calls == [] # git is never touched when --source is given
|
||||
|
||||
|
||||
@@ -186,9 +193,10 @@ def test_resolve_sources_db_rows_win_over_env(
|
||||
sources_dir=str(tmp_path / "bor"),
|
||||
)
|
||||
|
||||
sources = import_docs._resolve_sources(None, settings)
|
||||
sources, ignore_map = import_docs._resolve_sources(None, settings)
|
||||
|
||||
assert sources == [tmp_path / "bor" / "only"]
|
||||
assert ignore_map == {} # phase 89: no row carries a list → empty map
|
||||
assert calls == [("https://db.example/only.git", tmp_path / "bor" / "only")]
|
||||
|
||||
|
||||
@@ -197,8 +205,102 @@ def test_resolve_sources_defaults_when_nothing_configured(
|
||||
) -> None:
|
||||
# Both origins empty (the resolver's ``([], "env")``) → legacy dirs.
|
||||
monkeypatch.setattr(import_docs, "effective_sources", lambda db: ([], "env"))
|
||||
sources = import_docs._resolve_sources(None, _settings())
|
||||
sources, ignore_map = import_docs._resolve_sources(None, _settings())
|
||||
assert sources == [p.expanduser() for p in import_docs.DEFAULT_SOURCES]
|
||||
assert ignore_map == {} # phase 89: the legacy fallback has no rows
|
||||
|
||||
|
||||
def test_resolve_sources_rows_branch_builds_ignore_map(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
"""Phase 89: the rows branch returns each row's ignore list keyed by
|
||||
the resolved root string — the local row's directory, the git row's
|
||||
checkout dir; a row without a list contributes nothing to the map."""
|
||||
calls, fake = _fake_clone_factory()
|
||||
monkeypatch.setattr(import_docs, "clone_or_pull", fake)
|
||||
local_dir = tmp_path / "LocalDocs"
|
||||
local_dir.mkdir()
|
||||
monkeypatch.setattr(
|
||||
import_docs,
|
||||
"effective_sources",
|
||||
lambda db: (
|
||||
[
|
||||
_git_row("https://db.example/only.git"),
|
||||
_local_row(str(local_dir), ignore_paths=["ignore/"]),
|
||||
],
|
||||
"db",
|
||||
),
|
||||
)
|
||||
settings = _settings(sources_dir=str(tmp_path / "bor"))
|
||||
|
||||
sources, ignore_map = import_docs._resolve_sources(None, settings)
|
||||
|
||||
assert sources == [tmp_path / "bor" / "only", local_dir]
|
||||
# Keyed by the SAME string the importer sees (the root, not the name).
|
||||
assert ignore_map == {str(local_dir): ["ignore/"]}
|
||||
|
||||
|
||||
def test_resolve_sources_two_rows_sharing_root_string_extend(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
"""Phase 89 collision rule: two rows resolving to the SAME root
|
||||
string (the sibling/repo-name edge — ``…/shared`` and
|
||||
``…/shared.git``) get the UNION of their lists (extend, not
|
||||
replace), in row order."""
|
||||
calls, fake = _fake_clone_factory()
|
||||
monkeypatch.setattr(import_docs, "clone_or_pull", fake)
|
||||
monkeypatch.setattr(
|
||||
import_docs,
|
||||
"effective_sources",
|
||||
lambda db: (
|
||||
[
|
||||
_git_row("https://a.example/shared", ignore_paths=["a/"]),
|
||||
_git_row("https://a.example/shared.git", ignore_paths=["b"]),
|
||||
],
|
||||
"db",
|
||||
),
|
||||
)
|
||||
settings = _settings(sources_dir=str(tmp_path / "bor"))
|
||||
|
||||
sources, ignore_map = import_docs._resolve_sources(None, settings)
|
||||
|
||||
shared = str(tmp_path / "bor" / "shared")
|
||||
assert sources == [tmp_path / "bor" / "shared", tmp_path / "bor" / "shared"]
|
||||
assert ignore_map == {shared: ["a/", "b"]} # union, row order
|
||||
|
||||
|
||||
def test_main_rows_branch_passes_ignore_map_to_import(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
capsys: pytest.CaptureFixture[str],
|
||||
) -> None:
|
||||
"""Phase 89: a ``kind=local`` row carrying ``ignore_paths`` →
|
||||
``main`` passes the per-root map to ``import_sources`` (keyed by
|
||||
the directory string, prune flag unchanged)."""
|
||||
settings = _settings(sources_dir=str(tmp_path / "bor"))
|
||||
monkeypatch.setattr(import_docs, "get_settings", lambda: settings)
|
||||
local_dir = tmp_path / "LocalDocs"
|
||||
local_dir.mkdir()
|
||||
(local_dir / "keep.md").write_text("# Keep\nin scope\n", encoding="utf-8")
|
||||
(local_dir / "ignore").mkdir()
|
||||
(local_dir / "ignore" / "secret.md").write_text("# Secret\nignored\n",
|
||||
encoding="utf-8")
|
||||
monkeypatch.setattr(
|
||||
import_docs,
|
||||
"effective_sources",
|
||||
lambda db: ([_local_row(str(local_dir), ignore_paths=["ignore/"])], "db"),
|
||||
)
|
||||
fake_import = FakeImportSources()
|
||||
monkeypatch.setattr(import_docs, "import_sources", fake_import)
|
||||
_stub_bump(monkeypatch)
|
||||
|
||||
rc = import_docs.main([])
|
||||
|
||||
assert rc == 0
|
||||
call = fake_import.calls[0]
|
||||
assert call["sources"] == [local_dir]
|
||||
assert call["ignore_by_root"] == {str(local_dir): ["ignore/"]}
|
||||
assert call["prune"] is False # the CLI's no-prune default is unchanged
|
||||
|
||||
|
||||
# --- main() ----------------------------------------------------------------
|
||||
@@ -306,9 +408,10 @@ def test_resolve_sources_mixed_git_and_local(
|
||||
sources_dir=str(tmp_path / "bor"),
|
||||
)
|
||||
|
||||
sources = import_docs._resolve_sources(None, settings)
|
||||
sources, ignore_map = import_docs._resolve_sources(None, settings)
|
||||
|
||||
assert sources == [tmp_path / "bor" / "only", local_dir]
|
||||
assert ignore_map == {} # phase 89: neither row carries a list
|
||||
assert calls == [("https://db.example/only.git", tmp_path / "bor" / "only")]
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user