phase: 89_source_ignore_paths
All verification complete — TODO.md was already cleared in the roadmap commit; the two extra unit-test diffs are necessary fake-signature adaptations for the new keywords. Everything is green, no fixes were needed. ## Phase 89 — final verification pass: ALL GREEN **Verified (all 6 task files present in `complete/`):** - `git_sources.ignore_paths` JSONB column + migration 0013; `alembic downgrade -1 && upgrade head` round-trips (head `0013`) - Importer: `normalize_ignore_path`/`is_ignored`/`_ignore_for_root`, `ignore` in walk + progress pre-walk, `ignore_by_root` in `import_sources` - API: GET/POST carry list; admin-only `PATCH` (replace, 404/422 fixed details, anonymous 403) - Pipelines wired: `_run_sync`, `_run_upload` re-upload, `scripts/import_docs.py` - Sources-page box: dialog, §7.4 save lifecycle, `N ignored` tag, a11y; env rows get no box **Test/lint results:** - `uv run pytest` → 1808 passed - `uv run pytest --cov=app --cov-report=term-missing` → TOTAL **99%** (>90%) - `uv run pytest tests/e2e/test_source_ignore_paths.py -v --no-cov` → 6 passed (isolated, DB up) - Regressions in isolation: `test_git_sources_admin` 6, `test_archive_upload_sources` 5, `test_sync_button` 3, `test_smoke` 3 — all passed - `uv run ruff check . && uv run pyright` → clean (0 errors) **Completion criteria:** box→PATCH 200→count+GET round-trip ✅ · sync excludes `ignore/` (no docs/chunks/embeddings/summaries) + prunes newly-ignored (pruned==2) ✅ · no-mid-path rule E2E ✅ · PATCH 404/422/replace/clear/403 ✅ · full gate green ✅ · commit + phase move left to harness per rules. **Deviations:** none blocking — E2E pins `files == 4` (overview's "5" was an off-by-one vs its own 6-file tree, documented in-test); `tests/unit/test_importer.py` + `test_sync_button.py` test-double fakes extended for the new keywords (needed for the suite to stay green). **Next pending phase:** none — `todo/` holds only this phase.
This commit is contained in:
@@ -0,0 +1,180 @@
|
||||
"""Unit: the phase-89 ignore-path matcher (source-level pins).
|
||||
|
||||
Phase 89 (TODO.md L3 — "The ignore is just a prefix ignore."): each
|
||||
stored source carries a list of source-relative path prefixes; a file
|
||||
is ignored when its source-relative POSIX path (the ``documents.path``
|
||||
string — NO leading slash) STARTS WITH a normalized entry. The spec's
|
||||
own examples are pinned verbatim:
|
||||
|
||||
* ``"/my/files/"``, ``"my/files/"`` and ``"my/files"`` all normalize to
|
||||
``"my/files"`` (whitespace + ALL leading/trailing slashes stripped);
|
||||
* ``"myfile.txt"`` matches ``"myfile.txt"`` but NOT
|
||||
``"some/path/myfile.txt"`` — no mid-path matching, no globs;
|
||||
* raw string prefix, deliberately NO component boundary (A1,
|
||||
owner-confirmed 2026-09-08): ``"my/files"`` also matches
|
||||
``"my/files2/x.md"`` and ``"a"`` matches ``"ab.md"``.
|
||||
|
||||
Normalization happens exactly once — in ``_ignore_for_root``, the
|
||||
single choke point — so ``iter_importable_files`` receives
|
||||
ALREADY-normalized tuples (that contract is pinned too: a raw box line
|
||||
does NOT match at the walk level). The DB-facing behavior (ignored
|
||||
files never indexed/embedded/summarized; A2 prune) is integration-
|
||||
gated by ``tests/integration/test_importer_ignore.py``.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.rag.importer import (
|
||||
_ignore_for_root,
|
||||
is_ignored,
|
||||
iter_importable_files,
|
||||
normalize_ignore_path,
|
||||
)
|
||||
|
||||
#: The walk-level fixture uses an explicit extension set (not the A9
|
||||
#: config default) so the test pins the matcher, not the config.
|
||||
EXTS = frozenset({".md", ".txt", ".yaml"})
|
||||
|
||||
|
||||
# --- normalize_ignore_path -------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("raw", "expected"),
|
||||
[
|
||||
("/my/files/", "my/files"), # spec: all three spellings are equal
|
||||
("my/files/", "my/files"),
|
||||
("my/files", "my/files"),
|
||||
(" my/files ", "my/files"), # surrounding whitespace trimmed
|
||||
("//", ""), # nothing left after stripping
|
||||
("", ""),
|
||||
(" ", ""),
|
||||
],
|
||||
)
|
||||
def test_normalize_ignore_path(raw: str, expected: str) -> None:
|
||||
assert normalize_ignore_path(raw) == expected
|
||||
|
||||
|
||||
# --- is_ignored: the spec's own examples, verbatim --------------------------
|
||||
|
||||
|
||||
def test_is_ignored_myfile_txt_root_level_matches() -> None:
|
||||
# Spec verbatim: "myfile.txt" would match "/myfile.txt" — the leading
|
||||
# slash never exists in documents.path, so the root-level file matches.
|
||||
assert is_ignored("myfile.txt", ("myfile.txt",)) is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"rel",
|
||||
[
|
||||
"my/files/a.md", # spec: ignores everything under the prefix
|
||||
"my/files/sub/b.md", # …and arbitrarily deep
|
||||
"my/files", # a file literally named after the prefix
|
||||
],
|
||||
)
|
||||
def test_is_ignored_prefix_matches(rel: str) -> None:
|
||||
assert is_ignored(rel, ("my/files",)) is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"rel",
|
||||
[
|
||||
"some/path/myfile.txt", # spec verbatim: NO mid-path matching —
|
||||
# the string simply does not start with the pattern
|
||||
"xmyfile.txt", # no suffix/partial matching either
|
||||
],
|
||||
)
|
||||
def test_is_ignored_prefix_no_match(rel: str) -> None:
|
||||
assert is_ignored(rel, ("myfile.txt",)) is False
|
||||
|
||||
|
||||
def test_is_ignored_root_level_dir_is_prefix() -> None:
|
||||
# A root-level dir IS prefix matching: "files" covers files/**.
|
||||
assert is_ignored("files/x.md", ("files",)) is True
|
||||
# …but not a same-named dir deeper down (no mid-path matching).
|
||||
assert is_ignored("some/files/x.md", ("files",)) is False
|
||||
|
||||
|
||||
# --- is_ignored: the raw-prefix A1 edge (documented, owner-confirmed) -------
|
||||
|
||||
|
||||
def test_is_ignored_raw_prefix_no_component_boundary() -> None:
|
||||
# A1: raw string startswith — "my/files" ALSO ignores "my/files2/x.md".
|
||||
assert is_ignored("my/files2/x.md", ("my/files",)) is True
|
||||
assert is_ignored("ab.md", ("a",)) is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize("rel", ["a.md", "some/a.md", "anything"])
|
||||
def test_is_ignored_empty_prefixes_never_match(rel: str) -> None:
|
||||
assert is_ignored(rel, ()) is False
|
||||
|
||||
|
||||
# --- iter_importable_files: the walk-level skip ------------------------------
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def tree(tmp_path: Path) -> Path:
|
||||
root = tmp_path / "tree"
|
||||
(root / "ignore" / "deep").mkdir(parents=True)
|
||||
(root / "keep.md").write_text("keep\n", encoding="utf-8")
|
||||
(root / "ignore" / "secret.md").write_text("secret\n", encoding="utf-8")
|
||||
(root / "ignore" / "deep" / "x.yaml").write_text("x: 1\n", encoding="utf-8")
|
||||
(root / "notes.txt").write_text("notes\n", encoding="utf-8")
|
||||
(root / "myfiles.md").write_text("myfiles\n", encoding="utf-8")
|
||||
return root
|
||||
|
||||
|
||||
def _rels(paths: list[Path], root: Path) -> set[str]:
|
||||
return {p.relative_to(root).as_posix() for p in paths}
|
||||
|
||||
|
||||
ALL_FIVE = {
|
||||
"keep.md",
|
||||
"ignore/secret.md",
|
||||
"ignore/deep/x.yaml",
|
||||
"notes.txt",
|
||||
"myfiles.md",
|
||||
}
|
||||
|
||||
|
||||
def test_iter_default_ignore_keeps_everything(tree: Path) -> None:
|
||||
# Default ``ignore=()`` — every existing caller byte-identical.
|
||||
assert _rels(iter_importable_files(tree, EXTS), tree) == ALL_FIVE
|
||||
|
||||
|
||||
def test_iter_ignore_prefix_skips_matching_files(tree: Path) -> None:
|
||||
got = _rels(iter_importable_files(tree, EXTS, ignore=("ignore",)), tree)
|
||||
assert got == {"keep.md", "notes.txt", "myfiles.md"}
|
||||
|
||||
|
||||
def test_iter_ignore_receives_already_normalized_tuples(tree: Path) -> None:
|
||||
# Contract pin: normalization is the caller's job at THIS level — it
|
||||
# happens in _ignore_for_root, not here. A raw box line does NOT
|
||||
# match, so all five files still come back.
|
||||
got = _rels(iter_importable_files(tree, EXTS, ignore=("//ignore/ ",)), tree)
|
||||
assert got == ALL_FIVE
|
||||
|
||||
|
||||
# --- _ignore_for_root: the single normalization choke point ------------------
|
||||
|
||||
|
||||
def test_ignore_for_root_normalizes_and_drops_empties(tmp_path: Path) -> None:
|
||||
root = tmp_path / "src"
|
||||
root.mkdir()
|
||||
got = _ignore_for_root(root, {str(root): ["/a/", "b//", "", " "]})
|
||||
assert got == ("a", "b")
|
||||
|
||||
|
||||
def test_ignore_for_root_missing_root_yields_empty(tmp_path: Path) -> None:
|
||||
root = tmp_path / "src"
|
||||
root.mkdir()
|
||||
assert _ignore_for_root(root, {"/other/root": ["a"]}) == ()
|
||||
|
||||
|
||||
def test_ignore_for_root_none_map_yields_empty(tmp_path: Path) -> None:
|
||||
root = tmp_path / "src"
|
||||
root.mkdir()
|
||||
assert _ignore_for_root(root, None) == ()
|
||||
Reference in New Issue
Block a user