phase: 105_hidden_folders_toggle

All completion criteria verified. Everything is green. **Phase 105 final verification pass — all criteria verified** - Verified the full implementation in the working tree: `git_sources.include_hidden` column + alembic `0019` (dev DB at head, column present), `iter_importable_files`/`import_sources` flag support with `str(root)`-keyed map used by both walk and progress pre-walk, `GitSourcePatchIn` rename with optional fields, sync/CLI pipeline wiring (OR-collision), and the per-row "Hidden" checkbox + tag + error line on the Sources page - Unit + integration: `uv run pytest` → exit 0 (2148 tests collected, all pass; this sandbox occasionally swallows pytest's final status line — exit codes verified) - Coverage: `uv run pytest --cov=app --cov-report=term-missing` → **TOTAL 99%** (3879 stmts, 15 miss) — >90% gate ✓ - Dedicated E2E: `uv run pytest tests/e2e/test_hidden_folders_toggle.py -v --no-cov` → **6 passed in 23.20s** (DB up, in isolation) - Regression E2E in isolation: `test_source_ignore_paths` 6 passed, `test_git_sources_admin` 6 passed, `test_local_directory_sources` 3 passed, `test_sync_button` 3 passed, `test_smoke` 3 passed - Lint/types: `uv run ruff check .` + `uv run pyright` → clean (0 errors/warnings) **Completion criteria:** (1) checkbox persists via PATCH 200 → "hidden on" tag + GET round-trips `include_hidden: true`; failure path reverts box + `role="alert"` canned message ✓; (2) flag OFF byte-identical (only `visible.md` indexed), ON indexes `.hidden/note.md` into the KB catalog, `EXCLUDED_DIRS` excluded both states ✓; (3) A2: flag OFF → `detail.pruned==1`, doc gone from catalog ✓; (4) PATCH bool-only/list-only/both/neither no-op, phase-89 fixed 422s unchanged, 404, anonymous 403 (incl. bool-only body) ✓; (5) env-fallback rows render no checkbox, WCAG-clean (aria-label, keyboard focus, visible label, text tag) ✓; (6) full gate green ✓; (7) commit left to the harness per instructions (no `git add`/`commit` run; phase files untouched). **Deviations:** none — no defects found; no code changes were needed on this pass. **Next pending phase:** `.agents/phases/todo/98_sync_summary_visibility`.
2026-09-12 21:26:30 -04:00
parent ecc921098a
commit d731169b8b
49 changed files with 3553 additions and 93 deletions
@@ -0,0 +1,161 @@
+"""Unit: the phase-105 hidden-folders flag in ``iter_importable_files``.
+
+Phase 105 (TODO.md L3 — "…a toggle per input (next to the ignores
+button) to allow indexing hidden .folders."): ``include_hidden`` lifts
+ONLY the dot-prefixed-component skip for one source. A1
+(owner-confirmed 2026-09-14): when True, files inside hidden folders
+AND hidden files with an importable extension become importable;
+``EXCLUDED_DIRS`` (``.venv``, ``node_modules``, ``.git``, …) stay
+excluded in BOTH states; the extension filter always applies; the
+phase-89 *ignore* tuple composes additively. A4: the default
+(``False``) is byte-identical to the pre-phase-105 walk — pinned here
+against a tmp fixture tree. The DB-facing behavior (hidden files
+indexed / pruned through ``import_sources``) is integration-gated by
+``tests/integration/test_importer_include_hidden.py``.
+"""
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from app.rag.importer import _include_hidden_for_root, iter_importable_files
+
+#: Explicit extension set (not the A9 config default) — the tests pin
+#: the walk rules, not the config.
+EXTS = frozenset({".md"})
+
+
+@pytest.fixture()
+def tree(tmp_path: Path) -> Path:
+    """The task's fixture layout (plus the no-extension ``.env``).
+
+    visible.md            .hidden/note.md        .notes.md
+    .venv/junk.md         node_modules/x.md      .hidden/.deep.md
+    keep/ok.md            .env
+    """
+    root = tmp_path / "HiddenFix"
+    (root / ".hidden").mkdir(parents=True)
+    (root / ".venv").mkdir()
+    (root / "node_modules").mkdir()
+    (root / "keep").mkdir()
+    (root / "visible.md").write_text("visible\n", encoding="utf-8")
+    (root / ".hidden" / "note.md").write_text("note\n", encoding="utf-8")
+    (root / ".notes.md").write_text("notes\n", encoding="utf-8")
+    (root / ".venv" / "junk.md").write_text("junk\n", encoding="utf-8")
+    (root / "node_modules" / "x.md").write_text("x\n", encoding="utf-8")
+    (root / ".hidden" / ".deep.md").write_text("deep\n", encoding="utf-8")
+    (root / "keep" / "ok.md").write_text("ok\n", encoding="utf-8")
+    (root / ".env").write_text("SECRET=1\n", encoding="utf-8")
+    return root
+
+
+def _rels(paths: list[Path], root: Path) -> set[str]:
+    return {p.relative_to(root).as_posix() for p in paths}
+
+
+# --- default (flag False): today's behavior, byte-identical -----------------
+
+
+def test_iter_default_flag_skips_all_hidden(tree: Path) -> None:
+    # A4: the default is the pre-phase-105 walk — hidden dir, hidden
+    # file, and the excluded dirs are all absent. Sorted order pinned
+    # verbatim (not just the set).
+    got = iter_importable_files(tree, EXTS)
+    assert got == [tree / "keep" / "ok.md", tree / "visible.md"]
+
+
+def test_iter_default_flag_rejects_explicit_false(tree: Path) -> None:
+    assert iter_importable_files(tree, EXTS, include_hidden=False) == [
+        tree / "keep" / "ok.md",
+        tree / "visible.md",
+    ]
+
+
+# --- flag True (A1) -----------------------------------------------------------
+
+
+def test_iter_flag_on_admits_hidden_dirs_and_hidden_files(tree: Path) -> None:
+    got = _rels(iter_importable_files(tree, EXTS, include_hidden=True), tree)
+    assert got == {
+        ".hidden/note.md",  # inside a hidden dir
+        ".notes.md",  # hidden file with an importable extension
+        ".hidden/.deep.md",  # both at once
+        "visible.md",
+        "keep/ok.md",
+    }
+
+
+def test_iter_flag_on_keeps_excluded_dirs_out(tree: Path) -> None:
+    # A1: EXCLUDED_DIRS are skipped in BOTH states — caches are never
+    # content, even with the flag on.
+    got = _rels(iter_importable_files(tree, EXTS, include_hidden=True), tree)
+    assert ".venv/junk.md" not in got
+    assert "node_modules/x.md" not in got
+
+
+def test_iter_flag_on_keeps_extension_filter_in_force(tree: Path) -> None:
+    # ``.env`` has no A9 extension — the extension filter is the real
+    # content gate, so a secret-flavoured file is never listed.
+    for flag in (False, True):
+        got = _rels(iter_importable_files(tree, EXTS, include_hidden=flag), tree)
+        assert ".env" not in got
+
+
+# --- composition with the phase-89 ignore tuple -------------------------------
+
+
+def test_iter_flag_on_ignore_composes_additively(tree: Path) -> None:
+    # The ignored prefix still bites when the flag is ON; everything
+    # else hidden is admitted.
+    got = _rels(
+        iter_importable_files(tree, EXTS, include_hidden=True, ignore=(".hidden",)),
+        tree,
+    )
+    assert got == {".notes.md", "visible.md", "keep/ok.md"}
+
+
+def test_iter_ignore_only_flag_off(tree: Path) -> None:
+    # The pre-phase-105 combination still works untouched: ignore
+    # prefix on a visible subtree, flag at its default.
+    got = _rels(iter_importable_files(tree, EXTS, ignore=("keep",)), tree)
+    assert got == {"visible.md"}
+
+
+# --- _include_hidden_for_root: the single read point --------------------------
+
+
+def test_include_hidden_for_root_none_map_is_false(tmp_path: Path) -> None:
+    root = tmp_path / "src"
+    root.mkdir()
+    assert _include_hidden_for_root(root, None) is False
+
+
+def test_include_hidden_for_root_unlisted_root_is_false(tmp_path: Path) -> None:
+    root = tmp_path / "src"
+    root.mkdir()
+    assert _include_hidden_for_root(root, {"/other/root": True}) is False
+
+
+def test_include_hidden_for_root_listed_true_is_true(tmp_path: Path) -> None:
+    root = tmp_path / "src"
+    root.mkdir()
+    assert _include_hidden_for_root(root, {str(root): True}) is True
+
+
+def test_include_hidden_for_root_listed_false_is_false(tmp_path: Path) -> None:
+    root = tmp_path / "src"
+    root.mkdir()
+    assert _include_hidden_for_root(root, {str(root): False}) is False
+
+
+def test_include_hidden_for_root_keys_by_str_root(tmp_path: Path) -> None:
+    # Two distinct Path objects whose ``str()`` agrees — the answer must
+    # be the same (the ``_ignore_for_root`` keying convention, phase 89).
+    key = str(tmp_path / "src")
+    a = tmp_path / "src"
+    b = tmp_path / "src" / "."  # different object, same str()
+    assert str(a) == str(b) == key
+    mp = {key: True}
+    assert _include_hidden_for_root(a, mp) is True
+    assert _include_hidden_for_root(b, mp) is True