feat(rag): hybrid FTS+vector retrieval and multi-format ingestion — name-your-tool questions find the right document

tests/unit/test_importer.py

@@ -16,11 +16,15 @@ from app.models import Chunk, Document
 from app.rag.importer import (
     EXCLUDED_DIRS,
     import_sources,
-    iter_markdown_files,
+    iter_importable_files,
 )
 from app.rag.llm import EmbeddingError
 from tests.fakes import FakeEmbedder
 
+#: A9 default extension set as dotted suffixes (what the importer passes to
+#: the walker when no override is configured).
+DEFAULT_EXTS = frozenset({".md", ".markdown", ".txt", ".yaml", ".yml", ".json", ".py"})
+
 
 class _PoisonEmbedder(FakeEmbedder):
     """Fails (like a real endpoint) on any text containing 'poison'."""
@@ -50,7 +54,9 @@ def _cleanup_source(db, source: str) -> None:
     db.commit()
 
 
-def test_iter_markdown_files_excludes_noncontent_dirs(tmp_path: Path) -> None:
+def test_iter_importable_files_excludes_noncontent_dirs_and_hidden(tmp_path: Path) -> None:
+    """Well-known non-content dirs, hidden (dot-) dirs/files, and non-A9
+    extensions are all skipped; the A9 formats pass."""
     root = tmp_path / "proj"
     for d in (
         "notes/sub",
@@ -61,11 +67,20 @@ def test_iter_markdown_files_excludes_noncontent_dirs(tmp_path: Path) -> None:
         ".pytest_cache",
         "dist",
         "build",
+        ".esphome/.espressif",  # vendored hidden cache — the real A9 case
     ):
         (root / d).mkdir(parents=True)
     files = {
+        # content that must be found:
         "README.md": "readme",
         "notes/sub/deep.md": "deep",
+        "compose.yaml": "services: {}",
+        "legacy.YML": "a: b",  # case-insensitive suffix
+        "notes/sub/agent.py": "x = 1",
+        "config.json": "{}",
+        "README.txt": "plain",
+        "notes/sub/deep.markdown": "md2",
+        # must be skipped:
         ".venv/lib/junk.md": "junk",
         "node_modules/x/j.md": "j",
         ".git/c.md": "g",
@@ -73,17 +88,40 @@ def test_iter_markdown_files_excludes_noncontent_dirs(tmp_path: Path) -> None:
         ".pytest_cache/c.md": "pc",
         "dist/d.md": "d",
         "build/b.md": "b",
+        ".esphome/.espressif/secret.md": "vendor",
+        ".secret.md": "hidden file",  # dot-prefixed FILE, not just dir
+        "notes/sub/notes.csv": "a,b",  # not an A9 format
+        "notes/sub/file.md.bak": "x",
     }
     for rel, text in files.items():
         (root / rel).write_text(text)
-    (root / "notes" / "not-md.txt").write_text("skip me")
 
-    found = {p.relative_to(root).as_posix() for p in iter_markdown_files(root)}
-    assert found == {"README.md", "notes/sub/deep.md"}
+    found = {p.relative_to(root).as_posix() for p in iter_importable_files(root, DEFAULT_EXTS)}
+    assert found == {
+        "README.md",
+        "notes/sub/deep.md",
+        "compose.yaml",
+        "legacy.YML",
+        "notes/sub/agent.py",
+        "config.json",
+        "README.txt",
+        "notes/sub/deep.markdown",
+    }
 
 
-def test_iter_markdown_files_missing_dir_yields_nothing(tmp_path: Path) -> None:
-    assert iter_markdown_files(tmp_path / "definitely-missing") == []
+def test_iter_importable_files_missing_dir_yields_nothing(tmp_path: Path) -> None:
+    assert iter_importable_files(tmp_path / "definitely-missing", DEFAULT_EXTS) == []
+
+
+def test_iter_importable_files_respects_custom_extension_filter(tmp_path: Path) -> None:
+    """A narrower filter (e.g. md only) excludes the other A9 formats."""
+    root = tmp_path / "filtered"
+    root.mkdir()
+    (root / "a.md").write_text("a")
+    (root / "b.yaml").write_text("a: b")
+    (root / "c.py").write_text("x = 1")
+    found = {p.name for p in iter_importable_files(root, frozenset([".md"]))}
+    assert found == {"a.md"}
 
 
 def test_excluded_dirs_match_plan_anchor_a9() -> None:
@@ -277,3 +315,65 @@ def test_chunk_positions_and_titles(db, tmp_path: Path) -> None:
         assert positions == list(range(len(doc.chunks))) and len(doc.chunks) >= 2
     finally:
         _cleanup_source(db, root.name)
+
+
+def test_multi_format_import_counts_per_format_and_titles_stem(db, tmp_path: Path) -> None:
+    """A9 formats all import; the summary records per-format counts, and
+    non-markdown titles come from the file stem (a ``#`` line is a comment
+    there, not a heading)."""
+    root = tmp_path / "multi"
+    (root / "svc").mkdir(parents=True)
+    (root / "guide.md").write_text("# Real Heading\n\nbody\n")
+    (root / "svc" / "compose.yaml").write_text("# a comment\nservices:\n  gitlab: {}\n")
+    (root / "svc" / "agent.py").write_text("# docstring-like comment\ndef ping():\n    return 1\n")
+    (root / "inventory.json").write_text('{"hosts": []}\n')
+    (root / "notes.txt").write_text("plain text notes\n")
+    llm = FakeEmbedder()
+    try:
+        summary = asyncio.run(import_sources([root], llm, session=db))
+        assert summary.files == 5
+        assert summary.added == 5
+        assert summary.formats == {"md": 1, "yaml": 1, "py": 1, "json": 1, "txt": 1}
+        # PLAN §9 summary line: counts, highest first, ext:name pairs.
+        assert summary.format_counts() == "json:1,md:1,py:1,txt:1,yaml:1"
+
+        titles = {
+            d.path: d.title
+            for d in db.scalars(select(Document).where(Document.source == root.name)).all()
+        }
+        assert titles["guide.md"] == "Real Heading"  # markdown keeps the H1
+        assert titles["svc/compose.yaml"] == "compose"  # …comment is not a heading
+        assert titles["svc/agent.py"] == "agent"
+        assert titles["inventory.json"] == "inventory"
+        assert titles["notes.txt"] == "notes"
+    finally:
+        _cleanup_source(db, root.name)
+
+
+def test_prune_removes_files_now_excluded_by_format_filter(db, tmp_path: Path) -> None:
+    """Previously-imported junk leaves the index: a file that no longer
+    matches the A9 extension filter is pruned on the next ``prune=True`` run.
+    This is how dot-dir READMEs imported before the scope fix get cleaned up."""
+    root = tmp_path / "cleanup"
+    root.mkdir()
+    (root / "keep.md").write_text("# Keep\n\nkept\n")
+    (root / "junk.md.bak").write_text("old junk that was once imported\n")
+    llm = FakeEmbedder()
+    try:
+        # Seed: import both files as if they were valid at the time.
+        (root / "junk.md").write_text("old junk that was once imported\n")
+        (root / "junk.md.bak").unlink()
+        asyncio.run(import_sources([root], llm, session=db))
+        # Rename the junk out of the A9 formats, then prune.
+        (root / "junk.md").rename(root / "junk.md.bak")
+        summary = asyncio.run(import_sources([root], llm, session=db, prune=True))
+        assert summary.pruned == 1
+        assert summary.unchanged == 1  # keep.md survived
+        assert db.scalar(
+            select(Document).where(Document.source == root.name, Document.path == "junk.md")
+        ) is None
+        assert db.scalar(
+            select(Document).where(Document.source == root.name, Document.path == "keep.md")
+        ) is not None
+    finally:
+        _cleanup_source(db, root.name)