feat(rag): hybrid FTS+vector retrieval and multi-format ingestion — name-your-tool questions find the right document

app/rag/importer.py

@@ -1,23 +1,30 @@
 """Knowledge-base importer (PLAN §5 / §9 / §11).
 
-Walks ``*.md`` files (A9 exclusion list), diffs by sha256 against
-``documents.content_hash`` and, for every new or changed file, runs the
-two-phase upsert:
+Walks the A9-format files (``md, markdown, txt, yaml, yml, json, py`` by
+default — ``BOR_IMPORT_EXTENSIONS``; case-insensitive), diffs by sha256
+against ``documents.content_hash`` and, for every new or changed file, runs
+the two-phase upsert:
 
 1. upsert the document row and replace its chunk rows (embeddings NULL)
 2. embed the new chunks in batches and attach the vectors
 3. commit — one transaction per file, so a failed embedding leaves the
    database untouched and the file is simply retried on the next run
 
+Scope (A9, revised 2026-08-21): any path containing a dot-prefixed
+component (hidden dirs — vendored caches like ``.esphome/.espressif/**`` —
+or hidden files) is skipped, plus the well-known exclusion list.
+
 ``prune=True`` deletes documents (of the imported sources only) whose files
-no longer exist. Per-file logging uses the verbs
-``added | updated | unchanged | pruned`` plus a summary line (PLAN §9).
+no longer exist **or no longer match the format filter** — this is how
+previously-imported junk (e.g. dot-dir READMEs) leaves the index. Per-file
+logging uses the verbs ``added | updated | unchanged | pruned`` plus a
+summary line with per-format counts (PLAN §9).
 """
 from __future__ import annotations
 
 import hashlib
 import logging
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from datetime import UTC, datetime
 from pathlib import Path
 from typing import Protocol
@@ -28,7 +35,7 @@ from sqlalchemy.orm import Session
 from app.config import Settings
 from app.db import SessionLocal
 from app.models import Chunk, Document
-from app.rag.chunker import chunk_markdown, extract_title
+from app.rag.chunker import chunk_document, extract_title
 from app.rag.llm import EmbeddingError
 
 logger = logging.getLogger("app.importer")
@@ -60,11 +67,20 @@ class ImportSummary:
     errors: int = 0
     chunks: int = 0
     embed_batches: int = 0
+    #: Files walked, keyed by lowercased extension (``md``, ``yaml``, …).
+    formats: dict[str, int] = field(default_factory=dict)
+
+    def format_counts(self) -> str:
+        """``md:203,yaml:267,py:14`` — highest count first (PLAN §9)."""
+        if not self.formats:
+            return "none"
+        ordered = sorted(self.formats.items(), key=lambda kv: (-kv[1], kv[0]))
+        return ",".join(f"{ext}:{count}" for ext, count in ordered)
 
     def log(self) -> None:
         logger.info(
             "import: summary files=%d added=%d updated=%d unchanged=%d pruned=%d "
-            "errors=%d chunks=%d embed_batches=%d",
+            "errors=%d chunks=%d embed_batches=%d formats=%s",
             self.files,
             self.added,
             self.updated,
@@ -73,17 +89,32 @@ class ImportSummary:
             self.errors,
             self.chunks,
             self.embed_batches,
+            self.format_counts(),
         )
 
 
-def iter_markdown_files(root: Path, excluded: frozenset[str] = EXCLUDED_DIRS) -> list[Path]:
-    """All ``*.md`` files under *root* (sorted), skipping excluded dirs (A9)."""
+def iter_importable_files(
+    root: Path,
+    extensions: frozenset[str],
+    excluded: frozenset[str] = EXCLUDED_DIRS,
+) -> list[Path]:
+    """All importable files under *root* (sorted), per the A9 scope rules.
+
+    *extensions* is a set of lowercased dotted suffixes (``{'.md', '.py'}``).
+    Skips: any path with a dot-prefixed component (hidden dirs/files —
+    vendored caches like ``.esphome/.espressif/**``) and the well-known
+    non-content directories in *excluded*.
+    """
     if not root.is_dir():
         return []
     files: list[Path] = []
-    for path in sorted(root.rglob("*.md")):
+    for path in sorted(root.rglob("*")):
+        if not path.is_file():
+            continue
         rel = path.relative_to(root)
-        if any(part in excluded for part in rel.parts[:-1]):
+        if any(part.startswith(".") or part in excluded for part in rel.parts):
+            continue
+        if path.suffix.lower() not in extensions:
             continue
         files.append(path)
     return files
@@ -97,7 +128,7 @@ async def import_sources(
     limit: int | None = None,
     session: Session | None = None,
 ) -> ImportSummary:
-    """Import every ``*.md`` under *sources* (see module docstring).
+    """Import every A9-format file under *sources* (see module docstring).
 
     ``session`` may be supplied (tests); a private one is opened and closed
     otherwise. ``limit`` caps the number of files processed (debug only) and
@@ -120,12 +151,14 @@ async def import_sources(
                 break
             source = root.name
             source_names.add(source)
-            for path in iter_markdown_files(root):
+            for path in iter_importable_files(root, llm.settings.import_extension_set):
                 if limit is not None and summary.files >= limit:
                     break
                 rel = path.relative_to(root).as_posix()
                 seen.add((source, rel))
                 summary.files += 1
+                ext = path.suffix.lower().lstrip(".") or "unknown"
+                summary.formats[ext] = summary.formats.get(ext, 0) + 1
                 try:
                     await _index_file(
                         session, source=source, rel=rel, full_path=path, llm=llm,
@@ -172,7 +205,12 @@ async def _index_file(
         return
 
     verb = "updated" if doc is not None else "added"
-    title = extract_title(content, fallback=full_path.stem)
+    # A ``#`` line is a real heading in markdown but a comment in every
+    # other format — titles for those come from the file stem.
+    if full_path.suffix.lower() in (".md", ".markdown"):
+        title = extract_title(content, fallback=full_path.stem)
+    else:
+        title = full_path.stem
     if doc is None:
         doc = Document(
             source=source,
@@ -200,7 +238,7 @@ async def _index_file(
     # policy stays intact for the rest of the KB.
     target = max(400, settings.chunk_target_chars)
     while True:
-        chunks_text = chunk_markdown(content, target, settings.chunk_overlap_chars)
+        chunks_text = chunk_document(content, rel, target, settings.chunk_overlap_chars)
         doc.chunks = [
             Chunk(document_id=doc.id, position=i, content=c) for i, c in enumerate(chunks_text)
         ]