feat(rag): hybrid FTS+vector retrieval and multi-format ingestion — name-your-tool questions find the right document

scripts/import_docs.py

@@ -1,16 +1,20 @@
-"""Import markdown directories into the Brain of Reese knowledge base.
+"""Import A9-format directories into the Brain of Reese knowledge base.
 
 Examples::
 
     uv run python -m scripts.import_docs                      # ~/Homelab + ~/Deployments
     uv run python -m scripts.import_docs --source ~/OtherDocs # extra dir (repeatable)
-    uv run python -m scripts.import_docs --prune              # also drop deleted files
+    uv run python -m scripts.import_docs --prune              # also drop deleted/out-of-scope files
     uv run python -m scripts.import_docs --limit 5            # debug: first 5 files only
 
-Only ``*.md`` files are imported; non-content dirs (``.venv``,
+Imported formats (PLAN anchor A9, revised): ``md, markdown, txt, yaml,
+yml, json, py`` (case-insensitive; narrow with ``BOR_IMPORT_EXTENSIONS``).
+Any path with a dot-prefixed component (hidden files/dirs — vendored
+caches) is skipped, along with non-content dirs (``.venv``,
 ``node_modules``, ``.git``, ``__pycache__``, ``.pytest_cache``, ``dist``,
-``build``) are skipped (PLAN anchor A9). Re-runs are cheap: files are
-diffed by sha256 and unchanged ones are not re-embedded.
+``build``). Re-runs are cheap: files are diffed by sha256 and unchanged
+ones are not re-embedded; ``--prune`` also drops documents whose files no
+longer match the format filter.
 """
 from __future__ import annotations
 
@@ -31,7 +35,7 @@ DEFAULT_SOURCES: list[Path] = [Path("~/Homelab"), Path("~/Deployments")]
 def build_parser() -> argparse.ArgumentParser:
     p = argparse.ArgumentParser(
         prog="python -m scripts.import_docs",
-        description="Import *.md files into the Brain of Reese knowledge base.",
+        description="Import A9-format files (md/txt/yaml/json/py) into the knowledge base.",
     )
     p.add_argument(
         "--source",
@@ -43,7 +47,7 @@ def build_parser() -> argparse.ArgumentParser:
     p.add_argument(
         "--prune",
         action="store_true",
-        help="also delete documents whose files no longer exist",
+        help="also delete documents whose files no longer exist or match the format filter",
     )
     p.add_argument(
         "--limit",
@@ -74,7 +78,7 @@ def main(argv: list[str] | None = None) -> int:
         f"import_docs: files={summary.files} added={summary.added} "
         f"updated={summary.updated} unchanged={summary.unchanged} "
         f"pruned={summary.pruned} errors={summary.errors} chunks={summary.chunks} "
-        f"embed_batches={summary.embed_batches}"
+        f"embed_batches={summary.embed_batches} formats={summary.format_counts()}"
     )
     # Non-zero if any file failed, so cron/CI notice — the rest of the KB
     # was imported and the failed files are retried on the next run.