"""Import A9-format directories into the Brain of Reese knowledge base. Examples:: uv run python -m scripts.import_docs # ~/Homelab + ~/Deployments uv run python -m scripts.import_docs --source ~/OtherDocs # extra dir (repeatable) uv run python -m scripts.import_docs --prune # also drop deleted/out-of-scope files uv run python -m scripts.import_docs --limit 5 # debug: first 5 files only Imported formats (PLAN anchor A9, revised): ``md, markdown, txt, yaml, yml, json, py`` (case-insensitive; narrow with ``BOR_IMPORT_EXTENSIONS``). Any path with a dot-prefixed component (hidden files/dirs — vendored caches) is skipped, along with non-content dirs (``.venv``, ``node_modules``, ``.git``, ``__pycache__``, ``.pytest_cache``, ``dist``, ``build``). Re-runs are cheap: files are diffed by sha256 and unchanged ones are not re-embedded; ``--prune`` also drops documents whose files no longer match the format filter. """ from __future__ import annotations import argparse import asyncio import sys from pathlib import Path from app.config import get_settings from app.core.debugging import configure_debugging from app.core.logging import configure_logging from app.rag.importer import import_sources from app.rag.llm import LLMClient DEFAULT_SOURCES: list[Path] = [Path("~/Homelab"), Path("~/Deployments")] def build_parser() -> argparse.ArgumentParser: p = argparse.ArgumentParser( prog="python -m scripts.import_docs", description="Import A9-format files (md/txt/yaml/json/py) into the knowledge base.", ) p.add_argument( "--source", action="append", type=Path, metavar="PATH", help="directory to import (repeatable; default: ~/Homelab ~/Deployments)", ) p.add_argument( "--prune", action="store_true", help="also delete documents whose files no longer exist or match the format filter", ) p.add_argument( "--limit", type=int, default=None, metavar="N", help="only process the first N files (debug; disables --prune)", ) return p def main(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) configure_logging(get_settings().log_level) configure_debugging() sources = [path.expanduser() for path in (args.source or DEFAULT_SOURCES)] missing = [s for s in sources if not s.is_dir()] for s in missing: print(f"import_docs: source dir not found: {s}", file=sys.stderr) if all(not s.is_dir() for s in sources): print("import_docs: no source directories found — nothing to do.", file=sys.stderr) return 1 llm = LLMClient() summary = asyncio.run(import_sources(sources, llm, prune=args.prune, limit=args.limit)) print( f"import_docs: files={summary.files} added={summary.added} " f"updated={summary.updated} unchanged={summary.unchanged} " f"pruned={summary.pruned} errors={summary.errors} chunks={summary.chunks} " f"embed_batches={summary.embed_batches} formats={summary.format_counts()}" ) # Non-zero if any file failed, so cron/CI notice — the rest of the KB # was imported and the failed files are retried on the next run. return 1 if summary.errors else 0 if __name__ == "__main__": sys.exit(main())