90 lines
3.3 KiB
Python
90 lines
3.3 KiB
Python
"""Import A9-format directories into the Brain of Reese knowledge base.
|
|
|
|
Examples::
|
|
|
|
uv run python -m scripts.import_docs # ~/Homelab + ~/Deployments
|
|
uv run python -m scripts.import_docs --source ~/OtherDocs # extra dir (repeatable)
|
|
uv run python -m scripts.import_docs --prune # also drop deleted/out-of-scope files
|
|
uv run python -m scripts.import_docs --limit 5 # debug: first 5 files only
|
|
|
|
Imported formats (PLAN anchor A9, revised): ``md, markdown, txt, yaml,
|
|
yml, json, py`` (case-insensitive; narrow with ``BOR_IMPORT_EXTENSIONS``).
|
|
Any path with a dot-prefixed component (hidden files/dirs — vendored
|
|
caches) is skipped, along with non-content dirs (``.venv``,
|
|
``node_modules``, ``.git``, ``__pycache__``, ``.pytest_cache``, ``dist``,
|
|
``build``). Re-runs are cheap: files are diffed by sha256 and unchanged
|
|
ones are not re-embedded; ``--prune`` also drops documents whose files no
|
|
longer match the format filter.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import asyncio
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
from app.config import get_settings
|
|
from app.core.debugging import configure_debugging
|
|
from app.core.logging import configure_logging
|
|
from app.rag.importer import import_sources
|
|
from app.rag.llm import LLMClient
|
|
|
|
DEFAULT_SOURCES: list[Path] = [Path("~/Homelab"), Path("~/Deployments")]
|
|
|
|
|
|
def build_parser() -> argparse.ArgumentParser:
|
|
p = argparse.ArgumentParser(
|
|
prog="python -m scripts.import_docs",
|
|
description="Import A9-format files (md/txt/yaml/json/py) into the knowledge base.",
|
|
)
|
|
p.add_argument(
|
|
"--source",
|
|
action="append",
|
|
type=Path,
|
|
metavar="PATH",
|
|
help="directory to import (repeatable; default: ~/Homelab ~/Deployments)",
|
|
)
|
|
p.add_argument(
|
|
"--prune",
|
|
action="store_true",
|
|
help="also delete documents whose files no longer exist or match the format filter",
|
|
)
|
|
p.add_argument(
|
|
"--limit",
|
|
type=int,
|
|
default=None,
|
|
metavar="N",
|
|
help="only process the first N files (debug; disables --prune)",
|
|
)
|
|
return p
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
args = build_parser().parse_args(argv)
|
|
configure_logging(get_settings().log_level)
|
|
configure_debugging()
|
|
|
|
sources = [path.expanduser() for path in (args.source or DEFAULT_SOURCES)]
|
|
missing = [s for s in sources if not s.is_dir()]
|
|
for s in missing:
|
|
print(f"import_docs: source dir not found: {s}", file=sys.stderr)
|
|
if all(not s.is_dir() for s in sources):
|
|
print("import_docs: no source directories found — nothing to do.", file=sys.stderr)
|
|
return 1
|
|
|
|
llm = LLMClient()
|
|
summary = asyncio.run(import_sources(sources, llm, prune=args.prune, limit=args.limit))
|
|
print(
|
|
f"import_docs: files={summary.files} added={summary.added} "
|
|
f"updated={summary.updated} unchanged={summary.unchanged} "
|
|
f"pruned={summary.pruned} errors={summary.errors} chunks={summary.chunks} "
|
|
f"embed_batches={summary.embed_batches} formats={summary.format_counts()}"
|
|
)
|
|
# Non-zero if any file failed, so cron/CI notice — the rest of the KB
|
|
# was imported and the failed files are retried on the next run.
|
|
return 1 if summary.errors else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|