ducoterra
99c48cbe06
Phase 02 (story: import documents):
- fence-aware markdown chunker (heading sections, 200-char overlap,
heading anchor on every chunk, 1200-char hard cap, fence blocks
kept atomic and split under the cap)
- LLMClient over aipi (LiteLLM) reusing the openai client's httpx
transport to send a clean {model, input} payload — the openai SDK
injects encoding_format, which aipi's openai_like group rejects;
token-budget batching + halving retry for the endpoint's
~1024-token per-request input cap
- two-phase per-file upsert importer: sha256 delta (unchanged skip),
atomic commit, A9 exclusion walk, per-source prune, per-file error
tolerance (rollback + log + continue, non-zero CLI exit), adaptive
re-chunk at half target for URL-dense files the endpoint rejects
- scripts/import_docs CLI (repeatable --source, --prune, --limit,
defaults ~/Homelab + ~/Deployments)
- GET /api/docs with per-doc chunk counts; Sources page wired to the
real endpoint (stat cards, full-width a11y table, designed empty
state, DOM-built rows — no innerHTML)
- tests: 63 passed (chunker/llm/importer units, docs API + importer
integration), story E2E 3/3 (real endpoints, in-thread import);
app/ coverage 98%
- real KB imported: 672 docs / 8969 chunks in ~3m, idempotent
re-run (672 unchanged, 0 batches)
- harness: .agent/validate.sh now gates through uv (pytest +
coverage >90% + ruff + pyright) instead of system python3
86 lines
3.0 KiB
Python
86 lines
3.0 KiB
Python
"""Import markdown directories into the Brain of Reese knowledge base.
|
|
|
|
Examples::
|
|
|
|
uv run python -m scripts.import_docs # ~/Homelab + ~/Deployments
|
|
uv run python -m scripts.import_docs --source ~/OtherDocs # extra dir (repeatable)
|
|
uv run python -m scripts.import_docs --prune # also drop deleted files
|
|
uv run python -m scripts.import_docs --limit 5 # debug: first 5 files only
|
|
|
|
Only ``*.md`` files are imported; non-content dirs (``.venv``,
|
|
``node_modules``, ``.git``, ``__pycache__``, ``.pytest_cache``, ``dist``,
|
|
``build``) are skipped (PLAN anchor A9). Re-runs are cheap: files are
|
|
diffed by sha256 and unchanged ones are not re-embedded.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import asyncio
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
from app.config import get_settings
|
|
from app.core.debugging import configure_debugging
|
|
from app.core.logging import configure_logging
|
|
from app.rag.importer import import_sources
|
|
from app.rag.llm import LLMClient
|
|
|
|
DEFAULT_SOURCES: list[Path] = [Path("~/Homelab"), Path("~/Deployments")]
|
|
|
|
|
|
def build_parser() -> argparse.ArgumentParser:
|
|
p = argparse.ArgumentParser(
|
|
prog="python -m scripts.import_docs",
|
|
description="Import *.md files into the Brain of Reese knowledge base.",
|
|
)
|
|
p.add_argument(
|
|
"--source",
|
|
action="append",
|
|
type=Path,
|
|
metavar="PATH",
|
|
help="directory to import (repeatable; default: ~/Homelab ~/Deployments)",
|
|
)
|
|
p.add_argument(
|
|
"--prune",
|
|
action="store_true",
|
|
help="also delete documents whose files no longer exist",
|
|
)
|
|
p.add_argument(
|
|
"--limit",
|
|
type=int,
|
|
default=None,
|
|
metavar="N",
|
|
help="only process the first N files (debug; disables --prune)",
|
|
)
|
|
return p
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
args = build_parser().parse_args(argv)
|
|
configure_logging(get_settings().log_level)
|
|
configure_debugging()
|
|
|
|
sources = [path.expanduser() for path in (args.source or DEFAULT_SOURCES)]
|
|
missing = [s for s in sources if not s.is_dir()]
|
|
for s in missing:
|
|
print(f"import_docs: source dir not found: {s}", file=sys.stderr)
|
|
if all(not s.is_dir() for s in sources):
|
|
print("import_docs: no source directories found — nothing to do.", file=sys.stderr)
|
|
return 1
|
|
|
|
llm = LLMClient()
|
|
summary = asyncio.run(import_sources(sources, llm, prune=args.prune, limit=args.limit))
|
|
print(
|
|
f"import_docs: files={summary.files} added={summary.added} "
|
|
f"updated={summary.updated} unchanged={summary.unchanged} "
|
|
f"pruned={summary.pruned} errors={summary.errors} chunks={summary.chunks} "
|
|
f"embed_batches={summary.embed_batches}"
|
|
)
|
|
# Non-zero if any file failed, so cron/CI notice — the rest of the KB
|
|
# was imported and the failed files are retried on the next run.
|
|
return 1 if summary.errors else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|