feat(rag): index markdown KB — chunker, embed client, delta importer, Sources page

Phase 02 (story: import documents):

- fence-aware markdown chunker (heading sections, 200-char overlap,
  heading anchor on every chunk, 1200-char hard cap, fence blocks
  kept atomic and split under the cap)
- LLMClient over aipi (LiteLLM) reusing the openai client's httpx
  transport to send a clean {model, input} payload — the openai SDK
  injects encoding_format, which aipi's openai_like group rejects;
  token-budget batching + halving retry for the endpoint's
  ~1024-token per-request input cap
- two-phase per-file upsert importer: sha256 delta (unchanged skip),
  atomic commit, A9 exclusion walk, per-source prune, per-file error
  tolerance (rollback + log + continue, non-zero CLI exit), adaptive
  re-chunk at half target for URL-dense files the endpoint rejects
- scripts/import_docs CLI (repeatable --source, --prune, --limit,
  defaults ~/Homelab + ~/Deployments)
- GET /api/docs with per-doc chunk counts; Sources page wired to the
  real endpoint (stat cards, full-width a11y table, designed empty
  state, DOM-built rows — no innerHTML)
- tests: 63 passed (chunker/llm/importer units, docs API + importer
  integration), story E2E 3/3 (real endpoints, in-thread import);
  app/ coverage 98%
- real KB imported: 672 docs / 8969 chunks in ~3m, idempotent
  re-run (672 unchanged, 0 batches)
- harness: .agent/validate.sh now gates through uv (pytest +
  coverage >90% + ruff + pyright) instead of system python3

scripts/import_docs.py

@@ -0,0 +1,85 @@
+"""Import markdown directories into the Brain of Reese knowledge base.
+
+Examples::
+
+    uv run python -m scripts.import_docs                      # ~/Homelab + ~/Deployments
+    uv run python -m scripts.import_docs --source ~/OtherDocs # extra dir (repeatable)
+    uv run python -m scripts.import_docs --prune              # also drop deleted files
+    uv run python -m scripts.import_docs --limit 5            # debug: first 5 files only
+
+Only ``*.md`` files are imported; non-content dirs (``.venv``,
+``node_modules``, ``.git``, ``__pycache__``, ``.pytest_cache``, ``dist``,
+``build``) are skipped (PLAN anchor A9). Re-runs are cheap: files are
+diffed by sha256 and unchanged ones are not re-embedded.
+"""
+from __future__ import annotations
+
+import argparse
+import asyncio
+import sys
+from pathlib import Path
+
+from app.config import get_settings
+from app.core.debugging import configure_debugging
+from app.core.logging import configure_logging
+from app.rag.importer import import_sources
+from app.rag.llm import LLMClient
+
+DEFAULT_SOURCES: list[Path] = [Path("~/Homelab"), Path("~/Deployments")]
+
+
+def build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(
+        prog="python -m scripts.import_docs",
+        description="Import *.md files into the Brain of Reese knowledge base.",
+    )
+    p.add_argument(
+        "--source",
+        action="append",
+        type=Path,
+        metavar="PATH",
+        help="directory to import (repeatable; default: ~/Homelab ~/Deployments)",
+    )
+    p.add_argument(
+        "--prune",
+        action="store_true",
+        help="also delete documents whose files no longer exist",
+    )
+    p.add_argument(
+        "--limit",
+        type=int,
+        default=None,
+        metavar="N",
+        help="only process the first N files (debug; disables --prune)",
+    )
+    return p
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+    configure_logging(get_settings().log_level)
+    configure_debugging()
+
+    sources = [path.expanduser() for path in (args.source or DEFAULT_SOURCES)]
+    missing = [s for s in sources if not s.is_dir()]
+    for s in missing:
+        print(f"import_docs: source dir not found: {s}", file=sys.stderr)
+    if all(not s.is_dir() for s in sources):
+        print("import_docs: no source directories found — nothing to do.", file=sys.stderr)
+        return 1
+
+    llm = LLMClient()
+    summary = asyncio.run(import_sources(sources, llm, prune=args.prune, limit=args.limit))
+    print(
+        f"import_docs: files={summary.files} added={summary.added} "
+        f"updated={summary.updated} unchanged={summary.unchanged} "
+        f"pruned={summary.pruned} errors={summary.errors} chunks={summary.chunks} "
+        f"embed_batches={summary.embed_batches}"
+    )
+    # Non-zero if any file failed, so cron/CI notice — the rest of the KB
+    # was imported and the failed files are retried on the next run.
+    return 1 if summary.errors else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())