"""Import A9-format directories into the Brain of Reese knowledge base. Examples:: uv run python -m scripts.import_docs # BOR_GIT_SOURCES, else ~/Homelab + ~/Deployments uv run python -m scripts.import_docs --source ~/OtherDocs # explicit dir(s); always wins uv run python -m scripts.import_docs --prune # also drop deleted/out-of-scope files uv run python -m scripts.import_docs --limit 5 # debug: first 5 files only Source resolution (phase 28, extended in phases 35 and 38), in precedence order: 1. ``--source PATH`` — explicit manual directories (repeatable) always win; git and local sources are ignored when this flag is used. 2. The effective sources — the admin-managed ``git_sources`` table rows (both kinds), else the ``BOR_GIT_SOURCES`` (comma-separated) git-only fallback (:func:`app.rag.git_sources.effective_sources`, the same shared resolver the in-app Sync button uses). Git rows are cloned (first run, shallow ``--depth 1``) or fast-forwarded (``git pull --ff-only``) into ``BOR_SOURCES_DIR//`` (default ``~/bor-sources``); local rows are the existing directories themselves, walked directly. A failing clone/pull — or a local directory that is missing at run time — aborts the whole run *before* anything is imported. 3. Fallback — the legacy ``DEFAULT_SOURCES`` (``~/Homelab`` + ``~/Deployments``), kept for backwards compatibility (reached only while both the table and ``BOR_GIT_SOURCES`` are empty). Phase 89: resolution also returns each row's ignore paths, keyed by the resolved root string (the importer normalizes them); manual ``--source`` dirs and the legacy fallback have no rows, so they import with no ignore. Imported formats (PLAN anchor A9, revised; phase 56): the A9 family by default — ``md, markdown, txt, yaml, yml, json, py`` plus the quadlet family and ``j2`` (case-insensitive). ``BOR_IMPORT_EXTENSIONS`` may add ANY well-formed extension or narrow the list (the A9 family is the default, not a ceiling — owner permission 2026-08-31). Any path with a dot-prefixed component (hidden files/dirs — vendored caches) is skipped, along with non-content dirs (``.venv``, ``node_modules``, ``.git``, ``__pycache``, ``.pytest_cache``, ``dist``, ``build``). Re-runs are cheap: files are diffed by sha256 and unchanged ones are not re-embedded; ``--prune`` also drops documents whose files no longer match the format filter. After a run that **changed** the knowledge base (at least one document added or updated — or no outline stored yet), the single ``kb_overview`` row is regenerated with the ``lite`` model (phase 31): the plain-text outline of the KB's basic categories that every chat turn injects into the system prompt as ````. The regeneration is **best-effort and change-gated** — unchanged re-imports and ``--limit`` debug runs never burn a ``lite`` call, and a ``lite`` failure only reports ``overview=failed`` on the summary line: the import's exit code is about files, and the previous outline stays (an old outline is better than none). The stored folder summaries (phase 94 — the drill-down ``ls``'s per-level descriptions, the ``folder_summaries`` table) regenerate in the same run under the same gate: a changed KB (added + updated > 0), or an empty table after an unchanged walk (the first full run after migration 0017, or after a ``--limit`` first walk that skipped them). Same contract — **best-effort, per-folder fail-soft**: a ``lite`` failure keeps the failed folders' previous rows and only counts into the stats; ``--limit`` debug runs skip them entirely (never burning a ``lite`` call). The stats land on the summary line as ``folder_summaries=//`` (or ``folder_summaries=skipped`` when the gate did not fire), and the rows commit in the run's own short-lived session (the phase-53 convention — a failed commit rolls them back with it). A run that **changed** the knowledge base (added + updated + pruned > 0 — phase 53, task 02) also advances the single-row ``sources_meta`` version exactly once (``sources_version=`` on the summary line): the generation saved chats are stamped against, so a sync can no longer silently invalidate a stored answer. The gate is deliberately broader than the overview's — a pruned document can invalidate a saved answer that cited it — and ``--limit`` debug runs (an incomplete walk is debug-only, mirroring the ``--limit`` overview skip) and unchanged re-runs never bump (the line carries ``sources_version=skipped``). """ from __future__ import annotations import argparse import asyncio import logging import re import sys from pathlib import Path from app.config import Settings, get_settings from app.core.debugging import configure_debugging from app.core.logging import configure_logging from app.db import SessionLocal from app.models import KbOverview from app.rag.folder_summaries import folder_summary_table_empty, generate_folder_summaries from app.rag.git_sources import effective_sources from app.rag.importer import ImportSummary, import_sources from app.rag.llm import LLMClient from app.rag.overview import regenerate_overview from app.rag.sources_meta import bump_sources_version from scripts.git_sync import GitSyncError, clone_or_pull logger = logging.getLogger("scripts.import_docs") DEFAULT_SOURCES: list[Path] = [Path("~/Homelab"), Path("~/Deployments")] def build_parser() -> argparse.ArgumentParser: p = argparse.ArgumentParser( prog="python -m scripts.import_docs", description="Import A9-format files (md/txt/yaml/json/py) into the knowledge base.", ) p.add_argument( "--source", action="append", type=Path, metavar="PATH", help=( "directory to import (repeatable; always wins over the git_sources " "DB rows and BOR_GIT_SOURCES; default when neither is given: " "~/Homelab ~/Deployments)" ), ) p.add_argument( "--prune", action="store_true", help="also delete documents whose files no longer exist or match the format filter", ) p.add_argument( "--limit", type=int, default=None, metavar="N", help="only process the first N files (debug; disables --prune)", ) return p def repo_name(url: str) -> str: """Local directory name for a git URL (phase 28). Strips a trailing ``.git`` and takes the basename after the last ``/`` (``:`` for scp-style ``git@host:repo.git`` URLs); falls back to a slug of the whole URL when no usable basename remains. """ name = url.strip() if name.endswith(".git"): name = name[: -len(".git")] base = name.rsplit("/", 1)[-1].rsplit(":", 1)[-1].strip() if base: return base slug = re.sub(r"[^A-Za-z0-9]+", "-", name).strip("-") return slug or "repo" def _resolve_sources( cli_sources: list[Path] | None, settings: Settings ) -> tuple[list[Path], dict[str, list[str]]]: """Resolve the directories to import (phase 28, extended in phases 35 and 38; per-root ignore maps, phase 89). Precedence: ``--source`` (explicit manual paths — always wins) > the effective sources — the ``git_sources`` DB rows (git + local), else the ``BOR_GIT_SOURCES`` git-only fallback (:func:`app.rag.git_sources.effective_sources`; the import needs the database anyway, so resolution opens a short session and there is no DB-down branch) — git rows cloned/pulled via :func:`scripts.git_sync.clone_or_pull` into ``BOR_SOURCES_DIR//``, local rows walked directly (the stored directory, re-verified ``.is_dir()`` at run time) > the legacy ``DEFAULT_SOURCES``. Returns ``(sources, ignore_by_root)`` (phase 89): the map is keyed by the resolved root string, exactly as the importer sees it (two rows sharing a root string get the union — extend, not replace); manual ``--source`` dirs and the legacy fallback have no rows, so they import with an empty map (no ignore). A :class:`GitSyncError` from a failing clone/pull — or a missing local directory (``local source missing: ``) — propagates to :func:`main`, which aborts the run before importing anything. """ if cli_sources: return [path.expanduser() for path in cli_sources], {} db = SessionLocal() try: rows, origin = effective_sources(db) finally: db.close() if rows: git_count = sum(1 for row in rows if row.kind == "git") logger.info( "sources: %d repo(s) git=%d local=%d origin=%s", len(rows), git_count, len(rows) - git_count, origin, ) sources_root = Path(settings.sources_dir).expanduser() sources: list[Path] = [] ignore_by_root: dict[str, list[str]] = {} for row in rows: if row.kind == "git": root = clone_or_pull(row.url, sources_root / repo_name(row.url)) else: # kind=local — the stored expanded path (phase 38 also # mirrors it in the NOT-NULL ``url`` location column, the # ``or`` keeps the type checker honest); a missing # directory aborts before importing, the same pre-import # fail-loud as a failing git clone. root = Path(row.path or row.url).expanduser() if not root.is_dir(): raise GitSyncError(f"local source missing: {root}") sources.append(root) # Phase 89: the row's ignore list, keyed by the SAME root # string the importer sees; two rows sharing a root string # get the union (extend, not replace) — the sibling/repo-name # edge. if row.ignore_paths: ignore_by_root.setdefault(str(root), []).extend(row.ignore_paths) return sources, ignore_by_root return [path.expanduser() for path in DEFAULT_SOURCES], {} def _overview_row_exists() -> bool: """Whether the single ``kb_overview`` row (id = 1) is present. One indexed PK lookup (phase 31, task 04): a missing outline after an unchanged re-import — e.g. the first run after migration 0005 — still gets a fresh outline, while a present one is left untouched until the KB actually changes. """ with SessionLocal() as session: return session.get(KbOverview, 1) is not None def _folder_summaries_table_empty() -> bool: """Whether the ``folder_summaries`` table holds any row (phase 94). The ``_overview_row_exists`` pattern extended to a table-emptiness check (one bounded ``LIMIT 1`` probe): an empty table after an unchanged re-import — e.g. the first full run after migration 0017, or after a ``--limit`` first walk that skipped generation — still gets a fresh batch of folder summaries, while a populated table is left untouched until the KB actually changes. """ with SessionLocal() as session: return folder_summary_table_empty(session) def main(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) settings = get_settings() configure_logging(settings.log_level) configure_debugging() # Git sources resolve (and clone/pull) *before* any import: a failing # repo aborts the run with a non-zero exit, naming the failure — a bad # URL must never silently import partial junk. The second element is # the phase-89 per-root ignore map (empty for manual/fallback paths). try: sources, ignore_by_root = _resolve_sources(args.source, settings) except GitSyncError as e: print(f"import_docs: source sync failed: {e}", file=sys.stderr) return 1 logger.info( "import_docs: importing %d source dir(s): %s", len(sources), ", ".join(str(s) for s in sources), ) missing = [s for s in sources if not s.is_dir()] for s in missing: print(f"import_docs: source dir not found: {s}", file=sys.stderr) if all(not s.is_dir() for s in sources): print("import_docs: no source directories found — nothing to do.", file=sys.stderr) return 1 llm = LLMClient() async def _run() -> tuple[ImportSummary, str, str, dict[str, int] | None]: """Import, then (change-gated) advance the sources version, refresh the stored KB overview, and regenerate the stored folder summaries. One event loop, one ``LLMClient`` (phase 31, task 04): the outline that every chat prompt injects as ```` is regenerated only when this run added/updated at least one document — or when no outline exists yet after a walk that actually saw files (e.g. the first run after migration 0005). ``--limit`` debug runs (an incomplete walk must not rewrite the outline — mirrors the ``--prune``-with-``--limit`` guard) and unchanged re-imports never burn a ``lite`` call, and a ``lite`` failure only flips the status token (``failed``) — the import's exit code is unchanged. The sources version (phase 53, task 02) advances exactly once per run that changed the KB — change-gated on ``added + updated + pruned > 0`` (a pruned document can invalidate a saved answer that cited it, so the gate is deliberately broader than the overview's). ``--limit`` debug runs and unchanged re-runs never bump. The returned token is the new version, or ``"skipped"``. The folder summaries (phase 94, task 02) follow the same gate — a changed KB, or an empty ``folder_summaries`` table after an unchanged walk (the first full run after migration 0017, or after a ``--limit`` first walk) — with per-folder fail-soft inside the generator (a ``lite`` failure keeps the failed folders' previous rows). The generator only flushes: this run's own short-lived session commits (the phase-53 convention), and the stats land on the summary line as ``folder_summaries=//`` (``None`` — rendered ``skipped`` — when the gate did not fire). """ summary = await import_sources( sources, llm, prune=args.prune, limit=args.limit, ignore_by_root=ignore_by_root, ) if args.limit is not None: # An incomplete walk is debug-only — it must never advance # the generation (mirrors the --limit overview skip below). sources_version = "skipped" elif summary.added + summary.updated + summary.pruned > 0: # The KB changed — advance the saved-chat invalidation # marker exactly once, in its own short session (the # best-effort overview below runs in a separate one, so a # failed outline never rolls the bump back). session = SessionLocal() try: new_version = bump_sources_version(session) session.commit() finally: session.close() sources_version = str(new_version) logger.info("sources: version bumped to %d", new_version) else: sources_version = "skipped" logger.info("sources: version bump skipped (KB unchanged)") if args.limit is not None: # An incomplete walk must never rewrite the outline or the # folder summaries (the --limit skip, mirrored above for the # sources version). logger.info("overview: skipped (--limit)") return summary, "skipped", sources_version, None changed = summary.added + summary.updated > 0 overview_due = changed folders_due = changed if not changed: if summary.files == 0: logger.info("overview: skipped (nothing imported)") return summary, "skipped", sources_version, None # The unchanged-walk first-run triggers: the overview when # no row exists yet (the first run after migration 0005), # the folder summaries when the table is empty (the first # full run after migration 0017, or after a --limit first # walk that skipped them). overview_due = not _overview_row_exists() folders_due = _folder_summaries_table_empty() if not overview_due and not folders_due: logger.info("overview: skipped (KB unchanged)") return summary, "skipped", sources_version, None overview_status = "skipped" if overview_due: ok = await regenerate_overview(llm) overview_status = "updated" if ok else "failed" else: logger.info("overview: skipped (KB unchanged)") # Phase 94 (task 02): the folder summaries — same event loop + # LLMClient (phase 31 convention), per-folder fail-soft inside # the generator (a ``lite`` failure never flips the exit code). # It only flushes — this run's own short-lived session commits # (the phase-53 convention), so a failed commit rolls the # summaries back with it. folder_stats: dict[str, int] | None = None if folders_due: session = SessionLocal() try: folder_stats = await generate_folder_summaries(session, llm) session.commit() finally: session.close() return summary, overview_status, sources_version, folder_stats summary, overview_status, sources_version, folder_stats = asyncio.run(_run()) folder_token = ( "skipped" if folder_stats is None else f"{folder_stats['generated']}/{folder_stats['failed']}/{folder_stats['pruned']}" ) print( f"import_docs: files={summary.files} added={summary.added} " f"updated={summary.updated} unchanged={summary.unchanged} " f"pruned={summary.pruned} errors={summary.errors} chunks={summary.chunks} " f"embed_batches={summary.embed_batches} summaries={summary.summaries} " f"summary_errors={summary.summary_errors} formats={summary.format_counts()} " f"overview={overview_status} sources_version={sources_version} " f"folder_summaries={folder_token}" ) # Non-zero if any file failed, so cron/CI notice — the rest of the KB # was imported and the failed files are retried on the next run. The # overview and the folder summaries are best-effort: a failed outline # or a failed folder batch never changes the exit code. return 1 if summary.errors else 0 if __name__ == "__main__": sys.exit(main())