"""Import A9-format directories into the Brain of Reese knowledge base. Examples:: uv run python -m scripts.import_docs # BOR_GIT_SOURCES, else ~/Homelab + ~/Deployments uv run python -m scripts.import_docs --source ~/OtherDocs # explicit dir(s); always wins uv run python -m scripts.import_docs --prune # also drop deleted/out-of-scope files uv run python -m scripts.import_docs --limit 5 # debug: first 5 files only Source resolution (phase 28, extended in phases 35 and 38), in precedence order: 1. ``--source PATH`` — explicit manual directories (repeatable) always win; git and local sources are ignored when this flag is used. 2. The effective sources — the admin-managed ``git_sources`` table rows (both kinds), else the ``BOR_GIT_SOURCES`` (comma-separated) git-only fallback (:func:`app.rag.git_sources.effective_sources`, the same shared resolver the in-app Sync button uses). Git rows are cloned (first run, shallow ``--depth 1``) or fast-forwarded (``git pull --ff-only``) into ``BOR_SOURCES_DIR//`` (default ``~/bor-sources``); local rows are the existing directories themselves, walked directly. A failing clone/pull — or a local directory that is missing at run time — aborts the whole run *before* anything is imported. 3. Fallback — the legacy ``DEFAULT_SOURCES`` (``~/Homelab`` + ``~/Deployments``), kept for backwards compatibility (reached only while both the table and ``BOR_GIT_SOURCES`` are empty). Phase 89: resolution also returns each row's ignore paths, keyed by the resolved root string (the importer normalizes them); manual ``--source`` dirs and the legacy fallback have no rows, so they import with no ignore. Phase 105 extends the same resolution with each row's ``include_hidden`` flag — a second per-root map keyed by the same root strings (the importer reads it per root); manual ``--source`` dirs and the legacy fallback have no rows, so they import with the empty map (hidden paths skipped — A4). Imported formats (PLAN anchor A9, revised; phase 56): the A9 family by default — ``md, markdown, txt, yaml, yml, json, py`` plus the quadlet family and ``j2`` (case-insensitive). ``BOR_IMPORT_EXTENSIONS`` may add ANY well-formed extension or narrow the list (the A9 family is the default, not a ceiling — owner permission 2026-08-31). Any path with a dot-prefixed component (hidden files/dirs — vendored caches) is skipped, along with non-content dirs (``.venv``, ``node_modules``, ``.git``, ``__pycache``, ``.pytest_cache``, ``dist``, ``build``). Re-runs are cheap: files are diffed by sha256 and unchanged ones are not re-embedded; ``--prune`` also drops documents whose files no longer match the format filter. After a run that **changed** the knowledge base (at least one document added or updated — or no outline stored yet), the single ``kb_overview`` row is regenerated with the ``lite`` model (phase 31): the plain-text outline of the KB's basic categories that every chat turn injects into the system prompt as ````. The regeneration is **best-effort and change-gated** — unchanged re-imports and ``--limit`` debug runs never burn a ``lite`` call, and a ``lite`` failure only reports ``overview=failed`` on the summary line: the import's exit code is about files, and the previous outline stays (an old outline is better than none). The stored folder summaries (phase 94 — the drill-down ``ls``'s per-level descriptions, the ``folder_summaries`` table) regenerate in the same run under the same gate: a changed KB (added + updated > 0 — full regeneration), or, after an unchanged walk, a GAP — a candidate folder (≥ 2 docs) with no stored row (phase 96: this subsumes the old table-empty trigger exactly — an empty table leaves EVERY candidate missing, as after the first full run after migration 0017 or a ``--limit`` first walk that skipped them — and catches the single row an exhausted one-shot retry lost mid-run). A gap after an unchanged walk fills ONLY the missing rows (``only_missing`` — every other row stays byte-identical, summary text AND ``updated_at``), and the run's stats token carries `` (gap-fill)`` behind the numbers. Same contract — **best-effort, per-folder fail-soft**: a ``lite`` failure keeps the failed folders' previous rows (or leaves the row absent) and only counts into the stats; ``--limit`` debug runs skip them entirely (never burning a ``lite`` call). The stats land on the summary line as ``folder_summaries=//`` (`` (gap-fill)`` appended after the stats when the run took the targeted-fill path; ``folder_summaries=skipped`` when the gate did not fire), and the rows commit in the run's own short-lived session (the phase-53 convention — a failed commit rolls them back with it). A run that **changed** the knowledge base (added + updated + pruned > 0 — phase 53, task 02) also advances the single-row ``sources_meta`` version exactly once (``sources_version=`` on the summary line): the generation saved chats are stamped against, so a sync can no longer silently invalidate a stored answer. The gate is deliberately broader than the overview's — a pruned document can invalidate a saved answer that cited it — and ``--limit`` debug runs (an incomplete walk is debug-only, mirroring the ``--limit`` overview skip) and unchanged re-runs never bump (the line carries ``sources_version=skipped``). """ from __future__ import annotations import argparse import asyncio import logging import re import sys from pathlib import Path from app.config import Settings, get_settings from app.core.debugging import configure_debugging from app.core.logging import configure_logging from app.db import SessionLocal from app.models import KbOverview from app.rag.folder_summaries import generate_folder_summaries, missing_folder_summaries from app.rag.git_sources import effective_sources from app.rag.importer import ImportSummary, import_sources from app.rag.llm import LLMClient from app.rag.overview import regenerate_overview from app.rag.sources_meta import bump_sources_version from scripts.git_sync import GitSyncError, clone_or_pull logger = logging.getLogger("scripts.import_docs") DEFAULT_SOURCES: list[Path] = [Path("~/Homelab"), Path("~/Deployments")] def build_parser() -> argparse.ArgumentParser: p = argparse.ArgumentParser( prog="python -m scripts.import_docs", description="Import A9-format files (md/txt/yaml/json/py) into the knowledge base.", ) p.add_argument( "--source", action="append", type=Path, metavar="PATH", help=( "directory to import (repeatable; always wins over the git_sources " "DB rows and BOR_GIT_SOURCES; default when neither is given: " "~/Homelab ~/Deployments)" ), ) p.add_argument( "--prune", action="store_true", help="also delete documents whose files no longer exist or match the format filter", ) p.add_argument( "--limit", type=int, default=None, metavar="N", help="only process the first N files (debug; disables --prune)", ) return p def repo_name(url: str) -> str: """Local directory name for a git URL (phase 28). Strips a trailing ``.git`` and takes the basename after the last ``/`` (``:`` for scp-style ``git@host:repo.git`` URLs); falls back to a slug of the whole URL when no usable basename remains. """ name = url.strip() if name.endswith(".git"): name = name[: -len(".git")] base = name.rsplit("/", 1)[-1].rsplit(":", 1)[-1].strip() if base: return base slug = re.sub(r"[^A-Za-z0-9]+", "-", name).strip("-") return slug or "repo" def _resolve_sources( cli_sources: list[Path] | None, settings: Settings ) -> tuple[list[Path], dict[str, list[str]], dict[str, bool]]: """Resolve the directories to import (phase 28, extended in phases 35 and 38; per-root ignore maps, phase 89; per-root hidden-folders flag maps, phase 105). Precedence: ``--source`` (explicit manual paths — always wins) > the effective sources — the ``git_sources`` DB rows (git + local), else the ``BOR_GIT_SOURCES`` git-only fallback (:func:`app.rag.git_sources.effective_sources`; the import needs the database anyway, so resolution opens a short session and there is no DB-down branch) — git rows cloned/pulled via :func:`scripts.git_sync.clone_or_pull` into ``BOR_SOURCES_DIR//``, local rows walked directly (the stored directory, re-verified ``.is_dir()`` at run time) > the legacy ``DEFAULT_SOURCES``. Returns ``(sources, ignore_by_root, include_hidden_by_root)`` (phase 89; phase 105 adds the per-root flag map — the flag is stored per row, manual ``--source`` dirs and the legacy fallback have no rows and import with the empty map: hidden paths skipped, A4): both maps are keyed by the resolved root string, exactly as the importer sees it (two rows sharing a root string get the union — extend, not replace — for the ignore lists, and the OR of their flags for the hidden map); manual ``--source`` dirs and the legacy fallback have no rows, so they import with empty maps (no ignore, hidden skipped). A :class:`GitSyncError` from a failing clone/pull — or a missing local directory (``local source missing: ``) — propagates to :func:`main`, which aborts the run before importing anything. """ if cli_sources: return [path.expanduser() for path in cli_sources], {}, {} db = SessionLocal() try: rows, origin = effective_sources(db) finally: db.close() if rows: git_count = sum(1 for row in rows if row.kind == "git") logger.info( "sources: %d repo(s) git=%d local=%d origin=%s", len(rows), git_count, len(rows) - git_count, origin, ) sources_root = Path(settings.sources_dir).expanduser() sources: list[Path] = [] ignore_by_root: dict[str, list[str]] = {} include_hidden_by_root: dict[str, bool] = {} for row in rows: if row.kind == "git": root = clone_or_pull(row.url, sources_root / repo_name(row.url)) else: # kind=local — the stored expanded path (phase 38 also # mirrors it in the NOT-NULL ``url`` location column, the # ``or`` keeps the type checker honest); a missing # directory aborts before importing, the same pre-import # fail-loud as a failing git clone. root = Path(row.path or row.url).expanduser() if not root.is_dir(): raise GitSyncError(f"local source missing: {root}") sources.append(root) # Phase 89: the row's ignore list, keyed by the SAME root # string the importer sees; two rows sharing a root string # get the union (extend, not replace) — the sibling/repo-name # edge. if row.ignore_paths: ignore_by_root.setdefault(str(root), []).extend(row.ignore_paths) # Phase 105 (A1/A4): the row's hidden-folders flag, keyed by # the SAME root string the importer sees; a shared-root # collision ORs — if EITHER row says "index hidden", the # root does (the ignore-map union's boolean mirror). include_hidden_by_root[str(root)] = ( include_hidden_by_root.get(str(root), False) or bool(row.include_hidden) ) return sources, ignore_by_root, include_hidden_by_root return [path.expanduser() for path in DEFAULT_SOURCES], {}, {} def _overview_row_exists() -> bool: """Whether the single ``kb_overview`` row (id = 1) is present. One indexed PK lookup (phase 31, task 04): a missing outline after an unchanged re-import — e.g. the first run after migration 0005 — still gets a fresh outline, while a present one is left untouched until the KB actually changes. """ with SessionLocal() as session: return session.get(KbOverview, 1) is not None def _folder_summaries_gap() -> list[tuple[str, str]]: """The folder-summary gaps (phase 96, task 03): the candidate folders (≥ 2 docs) with no stored row, sorted. The unchanged-walk self-heal trigger, replacing the phase-94 table-emptiness probe (which the gap subsumes exactly: an empty table leaves every candidate missing, so the targeted fill over all candidates IS a full generation — the first full run after migration 0017, or after a ``--limit`` first walk that skipped generation, still generates — and a single row an exhausted one-shot retry lost mid-run is healed on the next sync). """ with SessionLocal() as session: return missing_folder_summaries(session) def main(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) settings = get_settings() configure_logging(settings.log_level) configure_debugging() # Git sources resolve (and clone/pull) *before* any import: a failing # repo aborts the run with a non-zero exit, naming the failure — a bad # URL must never silently import partial junk. The second element is # the phase-89 per-root ignore map and the third the phase-105 # per-root hidden-folders flag map (both empty for manual/fallback # paths). try: sources, ignore_by_root, include_hidden_by_root = _resolve_sources( args.source, settings ) except GitSyncError as e: print(f"import_docs: source sync failed: {e}", file=sys.stderr) return 1 logger.info( "import_docs: importing %d source dir(s): %s", len(sources), ", ".join(str(s) for s in sources), ) missing = [s for s in sources if not s.is_dir()] for s in missing: print(f"import_docs: source dir not found: {s}", file=sys.stderr) if all(not s.is_dir() for s in sources): print("import_docs: no source directories found — nothing to do.", file=sys.stderr) return 1 llm = LLMClient() async def _run( ) -> tuple[ImportSummary, str, str, dict[str, int] | None, bool]: """Import, then (change-gated) advance the sources version, refresh the stored KB overview, and regenerate the stored folder summaries. One event loop, one ``LLMClient`` (phase 31, task 04): the outline that every chat prompt injects as ```` is regenerated only when this run added/updated at least one document — or when no outline exists yet after a walk that actually saw files (e.g. the first run after migration 0005). ``--limit`` debug runs (an incomplete walk must not rewrite the outline — mirrors the ``--prune``-with-``--limit`` guard) and unchanged re-imports never burn a ``lite`` call, and a ``lite`` failure only flips the status token (``failed``) — the import's exit code is unchanged. The sources version (phase 53, task 02) advances exactly once per run that changed the KB — change-gated on ``added + updated + pruned > 0`` (a pruned document can invalidate a saved answer that cited it, so the gate is deliberately broader than the overview's). ``--limit`` debug runs and unchanged re-runs never bump. The returned token is the new version, or ``"skipped"``. The folder summaries (phase 94, task 02; phase 96, task 03) follow the same gate — a changed KB (full regeneration), or, after an unchanged walk, a GAP: a candidate folder (≥ 2 docs) with no stored row (the subsumed table-empty trigger — an empty table leaves every candidate missing — plus a row an exhausted one-shot retry lost) — with per-folder fail-soft inside the generator (a ``lite`` failure keeps the failed folders' previous rows or leaves the row absent). The gap path passes ``only_missing=True`` (existing rows stay byte-identical) and its stats token gains the `` (gap-fill)`` suffix on the summary line. The generator only flushes: this run's own short-lived session commits (the phase-53 convention), and the stats land on the summary line as ``folder_summaries=//`` (``None`` — rendered ``skipped`` — when the gate did not fire). The returned fifth element names the mode the stats were taken in (the `` (gap-fill)`` suffix trigger — ``True`` only when the unchanged-walk gap fired the targeted fill). """ summary = await import_sources( sources, llm, prune=args.prune, limit=args.limit, ignore_by_root=ignore_by_root, include_hidden_by_root=include_hidden_by_root, ) if args.limit is not None: # An incomplete walk is debug-only — it must never advance # the generation (mirrors the --limit overview skip below). sources_version = "skipped" elif summary.added + summary.updated + summary.pruned > 0: # The KB changed — advance the saved-chat invalidation # marker exactly once, in its own short session (the # best-effort overview below runs in a separate one, so a # failed outline never rolls the bump back). session = SessionLocal() try: new_version = bump_sources_version(session) session.commit() finally: session.close() sources_version = str(new_version) logger.info("sources: version bumped to %d", new_version) else: sources_version = "skipped" logger.info("sources: version bump skipped (KB unchanged)") if args.limit is not None: # An incomplete walk must never rewrite the outline or the # folder summaries (the --limit skip, mirrored above for the # sources version). logger.info("overview: skipped (--limit)") return summary, "skipped", sources_version, None, False changed = summary.added + summary.updated > 0 overview_due = changed folders_due = changed folder_gap_fill = False if not changed: if summary.files == 0: logger.info("overview: skipped (nothing imported)") return summary, "skipped", sources_version, None, False # The unchanged-walk triggers: the overview when no row # exists yet (the first run after migration 0005), the # folder summaries when the table has a GAP — a candidate # folder (>= 2 docs) with no stored row (phase 96, task # 03; the old table-empty trigger is the special case # where every candidate is missing). The gap path is # ALWAYS the targeted fill: only the missing rows # regenerate (only_missing=True), every other row stays # byte-identical. overview_due = not _overview_row_exists() folders_due = bool(_folder_summaries_gap()) folder_gap_fill = folders_due if not overview_due and not folders_due: logger.info("overview: skipped (KB unchanged)") return summary, "skipped", sources_version, None, False overview_status = "skipped" if overview_due: ok = await regenerate_overview(llm) overview_status = "updated" if ok else "failed" else: logger.info("overview: skipped (KB unchanged)") # Phase 94 (task 02): the folder summaries — same event loop + # LLMClient (phase 31 convention), per-folder fail-soft inside # the generator (a ``lite`` failure never flips the exit code). # It only flushes — this run's own short-lived session commits # (the phase-53 convention), so a failed commit rolls the # summaries back with it. folder_stats: dict[str, int] | None = None if folders_due: # Phase 96 (task 03): a changed-KB run is a full # regeneration (today's behavior, byte-identical); the # unchanged-walk gap run is the targeted fill (only the # missing candidates burn a lite call). session = SessionLocal() try: folder_stats = await generate_folder_summaries( session, llm, only_missing=folder_gap_fill ) session.commit() finally: session.close() return ( summary, overview_status, sources_version, folder_stats, folder_gap_fill, ) ( summary, overview_status, sources_version, folder_stats, folder_gap_fill, ) = asyncio.run(_run()) folder_token = ( "skipped" if folder_stats is None else f"{folder_stats['generated']}/{folder_stats['failed']}/{folder_stats['pruned']}" ) if folder_stats is not None and folder_gap_fill: # PLAN §9 greppable-cron-safe line — the line-extension house # rule: the targeted fill (phase 96, task 03) is named on the # summary line; the full-regeneration token stays # byte-identical to phase 94. folder_token += " (gap-fill)" print( f"import_docs: files={summary.files} added={summary.added} " f"updated={summary.updated} unchanged={summary.unchanged} " f"pruned={summary.pruned} errors={summary.errors} chunks={summary.chunks} " f"embed_batches={summary.embed_batches} summaries={summary.summaries} " f"summary_errors={summary.summary_errors} formats={summary.format_counts()} " f"overview={overview_status} sources_version={sources_version} " f"folder_summaries={folder_token}" ) # Non-zero if any file failed, so cron/CI notice — the rest of the KB # was imported and the failed files are retried on the next run. The # overview and the folder summaries are best-effort: a failed outline # or a failed folder batch never changes the exit code. return 1 if summary.errors else 0 if __name__ == "__main__": sys.exit(main())