brain-of-reese/scripts/import_docs.py

"""Import A9-format directories into the Brain of Reese knowledge base.

Examples::

    uv run python -m scripts.import_docs      # BOR_GIT_SOURCES, else ~/Homelab + ~/Deployments
    uv run python -m scripts.import_docs --source ~/OtherDocs # explicit dir(s); always wins
    uv run python -m scripts.import_docs --prune              # also drop deleted/out-of-scope files
    uv run python -m scripts.import_docs --limit 5            # debug: first 5 files only

Source resolution (phase 28, extended in phases 35 and 38), in
precedence order:

1. ``--source PATH`` — explicit manual directories (repeatable) always
   win; git and local sources are ignored when this flag is used.
2. The effective sources — the admin-managed ``git_sources`` table rows
   (both kinds), else the ``BOR_GIT_SOURCES`` (comma-separated) git-only
   fallback (:func:`app.rag.git_sources.effective_sources`, the same
   shared resolver the in-app Sync button uses). Git rows are cloned
   (first run, shallow ``--depth 1``) or fast-forwarded
   (``git pull --ff-only``) into ``BOR_SOURCES_DIR/<repo-name>/``
   (default ``~/bor-sources``); local rows are the existing directories
   themselves, walked directly. A failing clone/pull — or a local
   directory that is missing at run time — aborts the whole run
   *before* anything is imported.
3. Fallback — the legacy ``DEFAULT_SOURCES`` (``~/Homelab`` +
   ``~/Deployments``), kept for backwards compatibility (reached only
   while both the table and ``BOR_GIT_SOURCES`` are empty).

Phase 89: resolution also returns each row's ignore paths, keyed by
the resolved root string (the importer normalizes them); manual
``--source`` dirs and the legacy fallback have no rows, so they import
with no ignore.

Imported formats (PLAN anchor A9, revised; phase 56): the A9 family by
default — ``md, markdown, txt, yaml, yml, json, py`` plus the quadlet
family and ``j2`` (case-insensitive). ``BOR_IMPORT_EXTENSIONS`` may add
ANY well-formed extension or narrow the list (the A9 family is the
default, not a ceiling — owner permission 2026-08-31).
Any path with a dot-prefixed component (hidden files/dirs — vendored
caches) is skipped, along with non-content dirs (``.venv``,
``node_modules``, ``.git``, ``__pycache``, ``.pytest_cache``, ``dist``,
``build``). Re-runs are cheap: files are diffed by sha256 and unchanged
ones are not re-embedded; ``--prune`` also drops documents whose files no
longer match the format filter.

After a run that **changed** the knowledge base (at least one document
added or updated — or no outline stored yet), the single ``kb_overview``
row is regenerated with the ``lite`` model (phase 31): the plain-text
outline of the KB's basic categories that every chat turn injects into
the system prompt as ``<knowledge_base>``. The regeneration is
**best-effort and change-gated** — unchanged re-imports and ``--limit``
debug runs never burn a ``lite`` call, and a ``lite`` failure only
reports ``overview=failed`` on the summary line: the import's exit code
is about files, and the previous outline stays (an old outline is better
than none).

A run that **changed** the knowledge base (added + updated + pruned > 0
— phase 53, task 02) also advances the single-row ``sources_meta``
version exactly once (``sources_version=<n>`` on the summary line): the
generation saved chats are stamped against, so a sync can no longer
silently invalidate a stored answer. The gate is deliberately broader
than the overview's — a pruned document can invalidate a saved answer
that cited it — and ``--limit`` debug runs (an incomplete walk is
debug-only, mirroring the ``--limit`` overview skip) and unchanged
re-runs never bump (the line carries ``sources_version=skipped``).
"""
from __future__ import annotations

import argparse
import asyncio
import logging
import re
import sys
from pathlib import Path

from app.config import Settings, get_settings
from app.core.debugging import configure_debugging
from app.core.logging import configure_logging
from app.db import SessionLocal
from app.models import KbOverview
from app.rag.git_sources import effective_sources
from app.rag.importer import ImportSummary, import_sources
from app.rag.llm import LLMClient
from app.rag.overview import regenerate_overview
from app.rag.sources_meta import bump_sources_version
from scripts.git_sync import GitSyncError, clone_or_pull

logger = logging.getLogger("scripts.import_docs")

DEFAULT_SOURCES: list[Path] = [Path("~/Homelab"), Path("~/Deployments")]


def build_parser() -> argparse.ArgumentParser:
    p = argparse.ArgumentParser(
        prog="python -m scripts.import_docs",
        description="Import A9-format files (md/txt/yaml/json/py) into the knowledge base.",
    )
    p.add_argument(
        "--source",
        action="append",
        type=Path,
        metavar="PATH",
        help=(
            "directory to import (repeatable; always wins over the git_sources "
            "DB rows and BOR_GIT_SOURCES; default when neither is given: "
            "~/Homelab ~/Deployments)"
        ),
    )
    p.add_argument(
        "--prune",
        action="store_true",
        help="also delete documents whose files no longer exist or match the format filter",
    )
    p.add_argument(
        "--limit",
        type=int,
        default=None,
        metavar="N",
        help="only process the first N files (debug; disables --prune)",
    )
    return p


def repo_name(url: str) -> str:
    """Local directory name for a git URL (phase 28).

    Strips a trailing ``.git`` and takes the basename after the last ``/``
    (``:`` for scp-style ``git@host:repo.git`` URLs); falls back to a slug
    of the whole URL when no usable basename remains.
    """
    name = url.strip()
    if name.endswith(".git"):
        name = name[: -len(".git")]
    base = name.rsplit("/", 1)[-1].rsplit(":", 1)[-1].strip()
    if base:
        return base
    slug = re.sub(r"[^A-Za-z0-9]+", "-", name).strip("-")
    return slug or "repo"


def _resolve_sources(
    cli_sources: list[Path] | None, settings: Settings
) -> tuple[list[Path], dict[str, list[str]]]:
    """Resolve the directories to import (phase 28, extended in phases
    35 and 38; per-root ignore maps, phase 89).

    Precedence: ``--source`` (explicit manual paths — always wins) >
    the effective sources — the ``git_sources`` DB rows (git + local),
    else the ``BOR_GIT_SOURCES`` git-only fallback
    (:func:`app.rag.git_sources.effective_sources`; the import needs the
    database anyway, so resolution opens a short session and there is
    no DB-down branch) — git rows cloned/pulled via
    :func:`scripts.git_sync.clone_or_pull` into
    ``BOR_SOURCES_DIR/<repo-name>/``, local rows walked directly (the
    stored directory, re-verified ``.is_dir()`` at run time) > the
    legacy ``DEFAULT_SOURCES``.

    Returns ``(sources, ignore_by_root)`` (phase 89): the map is keyed
    by the resolved root string, exactly as the importer sees it (two
    rows sharing a root string get the union — extend, not replace);
    manual ``--source`` dirs and the legacy fallback have no rows, so
    they import with an empty map (no ignore).

    A :class:`GitSyncError` from a failing clone/pull — or a missing
    local directory (``local source missing: <path>``) — propagates to
    :func:`main`, which aborts the run before importing anything.
    """
    if cli_sources:
        return [path.expanduser() for path in cli_sources], {}
    db = SessionLocal()
    try:
        rows, origin = effective_sources(db)
    finally:
        db.close()
    if rows:
        git_count = sum(1 for row in rows if row.kind == "git")
        logger.info(
            "sources: %d repo(s) git=%d local=%d origin=%s",
            len(rows), git_count, len(rows) - git_count, origin,
        )
        sources_root = Path(settings.sources_dir).expanduser()
        sources: list[Path] = []
        ignore_by_root: dict[str, list[str]] = {}
        for row in rows:
            if row.kind == "git":
                root = clone_or_pull(row.url, sources_root / repo_name(row.url))
            else:
                # kind=local — the stored expanded path (phase 38 also
                # mirrors it in the NOT-NULL ``url`` location column, the
                # ``or`` keeps the type checker honest); a missing
                # directory aborts before importing, the same pre-import
                # fail-loud as a failing git clone.
                root = Path(row.path or row.url).expanduser()
                if not root.is_dir():
                    raise GitSyncError(f"local source missing: {root}")
            sources.append(root)
            # Phase 89: the row's ignore list, keyed by the SAME root
            # string the importer sees; two rows sharing a root string
            # get the union (extend, not replace) — the sibling/repo-name
            # edge.
            if row.ignore_paths:
                ignore_by_root.setdefault(str(root), []).extend(row.ignore_paths)
        return sources, ignore_by_root
    return [path.expanduser() for path in DEFAULT_SOURCES], {}


def _overview_row_exists() -> bool:
    """Whether the single ``kb_overview`` row (id = 1) is present.

    One indexed PK lookup (phase 31, task 04): a missing outline after an
    unchanged re-import — e.g. the first run after migration 0005 — still
    gets a fresh outline, while a present one is left untouched until the
    KB actually changes.
    """
    with SessionLocal() as session:
        return session.get(KbOverview, 1) is not None


def main(argv: list[str] | None = None) -> int:
    args = build_parser().parse_args(argv)
    settings = get_settings()
    configure_logging(settings.log_level)
    configure_debugging()

    # Git sources resolve (and clone/pull) *before* any import: a failing
    # repo aborts the run with a non-zero exit, naming the failure — a bad
    # URL must never silently import partial junk. The second element is
    # the phase-89 per-root ignore map (empty for manual/fallback paths).
    try:
        sources, ignore_by_root = _resolve_sources(args.source, settings)
    except GitSyncError as e:
        print(f"import_docs: source sync failed: {e}", file=sys.stderr)
        return 1

    logger.info(
        "import_docs: importing %d source dir(s): %s",
        len(sources),
        ", ".join(str(s) for s in sources),
    )
    missing = [s for s in sources if not s.is_dir()]
    for s in missing:
        print(f"import_docs: source dir not found: {s}", file=sys.stderr)
    if all(not s.is_dir() for s in sources):
        print("import_docs: no source directories found — nothing to do.", file=sys.stderr)
        return 1

    llm = LLMClient()

    async def _run() -> tuple[ImportSummary, str, str]:
        """Import, then (change-gated) advance the sources version and
        refresh the stored KB overview.

        One event loop, one ``LLMClient`` (phase 31, task 04): the
        outline that every chat prompt injects as ``<knowledge_base>`` is
        regenerated only when this run added/updated at least one
        document — or when no outline exists yet after a walk that
        actually saw files (e.g. the first run after migration 0005).
        ``--limit`` debug runs (an incomplete walk must not rewrite the
        outline — mirrors the ``--prune``-with-``--limit`` guard) and
        unchanged re-imports never burn a ``lite`` call, and a ``lite``
        failure only flips the status token (``failed``) — the import's
        exit code is unchanged.

        The sources version (phase 53, task 02) advances exactly once
        per run that changed the KB — change-gated on
        ``added + updated + pruned > 0`` (a pruned document can
        invalidate a saved answer that cited it, so the gate is
        deliberately broader than the overview's). ``--limit`` debug
        runs and unchanged re-runs never bump. The returned token is
        the new version, or ``"skipped"``.
        """
        summary = await import_sources(
            sources, llm, prune=args.prune, limit=args.limit,
            ignore_by_root=ignore_by_root,
        )
        if args.limit is not None:
            # An incomplete walk is debug-only — it must never advance
            # the generation (mirrors the --limit overview skip below).
            sources_version = "skipped"
        elif summary.added + summary.updated + summary.pruned > 0:
            # The KB changed — advance the saved-chat invalidation
            # marker exactly once, in its own short session (the
            # best-effort overview below runs in a separate one, so a
            # failed outline never rolls the bump back).
            session = SessionLocal()
            try:
                new_version = bump_sources_version(session)
                session.commit()
            finally:
                session.close()
            sources_version = str(new_version)
            logger.info("sources: version bumped to %d", new_version)
        else:
            sources_version = "skipped"
            logger.info("sources: version bump skipped (KB unchanged)")
        if args.limit is not None:
            logger.info("overview: skipped (--limit)")
            return summary, "skipped", sources_version
        if summary.added + summary.updated == 0:
            if summary.files == 0:
                logger.info("overview: skipped (nothing imported)")
                return summary, "skipped", sources_version
            if _overview_row_exists():
                logger.info("overview: skipped (KB unchanged)")
                return summary, "skipped", sources_version
            # No outline yet after an unchanged re-import (e.g. the first
            # run after migration 0005) — fall through and generate one.
        ok = await regenerate_overview(llm)
        return summary, "updated" if ok else "failed", sources_version

    summary, overview_status, sources_version = asyncio.run(_run())
    print(
        f"import_docs: files={summary.files} added={summary.added} "
        f"updated={summary.updated} unchanged={summary.unchanged} "
        f"pruned={summary.pruned} errors={summary.errors} chunks={summary.chunks} "
        f"embed_batches={summary.embed_batches} summaries={summary.summaries} "
        f"summary_errors={summary.summary_errors} formats={summary.format_counts()} "
        f"overview={overview_status} sources_version={sources_version}"
    )
    # Non-zero if any file failed, so cron/CI notice — the rest of the KB
    # was imported and the failed files are retried on the next run. The
    # overview is best-effort: a failed outline never changes the exit code.
    return 1 if summary.errors else 0


if __name__ == "__main__":
    sys.exit(main())