brain-of-reese/tests/integration/test_import_docs_git.py

"""Integration test: ``import_docs`` source resolution (phase 28, task
03; phase 35 re-points at the shared resolver; phase 38 adds the local
kind).

Drives ``scripts.import_docs`` end to end with a fake ``clone_or_pull``
(no real git, no network) and a recording fake ``import_sources`` (no
real DB), covering: the phase-53 version bump is stubbed in the
``main()`` tests the same way (the ``sources_version=`` summary token
is asserted against the canned value).

- Effective sources set (phase 35: the shared resolver — stubbed here,
  keeping this file's no-real-DB style) → each git URL is cloned/pulled
  into ``BOR_SOURCES_DIR/<repo-name>/``; local rows are their existing
  directories, walked directly; exactly those dirs are imported.
- DB rows (both kinds) win over ``BOR_GIT_SOURCES`` (the resolver's
  ``db`` origin — the env list is ignored; the env fallback stays
  git-only).
- ``--source`` still wins over the DB rows (no git at all, no resolver
  call).
- No sources configured + no ``--source`` → the legacy
  ``DEFAULT_SOURCES``.
- A failing git sync → exit code 1, an error naming the failing repo on
  stderr, and **zero** import attempts.
- A missing local directory → the same pre-import fail-loud: exit code
  1, ``local source missing: <path>`` on stderr, zero import attempts.
"""
from __future__ import annotations

import re
from pathlib import Path

import pytest

from app.config import Settings
from app.models import GitSource
from app.rag.importer import ImportSummary
from scripts import import_docs
from scripts.git_sync import GitSyncError


def _settings(git_sources: str = "", sources_dir: str = "~/bor-sources") -> Settings:
    """Fresh settings (no .env file); explicit kwargs beat any env leaks."""
    return Settings(_env_file=None, git_sources=git_sources, sources_dir=sources_dir)  # pyright: ignore[reportCallIssue]


def _git_row(url: str, ignore_paths: list[str] | None = None) -> GitSource:
    return GitSource(url=url, kind="git", ignore_paths=ignore_paths or [])


def _local_row(path: str, ignore_paths: list[str] | None = None) -> GitSource:
    """A local row as the phase-38 API stores it: the expanded path in
    both ``path`` and the NOT-NULL ``url`` location column (plus the
    phase-89 ignore list, empty by default)."""
    return GitSource(url=path, kind="local", path=path, ignore_paths=ignore_paths or [])


class FakeImportSources:
    """Records every ``import_sources`` call instead of touching a DB."""

    def __init__(self) -> None:
        self.calls: list[dict] = []

    async def __call__(
        self,
        sources: list[Path],
        llm: object,
        *,
        prune: bool = False,
        limit: int | None = None,
        ignore_by_root: dict[str, list[str]] | None = None,  # phase 89
    ) -> ImportSummary:
        self.calls.append(
            {"sources": list(sources), "prune": prune, "limit": limit,
             "ignore_by_root": ignore_by_root}
        )
        return ImportSummary(files=1, added=1)


def _fake_clone_factory() -> tuple[list[tuple[str, Path]], object]:
    """A ``clone_or_pull`` that materialises a checkout with one .md file."""
    calls: list[tuple[str, Path]] = []

    def fake_clone_or_pull(url: str, dest: Path | str) -> Path:
        dest = Path(dest)
        dest.mkdir(parents=True, exist_ok=True)
        (dest / "notes.md").write_text(f"# {dest.name}\ncontent for the KB\n", encoding="utf-8")
        calls.append((url, dest))
        return dest

    return calls, fake_clone_or_pull


def _stub_bump(monkeypatch: pytest.MonkeyPatch) -> list[None]:
    """Stub the phase-53 version bump (this file keeps its no-real-DB
    style for the counter — the fake import already avoids the KB
    tables). Returns the call record; the canned new version is 1."""
    bumps: list[None] = []

    def fake_bump(session: object) -> int:
        bumps.append(None)
        return 1

    monkeypatch.setattr(import_docs, "bump_sources_version", fake_bump)
    return bumps


def _stub_folder_summaries(monkeypatch: pytest.MonkeyPatch) -> list[dict]:
    """Stub the phase-94 folder-summary regeneration (this file keeps
    its no-real-DB / no-network style — the real generator would read
    the global ``documents`` table and burn ``lite`` calls). Returns
    the call record; the canned stats are the zero dict."""
    calls: list[dict] = []

    async def fake_generate(
        db: object, llm: object, *, skip: bool = False, only_missing: bool = False
    ) -> dict[str, int]:
        calls.append({"skip": skip, "only_missing": only_missing})
        return {"generated": 0, "failed": 0, "pruned": 0}

    monkeypatch.setattr(import_docs, "generate_folder_summaries", fake_generate)
    return calls


# --- repo_name -------------------------------------------------------------


@pytest.mark.parametrize(
    ("url", "name"),
    [
        ("https://github.com/user/homelab.git", "homelab"),
        ("https://github.com/user/homelab", "homelab"),
        ("git@github.com:user/homelab.git", "homelab"),
        ("git@github.com:homelab.git", "homelab"),
        ("ssh://git@host:2222/group/deployments.git", "deployments"),
        ("https://git.reeseapps.com:8443/proj/notes", "notes"),
    ],
)
def test_repo_name(url: str, name: str) -> None:
    assert import_docs.repo_name(url) == name


def test_repo_name_slug_fallback() -> None:
    # No usable basename (path ends in the .git suffix itself) → slug.
    assert import_docs.repo_name("https://host/.git") == "https-host"


# --- _resolve_sources ------------------------------------------------------


def test_resolve_sources_git_urls_cloned_into_sources_dir(
    monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
    calls, fake = _fake_clone_factory()
    monkeypatch.setattr(import_docs, "clone_or_pull", fake)
    # Phase 35: resolution goes through the shared resolver (stubbed —
    # this file keeps its no-real-DB style); the rows are the env list,
    # surfaced as synthetic git rows.
    monkeypatch.setattr(
        import_docs,
        "effective_sources",
        lambda db: (
            [_git_row("https://host/a/homelab.git"), _git_row("git@host:user/deploy.git")],
            "env",
        ),
    )
    settings = _settings(
        git_sources="https://host/a/homelab.git, git@host:user/deploy.git ,",
        sources_dir=str(tmp_path / "bor"),
    )

    sources, ignore_map = import_docs._resolve_sources(None, settings)

    assert sources == [tmp_path / "bor" / "homelab", tmp_path / "bor" / "deploy"]
    assert ignore_map == {}  # phase 89: no row carries a list → empty map
    assert calls == [
        ("https://host/a/homelab.git", tmp_path / "bor" / "homelab"),
        ("git@host:user/deploy.git", tmp_path / "bor" / "deploy"),
    ]


def test_resolve_sources_cli_source_wins(
    monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
    calls, fake = _fake_clone_factory()
    monkeypatch.setattr(import_docs, "clone_or_pull", fake)
    settings = _settings(git_sources="https://host/a/repo.git")
    manual = tmp_path / "Manual"

    sources, ignore_map = import_docs._resolve_sources([manual], settings)

    assert sources == [manual]
    assert ignore_map == {}  # phase 89: manual dirs have no rows → no ignore
    assert calls == []  # git is never touched when --source is given


def test_resolve_sources_db_rows_win_over_env(
    monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
    """Phase 35: the resolver's ``db`` origin (table has rows) — the
    ``BOR_GIT_SOURCES`` list must be ignored; only the DB repo is cloned."""
    calls, fake = _fake_clone_factory()
    monkeypatch.setattr(import_docs, "clone_or_pull", fake)
    monkeypatch.setattr(
        import_docs,
        "effective_sources",
        lambda db: ([_git_row("https://db.example/only.git")], "db"),
    )
    settings = _settings(
        git_sources="https://env.example/ignored.git",
        sources_dir=str(tmp_path / "bor"),
    )

    sources, ignore_map = import_docs._resolve_sources(None, settings)

    assert sources == [tmp_path / "bor" / "only"]
    assert ignore_map == {}  # phase 89: no row carries a list → empty map
    assert calls == [("https://db.example/only.git", tmp_path / "bor" / "only")]


def test_resolve_sources_defaults_when_nothing_configured(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    # Both origins empty (the resolver's ``([], "env")``) → legacy dirs.
    monkeypatch.setattr(import_docs, "effective_sources", lambda db: ([], "env"))
    sources, ignore_map = import_docs._resolve_sources(None, _settings())
    assert sources == [p.expanduser() for p in import_docs.DEFAULT_SOURCES]
    assert ignore_map == {}  # phase 89: the legacy fallback has no rows


def test_resolve_sources_rows_branch_builds_ignore_map(
    monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
    """Phase 89: the rows branch returns each row's ignore list keyed by
    the resolved root string — the local row's directory, the git row's
    checkout dir; a row without a list contributes nothing to the map."""
    calls, fake = _fake_clone_factory()
    monkeypatch.setattr(import_docs, "clone_or_pull", fake)
    local_dir = tmp_path / "LocalDocs"
    local_dir.mkdir()
    monkeypatch.setattr(
        import_docs,
        "effective_sources",
        lambda db: (
            [
                _git_row("https://db.example/only.git"),
                _local_row(str(local_dir), ignore_paths=["ignore/"]),
            ],
            "db",
        ),
    )
    settings = _settings(sources_dir=str(tmp_path / "bor"))

    sources, ignore_map = import_docs._resolve_sources(None, settings)

    assert sources == [tmp_path / "bor" / "only", local_dir]
    # Keyed by the SAME string the importer sees (the root, not the name).
    assert ignore_map == {str(local_dir): ["ignore/"]}


def test_resolve_sources_two_rows_sharing_root_string_extend(
    monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
    """Phase 89 collision rule: two rows resolving to the SAME root
    string (the sibling/repo-name edge — ``…/shared`` and
    ``…/shared.git``) get the UNION of their lists (extend, not
    replace), in row order."""
    calls, fake = _fake_clone_factory()
    monkeypatch.setattr(import_docs, "clone_or_pull", fake)
    monkeypatch.setattr(
        import_docs,
        "effective_sources",
        lambda db: (
            [
                _git_row("https://a.example/shared", ignore_paths=["a/"]),
                _git_row("https://a.example/shared.git", ignore_paths=["b"]),
            ],
            "db",
        ),
    )
    settings = _settings(sources_dir=str(tmp_path / "bor"))

    sources, ignore_map = import_docs._resolve_sources(None, settings)

    shared = str(tmp_path / "bor" / "shared")
    assert sources == [tmp_path / "bor" / "shared", tmp_path / "bor" / "shared"]
    assert ignore_map == {shared: ["a/", "b"]}  # union, row order


def test_main_rows_branch_passes_ignore_map_to_import(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
    capsys: pytest.CaptureFixture[str],
) -> None:
    """Phase 89: a ``kind=local`` row carrying ``ignore_paths`` →
    ``main`` passes the per-root map to ``import_sources`` (keyed by
    the directory string, prune flag unchanged)."""
    settings = _settings(sources_dir=str(tmp_path / "bor"))
    monkeypatch.setattr(import_docs, "get_settings", lambda: settings)
    local_dir = tmp_path / "LocalDocs"
    local_dir.mkdir()
    (local_dir / "keep.md").write_text("# Keep\nin scope\n", encoding="utf-8")
    (local_dir / "ignore").mkdir()
    (local_dir / "ignore" / "secret.md").write_text("# Secret\nignored\n",
                                                    encoding="utf-8")
    monkeypatch.setattr(
        import_docs,
        "effective_sources",
        lambda db: ([_local_row(str(local_dir), ignore_paths=["ignore/"])], "db"),
    )
    fake_import = FakeImportSources()
    monkeypatch.setattr(import_docs, "import_sources", fake_import)
    _stub_bump(monkeypatch)
    _stub_folder_summaries(monkeypatch)

    rc = import_docs.main([])

    assert rc == 0
    call = fake_import.calls[0]
    assert call["sources"] == [local_dir]
    assert call["ignore_by_root"] == {str(local_dir): ["ignore/"]}
    assert call["prune"] is False  # the CLI's no-prune default is unchanged


# --- main() ----------------------------------------------------------------


def test_main_git_sources_clone_then_import(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
    capsys: pytest.CaptureFixture[str],
) -> None:
    settings = _settings(
        git_sources="https://host/a/homelab.git,https://host/a/deploy.git",
        sources_dir=str(tmp_path / "bor"),
    )
    monkeypatch.setattr(import_docs, "get_settings", lambda: settings)
    monkeypatch.setattr(
        import_docs,
        "effective_sources",
        lambda db: (
            [_git_row("https://host/a/homelab.git"), _git_row("https://host/a/deploy.git")],
            "env",
        ),
    )
    calls, fake = _fake_clone_factory()
    monkeypatch.setattr(import_docs, "clone_or_pull", fake)
    fake_import = FakeImportSources()
    monkeypatch.setattr(import_docs, "import_sources", fake_import)
    bumps = _stub_bump(monkeypatch)
    _stub_folder_summaries(monkeypatch)

    rc = import_docs.main([])

    assert rc == 0
    assert [(url, dest) for url, dest in calls] == [
        ("https://host/a/homelab.git", tmp_path / "bor" / "homelab"),
        ("https://host/a/deploy.git", tmp_path / "bor" / "deploy"),
    ]
    assert len(fake_import.calls) == 1
    # The cloned checkouts are exactly what gets imported.
    assert fake_import.calls[0]["sources"] == [
        tmp_path / "bor" / "homelab",
        tmp_path / "bor" / "deploy",
    ]
    for dest in (tmp_path / "bor" / "homelab", tmp_path / "bor" / "deploy"):
        assert (dest / "notes.md").is_file()
    # The final summary print reflects the import (added > 0).
    out = capsys.readouterr().out
    assert "added=1" in out
    # Phase 53: a KB-changing run bumps the sources version exactly
    # once and reports it (stubbed — this file keeps its no-real-DB
    # style for the counter, like the fake import above).
    assert len(bumps) == 1
    assert "sources_version=1" in out


def test_main_cli_source_still_imports_manual_dir(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
    capsys: pytest.CaptureFixture[str],
) -> None:
    manual = tmp_path / "manual"
    manual.mkdir()
    (manual / "a.md").write_text("# A\nhi\n", encoding="utf-8")
    # BOR_GIT_SOURCES is set but must be ignored — --source always wins.
    settings = _settings(git_sources="https://host/a/repo.git")
    monkeypatch.setattr(import_docs, "get_settings", lambda: settings)
    calls, fake = _fake_clone_factory()
    monkeypatch.setattr(import_docs, "clone_or_pull", fake)
    fake_import = FakeImportSources()
    monkeypatch.setattr(import_docs, "import_sources", fake_import)
    bumps = _stub_bump(monkeypatch)
    _stub_folder_summaries(monkeypatch)

    rc = import_docs.main(["--source", str(manual)])

    assert rc == 0
    assert calls == []
    assert fake_import.calls[0]["sources"] == [manual]
    assert fake_import.calls[0]["prune"] is False
    # Phase 53: a manual --source run that changes the KB bumps exactly
    # once (the CLI is the other canonical sync path).
    assert len(bumps) == 1
    assert "sources_version=1" in capsys.readouterr().out


def test_resolve_sources_mixed_git_and_local(
    monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
    """Phase 38: DB rows of both kinds — the git row is cloned into
    ``BOR_SOURCES_DIR``, the local row is its existing directory itself
    (no clone), in row order; the env list is ignored."""
    calls, fake = _fake_clone_factory()
    monkeypatch.setattr(import_docs, "clone_or_pull", fake)
    local_dir = tmp_path / "LocalDocs"
    local_dir.mkdir()
    (local_dir / "a.md").write_text("# A\nlocal fixture\n", encoding="utf-8")
    monkeypatch.setattr(
        import_docs,
        "effective_sources",
        lambda db: (
            [_git_row("https://db.example/only.git"), _local_row(str(local_dir))],
            "db",
        ),
    )
    settings = _settings(
        git_sources="https://env.example/ignored.git",
        sources_dir=str(tmp_path / "bor"),
    )

    sources, ignore_map = import_docs._resolve_sources(None, settings)

    assert sources == [tmp_path / "bor" / "only", local_dir]
    assert ignore_map == {}  # phase 89: neither row carries a list
    assert calls == [("https://db.example/only.git", tmp_path / "bor" / "only")]


def test_resolve_sources_missing_local_dir_aborts(
    monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
    """Phase 38: a local row whose directory is gone at run time →
    ``GitSyncError`` naming the path, before any import (the same
    pre-import fail-loud as a failing git clone)."""
    monkeypatch.setattr(import_docs, "clone_or_pull", _fake_clone_factory()[1])
    missing = tmp_path / "Gone"
    monkeypatch.setattr(
        import_docs,
        "effective_sources",
        lambda db: ([_local_row(str(missing))], "db"),
    )
    settings = _settings(sources_dir=str(tmp_path / "bor"))

    with pytest.raises(GitSyncError, match=f"local source missing: {re.escape(str(missing))}"):
        import_docs._resolve_sources(None, settings)


def test_main_git_failure_aborts_before_import(
    monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str]
) -> None:
    settings = _settings(
        git_sources="https://host/a/bad.git",
        sources_dir=str(tmp_path / "bor"),
    )
    monkeypatch.setattr(import_docs, "get_settings", lambda: settings)
    monkeypatch.setattr(
        import_docs,
        "effective_sources",
        lambda db: ([_git_row("https://host/a/bad.git")], "env"),
    )

    def failing_clone(url: str, dest: Path | str) -> Path:
        raise GitSyncError(
            f"git clone --depth 1 {url} failed (exit 128): "
            "fatal: repository not found"
        )

    monkeypatch.setattr(import_docs, "clone_or_pull", failing_clone)
    fake_import = FakeImportSources()
    monkeypatch.setattr(import_docs, "import_sources", fake_import)

    rc = import_docs.main([])

    assert rc == 1
    err = capsys.readouterr().err
    assert "import_docs: source sync failed" in err
    assert "bad.git" in err  # the failing repo is named
    assert fake_import.calls == []  # no partial import
    assert not (tmp_path / "bor").exists()


def test_main_missing_local_dir_aborts_before_import(
    monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str]
) -> None:
    """Phase 38: a DB local row whose directory is missing → exit code
    1, ``local source missing: <path>`` on stderr, zero import attempts
    (no ``--source`` given, so the DB row is what should have been
    imported)."""
    settings = _settings(sources_dir=str(tmp_path / "bor"))
    monkeypatch.setattr(import_docs, "get_settings", lambda: settings)
    missing = tmp_path / "Gone"
    monkeypatch.setattr(
        import_docs,
        "effective_sources",
        lambda db: ([_local_row(str(missing))], "db"),
    )
    fake_import = FakeImportSources()
    monkeypatch.setattr(import_docs, "import_sources", fake_import)

    rc = import_docs.main([])

    assert rc == 1
    err = capsys.readouterr().err
    assert "import_docs: source sync failed" in err
    assert f"local source missing: {missing}" in err  # the path is named
    assert fake_import.calls == []  # no partial import