feat(rag): git-based import sources — BOR_GIT_SOURCES repos cloned (first run, --depth 1) or pulled (--ff-only) into BOR_SOURCES_DIR/<repo>/ then indexed; --source still wins; a failed sync aborts before importing anything
This commit is contained in:
@@ -0,0 +1,212 @@
|
||||
"""Integration test: ``import_docs`` git-source resolution (phase 28, task 03).
|
||||
|
||||
Drives ``scripts.import_docs`` end to end with a fake ``clone_or_pull`` (no
|
||||
real git, no network) and a recording fake ``import_sources`` (no real
|
||||
DB), covering:
|
||||
|
||||
- ``BOR_GIT_SOURCES`` set → each URL is cloned/pulled into
|
||||
``BOR_SOURCES_DIR/<repo-name>/`` and exactly those dirs are imported.
|
||||
- ``--source`` still wins over ``BOR_GIT_SOURCES`` (no git at all).
|
||||
- No git sources + no ``--source`` → the legacy ``DEFAULT_SOURCES``.
|
||||
- A failing git sync → exit code 1, an error naming the failing repo on
|
||||
stderr, and **zero** import attempts.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.config import Settings
|
||||
from app.rag.importer import ImportSummary
|
||||
from scripts import import_docs
|
||||
from scripts.git_sync import GitSyncError
|
||||
|
||||
|
||||
def _settings(git_sources: str = "", sources_dir: str = "~/bor-sources") -> Settings:
|
||||
"""Fresh settings (no .env file); explicit kwargs beat any env leaks."""
|
||||
return Settings(_env_file=None, git_sources=git_sources, sources_dir=sources_dir) # pyright: ignore[reportCallIssue]
|
||||
|
||||
|
||||
class FakeImportSources:
|
||||
"""Records every ``import_sources`` call instead of touching a DB."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.calls: list[dict] = []
|
||||
|
||||
async def __call__(
|
||||
self,
|
||||
sources: list[Path],
|
||||
llm: object,
|
||||
*,
|
||||
prune: bool = False,
|
||||
limit: int | None = None,
|
||||
) -> ImportSummary:
|
||||
self.calls.append({"sources": list(sources), "prune": prune, "limit": limit})
|
||||
return ImportSummary(files=1, added=1)
|
||||
|
||||
|
||||
def _fake_clone_factory() -> tuple[list[tuple[str, Path]], object]:
|
||||
"""A ``clone_or_pull`` that materialises a checkout with one .md file."""
|
||||
calls: list[tuple[str, Path]] = []
|
||||
|
||||
def fake_clone_or_pull(url: str, dest: Path | str) -> Path:
|
||||
dest = Path(dest)
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
(dest / "notes.md").write_text(f"# {dest.name}\ncontent for the KB\n", encoding="utf-8")
|
||||
calls.append((url, dest))
|
||||
return dest
|
||||
|
||||
return calls, fake_clone_or_pull
|
||||
|
||||
|
||||
# --- repo_name -------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("url", "name"),
|
||||
[
|
||||
("https://github.com/user/homelab.git", "homelab"),
|
||||
("https://github.com/user/homelab", "homelab"),
|
||||
("git@github.com:user/homelab.git", "homelab"),
|
||||
("git@github.com:homelab.git", "homelab"),
|
||||
("ssh://git@host:2222/group/deployments.git", "deployments"),
|
||||
("https://git.reeseapps.com:8443/proj/notes", "notes"),
|
||||
],
|
||||
)
|
||||
def test_repo_name(url: str, name: str) -> None:
|
||||
assert import_docs.repo_name(url) == name
|
||||
|
||||
|
||||
def test_repo_name_slug_fallback() -> None:
|
||||
# No usable basename (path ends in the .git suffix itself) → slug.
|
||||
assert import_docs.repo_name("https://host/.git") == "https-host"
|
||||
|
||||
|
||||
# --- _resolve_sources ------------------------------------------------------
|
||||
|
||||
|
||||
def test_resolve_sources_git_urls_cloned_into_sources_dir(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
calls, fake = _fake_clone_factory()
|
||||
monkeypatch.setattr(import_docs, "clone_or_pull", fake)
|
||||
settings = _settings(
|
||||
git_sources="https://host/a/homelab.git, git@host:user/deploy.git ,",
|
||||
sources_dir=str(tmp_path / "bor"),
|
||||
)
|
||||
|
||||
sources = import_docs._resolve_sources(None, settings)
|
||||
|
||||
assert sources == [tmp_path / "bor" / "homelab", tmp_path / "bor" / "deploy"]
|
||||
assert calls == [
|
||||
("https://host/a/homelab.git", tmp_path / "bor" / "homelab"),
|
||||
("git@host:user/deploy.git", tmp_path / "bor" / "deploy"),
|
||||
]
|
||||
|
||||
|
||||
def test_resolve_sources_cli_source_wins(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
calls, fake = _fake_clone_factory()
|
||||
monkeypatch.setattr(import_docs, "clone_or_pull", fake)
|
||||
settings = _settings(git_sources="https://host/a/repo.git")
|
||||
manual = tmp_path / "Manual"
|
||||
|
||||
sources = import_docs._resolve_sources([manual], settings)
|
||||
|
||||
assert sources == [manual]
|
||||
assert calls == [] # git is never touched when --source is given
|
||||
|
||||
|
||||
def test_resolve_sources_defaults_when_nothing_configured() -> None:
|
||||
sources = import_docs._resolve_sources(None, _settings())
|
||||
assert sources == [p.expanduser() for p in import_docs.DEFAULT_SOURCES]
|
||||
|
||||
|
||||
# --- main() ----------------------------------------------------------------
|
||||
|
||||
|
||||
def test_main_git_sources_clone_then_import(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
capsys: pytest.CaptureFixture[str],
|
||||
) -> None:
|
||||
settings = _settings(
|
||||
git_sources="https://host/a/homelab.git,https://host/a/deploy.git",
|
||||
sources_dir=str(tmp_path / "bor"),
|
||||
)
|
||||
monkeypatch.setattr(import_docs, "get_settings", lambda: settings)
|
||||
calls, fake = _fake_clone_factory()
|
||||
monkeypatch.setattr(import_docs, "clone_or_pull", fake)
|
||||
fake_import = FakeImportSources()
|
||||
monkeypatch.setattr(import_docs, "import_sources", fake_import)
|
||||
|
||||
rc = import_docs.main([])
|
||||
|
||||
assert rc == 0
|
||||
assert [(url, dest) for url, dest in calls] == [
|
||||
("https://host/a/homelab.git", tmp_path / "bor" / "homelab"),
|
||||
("https://host/a/deploy.git", tmp_path / "bor" / "deploy"),
|
||||
]
|
||||
assert len(fake_import.calls) == 1
|
||||
# The cloned checkouts are exactly what gets imported.
|
||||
assert fake_import.calls[0]["sources"] == [
|
||||
tmp_path / "bor" / "homelab",
|
||||
tmp_path / "bor" / "deploy",
|
||||
]
|
||||
for dest in (tmp_path / "bor" / "homelab", tmp_path / "bor" / "deploy"):
|
||||
assert (dest / "notes.md").is_file()
|
||||
# The final summary print reflects the import (added > 0).
|
||||
assert "added=1" in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_main_cli_source_still_imports_manual_dir(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
manual = tmp_path / "manual"
|
||||
manual.mkdir()
|
||||
(manual / "a.md").write_text("# A\nhi\n", encoding="utf-8")
|
||||
# BOR_GIT_SOURCES is set but must be ignored — --source always wins.
|
||||
settings = _settings(git_sources="https://host/a/repo.git")
|
||||
monkeypatch.setattr(import_docs, "get_settings", lambda: settings)
|
||||
calls, fake = _fake_clone_factory()
|
||||
monkeypatch.setattr(import_docs, "clone_or_pull", fake)
|
||||
fake_import = FakeImportSources()
|
||||
monkeypatch.setattr(import_docs, "import_sources", fake_import)
|
||||
|
||||
rc = import_docs.main(["--source", str(manual)])
|
||||
|
||||
assert rc == 0
|
||||
assert calls == []
|
||||
assert fake_import.calls[0]["sources"] == [manual]
|
||||
assert fake_import.calls[0]["prune"] is False
|
||||
|
||||
|
||||
def test_main_git_failure_aborts_before_import(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str]
|
||||
) -> None:
|
||||
settings = _settings(
|
||||
git_sources="https://host/a/bad.git",
|
||||
sources_dir=str(tmp_path / "bor"),
|
||||
)
|
||||
monkeypatch.setattr(import_docs, "get_settings", lambda: settings)
|
||||
|
||||
def failing_clone(url: str, dest: Path | str) -> Path:
|
||||
raise GitSyncError(
|
||||
f"git clone --depth 1 {url} failed (exit 128): "
|
||||
"fatal: repository not found"
|
||||
)
|
||||
|
||||
monkeypatch.setattr(import_docs, "clone_or_pull", failing_clone)
|
||||
fake_import = FakeImportSources()
|
||||
monkeypatch.setattr(import_docs, "import_sources", fake_import)
|
||||
|
||||
rc = import_docs.main([])
|
||||
|
||||
assert rc == 1
|
||||
err = capsys.readouterr().err
|
||||
assert "import_docs: git sync failed" in err
|
||||
assert "bad.git" in err # the failing repo is named
|
||||
assert fake_import.calls == [] # no partial import
|
||||
assert not (tmp_path / "bor").exists()
|
||||
@@ -92,6 +92,53 @@ def test_import_extensions_rejects_empty(monkeypatch) -> None:
|
||||
_settings()
|
||||
|
||||
|
||||
def test_git_sources_default_empty_and_sources_dir_default() -> None:
|
||||
"""Phase 28: no git sources by default (backwards-compatible with the
|
||||
``--source`` / ``DEFAULT_SOURCES`` fallback); the clone location stays
|
||||
a raw string (``~`` is expanded by the import script, not the setting)."""
|
||||
s = _settings()
|
||||
assert s.git_sources == ""
|
||||
assert s.git_source_list == []
|
||||
assert s.sources_dir == "~/bor-sources"
|
||||
|
||||
|
||||
def test_git_sources_env_override_parses_comma_separated_list(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""``BOR_GIT_SOURCES`` is a raw CSV: entries are trimmed and empty
|
||||
entries dropped; URLs are stored untouched (no scheme parsing here)."""
|
||||
monkeypatch.setenv(
|
||||
"BOR_GIT_SOURCES",
|
||||
"https://github.com/user/homelab.git, "
|
||||
" git@github.com:user/deployments.git ,, https://git.reeseapps.com/x/y.git ",
|
||||
)
|
||||
s = _settings()
|
||||
# The raw CSV string is preserved untouched (no parsing in the setting).
|
||||
assert s.git_sources == (
|
||||
"https://github.com/user/homelab.git, "
|
||||
" git@github.com:user/deployments.git ,, https://git.reeseapps.com/x/y.git "
|
||||
)
|
||||
assert s.git_source_list == [
|
||||
"https://github.com/user/homelab.git",
|
||||
"git@github.com:user/deployments.git",
|
||||
"https://git.reeseapps.com/x/y.git",
|
||||
]
|
||||
|
||||
|
||||
def test_git_sources_whitespace_only_yields_empty_list(monkeypatch) -> None:
|
||||
"""A configured-but-blank value behaves the same as unset: no git
|
||||
sources, so the script falls back to its legacy local defaults."""
|
||||
monkeypatch.setenv("BOR_GIT_SOURCES", " , , ")
|
||||
s = _settings()
|
||||
assert s.git_source_list == []
|
||||
|
||||
|
||||
def test_sources_dir_env_override_is_raw_string(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("BOR_SOURCES_DIR", "/data/bor/sources")
|
||||
s = _settings()
|
||||
assert s.sources_dir == "/data/bor/sources"
|
||||
|
||||
|
||||
def test_suggestions_default_is_three_plus_real_questions() -> None:
|
||||
s = _settings()
|
||||
assert len(s.suggestions) >= 3
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
"""Unit tests: git clone/pull utility (phase 28).
|
||||
|
||||
``clone_or_pull`` dispatches to ``git clone --depth 1`` (fresh dest) or
|
||||
``git pull --ff-only`` (existing checkout) with ``subprocess`` fully
|
||||
mocked — the real git CLI is never invoked, so the tests run anywhere.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
import scripts.git_sync as git_sync
|
||||
from scripts.git_sync import GitSyncError, clone_or_pull
|
||||
|
||||
|
||||
class _FakeProc:
|
||||
def __init__(self, returncode: int = 0, stdout: str = "", stderr: str = "") -> None:
|
||||
self.returncode = returncode
|
||||
self.stdout = stdout
|
||||
self.stderr = stderr
|
||||
|
||||
|
||||
def _fake_run(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
returncode: int = 0,
|
||||
stdout: str = "",
|
||||
stderr: str = "",
|
||||
missing_git: bool = False,
|
||||
) -> list[dict[str, object]]:
|
||||
"""Monkeypatch scripts.git_sync.subprocess.run; record each call."""
|
||||
calls: list[dict[str, object]] = []
|
||||
|
||||
def fake_run(argv: list[str], cwd: Path | None = None, **kwargs: object) -> _FakeProc:
|
||||
calls.append({"argv": list(argv), "cwd": cwd, **kwargs})
|
||||
if missing_git:
|
||||
raise FileNotFoundError("git")
|
||||
return _FakeProc(returncode, stdout, stderr)
|
||||
|
||||
monkeypatch.setattr(subprocess, "run", fake_run)
|
||||
return calls
|
||||
|
||||
|
||||
def test_clone_or_pull_clones_when_dest_has_no_git_dir(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""First run: dest absent → ``git clone --depth 1`` from the parent dir."""
|
||||
url = "https://github.com/user/homelab.git"
|
||||
dest = tmp_path / "homelab"
|
||||
calls = _fake_run(monkeypatch)
|
||||
|
||||
result = clone_or_pull(url, dest)
|
||||
|
||||
assert result == dest
|
||||
assert len(calls) == 1
|
||||
call = calls[0]
|
||||
assert call["argv"] == ["git", "clone", "--depth", "1", url, str(dest)]
|
||||
assert call["cwd"] == dest.parent
|
||||
assert call["capture_output"] is True
|
||||
assert call["text"] is True
|
||||
|
||||
|
||||
def test_clone_or_pull_pulls_when_git_dir_exists(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""Subsequent run: dest/.git present → ``git pull --ff-only`` in-place."""
|
||||
dest = tmp_path / "homelab"
|
||||
(dest / ".git").mkdir(parents=True)
|
||||
calls = _fake_run(monkeypatch)
|
||||
|
||||
result = clone_or_pull("https://github.com/user/homelab.git", dest)
|
||||
|
||||
assert result == dest
|
||||
assert calls[0]["argv"] == ["git", "pull", "--ff-only"]
|
||||
assert calls[0]["cwd"] == dest
|
||||
|
||||
|
||||
def test_clone_or_pull_creates_missing_parent_before_clone(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""The (nested) parent of the destination is mkdir'd before git runs."""
|
||||
dest = tmp_path / "nested" / "deeper" / "homelab"
|
||||
assert not dest.parent.exists()
|
||||
_fake_run(monkeypatch)
|
||||
|
||||
clone_or_pull("https://github.com/user/homelab.git", dest)
|
||||
|
||||
assert dest.parent.is_dir()
|
||||
|
||||
|
||||
def test_existing_dir_without_git_dir_is_treated_as_fresh(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""A leftover non-repo dir at the dest still takes the clone path —
|
||||
git fails loudly (non-zero exit) if the dir is not empty."""
|
||||
dest = tmp_path / "homelab"
|
||||
dest.mkdir()
|
||||
calls = _fake_run(monkeypatch)
|
||||
|
||||
clone_or_pull("https://github.com/user/homelab.git", dest)
|
||||
|
||||
assert calls[0]["argv"] == [
|
||||
"git",
|
||||
"clone",
|
||||
"--depth",
|
||||
"1",
|
||||
"https://github.com/user/homelab.git",
|
||||
str(dest),
|
||||
]
|
||||
|
||||
|
||||
def test_failing_git_raises_error_carrying_stderr(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""Non-zero exit → GitSyncError naming the subcommand, exit code, stderr."""
|
||||
dest = tmp_path / "homelab"
|
||||
_fake_run(monkeypatch, returncode=128, stderr="fatal: repository not found\n")
|
||||
|
||||
with pytest.raises(
|
||||
GitSyncError,
|
||||
match=r"git clone --depth 1 .* failed \(exit 128\): fatal: repository not found",
|
||||
):
|
||||
clone_or_pull("https://example.com/nope.git", dest)
|
||||
|
||||
# Same for the pull path (broken checkout, e.g. diverged history).
|
||||
(dest / ".git").mkdir(parents=True)
|
||||
_fake_run(monkeypatch, returncode=1, stderr="error: cannot pull with rebase")
|
||||
|
||||
with pytest.raises(
|
||||
GitSyncError, match=r"git pull --ff-only failed \(exit 1\): error: cannot pull with rebase"
|
||||
):
|
||||
clone_or_pull("https://example.com/homelab.git", dest)
|
||||
|
||||
|
||||
def test_missing_git_raises_named_error(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""No git binary on PATH → GitSyncError telling the user to install it."""
|
||||
_fake_run(monkeypatch, missing_git=True)
|
||||
|
||||
with pytest.raises(GitSyncError, match="git was not found on PATH"):
|
||||
clone_or_pull("https://example.com/homelab.git", tmp_path / "homelab")
|
||||
|
||||
|
||||
def test_run_captures_and_returns_stdout(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""_run returns the captured stdout on success (git output is not lost)."""
|
||||
calls = _fake_run(monkeypatch, stdout="From example.com\n + abc..def main")
|
||||
|
||||
assert git_sync._run(["git", "status"], cwd=tmp_path) == "From example.com\n + abc..def main"
|
||||
assert len(calls) == 1
|
||||
Reference in New Issue
Block a user