ducoterra
6bcee831ec
All green — verification complete. Final report: **Phase 107 — final verification pass (all tasks already in `complete/`)** - **Verified:** `clone_or_pull` full-history fix (no `--depth` clone; `--is-shallow-repository` probe → one-time `git fetch --unshallow` self-heal → `git pull --ff-only`; D12 fail-loud) in `scripts/git_sync.py`; E2E suite `tests/e2e/test_git_source_dates.py` present; all five stale shallow/tip-date narratives corrected (no residual claims repo-wide; `docs_push.py` `--depth` fetches untouched as designed). - **Unit+integration:** `uv run pytest tests/unit/test_git_sync.py tests/integration/test_git_file_dates.py tests/integration/test_import_docs_git.py tests/integration/test_sync_api.py -v --no-cov` → **66 passed** (D10 tip-date test replaced by `test_url_clone_yields_true_per_file_dates` + `test_existing_shallow_checkout_self_heals`; unit pins: clone argv w/o `--depth`, probe→unshallow→pull order, unshallow-failure → `GitSyncError`). - **New E2E in isolation:** `uv run pytest tests/e2e/test_git_source_dates.py -v --no-cov` → **4 passed** (old file 2020-06-15 ≠ tip file 2024-06-15 in `GET /api/docs`, Sources `Created`/`Updated` columns, viewer `Created` badge ISO title; folder `updated_at` subtree maxes). - **Phase-106 regressions in isolation:** `test_document_dates.py` → 6 passed; `test_sync_button.py` → 3 passed; `test_git_sources_admin.py` → 6 passed. - **Full gates:** `uv run pytest` → **2302 passed**; `uv run pytest --cov=app --cov-report=term-missing` → **TOTAL 99%** (>90%); `uv run ruff check . && uv run pyright` → **clean, 0 errors**. - **Completion criteria:** 1 ✅ non-shallow URL clone + true per-file dates · 2 ✅ self-heal / plain-pull / fail-loud · 3 ✅ both suites green, bug-pin replaced · 4 ✅ E2E green in isolation · 5 ✅ all regressions + full suite + coverage + lint · 6 ✅ narratives corrected · 7 — no commit made (harness override: changes left in working tree; task files already in `complete/`). - **No defects found; no deviations.** Next pending phase: **108_history_wire_check**.
735 lines
29 KiB
Python
735 lines
29 KiB
Python
"""Integration test: ``import_docs`` source resolution (phase 28, task
|
|
03; phase 35 re-points at the shared resolver; phase 38 adds the local
|
|
kind).
|
|
|
|
Drives ``scripts.import_docs`` end to end with a fake ``clone_or_pull``
|
|
(no real git, no network) and a recording fake ``import_sources`` (no
|
|
real DB), covering: the phase-53 version bump is stubbed in the
|
|
``main()`` tests the same way (the ``sources_version=`` summary token
|
|
is asserted against the canned value).
|
|
|
|
- Effective sources set (phase 35: the shared resolver — stubbed here,
|
|
keeping this file's no-real-DB style) → each git URL is cloned/pulled
|
|
into ``BOR_SOURCES_DIR/<repo-name>/``; local rows are their existing
|
|
directories, walked directly; exactly those dirs are imported.
|
|
- DB rows (both kinds) win over ``BOR_GIT_SOURCES`` (the resolver's
|
|
``db`` origin — the env list is ignored; the env fallback stays
|
|
git-only).
|
|
- ``--source`` still wins over the DB rows (no git at all, no resolver
|
|
call).
|
|
- No sources configured + no ``--source`` → the legacy
|
|
``DEFAULT_SOURCES``.
|
|
- A failing git sync → exit code 1, an error naming the failing repo on
|
|
stderr, and **zero** import attempts.
|
|
- A missing local directory → the same pre-import fail-loud: exit code
|
|
1, ``local source missing: <path>`` on stderr, zero import attempts.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from collections.abc import Iterator
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from sqlalchemy import text
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app.config import Settings
|
|
from app.models import GitSource
|
|
from app.rag.importer import ImportSummary
|
|
from scripts import import_docs
|
|
from scripts.git_sync import GitSyncError
|
|
from tests.fakes import FakeEmbedder
|
|
|
|
|
|
def _settings(git_sources: str = "", sources_dir: str = "~/bor-sources") -> Settings:
|
|
"""Fresh settings (no .env file); explicit kwargs beat any env leaks."""
|
|
return Settings(_env_file=None, git_sources=git_sources, sources_dir=sources_dir) # pyright: ignore[reportCallIssue]
|
|
|
|
|
|
def _git_row(url: str, ignore_paths: list[str] | None = None) -> GitSource:
|
|
return GitSource(url=url, kind="git", ignore_paths=ignore_paths or [])
|
|
|
|
|
|
def _local_row(
|
|
path: str,
|
|
ignore_paths: list[str] | None = None,
|
|
include_hidden: bool = False,
|
|
) -> GitSource:
|
|
"""A local row as the phase-38 API stores it: the expanded path in
|
|
both ``path`` and the NOT-NULL ``url`` location column (plus the
|
|
phase-89 ignore list and the phase-105 hidden-folders flag, both
|
|
defaulting off — A4)."""
|
|
return GitSource(
|
|
url=path, kind="local", path=path,
|
|
ignore_paths=ignore_paths or [], include_hidden=include_hidden,
|
|
)
|
|
|
|
|
|
@pytest.fixture()
|
|
def clean_documents(db: Session) -> Iterator[None]:
|
|
"""Phase 105: the real-import CLI tests write ``documents``/
|
|
``chunks`` (the canonical KB state) — global, truncated around
|
|
every such test."""
|
|
db.execute(text("TRUNCATE chunks, documents"))
|
|
db.commit()
|
|
yield
|
|
db.execute(text("TRUNCATE chunks, documents"))
|
|
db.commit()
|
|
|
|
|
|
class FakeImportSources:
|
|
"""Records every ``import_sources`` call instead of touching a DB."""
|
|
|
|
def __init__(self) -> None:
|
|
self.calls: list[dict] = []
|
|
|
|
async def __call__(
|
|
self,
|
|
sources: list[Path],
|
|
llm: object,
|
|
*,
|
|
prune: bool = False,
|
|
limit: int | None = None,
|
|
ignore_by_root: dict[str, list[str]] | None = None, # phase 89
|
|
include_hidden_by_root: dict[str, bool] | None = None, # phase 105
|
|
doc_dates_by_root: dict[str, dict[str, datetime]] | None = None, # phase 106
|
|
) -> ImportSummary:
|
|
self.calls.append(
|
|
{"sources": list(sources), "prune": prune, "limit": limit,
|
|
"ignore_by_root": ignore_by_root,
|
|
"include_hidden_by_root": include_hidden_by_root,
|
|
"doc_dates_by_root": doc_dates_by_root}
|
|
)
|
|
return ImportSummary(files=1, added=1)
|
|
|
|
|
|
def _fake_clone_factory() -> tuple[list[tuple[str, Path]], object]:
|
|
"""A ``clone_or_pull`` that materialises a checkout with one .md file."""
|
|
calls: list[tuple[str, Path]] = []
|
|
|
|
def fake_clone_or_pull(url: str, dest: Path | str) -> Path:
|
|
dest = Path(dest)
|
|
dest.mkdir(parents=True, exist_ok=True)
|
|
(dest / "notes.md").write_text(f"# {dest.name}\ncontent for the KB\n", encoding="utf-8")
|
|
calls.append((url, dest))
|
|
return dest
|
|
|
|
return calls, fake_clone_or_pull
|
|
|
|
|
|
def _stub_bump(monkeypatch: pytest.MonkeyPatch) -> list[None]:
|
|
"""Stub the phase-53 version bump (this file keeps its no-real-DB
|
|
style for the counter — the fake import already avoids the KB
|
|
tables). Returns the call record; the canned new version is 1."""
|
|
bumps: list[None] = []
|
|
|
|
def fake_bump(session: object) -> int:
|
|
bumps.append(None)
|
|
return 1
|
|
|
|
monkeypatch.setattr(import_docs, "bump_sources_version", fake_bump)
|
|
return bumps
|
|
|
|
|
|
def _stub_folder_summaries(monkeypatch: pytest.MonkeyPatch) -> list[dict]:
|
|
"""Stub the phase-94 folder-summary regeneration (this file keeps
|
|
its no-real-DB / no-network style — the real generator would read
|
|
the global ``documents`` table and burn ``lite`` calls). Returns
|
|
the call record; the canned stats are the zero dict."""
|
|
calls: list[dict] = []
|
|
|
|
async def fake_generate(
|
|
db: object, llm: object, *, skip: bool = False, only_missing: bool = False
|
|
) -> dict[str, int]:
|
|
calls.append({"skip": skip, "only_missing": only_missing})
|
|
return {"generated": 0, "failed": 0, "pruned": 0}
|
|
|
|
monkeypatch.setattr(import_docs, "generate_folder_summaries", fake_generate)
|
|
return calls
|
|
|
|
|
|
# --- repo_name -------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("url", "name"),
|
|
[
|
|
("https://github.com/user/homelab.git", "homelab"),
|
|
("https://github.com/user/homelab", "homelab"),
|
|
("git@github.com:user/homelab.git", "homelab"),
|
|
("git@github.com:homelab.git", "homelab"),
|
|
("ssh://git@host:2222/group/deployments.git", "deployments"),
|
|
("https://git.reeseapps.com:8443/proj/notes", "notes"),
|
|
],
|
|
)
|
|
def test_repo_name(url: str, name: str) -> None:
|
|
assert import_docs.repo_name(url) == name
|
|
|
|
|
|
def test_repo_name_slug_fallback() -> None:
|
|
# No usable basename (path ends in the .git suffix itself) → slug.
|
|
assert import_docs.repo_name("https://host/.git") == "https-host"
|
|
|
|
|
|
# --- _resolve_sources ------------------------------------------------------
|
|
|
|
|
|
def test_resolve_sources_git_urls_cloned_into_sources_dir(
|
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
) -> None:
|
|
calls, fake = _fake_clone_factory()
|
|
monkeypatch.setattr(import_docs, "clone_or_pull", fake)
|
|
# Phase 35: resolution goes through the shared resolver (stubbed —
|
|
# this file keeps its no-real-DB style); the rows are the env list,
|
|
# surfaced as synthetic git rows.
|
|
monkeypatch.setattr(
|
|
import_docs,
|
|
"effective_sources",
|
|
lambda db: (
|
|
[_git_row("https://host/a/homelab.git"), _git_row("git@host:user/deploy.git")],
|
|
"env",
|
|
),
|
|
)
|
|
settings = _settings(
|
|
git_sources="https://host/a/homelab.git, git@host:user/deploy.git ,",
|
|
sources_dir=str(tmp_path / "bor"),
|
|
)
|
|
|
|
sources, ignore_map, hidden_map, date_map = import_docs._resolve_sources(None, settings)
|
|
|
|
assert sources == [tmp_path / "bor" / "homelab", tmp_path / "bor" / "deploy"]
|
|
assert ignore_map == {} # phase 89: no row carries a list → empty map
|
|
# Phase 105: flag-off rows still contribute their root — with False
|
|
# (A4), keyed by the SAME root string the importer sees.
|
|
assert hidden_map == {
|
|
str(tmp_path / "bor" / "homelab"): False,
|
|
str(tmp_path / "bor" / "deploy"): False,
|
|
}
|
|
# Phase 106: git rows are listed with their checkout's date walk —
|
|
# the fake checkouts are not git repos, so the walk fails soft to
|
|
# ``{}`` (the importer would then take the mtime fallback).
|
|
assert date_map == {
|
|
str(tmp_path / "bor" / "homelab"): {},
|
|
str(tmp_path / "bor" / "deploy"): {},
|
|
}
|
|
assert calls == [
|
|
("https://host/a/homelab.git", tmp_path / "bor" / "homelab"),
|
|
("git@host:user/deploy.git", tmp_path / "bor" / "deploy"),
|
|
]
|
|
|
|
|
|
def test_resolve_sources_cli_source_wins(
|
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
) -> None:
|
|
calls, fake = _fake_clone_factory()
|
|
monkeypatch.setattr(import_docs, "clone_or_pull", fake)
|
|
settings = _settings(git_sources="https://host/a/repo.git")
|
|
manual = tmp_path / "Manual"
|
|
|
|
sources, ignore_map, hidden_map, date_map = import_docs._resolve_sources([manual], settings)
|
|
|
|
assert sources == [manual]
|
|
assert ignore_map == {} # phase 89: manual dirs have no rows → no ignore
|
|
assert hidden_map == {} # phase 105: manual dirs have no rows → hidden skipped
|
|
# Phase 106: manual dirs have no rows (no clone) → no date map
|
|
# entries (the importer's mtime fallback applies).
|
|
assert date_map == {}
|
|
assert calls == [] # git is never touched when --source is given
|
|
|
|
|
|
def test_resolve_sources_db_rows_win_over_env(
|
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
) -> None:
|
|
"""Phase 35: the resolver's ``db`` origin (table has rows) — the
|
|
``BOR_GIT_SOURCES`` list must be ignored; only the DB repo is cloned."""
|
|
calls, fake = _fake_clone_factory()
|
|
monkeypatch.setattr(import_docs, "clone_or_pull", fake)
|
|
monkeypatch.setattr(
|
|
import_docs,
|
|
"effective_sources",
|
|
lambda db: ([_git_row("https://db.example/only.git")], "db"),
|
|
)
|
|
settings = _settings(
|
|
git_sources="https://env.example/ignored.git",
|
|
sources_dir=str(tmp_path / "bor"),
|
|
)
|
|
|
|
sources, ignore_map, hidden_map, date_map = import_docs._resolve_sources(None, settings)
|
|
|
|
assert sources == [tmp_path / "bor" / "only"]
|
|
assert ignore_map == {} # phase 89: no row carries a list → empty map
|
|
# Phase 105: the default-flag row contributes its root with False (A4).
|
|
assert hidden_map == {str(tmp_path / "bor" / "only"): False}
|
|
# Phase 106: the git row's (fake, non-repo) checkout fails soft → {}.
|
|
assert date_map == {str(tmp_path / "bor" / "only"): {}}
|
|
assert calls == [("https://db.example/only.git", tmp_path / "bor" / "only")]
|
|
|
|
|
|
def test_resolve_sources_defaults_when_nothing_configured(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
# Both origins empty (the resolver's ``([], "env")``) → legacy dirs.
|
|
monkeypatch.setattr(import_docs, "effective_sources", lambda db: ([], "env"))
|
|
sources, ignore_map, hidden_map, date_map = import_docs._resolve_sources(None, _settings())
|
|
assert sources == [p.expanduser() for p in import_docs.DEFAULT_SOURCES]
|
|
assert ignore_map == {} # phase 89: the legacy fallback has no rows
|
|
assert hidden_map == {} # phase 105: the legacy fallback has no rows
|
|
# Phase 106: the legacy fallback has no rows (no clone) → mtime
|
|
# fallback for every file.
|
|
assert date_map == {}
|
|
|
|
|
|
def test_resolve_sources_rows_branch_builds_ignore_map(
|
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
) -> None:
|
|
"""Phase 89: the rows branch returns each row's ignore list keyed by
|
|
the resolved root string — the local row's directory, the git row's
|
|
checkout dir; a row without a list contributes nothing to the map."""
|
|
calls, fake = _fake_clone_factory()
|
|
monkeypatch.setattr(import_docs, "clone_or_pull", fake)
|
|
local_dir = tmp_path / "LocalDocs"
|
|
local_dir.mkdir()
|
|
monkeypatch.setattr(
|
|
import_docs,
|
|
"effective_sources",
|
|
lambda db: (
|
|
[
|
|
_git_row("https://db.example/only.git"),
|
|
_local_row(str(local_dir), ignore_paths=["ignore/"]),
|
|
],
|
|
"db",
|
|
),
|
|
)
|
|
settings = _settings(sources_dir=str(tmp_path / "bor"))
|
|
|
|
sources, ignore_map, hidden_map, date_map = import_docs._resolve_sources(None, settings)
|
|
|
|
assert sources == [tmp_path / "bor" / "only", local_dir]
|
|
# Keyed by the SAME string the importer sees (the root, not the name).
|
|
assert ignore_map == {str(local_dir): ["ignore/"]}
|
|
# Phase 105: both rows are flag-off → per-root False entries (A4).
|
|
assert hidden_map == {str(tmp_path / "bor" / "only"): False, str(local_dir): False}
|
|
# Phase 106: ONLY the git row is listed (local rows take the mtime
|
|
# fallback); the fake checkout's date walk fails soft to ``{}``.
|
|
assert date_map == {str(tmp_path / "bor" / "only"): {}}
|
|
|
|
|
|
def test_resolve_sources_two_rows_sharing_root_string_extend(
|
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
) -> None:
|
|
"""Phase 89 collision rule: two rows resolving to the SAME root
|
|
string (the sibling/repo-name edge — ``…/shared`` and
|
|
``…/shared.git``) get the UNION of their lists (extend, not
|
|
replace), in row order."""
|
|
calls, fake = _fake_clone_factory()
|
|
monkeypatch.setattr(import_docs, "clone_or_pull", fake)
|
|
monkeypatch.setattr(
|
|
import_docs,
|
|
"effective_sources",
|
|
lambda db: (
|
|
[
|
|
_git_row("https://a.example/shared", ignore_paths=["a/"]),
|
|
_git_row("https://a.example/shared.git", ignore_paths=["b"]),
|
|
],
|
|
"db",
|
|
),
|
|
)
|
|
settings = _settings(sources_dir=str(tmp_path / "bor"))
|
|
|
|
sources, ignore_map, hidden_map, date_map = import_docs._resolve_sources(None, settings)
|
|
|
|
shared = str(tmp_path / "bor" / "shared")
|
|
assert sources == [tmp_path / "bor" / "shared", tmp_path / "bor" / "shared"]
|
|
assert ignore_map == {shared: ["a/", "b"]} # union, row order
|
|
# Phase 105 collision: the shared root gets the OR of the flags —
|
|
# both rows off here, so one False entry for the one root string.
|
|
assert hidden_map == {shared: False}
|
|
# Phase 106 collision: both git rows resolve to the SAME root — one
|
|
# date walk for the one root string (last row's walk wins, both
|
|
# fail soft to ``{}`` for the fake checkout).
|
|
assert date_map == {shared: {}}
|
|
|
|
|
|
def test_main_rows_branch_passes_ignore_map_to_import(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
tmp_path: Path,
|
|
capsys: pytest.CaptureFixture[str],
|
|
) -> None:
|
|
"""Phase 89: a ``kind=local`` row carrying ``ignore_paths`` →
|
|
``main`` passes the per-root map to ``import_sources`` (keyed by
|
|
the directory string, prune flag unchanged)."""
|
|
settings = _settings(sources_dir=str(tmp_path / "bor"))
|
|
monkeypatch.setattr(import_docs, "get_settings", lambda: settings)
|
|
local_dir = tmp_path / "LocalDocs"
|
|
local_dir.mkdir()
|
|
(local_dir / "keep.md").write_text("# Keep\nin scope\n", encoding="utf-8")
|
|
(local_dir / "ignore").mkdir()
|
|
(local_dir / "ignore" / "secret.md").write_text("# Secret\nignored\n",
|
|
encoding="utf-8")
|
|
monkeypatch.setattr(
|
|
import_docs,
|
|
"effective_sources",
|
|
lambda db: ([_local_row(str(local_dir), ignore_paths=["ignore/"])], "db"),
|
|
)
|
|
fake_import = FakeImportSources()
|
|
monkeypatch.setattr(import_docs, "import_sources", fake_import)
|
|
_stub_bump(monkeypatch)
|
|
_stub_folder_summaries(monkeypatch)
|
|
|
|
rc = import_docs.main([])
|
|
|
|
assert rc == 0
|
|
call = fake_import.calls[0]
|
|
assert call["sources"] == [local_dir]
|
|
assert call["ignore_by_root"] == {str(local_dir): ["ignore/"]}
|
|
# Phase 105: the default-flag row passes the per-root map too — a
|
|
# False entry, not an absent key (the importer reads it per root).
|
|
assert call["include_hidden_by_root"] == {str(local_dir): False}
|
|
# Phase 106: the local-only resolution contributes no date map
|
|
# (no clone — the importer's mtime fallback applies).
|
|
assert call["doc_dates_by_root"] == {}
|
|
assert call["prune"] is False # the CLI's no-prune default is unchanged
|
|
|
|
|
|
# --- phase 105: per-row hidden-folders flag ---------------------------------
|
|
|
|
|
|
def test_main_rows_branch_passes_include_hidden_map_to_import(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
tmp_path: Path,
|
|
capsys: pytest.CaptureFixture[str],
|
|
) -> None:
|
|
"""Phase 105: a ``kind=local`` row with ``include_hidden=True`` →
|
|
``main`` passes the per-root flag map to ``import_sources`` (keyed
|
|
by the directory string — the map is built, not lost)."""
|
|
settings = _settings(sources_dir=str(tmp_path / "bor"))
|
|
monkeypatch.setattr(import_docs, "get_settings", lambda: settings)
|
|
local_dir = tmp_path / "LocalDocs"
|
|
local_dir.mkdir()
|
|
(local_dir / "visible.md").write_text("# Visible\nin scope\n", encoding="utf-8")
|
|
(local_dir / ".hidden").mkdir()
|
|
(local_dir / ".hidden" / "note.md").write_text("# Note\nhidden\n", encoding="utf-8")
|
|
monkeypatch.setattr(
|
|
import_docs,
|
|
"effective_sources",
|
|
lambda db: ([_local_row(str(local_dir), include_hidden=True)], "db"),
|
|
)
|
|
fake_import = FakeImportSources()
|
|
monkeypatch.setattr(import_docs, "import_sources", fake_import)
|
|
_stub_bump(monkeypatch)
|
|
_stub_folder_summaries(monkeypatch)
|
|
|
|
rc = import_docs.main([])
|
|
|
|
assert rc == 0
|
|
call = fake_import.calls[0]
|
|
assert call["sources"] == [local_dir]
|
|
assert call["include_hidden_by_root"] == {str(local_dir): True}
|
|
assert call["ignore_by_root"] == {} # the row carries no ignore list
|
|
assert call["prune"] is False # the CLI's no-prune default is unchanged
|
|
|
|
|
|
def _stub_overview(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""Stub the phase-31 overview regeneration (the CLI's real-import
|
|
tests: the deterministic ``FakeEmbedder`` must not burn its canned
|
|
``chat`` on the KB outline — the import is what is under test)."""
|
|
|
|
async def fake_overview(llm: object, session: object = None) -> bool:
|
|
return False
|
|
|
|
monkeypatch.setattr(import_docs, "regenerate_overview", fake_overview)
|
|
|
|
|
|
def _kb_docs(db: Session) -> set[tuple[str, str]]:
|
|
"""The (source, path) pairs of the ``documents`` table."""
|
|
rows = db.execute(text("SELECT source, path FROM documents")).all()
|
|
return {(row.source, row.path) for row in rows}
|
|
|
|
|
|
def test_main_local_row_include_hidden_true_indexes_hidden_file(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
tmp_path: Path,
|
|
db: Session,
|
|
capsys: pytest.CaptureFixture[str],
|
|
clean_documents: None,
|
|
) -> None:
|
|
"""Phase 105 (A1): the CLI's DB-row path with the flag ON — the
|
|
file inside the hidden folder is indexed, embedded, and counted
|
|
like any visible file (real import over a host temp dir, the
|
|
deterministic ``FakeEmbedder``; the regression this phase most
|
|
plausibly breaks — the CLI's map built but lost — would leave it
|
|
out)."""
|
|
settings = _settings(sources_dir=str(tmp_path / "bor"))
|
|
monkeypatch.setattr(import_docs, "get_settings", lambda: settings)
|
|
local_dir = tmp_path / "LocalDocs"
|
|
local_dir.mkdir()
|
|
(local_dir / "visible.md").write_text("# Visible\nin scope\n", encoding="utf-8")
|
|
(local_dir / ".hidden").mkdir()
|
|
(local_dir / ".hidden" / "note.md").write_text("# Note\nhidden\n", encoding="utf-8")
|
|
monkeypatch.setattr(
|
|
import_docs,
|
|
"effective_sources",
|
|
lambda db: ([_local_row(str(local_dir), include_hidden=True)], "db"),
|
|
)
|
|
monkeypatch.setattr(import_docs, "LLMClient", lambda: FakeEmbedder())
|
|
_stub_overview(monkeypatch)
|
|
_stub_bump(monkeypatch)
|
|
_stub_folder_summaries(monkeypatch)
|
|
|
|
rc = import_docs.main([])
|
|
|
|
assert rc == 0
|
|
# The hidden file has a documents row next to the visible one.
|
|
assert _kb_docs(db) == {
|
|
("LocalDocs", "visible.md"),
|
|
("LocalDocs", ".hidden/note.md"),
|
|
}
|
|
out = capsys.readouterr().out
|
|
assert "added=2" in out # both files were imported
|
|
|
|
|
|
def test_main_local_row_include_hidden_false_skips_hidden_file(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
tmp_path: Path,
|
|
db: Session,
|
|
capsys: pytest.CaptureFixture[str],
|
|
clean_documents: None,
|
|
) -> None:
|
|
"""Phase 105 (A4): the same fixture with the default flag (off) —
|
|
the hidden-folder file never enters the KB (the byte-identical
|
|
pre-phase-105 walk): no documents row, and the summary line counts
|
|
only the visible file."""
|
|
settings = _settings(sources_dir=str(tmp_path / "bor"))
|
|
monkeypatch.setattr(import_docs, "get_settings", lambda: settings)
|
|
local_dir = tmp_path / "LocalDocs"
|
|
local_dir.mkdir()
|
|
(local_dir / "visible.md").write_text("# Visible\nin scope\n", encoding="utf-8")
|
|
(local_dir / ".hidden").mkdir()
|
|
(local_dir / ".hidden" / "note.md").write_text("# Note\nhidden\n", encoding="utf-8")
|
|
monkeypatch.setattr(
|
|
import_docs,
|
|
"effective_sources",
|
|
lambda db: ([_local_row(str(local_dir))], "db"), # flag defaults to False
|
|
)
|
|
monkeypatch.setattr(import_docs, "LLMClient", lambda: FakeEmbedder())
|
|
_stub_overview(monkeypatch)
|
|
_stub_bump(monkeypatch)
|
|
_stub_folder_summaries(monkeypatch)
|
|
|
|
rc = import_docs.main([])
|
|
|
|
assert rc == 0
|
|
assert _kb_docs(db) == {("LocalDocs", "visible.md")}
|
|
out = capsys.readouterr().out
|
|
assert "added=1" in out # the hidden file was never walked
|
|
|
|
|
|
# --- main() ----------------------------------------------------------------
|
|
|
|
|
|
def test_main_git_sources_clone_then_import(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
tmp_path: Path,
|
|
capsys: pytest.CaptureFixture[str],
|
|
) -> None:
|
|
settings = _settings(
|
|
git_sources="https://host/a/homelab.git,https://host/a/deploy.git",
|
|
sources_dir=str(tmp_path / "bor"),
|
|
)
|
|
monkeypatch.setattr(import_docs, "get_settings", lambda: settings)
|
|
monkeypatch.setattr(
|
|
import_docs,
|
|
"effective_sources",
|
|
lambda db: (
|
|
[_git_row("https://host/a/homelab.git"), _git_row("https://host/a/deploy.git")],
|
|
"env",
|
|
),
|
|
)
|
|
calls, fake = _fake_clone_factory()
|
|
monkeypatch.setattr(import_docs, "clone_or_pull", fake)
|
|
fake_import = FakeImportSources()
|
|
monkeypatch.setattr(import_docs, "import_sources", fake_import)
|
|
bumps = _stub_bump(monkeypatch)
|
|
_stub_folder_summaries(monkeypatch)
|
|
|
|
rc = import_docs.main([])
|
|
|
|
assert rc == 0
|
|
assert [(url, dest) for url, dest in calls] == [
|
|
("https://host/a/homelab.git", tmp_path / "bor" / "homelab"),
|
|
("https://host/a/deploy.git", tmp_path / "bor" / "deploy"),
|
|
]
|
|
assert len(fake_import.calls) == 1
|
|
# The cloned checkouts are exactly what gets imported.
|
|
assert fake_import.calls[0]["sources"] == [
|
|
tmp_path / "bor" / "homelab",
|
|
tmp_path / "bor" / "deploy",
|
|
]
|
|
# Phase 106: the git rows' (fake, non-repo) checkouts fail soft to
|
|
# ``{}`` — but the roots ARE listed (the CLI feeds the map).
|
|
assert fake_import.calls[0]["doc_dates_by_root"] == {
|
|
str(tmp_path / "bor" / "homelab"): {},
|
|
str(tmp_path / "bor" / "deploy"): {},
|
|
}
|
|
for dest in (tmp_path / "bor" / "homelab", tmp_path / "bor" / "deploy"):
|
|
assert (dest / "notes.md").is_file()
|
|
# The final summary print reflects the import (added > 0).
|
|
out = capsys.readouterr().out
|
|
assert "added=1" in out
|
|
# Phase 53: a KB-changing run bumps the sources version exactly
|
|
# once and reports it (stubbed — this file keeps its no-real-DB
|
|
# style for the counter, like the fake import above).
|
|
assert len(bumps) == 1
|
|
assert "sources_version=1" in out
|
|
|
|
|
|
def test_main_cli_source_still_imports_manual_dir(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
tmp_path: Path,
|
|
capsys: pytest.CaptureFixture[str],
|
|
) -> None:
|
|
manual = tmp_path / "manual"
|
|
manual.mkdir()
|
|
(manual / "a.md").write_text("# A\nhi\n", encoding="utf-8")
|
|
# BOR_GIT_SOURCES is set but must be ignored — --source always wins.
|
|
settings = _settings(git_sources="https://host/a/repo.git")
|
|
monkeypatch.setattr(import_docs, "get_settings", lambda: settings)
|
|
calls, fake = _fake_clone_factory()
|
|
monkeypatch.setattr(import_docs, "clone_or_pull", fake)
|
|
fake_import = FakeImportSources()
|
|
monkeypatch.setattr(import_docs, "import_sources", fake_import)
|
|
bumps = _stub_bump(monkeypatch)
|
|
_stub_folder_summaries(monkeypatch)
|
|
|
|
rc = import_docs.main(["--source", str(manual)])
|
|
|
|
assert rc == 0
|
|
assert calls == []
|
|
assert fake_import.calls[0]["sources"] == [manual]
|
|
# Phase 106: manual --source has no rows (no clone) → no date map
|
|
# (the importer's mtime fallback applies).
|
|
assert fake_import.calls[0]["doc_dates_by_root"] == {}
|
|
assert fake_import.calls[0]["prune"] is False
|
|
# Phase 53: a manual --source run that changes the KB bumps exactly
|
|
# once (the CLI is the other canonical sync path).
|
|
assert len(bumps) == 1
|
|
assert "sources_version=1" in capsys.readouterr().out
|
|
|
|
|
|
def test_resolve_sources_mixed_git_and_local(
|
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
) -> None:
|
|
"""Phase 38: DB rows of both kinds — the git row is cloned into
|
|
``BOR_SOURCES_DIR``, the local row is its existing directory itself
|
|
(no clone), in row order; the env list is ignored."""
|
|
calls, fake = _fake_clone_factory()
|
|
monkeypatch.setattr(import_docs, "clone_or_pull", fake)
|
|
local_dir = tmp_path / "LocalDocs"
|
|
local_dir.mkdir()
|
|
(local_dir / "a.md").write_text("# A\nlocal fixture\n", encoding="utf-8")
|
|
monkeypatch.setattr(
|
|
import_docs,
|
|
"effective_sources",
|
|
lambda db: (
|
|
[_git_row("https://db.example/only.git"), _local_row(str(local_dir))],
|
|
"db",
|
|
),
|
|
)
|
|
settings = _settings(
|
|
git_sources="https://env.example/ignored.git",
|
|
sources_dir=str(tmp_path / "bor"),
|
|
)
|
|
|
|
sources, ignore_map, hidden_map, date_map = import_docs._resolve_sources(None, settings)
|
|
|
|
assert sources == [tmp_path / "bor" / "only", local_dir]
|
|
assert ignore_map == {} # phase 89: neither row carries a list
|
|
# Phase 105: both rows default-flag → per-root False entries (A4).
|
|
assert hidden_map == {str(tmp_path / "bor" / "only"): False, str(local_dir): False}
|
|
# Phase 106: only the git row is listed (local takes the mtime
|
|
# fallback); the fake checkout's date walk fails soft to ``{}``.
|
|
assert date_map == {str(tmp_path / "bor" / "only"): {}}
|
|
assert calls == [("https://db.example/only.git", tmp_path / "bor" / "only")]
|
|
|
|
|
|
def test_resolve_sources_missing_local_dir_aborts(
|
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
) -> None:
|
|
"""Phase 38: a local row whose directory is gone at run time →
|
|
``GitSyncError`` naming the path, before any import (the same
|
|
pre-import fail-loud as a failing git clone)."""
|
|
monkeypatch.setattr(import_docs, "clone_or_pull", _fake_clone_factory()[1])
|
|
missing = tmp_path / "Gone"
|
|
monkeypatch.setattr(
|
|
import_docs,
|
|
"effective_sources",
|
|
lambda db: ([_local_row(str(missing))], "db"),
|
|
)
|
|
settings = _settings(sources_dir=str(tmp_path / "bor"))
|
|
|
|
with pytest.raises(GitSyncError, match=f"local source missing: {re.escape(str(missing))}"):
|
|
import_docs._resolve_sources(None, settings)
|
|
|
|
|
|
def test_main_git_failure_aborts_before_import(
|
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str]
|
|
) -> None:
|
|
settings = _settings(
|
|
git_sources="https://host/a/bad.git",
|
|
sources_dir=str(tmp_path / "bor"),
|
|
)
|
|
monkeypatch.setattr(import_docs, "get_settings", lambda: settings)
|
|
monkeypatch.setattr(
|
|
import_docs,
|
|
"effective_sources",
|
|
lambda db: ([_git_row("https://host/a/bad.git")], "env"),
|
|
)
|
|
|
|
def failing_clone(url: str, dest: Path | str) -> Path:
|
|
raise GitSyncError(
|
|
f"git clone {url} failed (exit 128): "
|
|
"fatal: repository not found"
|
|
)
|
|
|
|
monkeypatch.setattr(import_docs, "clone_or_pull", failing_clone)
|
|
fake_import = FakeImportSources()
|
|
monkeypatch.setattr(import_docs, "import_sources", fake_import)
|
|
|
|
rc = import_docs.main([])
|
|
|
|
assert rc == 1
|
|
err = capsys.readouterr().err
|
|
assert "import_docs: source sync failed" in err
|
|
assert "bad.git" in err # the failing repo is named
|
|
assert fake_import.calls == [] # no partial import
|
|
assert not (tmp_path / "bor").exists()
|
|
|
|
|
|
def test_main_missing_local_dir_aborts_before_import(
|
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str]
|
|
) -> None:
|
|
"""Phase 38: a DB local row whose directory is missing → exit code
|
|
1, ``local source missing: <path>`` on stderr, zero import attempts
|
|
(no ``--source`` given, so the DB row is what should have been
|
|
imported)."""
|
|
settings = _settings(sources_dir=str(tmp_path / "bor"))
|
|
monkeypatch.setattr(import_docs, "get_settings", lambda: settings)
|
|
missing = tmp_path / "Gone"
|
|
monkeypatch.setattr(
|
|
import_docs,
|
|
"effective_sources",
|
|
lambda db: ([_local_row(str(missing))], "db"),
|
|
)
|
|
fake_import = FakeImportSources()
|
|
monkeypatch.setattr(import_docs, "import_sources", fake_import)
|
|
|
|
rc = import_docs.main([])
|
|
|
|
assert rc == 1
|
|
err = capsys.readouterr().err
|
|
assert "import_docs: source sync failed" in err
|
|
assert f"local source missing: {missing}" in err # the path is named
|
|
assert fake_import.calls == [] # no partial import
|