298 lines
11 KiB
Python
298 lines
11 KiB
Python
"""Integration test: ``import_docs`` regenerates the KB overview (phase 31, task 04).
|
|
|
|
Drives ``scripts.import_docs.main()`` end to end against the local compose
|
|
Postgres with a deterministic fake LLM (no live aipi, no git — explicit
|
|
``--source`` dirs and fresh settings, the phase 28 test's mocking style),
|
|
covering the change-gated overview trigger:
|
|
|
|
- a KB-changing import → ``kb_overview`` row written, the ``lite`` ``chat``
|
|
called exactly once, summary line ends ``overview=updated``;
|
|
- an unchanged re-import → ``chat`` **not** called again, ``overview=skipped``;
|
|
- a ``lite`` failure → exit code still ``0`` (the import itself was fine),
|
|
``overview=failed``, the previous row untouched;
|
|
- a ``--limit`` debug run with changes → ``overview=skipped``;
|
|
- an empty source run with no row → no row created, ``overview=skipped``.
|
|
|
|
Phase 53 (task 02): the sources-version bump sits alongside the
|
|
overview gate — a KB-changing run bumps ``sources_meta`` exactly once
|
|
(``sources_version=<n>`` on the summary line), including a **prune-only**
|
|
run: the invalidation gate ``added + updated + pruned > 0`` is
|
|
deliberately broader than the overview's (a pruned doc can invalidate a
|
|
saved answer that cited it, while the outline stays). ``--limit`` runs
|
|
and unchanged re-runs never bump (``sources_version=skipped``), and a
|
|
failed ``lite`` never rolls the bump back. The counter is pinned to the
|
|
migration-0010 seed (0) around every test by
|
|
:func:`_reset_sources_version`.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from collections.abc import Iterator
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from sqlalchemy import text
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app.config import Settings
|
|
from app.db import SessionLocal, db_available
|
|
from app.models import KbOverview
|
|
from app.rag.llm import LLMError
|
|
from app.rag.sources_meta import current_sources_version
|
|
from scripts import import_docs
|
|
from tests.fakes import FakeEmbedder
|
|
|
|
|
|
class FailingChatEmbedder(FakeEmbedder):
|
|
"""A ``lite`` model that always fails (drives the fail-soft path)."""
|
|
|
|
async def chat(self, messages: list[dict[str, str]], model: str | None = None) -> str:
|
|
self.chat_calls.append(list(messages))
|
|
raise LLMError("simulated lite-model failure (test sentinel)")
|
|
|
|
|
|
def _row(db: Session) -> KbOverview | None:
|
|
"""The stored ``kb_overview`` row (freshly reloaded)."""
|
|
db.expire_all()
|
|
return db.get(KbOverview, 1)
|
|
|
|
|
|
def _version(db: Session) -> int:
|
|
"""The ``sources_meta`` generation (phase 53; freshly reloaded)."""
|
|
db.expire_all()
|
|
return current_sources_version(db)
|
|
|
|
|
|
def _run_main(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
llm: FakeEmbedder,
|
|
argv: list[str],
|
|
capsys: pytest.CaptureFixture[str],
|
|
) -> tuple[int, str]:
|
|
"""Run ``import_docs.main`` with fresh settings, a fake LLM, and a
|
|
fail-loud git mock (``--source`` always wins, so git must stay idle)."""
|
|
monkeypatch.setattr(
|
|
import_docs,
|
|
"get_settings",
|
|
lambda: Settings(_env_file=None), # pyright: ignore[reportCallIssue]
|
|
)
|
|
|
|
def _no_git(url: str, dest: Path | str) -> Path:
|
|
raise AssertionError("git sync must not run with explicit --source")
|
|
|
|
monkeypatch.setattr(import_docs, "clone_or_pull", _no_git)
|
|
monkeypatch.setattr(import_docs, "LLMClient", lambda: llm)
|
|
rc = import_docs.main(argv)
|
|
return rc, capsys.readouterr().out
|
|
|
|
|
|
@pytest.fixture()
|
|
def src(tmp_path: Path) -> Path:
|
|
"""A source dir with two markdown docs (md → no summary chat calls)."""
|
|
root = tmp_path / "MyDocs"
|
|
root.mkdir()
|
|
(root / "alpha.md").write_text("# Alpha\n\nFirst document.\n", encoding="utf-8")
|
|
(root / "beta.md").write_text("# Beta\n\nSecond document.\n", encoding="utf-8")
|
|
return root
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _clean_kb(db: Session) -> Iterator[None]:
|
|
db.execute(text("TRUNCATE chunks, documents, kb_overview"))
|
|
db.commit()
|
|
yield
|
|
db.execute(text("TRUNCATE chunks, documents, kb_overview"))
|
|
db.commit()
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _reset_sources_version() -> Iterator[None]:
|
|
"""Phase 53: the sources version counter is global mutable state —
|
|
pin it to the migration-0010 seed (0) around every test so the bump
|
|
assertions start from a known generation (own session: the CLI
|
|
bumps through its own short-lived ``SessionLocal``). Skips like the
|
|
``db`` fixture when Postgres is down."""
|
|
if not db_available():
|
|
pytest.skip("Postgres not reachable — run `podman compose up -d db` first")
|
|
session = SessionLocal()
|
|
try:
|
|
session.execute(text("UPDATE sources_meta SET version = 0 WHERE id = 1"))
|
|
session.commit()
|
|
yield
|
|
finally:
|
|
session.execute(text("UPDATE sources_meta SET version = 0 WHERE id = 1"))
|
|
session.commit()
|
|
session.close()
|
|
|
|
|
|
def test_changed_import_writes_overview_row(
|
|
db: Session,
|
|
src: Path,
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
capsys: pytest.CaptureFixture[str],
|
|
) -> None:
|
|
llm = FakeEmbedder()
|
|
records: list[logging.LogRecord] = []
|
|
|
|
class _Sink(logging.Handler):
|
|
def emit(self, record: logging.LogRecord) -> None:
|
|
records.append(record)
|
|
|
|
overview_logger = logging.getLogger("app.rag.overview")
|
|
sink = _Sink()
|
|
overview_logger.addHandler(sink)
|
|
overview_logger.setLevel(logging.INFO)
|
|
try:
|
|
rc, out = _run_main(monkeypatch, llm, ["--source", str(src)], capsys)
|
|
finally:
|
|
overview_logger.removeHandler(sink)
|
|
|
|
assert rc == 0
|
|
assert "added=2" in out
|
|
assert out.rstrip().endswith("overview=updated sources_version=1")
|
|
assert _version(db) == 1 # phase 53: a changed import bumps exactly once
|
|
# Exactly one lite call — the overview itself (markdown files never
|
|
# get a summary, so nothing else may touch ``chat``).
|
|
assert len(llm.chat_calls) == 1
|
|
by_role = {m["role"]: m["content"] for m in llm.chat_calls[0]}
|
|
assert "KB_OVERVIEW_MODE" in by_role["system"]
|
|
# One line per doc: source — path — title (no summary for markdown).
|
|
assert "MyDocs — alpha.md — Alpha" in by_role["user"]
|
|
assert "MyDocs — beta.md — Beta" in by_role["user"]
|
|
# The model's outline lands in the single row.
|
|
row = _row(db)
|
|
assert row is not None
|
|
assert row.content == "Summary of MyDocs"
|
|
assert row.updated_at is not None
|
|
# The phase's required log line (PLAN §9).
|
|
assert any("overview: regenerated docs=2 chars=" in r.getMessage() for r in records)
|
|
|
|
|
|
def test_unchanged_reimport_does_not_call_lite(
|
|
db: Session,
|
|
src: Path,
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
capsys: pytest.CaptureFixture[str],
|
|
) -> None:
|
|
llm = FakeEmbedder()
|
|
rc, out = _run_main(monkeypatch, llm, ["--source", str(src)], capsys)
|
|
assert rc == 0
|
|
assert out.rstrip().endswith("overview=updated sources_version=1")
|
|
assert len(llm.chat_calls) == 1
|
|
assert _row(db) is not None
|
|
|
|
# Same hashes → no KB change → no lite call, previous outline kept.
|
|
rc, out = _run_main(monkeypatch, llm, ["--source", str(src)], capsys)
|
|
assert rc == 0
|
|
assert "unchanged=2" in out
|
|
assert out.rstrip().endswith("overview=skipped sources_version=skipped")
|
|
assert len(llm.chat_calls) == 1 # no new lite call
|
|
row = _row(db)
|
|
assert row is not None and row.content == "Summary of MyDocs"
|
|
assert _version(db) == 1 # phase 53: an unchanged re-run never bumps
|
|
|
|
|
|
def test_lite_failure_is_fail_soft(
|
|
db: Session,
|
|
src: Path,
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
capsys: pytest.CaptureFixture[str],
|
|
) -> None:
|
|
good = FakeEmbedder()
|
|
rc, out = _run_main(monkeypatch, good, ["--source", str(src)], capsys)
|
|
assert rc == 0
|
|
assert out.rstrip().endswith("overview=updated sources_version=1")
|
|
previous = _row(db)
|
|
assert previous is not None
|
|
previous_content = previous.content
|
|
|
|
# A KB-changing run whose ``lite`` model fails: the import still
|
|
# succeeds (exit 0) and the previous outline stays untouched.
|
|
(src / "alpha.md").write_text("# Alpha\n\nChanged content.\n", encoding="utf-8")
|
|
bad = FailingChatEmbedder()
|
|
rc, out = _run_main(monkeypatch, bad, ["--source", str(src)], capsys)
|
|
assert rc == 0 # a failed outline must not fail the import
|
|
assert "updated=1" in out
|
|
assert out.rstrip().endswith("overview=failed sources_version=2")
|
|
assert len(bad.chat_calls) == 1 # the (failed) attempt was made
|
|
row = _row(db)
|
|
assert row is not None
|
|
assert row.content == previous_content # previous row untouched
|
|
# Phase 53: the bump commits independently of the best-effort
|
|
# overview — a failed ``lite`` never rolls the version back.
|
|
assert _version(db) == 2
|
|
|
|
|
|
def test_limit_run_skips_overview(
|
|
db: Session,
|
|
src: Path,
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
capsys: pytest.CaptureFixture[str],
|
|
) -> None:
|
|
llm = FakeEmbedder()
|
|
rc, out = _run_main(monkeypatch, llm, ["--source", str(src)], capsys)
|
|
assert rc == 0
|
|
assert out.rstrip().endswith("overview=updated sources_version=1")
|
|
assert len(llm.chat_calls) == 1
|
|
|
|
# An incomplete walk must not rewrite the outline (mirrors the
|
|
# --prune-with---limit guard) — and must not advance the version.
|
|
(src / "alpha.md").write_text("# Alpha\n\nChanged content.\n", encoding="utf-8")
|
|
rc, out = _run_main(monkeypatch, llm, ["--source", str(src), "--limit", "1"], capsys)
|
|
assert rc == 0
|
|
assert "updated=1" in out
|
|
assert out.rstrip().endswith("overview=skipped sources_version=skipped")
|
|
assert len(llm.chat_calls) == 1 # --limit never burns a lite call
|
|
row = _row(db)
|
|
assert row is not None and row.content == "Summary of MyDocs"
|
|
assert _version(db) == 1 # phase 53: --limit debug runs never bump
|
|
|
|
|
|
def test_empty_source_without_row_creates_nothing(
|
|
db: Session,
|
|
tmp_path: Path,
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
capsys: pytest.CaptureFixture[str],
|
|
) -> None:
|
|
empty = tmp_path / "EmptyDocs"
|
|
empty.mkdir()
|
|
llm = FakeEmbedder()
|
|
|
|
rc, out = _run_main(monkeypatch, llm, ["--source", str(empty)], capsys)
|
|
|
|
assert rc == 0
|
|
assert "files=0" in out
|
|
assert out.rstrip().endswith("overview=skipped sources_version=skipped")
|
|
assert llm.chat_calls == [] # no KB → no outline, no wasted model call
|
|
assert _row(db) is None # nothing created
|
|
assert _version(db) == 0 # nothing changed → nothing bumped
|
|
|
|
|
|
def test_prune_only_run_bumps_sources_version(
|
|
db: Session,
|
|
src: Path,
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
capsys: pytest.CaptureFixture[str],
|
|
) -> None:
|
|
"""Phase 53 (task 02): the invalidation gate is deliberately broader
|
|
than the overview's — a prune-only run (added + updated == 0,
|
|
pruned > 0) advances the version (a pruned document can invalidate
|
|
a saved answer that cited it) while the outline stays.
|
|
"""
|
|
llm = FakeEmbedder()
|
|
rc, out = _run_main(monkeypatch, llm, ["--source", str(src)], capsys)
|
|
assert rc == 0
|
|
assert "added=2" in out
|
|
assert out.rstrip().endswith("overview=updated sources_version=1")
|
|
assert _version(db) == 1
|
|
|
|
# Delete one file; a --prune run drops exactly it: no add/update,
|
|
# but pruned=1 → the version still bumps while the overview skips.
|
|
(src / "alpha.md").unlink()
|
|
rc, out = _run_main(monkeypatch, llm, ["--source", str(src), "--prune"], capsys)
|
|
assert rc == 0
|
|
assert "pruned=1" in out
|
|
assert out.rstrip().endswith("overview=skipped sources_version=2")
|
|
assert _version(db) == 2 # the prune-only change bumped exactly once
|
|
assert _row(db) is not None # the outline row is untouched
|