feat(rag): lite-generated KB overview in the system prompt — stored single row, regenerated on import, <knowledge_base> section in HIGH+LOW prompts
This commit is contained in:
@@ -0,0 +1,222 @@
|
||||
"""Integration test: ``import_docs`` regenerates the KB overview (phase 31, task 04).
|
||||
|
||||
Drives ``scripts.import_docs.main()`` end to end against the local compose
|
||||
Postgres with a deterministic fake LLM (no live aipi, no git — explicit
|
||||
``--source`` dirs and fresh settings, the phase 28 test's mocking style),
|
||||
covering the change-gated overview trigger:
|
||||
|
||||
- a KB-changing import → ``kb_overview`` row written, the ``lite`` ``chat``
|
||||
called exactly once, summary line ends ``overview=updated``;
|
||||
- an unchanged re-import → ``chat`` **not** called again, ``overview=skipped``;
|
||||
- a ``lite`` failure → exit code still ``0`` (the import itself was fine),
|
||||
``overview=failed``, the previous row untouched;
|
||||
- a ``--limit`` debug run with changes → ``overview=skipped``;
|
||||
- an empty source run with no row → no row created, ``overview=skipped``.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import Settings
|
||||
from app.models import KbOverview
|
||||
from app.rag.llm import LLMError
|
||||
from scripts import import_docs
|
||||
from tests.fakes import FakeEmbedder
|
||||
|
||||
|
||||
class FailingChatEmbedder(FakeEmbedder):
|
||||
"""A ``lite`` model that always fails (drives the fail-soft path)."""
|
||||
|
||||
async def chat(self, messages: list[dict[str, str]], model: str | None = None) -> str:
|
||||
self.chat_calls.append(list(messages))
|
||||
raise LLMError("simulated lite-model failure (test sentinel)")
|
||||
|
||||
|
||||
def _row(db: Session) -> KbOverview | None:
|
||||
"""The stored ``kb_overview`` row (freshly reloaded)."""
|
||||
db.expire_all()
|
||||
return db.get(KbOverview, 1)
|
||||
|
||||
|
||||
def _run_main(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
llm: FakeEmbedder,
|
||||
argv: list[str],
|
||||
capsys: pytest.CaptureFixture[str],
|
||||
) -> tuple[int, str]:
|
||||
"""Run ``import_docs.main`` with fresh settings, a fake LLM, and a
|
||||
fail-loud git mock (``--source`` always wins, so git must stay idle)."""
|
||||
monkeypatch.setattr(
|
||||
import_docs,
|
||||
"get_settings",
|
||||
lambda: Settings(_env_file=None), # pyright: ignore[reportCallIssue]
|
||||
)
|
||||
|
||||
def _no_git(url: str, dest: Path | str) -> Path:
|
||||
raise AssertionError("git sync must not run with explicit --source")
|
||||
|
||||
monkeypatch.setattr(import_docs, "clone_or_pull", _no_git)
|
||||
monkeypatch.setattr(import_docs, "LLMClient", lambda: llm)
|
||||
rc = import_docs.main(argv)
|
||||
return rc, capsys.readouterr().out
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def src(tmp_path: Path) -> Path:
|
||||
"""A source dir with two markdown docs (md → no summary chat calls)."""
|
||||
root = tmp_path / "MyDocs"
|
||||
root.mkdir()
|
||||
(root / "alpha.md").write_text("# Alpha\n\nFirst document.\n", encoding="utf-8")
|
||||
(root / "beta.md").write_text("# Beta\n\nSecond document.\n", encoding="utf-8")
|
||||
return root
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clean_kb(db: Session) -> Iterator[None]:
|
||||
db.execute(text("TRUNCATE chunks, documents, kb_overview"))
|
||||
db.commit()
|
||||
yield
|
||||
db.execute(text("TRUNCATE chunks, documents, kb_overview"))
|
||||
db.commit()
|
||||
|
||||
|
||||
def test_changed_import_writes_overview_row(
|
||||
db: Session,
|
||||
src: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
capsys: pytest.CaptureFixture[str],
|
||||
) -> None:
|
||||
llm = FakeEmbedder()
|
||||
records: list[logging.LogRecord] = []
|
||||
|
||||
class _Sink(logging.Handler):
|
||||
def emit(self, record: logging.LogRecord) -> None:
|
||||
records.append(record)
|
||||
|
||||
overview_logger = logging.getLogger("app.rag.overview")
|
||||
sink = _Sink()
|
||||
overview_logger.addHandler(sink)
|
||||
overview_logger.setLevel(logging.INFO)
|
||||
try:
|
||||
rc, out = _run_main(monkeypatch, llm, ["--source", str(src)], capsys)
|
||||
finally:
|
||||
overview_logger.removeHandler(sink)
|
||||
|
||||
assert rc == 0
|
||||
assert "added=2" in out
|
||||
assert out.rstrip().endswith("overview=updated")
|
||||
# Exactly one lite call — the overview itself (markdown files never
|
||||
# get a summary, so nothing else may touch ``chat``).
|
||||
assert len(llm.chat_calls) == 1
|
||||
by_role = {m["role"]: m["content"] for m in llm.chat_calls[0]}
|
||||
assert "KB_OVERVIEW_MODE" in by_role["system"]
|
||||
# One line per doc: source — path — title (no summary for markdown).
|
||||
assert "MyDocs — alpha.md — Alpha" in by_role["user"]
|
||||
assert "MyDocs — beta.md — Beta" in by_role["user"]
|
||||
# The model's outline lands in the single row.
|
||||
row = _row(db)
|
||||
assert row is not None
|
||||
assert row.content == "Summary of MyDocs"
|
||||
assert row.updated_at is not None
|
||||
# The phase's required log line (PLAN §9).
|
||||
assert any("overview: regenerated docs=2 chars=" in r.getMessage() for r in records)
|
||||
|
||||
|
||||
def test_unchanged_reimport_does_not_call_lite(
|
||||
db: Session,
|
||||
src: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
capsys: pytest.CaptureFixture[str],
|
||||
) -> None:
|
||||
llm = FakeEmbedder()
|
||||
rc, out = _run_main(monkeypatch, llm, ["--source", str(src)], capsys)
|
||||
assert rc == 0
|
||||
assert out.rstrip().endswith("overview=updated")
|
||||
assert len(llm.chat_calls) == 1
|
||||
assert _row(db) is not None
|
||||
|
||||
# Same hashes → no KB change → no lite call, previous outline kept.
|
||||
rc, out = _run_main(monkeypatch, llm, ["--source", str(src)], capsys)
|
||||
assert rc == 0
|
||||
assert "unchanged=2" in out
|
||||
assert out.rstrip().endswith("overview=skipped")
|
||||
assert len(llm.chat_calls) == 1 # no new lite call
|
||||
row = _row(db)
|
||||
assert row is not None and row.content == "Summary of MyDocs"
|
||||
|
||||
|
||||
def test_lite_failure_is_fail_soft(
|
||||
db: Session,
|
||||
src: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
capsys: pytest.CaptureFixture[str],
|
||||
) -> None:
|
||||
good = FakeEmbedder()
|
||||
rc, out = _run_main(monkeypatch, good, ["--source", str(src)], capsys)
|
||||
assert rc == 0
|
||||
assert out.rstrip().endswith("overview=updated")
|
||||
previous = _row(db)
|
||||
assert previous is not None
|
||||
previous_content = previous.content
|
||||
|
||||
# A KB-changing run whose ``lite`` model fails: the import still
|
||||
# succeeds (exit 0) and the previous outline stays untouched.
|
||||
(src / "alpha.md").write_text("# Alpha\n\nChanged content.\n", encoding="utf-8")
|
||||
bad = FailingChatEmbedder()
|
||||
rc, out = _run_main(monkeypatch, bad, ["--source", str(src)], capsys)
|
||||
assert rc == 0 # a failed outline must not fail the import
|
||||
assert "updated=1" in out
|
||||
assert out.rstrip().endswith("overview=failed")
|
||||
assert len(bad.chat_calls) == 1 # the (failed) attempt was made
|
||||
row = _row(db)
|
||||
assert row is not None
|
||||
assert row.content == previous_content # previous row untouched
|
||||
|
||||
|
||||
def test_limit_run_skips_overview(
|
||||
db: Session,
|
||||
src: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
capsys: pytest.CaptureFixture[str],
|
||||
) -> None:
|
||||
llm = FakeEmbedder()
|
||||
rc, out = _run_main(monkeypatch, llm, ["--source", str(src)], capsys)
|
||||
assert rc == 0
|
||||
assert out.rstrip().endswith("overview=updated")
|
||||
assert len(llm.chat_calls) == 1
|
||||
|
||||
# An incomplete walk must not rewrite the outline (mirrors the
|
||||
# --prune-with---limit guard).
|
||||
(src / "alpha.md").write_text("# Alpha\n\nChanged content.\n", encoding="utf-8")
|
||||
rc, out = _run_main(monkeypatch, llm, ["--source", str(src), "--limit", "1"], capsys)
|
||||
assert rc == 0
|
||||
assert "updated=1" in out
|
||||
assert out.rstrip().endswith("overview=skipped")
|
||||
assert len(llm.chat_calls) == 1 # --limit never burns a lite call
|
||||
row = _row(db)
|
||||
assert row is not None and row.content == "Summary of MyDocs"
|
||||
|
||||
|
||||
def test_empty_source_without_row_creates_nothing(
|
||||
db: Session,
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
capsys: pytest.CaptureFixture[str],
|
||||
) -> None:
|
||||
empty = tmp_path / "EmptyDocs"
|
||||
empty.mkdir()
|
||||
llm = FakeEmbedder()
|
||||
|
||||
rc, out = _run_main(monkeypatch, llm, ["--source", str(empty)], capsys)
|
||||
|
||||
assert rc == 0
|
||||
assert "files=0" in out
|
||||
assert out.rstrip().endswith("overview=skipped")
|
||||
assert llm.chat_calls == [] # no KB → no outline, no wasted model call
|
||||
assert _row(db) is None # nothing created
|
||||
Reference in New Issue
Block a user