feat(rag): lite-generated KB overview in the system prompt — stored single row, regenerated on import, <knowledge_base> section in HIGH+LOW prompts
This commit is contained in:
@@ -0,0 +1,281 @@
|
||||
"""Phase 31 E2E (Playwright): the stored KB overview reaches every turn.
|
||||
|
||||
Story: ``.agent/user_stories/kb-overview-prompt.md``
|
||||
Run in isolation (DB must be up: ``podman compose up -d db``):
|
||||
|
||||
uv run pytest tests/e2e/test_kb_overview.py -v --no-cov
|
||||
|
||||
The story: a single-row ``kb_overview`` table holds a lite-generated
|
||||
plain-text outline of the KB's basic categories; every chat turn — HIGH
|
||||
(grounded) and LOW (deflected) alike — injects it into the system prompt
|
||||
as the ``<knowledge_base>`` section, so the agent knows roughly what the
|
||||
KB contains before retrieval returns. The mock LLM echoes the section's
|
||||
first bullet into its answer (`` (kb: <first bullet>)`` — the
|
||||
``(tuning: …)`` steering-echo precedent), which makes the prompt change
|
||||
observable in the rendered UI deterministically.
|
||||
|
||||
The row is seeded **directly in the DB** so the INJECTION path is what
|
||||
is under test (the import-time trigger is integration-tested in
|
||||
``tests/integration/test_import_docs_overview.py``). The outline is
|
||||
multi-line so the echo must pick the FIRST bullet — not the intro line,
|
||||
not a later line. The last test additionally runs the real
|
||||
``regenerate_overview`` against the mock to cover the mock's
|
||||
``KB_OVERVIEW_MODE`` generation branch end-to-end (stored outline is the
|
||||
byte-stable 8-token digest of the generator's document list).
|
||||
|
||||
Test → story mapping (Playwright Mapping Rule):
|
||||
1. ``test_on_topic_answer_echoes_kb_overview``
|
||||
2. ``test_deflected_answer_still_echoes_kb_overview``
|
||||
3. ``test_absent_row_yields_no_kb_echo``
|
||||
4. ``test_mock_generated_outline_is_stored_and_echoed``
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import re
|
||||
from pathlib import Path
|
||||
from threading import Thread
|
||||
from typing import Any
|
||||
|
||||
from playwright.sync_api import Page, expect
|
||||
from sqlalchemy import select, text
|
||||
|
||||
from app.config import Settings
|
||||
from app.db import SessionLocal
|
||||
from app.models import Document, KbOverview, QueryLog
|
||||
from app.rag.importer import ImportSummary, import_sources
|
||||
from app.rag.llm import LLMClient
|
||||
from app.rag.overview import build_overview_prompt, regenerate_overview
|
||||
from tests.e2e.mock_llm import TOKEN_RE
|
||||
|
||||
REPO = Path(__file__).resolve().parents[2]
|
||||
FIXTURES = REPO / "tests" / "fixtures" / "docs"
|
||||
QUESTION = "How is my Kubernetes cluster set up?"
|
||||
OFF_TOPIC = "How do I bake sourdough bread?"
|
||||
MOCK_ANSWER_MARKER = "Deterministic mock answer for E2E"
|
||||
|
||||
#: The stored outline: multi-line ``-`` bullets (the generator's locked
|
||||
#: format). The echo must pick the FIRST bullet — the parsing (skip the
|
||||
#: intro line, strip the dash) is part of the story.
|
||||
OVERVIEW = (
|
||||
"- Kubernetes cluster and node maintenance notes\n"
|
||||
"- Backup schedules and restore runbooks\n"
|
||||
"- Networking: static DNS and kafkabridge"
|
||||
)
|
||||
FIRST_BULLET = "Kubernetes cluster and node maintenance notes"
|
||||
KB_ECHO = f"(kb: {FIRST_BULLET})"
|
||||
|
||||
|
||||
# --- Importer + thread helpers (test_document_summaries.py pattern) ---
|
||||
|
||||
|
||||
async def _import_fixtures(mock_port: int) -> ImportSummary:
|
||||
kwargs: dict[str, Any] = {
|
||||
"_env_file": None,
|
||||
"llm_base_url": f"http://127.0.0.1:{mock_port}/v1",
|
||||
}
|
||||
settings = Settings(**kwargs) # pyright: ignore[reportCallIssue]
|
||||
return await import_sources([FIXTURES], LLMClient(settings))
|
||||
|
||||
|
||||
def _run_in_thread(coro: Any) -> Any:
|
||||
"""Run a coroutine on a worker thread.
|
||||
|
||||
Playwright's sync API keeps an asyncio loop running on the test thread,
|
||||
so ``asyncio.run`` cannot be called directly from a test body.
|
||||
"""
|
||||
box: dict[str, Any] = {}
|
||||
|
||||
def runner() -> None:
|
||||
try:
|
||||
box["value"] = asyncio.run(coro)
|
||||
except BaseException as e: # noqa: BLE001 — re-raised on the test thread
|
||||
box["error"] = e
|
||||
|
||||
t = Thread(target=runner)
|
||||
t.start()
|
||||
t.join()
|
||||
if "error" in box:
|
||||
raise box["error"]
|
||||
return box["value"]
|
||||
|
||||
|
||||
def _reset_db(
|
||||
mock_port: int, seed: bool, overview: str | None = None
|
||||
) -> ImportSummary | None:
|
||||
"""Truncate the KB (+ query log, steering, kb_overview), re-import the
|
||||
fixtures, and optionally seed the single ``kb_overview`` row."""
|
||||
with SessionLocal() as db:
|
||||
db.execute(
|
||||
text("TRUNCATE chunks, documents, query_log, steering_notes, kb_overview")
|
||||
)
|
||||
db.commit()
|
||||
if overview is not None:
|
||||
db.add(KbOverview(id=1, content=overview))
|
||||
db.commit()
|
||||
if not seed:
|
||||
return None
|
||||
return _run_in_thread(_import_fixtures(mock_port))
|
||||
|
||||
|
||||
def _ask(page: Page, app_url: str, question: str, done_text: str) -> Any:
|
||||
"""Submit *question* and wait until the streamed answer has fully
|
||||
landed.
|
||||
|
||||
``done_text`` is the deterministic last thing the answer contains for
|
||||
its path: the ``(kb: …)`` echo (grounded AND deflected turns with a
|
||||
stored row — it is appended after everything else) or the mock
|
||||
marker (the no-row control, where the echo is absent by design).
|
||||
"""
|
||||
page.set_default_timeout(30_000)
|
||||
page.goto(app_url)
|
||||
page.fill("#message-input", question)
|
||||
page.click("#send-btn")
|
||||
bubble = page.locator(".msg.brain .bubble")
|
||||
bubble.first.wait_for(state="visible", timeout=30_000)
|
||||
expect(bubble.first).to_contain_text(done_text, timeout=30_000)
|
||||
expect(page.locator("#send-btn")).to_be_enabled()
|
||||
expect(page.locator("#send-label")).to_have_text("Send")
|
||||
return bubble.first
|
||||
|
||||
|
||||
def _last_query_log() -> QueryLog:
|
||||
with SessionLocal() as db:
|
||||
rows = db.scalars(select(QueryLog)).all()
|
||||
assert len(rows) == 1, f"expected exactly one query_log row, got {len(rows)}"
|
||||
return rows[0]
|
||||
|
||||
|
||||
def _overview_row() -> KbOverview | None:
|
||||
with SessionLocal() as db:
|
||||
return db.get(KbOverview, 1)
|
||||
|
||||
|
||||
def _doc_rows() -> list[tuple[str, str, str, str | None]]:
|
||||
with SessionLocal() as db:
|
||||
rows = db.execute(
|
||||
select(Document.source, Document.path, Document.title, Document.summary)
|
||||
.order_by(Document.source, Document.path)
|
||||
).all()
|
||||
return [(source, path, title, summary) for source, path, title, summary in rows]
|
||||
|
||||
|
||||
# --- 1. Injection: grounded turn ------------------------------------------
|
||||
|
||||
|
||||
def test_on_topic_answer_echoes_kb_overview(
|
||||
page: Page, app_url: str, mock_llm: int, db_ready: None
|
||||
) -> None:
|
||||
"""On-topic question with a stored row: the rendered answer ENDS with
|
||||
the mock's echo of the ``<knowledge_base>`` section's first bullet —
|
||||
only possible if the section reached the LLM prompt."""
|
||||
summary = _reset_db(mock_llm, seed=True, overview=OVERVIEW)
|
||||
assert summary is not None and summary.added == 8 # A9 formats
|
||||
assert _overview_row() is not None # the row the turn must inject
|
||||
|
||||
bubble = _ask(page, app_url, QUESTION, KB_ECHO)
|
||||
|
||||
# The echo is the LAST thing in the bubble (the section was in the
|
||||
# HIGH prompt) and names the FIRST outline bullet (not the intro,
|
||||
# not a later line).
|
||||
expect(bubble).to_have_text(
|
||||
re.compile(rf"{re.escape(KB_ECHO)}\s*$"), timeout=30_000
|
||||
)
|
||||
# Grounded, not deflected — the overview does not change the gate.
|
||||
expect(page.locator(".msg.brain.is-deflected")).to_have_count(0)
|
||||
row = _last_query_log()
|
||||
assert row.question == QUESTION
|
||||
assert row.deflected is False
|
||||
|
||||
|
||||
# --- 2. Injection: deflected turn ------------------------------------------
|
||||
|
||||
|
||||
def test_deflected_answer_still_echoes_kb_overview(
|
||||
page: Page, app_url: str, mock_llm: int, db_ready: None
|
||||
) -> None:
|
||||
"""Off-topic question (LOW path): the deflected answer STILL carries
|
||||
the echo — the ``<knowledge_base>`` section is in the LOW prompt too
|
||||
(that is what lets the model offer real alternatives)."""
|
||||
_reset_db(mock_llm, seed=True, overview=OVERVIEW)
|
||||
|
||||
# The deflection answer carries no mock marker — the kb echo is its
|
||||
# deterministic end, so it doubles as the completion signal.
|
||||
bubble = _ask(page, app_url, OFF_TOPIC, KB_ECHO)
|
||||
|
||||
expect(page.locator(".msg.brain.is-deflected")).to_have_count(1)
|
||||
expect(bubble).to_have_text(re.compile(r"haven't done anything like that"))
|
||||
expect(bubble).to_have_text(
|
||||
re.compile(rf"{re.escape(KB_ECHO)}\s*$"), timeout=30_000
|
||||
)
|
||||
row = _last_query_log()
|
||||
assert row.question == OFF_TOPIC
|
||||
assert row.deflected is True
|
||||
|
||||
|
||||
# --- 3. Absence: no row → no echo ------------------------------------------
|
||||
|
||||
|
||||
def test_absent_row_yields_no_kb_echo(
|
||||
page: Page, app_url: str, mock_llm: int, db_ready: None
|
||||
) -> None:
|
||||
"""Delete the row: a fresh question's answer carries NO ``(kb: …)``
|
||||
suffix — the section is absent and the prompt behaves exactly like
|
||||
the pre-phase text (byte-identity is unit-asserted; this proves it
|
||||
end-to-end through the rendered answer)."""
|
||||
summary = _reset_db(mock_llm, seed=True, overview=OVERVIEW)
|
||||
assert summary is not None
|
||||
# Now delete the row (the absence control).
|
||||
with SessionLocal() as db:
|
||||
db.execute(text("TRUNCATE kb_overview"))
|
||||
db.commit()
|
||||
assert _overview_row() is None
|
||||
|
||||
bubble = _ask(page, app_url, QUESTION, MOCK_ANSWER_MARKER)
|
||||
|
||||
expect(bubble).to_contain_text(MOCK_ANSWER_MARKER)
|
||||
expect(bubble).not_to_contain_text("(kb:")
|
||||
row = _last_query_log()
|
||||
assert row.deflected is False # grounded either way — only the echo is gone
|
||||
|
||||
|
||||
# --- 4. Generation: KB_OVERVIEW_MODE branch through the real regenerator ----
|
||||
|
||||
|
||||
def test_mock_generated_outline_is_stored_and_echoed(
|
||||
page: Page, app_url: str, mock_llm: int, db_ready: None
|
||||
) -> None:
|
||||
"""The mock's ``KB_OVERVIEW_MODE`` branch is exercised end-to-end:
|
||||
the real ``regenerate_overview`` (``LLMClient`` pointed at the mock)
|
||||
stores the byte-stable 8-token digest of the generator's document
|
||||
list, and a chat turn echoes its first bullet."""
|
||||
summary = _reset_db(mock_llm, seed=True, overview=None)
|
||||
assert summary is not None and summary.added == 8
|
||||
assert _overview_row() is None # direct import never regenerates
|
||||
|
||||
kwargs: dict[str, Any] = {
|
||||
"_env_file": None,
|
||||
"llm_base_url": f"http://127.0.0.1:{mock_llm}/v1",
|
||||
}
|
||||
settings = Settings(**kwargs) # pyright: ignore[reportCallIssue]
|
||||
ok = _run_in_thread(regenerate_overview(LLMClient(settings)))
|
||||
assert ok is True
|
||||
|
||||
row = _overview_row()
|
||||
assert row is not None and row.id == 1
|
||||
# Byte-stable expectation: the mock digests the generator's user
|
||||
# message (the same document list, the same cap) to its first 8 tokens.
|
||||
_, user = build_overview_prompt(_doc_rows())
|
||||
expected = "Knowledge base outline:\n- " + " ".join(
|
||||
TOKEN_RE.findall(user.lower())[:8]
|
||||
)
|
||||
assert row.content == expected
|
||||
first_bullet = row.content.splitlines()[1].removeprefix("- ").strip()
|
||||
assert first_bullet # the digest line is non-empty
|
||||
|
||||
# A grounded chat turn echoes the STORED outline's first bullet.
|
||||
bubble = _ask(page, app_url, QUESTION, f"(kb: {first_bullet})")
|
||||
expect(bubble).to_have_text(
|
||||
re.compile(rf"\(kb: {re.escape(first_bullet)}\)\s*$"), timeout=30_000
|
||||
)
|
||||
expect(page.locator(".msg.brain.is-deflected")).to_have_count(0)
|
||||
Reference in New Issue
Block a user