feat(rag): stream grounded RAG answers over SSE with source citations

Phase 03 (Story: Chat RAG Answer — happy path): - app/rag/retriever.py: top-k cosine search + parent-doc selection with per-doc dedupe and BOR_MAX_CONTEXT_CHARS cap ([…truncated…] marker) - app/rag/prompts.py: locked persona + HIGH/DEFLECT prompt builders - app/rag/llm.py: LLMError + chat_stream (turbo, temp 0.4, max 700, stream) - app/api/chat.py: POST /api/chat SSE — delta* then done{deflected, sources, suggestions}; query_log row + PLAN §9 per-turn log line; structured error event on mid-stream failure, JSON 503 when DB down - frontend: SSE reader, live bubble streaming, source chips -> /sources.html, red role=alert banner, Send button state that always recovers - fix(scaffold): [hidden] { display: none !important } — .kb-banner's display:flex was overriding the hidden attribute (banner always visible) - tests: unit (retriever/prompts/sse/llm) + integration (real Postgres RAG turn, query_log, error + 503 paths, mid-turn failures) + Playwright story suite (grounded answer, log row, raw SSE shape); smoke placeholder test replaced with the real never-stale-button contract
2026-08-21 17:17:02 -04:00
parent 99c48cbe06
commit 396e4d47fb
14 changed files with 1266 additions and 59 deletions
@@ -0,0 +1,170 @@
+"""Phase 03 E2E (Playwright): the happy-path RAG chat turn.
+
+Story: ``.agent/user_stories/chat-rag-answer.md``
+Run in isolation (DB must be up: ``podman compose up -d db``):
+
+    uv run pytest tests/e2e/test_chat_rag.py -v --no-cov
+
+Seeding reuses the real importer against ``tests/fixtures/docs/`` with the
+deterministic mock embeddings (same pattern as the phase 02 story suite);
+the mock LLM answers on-topic questions by quoting the question and the
+document context, so the UI assertions are fully deterministic.
+"""
+from __future__ import annotations
+
+import asyncio
+import json
+from pathlib import Path
+from threading import Thread
+from typing import Any
+
+import httpx
+from playwright.sync_api import Page, expect
+from sqlalchemy import select, text
+
+from app.config import Settings
+from app.db import SessionLocal
+from app.models import QueryLog
+from app.rag.importer import ImportSummary, import_sources
+from app.rag.llm import LLMClient
+
+REPO = Path(__file__).resolve().parents[2]
+FIXTURES = REPO / "tests" / "fixtures" / "docs"
+QUESTION = "How is my Kubernetes cluster set up?"
+MOCK_ANSWER_MARKER = "Deterministic mock answer for E2E"
+
+
+async def _import_fixtures(mock_port: int) -> ImportSummary:
+    kwargs: dict[str, Any] = {"_env_file": None, "llm_base_url": f"http://127.0.0.1:{mock_port}/v1"}
+    settings = Settings(**kwargs)  # pyright: ignore[reportCallIssue]
+    return await import_sources([FIXTURES], LLMClient(settings))
+
+
+def _run_in_thread(coro: Any) -> Any:
+    """Run a coroutine on a worker thread.
+
+    Playwright's sync API keeps an asyncio loop running on the test thread,
+    so ``asyncio.run`` cannot be called directly from a test body.
+    """
+    box: dict[str, Any] = {}
+
+    def runner() -> None:
+        try:
+            box["value"] = asyncio.run(coro)
+        except BaseException as e:  # noqa: BLE001 — re-raised on the test thread
+            box["error"] = e
+
+    t = Thread(target=runner)
+    t.start()
+    t.join()
+    if "error" in box:
+        raise box["error"]
+    return box["value"]
+
+
+def _reset_db(mock_port: int, seed: bool) -> ImportSummary | None:
+    """Truncate the KB (and query log), then optionally re-import fixtures."""
+    with SessionLocal() as db:
+        db.execute(text("TRUNCATE chunks, documents, query_log"))
+        db.commit()
+    if not seed:
+        return None
+    return _run_in_thread(_import_fixtures(mock_port))
+
+
+def test_on_topic_question_streams_grounded_answer(
+    page: Page, app_url: str, mock_llm: int, db_ready: None
+) -> None:
+    summary = _reset_db(mock_llm, seed=True)
+    assert summary is not None and summary.added == 3
+    page.set_default_timeout(30_000)
+    page.goto(app_url)
+
+    # Healthy KB: the offline banner must stay hidden.
+    expect(page.locator("#kb-banner")).to_be_hidden()
+
+    page.fill("#message-input", QUESTION)
+    page.click("#send-btn")
+
+    # User bubble (right, brand) shows the question.
+    expect(page.locator(".msg.user .bubble")).to_contain_text(QUESTION)
+
+    # Brain bubble streams in: the mock quotes the question and ends with a
+    # deterministic marker — waiting on the marker proves content arrived.
+    bubble = page.locator(".msg.brain .bubble")
+    bubble.first.wait_for(state="visible", timeout=30_000)
+    expect(bubble.first).to_contain_text(QUESTION, timeout=30_000)
+    expect(bubble.first).to_contain_text(MOCK_ANSWER_MARKER, timeout=30_000)
+
+    # Grounded: a kubernetes.md source chip renders under the bubble
+    # (top-N docs can add more chips; the question's doc must be among them).
+    chip = page.locator(".msg.brain .source-chip", has_text="kubernetes.md")
+    expect(chip).to_have_count(1)
+    expect(chip.first).to_contain_text("kubernetes.md")
+    expect(chip.first).to_have_attribute("href", "/sources.html")
+
+    # Button recovers: enabled + "Send" (never stale).
+    expect(page.locator("#send-btn")).to_be_enabled()
+    expect(page.locator("#send-label")).to_have_text("Send")
+
+
+def test_chat_logs_query(page: Page, app_url: str, mock_llm: int, db_ready: None) -> None:
+    _reset_db(mock_llm, seed=True)
+    page.set_default_timeout(30_000)
+    page.goto(app_url)
+    page.fill("#message-input", QUESTION)
+    page.click("#send-btn")
+    expect(
+        page.locator(".msg.brain .source-chip", has_text="kubernetes.md")
+    ).to_have_count(1, timeout=30_000)
+
+    # App still healthy after the turn.
+    r = httpx.get(f"{app_url}/api/health", timeout=5)
+    assert r.status_code == 200
+    body = r.json()
+    assert body["status"] == "ok" and body["db"] == "up"
+
+    # Durable record: exactly one query_log row for the turn.
+    with SessionLocal() as db:
+        rows = db.scalars(select(QueryLog)).all()
+    assert len(rows) == 1
+    row = rows[0]
+    assert row.question == QUESTION
+    assert row.deflected is False
+    assert row.top_score > 0.0
+    assert row.chunk_hits >= 1
+    assert "docs/homelab/kubernetes.md" in row.sources
+    assert row.latency_ms >= 0
+
+
+def test_sse_stream_shape(app_url: str, mock_llm: int, db_ready: None) -> None:
+    """Raw transport contract (PLAN §4): delta events, then one done."""
+    _reset_db(mock_llm, seed=True)
+
+    frames: list[dict[str, Any]] = []
+    with httpx.stream(
+        "POST", f"{app_url}/api/chat", json={"message": QUESTION}, timeout=60.0
+    ) as r:
+        assert r.status_code == 200
+        assert r.headers["content-type"].startswith("text/event-stream")
+        buf = ""
+        for part in r.iter_text():
+            buf += part
+            while "\n\n" in buf:
+                frame, buf = buf.split("\n\n", 1)
+                if frame.strip().startswith("data:"):
+                    frames.append(json.loads(frame.strip().removeprefix("data:").strip()))
+    assert buf.strip() == ""  # stream ends cleanly on a frame boundary
+
+    deltas = [f for f in frames if f.get("type") == "delta"]
+    assert len(deltas) >= 2, "answer must arrive as multiple deltas (streamed)"
+    assert all(d.get("text") for d in deltas)
+    assert "".join(d["text"] for d in deltas)  # non-empty answer
+
+    done = [f for f in frames if f.get("type") == "done"]
+    assert len(done) == 1
+    assert frames[-1]["type"] == "done"  # done is the final event
+    assert done[0]["deflected"] is False
+    assert done[0]["suggestions"] == []
+    assert done[0]["sources"], "done must carry the cited sources"
+    assert any(s["path"] == "homelab/kubernetes.md" for s in done[0]["sources"])
@@ -1,12 +1,10 @@
-"""Phase 01 smoke E2E: the app boots, serves the local frontend, and the
-placeholder chat round-trips without a stale button.
+"""Phase 01 smoke E2E: the app boots, serves the local frontend, and a chat
+round-trip never leaves a stale button (answer or error banner, both fine).

 Run:  uv run pytest tests/e2e/test_smoke.py -v
 """
 from __future__ import annotations

-import re
-
 import httpx
 from playwright.sync_api import Page, expect

@@ -27,19 +25,21 @@ def test_index_page_loads_locally(page: Page, app_url: str) -> None:
    assert 'href="http' not in html.replace('href="http://www.w3.org', "")


-def test_placeholder_chat_roundtrip(page: Page, app_url: str) -> None:
+def test_chat_roundtrip_never_stale_button(page: Page, app_url: str) -> None:
    page.goto(app_url)
    page.locator("#message-input").fill("hello brain")
    page.locator("#send-btn").click()

-    # User bubble appears, then the Brain placeholder answer arrives.
+    # User bubble appears first.
    page.locator(".msg.user .bubble").first.wait_for(state="visible", timeout=10_000)
-    brain_bubble = page.locator(".msg.brain .bubble").first
-    brain_bubble.wait_for(state="visible", timeout=10_000)
-    # to_have_text retries until the async fetch resolves (no stale read).
-    expect(brain_bubble).to_have_text(re.compile("neurons"), timeout=10_000)
+    # Then either a streamed Brain answer (DB up) or an error banner
+    # (DB down) — but the turn must always complete.
+    page.wait_for_selector(
+        ".msg.brain .bubble, #kb-banner.is-error",
+        state="visible",
+        timeout=20_000,
+    )

    # Button is never left stuck: back to "Send" and enabled.
-    btn = page.locator("#send-btn")
-    assert btn.is_enabled()
-    assert "Send" in btn.inner_text()
+    expect(page.locator("#send-btn")).to_be_enabled(timeout=10_000)
+    expect(page.locator("#send-label")).to_have_text("Send", timeout=10_000)