Experimental/brain-of-reese
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(rag): feed whole matched documents to the LLM — no context truncation (A7 revised)

Browse Source
This commit is contained in:
ducoterra
2026-08-24 23:37:44 -04:00
parent d7a4064616
commit 1e6ae360e0
16 changed files with 923 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/e2e/mock_llm.py
+22
View File
@@ -26,6 +26,11 @@ Implements just enough of the aipi surface:
- system prompt containing ``<tuning>`` (phase 15, steering notes) ->
the composed answer ends with `` (tuning: <first note line>)`` —
makes prompt injection observable in the UI deterministically.
- user message containing ``show the end of your notes`` (phase 24,
whole-document context) -> the answer quotes the **last 160 chars of
the document context** — a tail echo, byte-stable across runs, so a
sentinel placed at the *end* of a document appears in the rendered
answer iff the whole document was in the prompt.
``max_tokens`` is honored deterministically (token ≈ whitespace word),
like a real endpoint: an answer longer than the cap is truncated. This
@@ -104,6 +109,13 @@ THINKING_TRIGGER = "think out loud"
SLOW_PRETOKEN_TRIGGER = "think out loud then hesitate"
PRE_CONTENT_PAUSE_S = 4.0
#: Phase 24 (whole-document-context story): a user message containing this
#: substring (case-insensitive) gets an answer quoting the TAIL of the
#: document context (see the module docstring). Verified 2026-08-24: no
#: existing E2E question or fixture file contains the phrase, so every
#: other suite is unaffected.
END_OF_NOTES_TRIGGER = "show the end of your notes"
def long_answer() -> str:
"""~900-word deterministic walkthrough (phase 11): numbered steps plus
@@ -151,6 +163,16 @@ def compose_answer(body: dict[str, Any]) -> str:
"kubernetes, backups, or deploying a new service — I know those inside out. "
"You've got this!"
)
elif END_OF_NOTES_TRIGGER in user.lower():
# Whole-document-context story (phase 24): echo the tail of the
# context. Byte-stable across runs — a sentinel on the document's
# last line appears in the answer iff the whole document was in
# the prompt. (The tail includes the closing </documents> —
# harmless for the E2E sentinel assertions.)
answer = (
f"…and the very end of my notes reads: “{_context(body)[-160:]}” "
"(Deterministic mock answer for E2E.)"
)
else:
ctx = _context(body)
snippet = ctx[:220].replace("\n", " ").strip()
tests/e2e/test_whole_document_context.py
+315
View File
@@ -0,0 +1,315 @@
"""Phase 24 E2E (Playwright): a matched document reaches the LLM whole.
Story: ``.agent/user_stories/whole-document-context.md``
Run in isolation (DB must be up: ``podman compose up -d db``):
uv run pytest tests/e2e/test_whole_document_context.py -v --no-cov
The mock LLM's tail-echo trigger (``END_OF_NOTES_TRIGGER``, see
``tests/e2e/mock_llm.py``) makes the model quote the last 160 chars of
the document context. A sentinel placed on the *last line* of a document
therefore appears in the rendered answer **iff the entire document was in
the prompt** — which is what makes the no-truncation contract (A7 revised,
owner permission 2026-08-24: matched parent documents are never cut)
provable end-to-end.
The oversized documents are seeded directly via SQLAlchemy (a
``documents`` row + 2–3 ``chunks`` rows whose embeddings are the mock's
own deterministic bag-of-words vectors, so the question's live mock
embedding genuinely overlaps — no fixture files added:
``tests/fixtures/docs/`` stays at its 8 files, other suites pin
``summary.added == 8``).
"""
from __future__ import annotations
import asyncio
import hashlib
from collections.abc import Callable, Sequence
from datetime import UTC, datetime
from pathlib import Path
from threading import Thread
from typing import Any
from playwright.sync_api import Page, expect
from sqlalchemy import select, text
from sqlalchemy.orm import Session
from app.config import Settings
from app.db import SessionLocal
from app.models import Chunk, Document, QueryLog
from app.rag.importer import ImportSummary, import_sources
from app.rag.llm import LLMClient
from app.rag.retriever import TRUNCATION_MARKER
from tests.e2e.mock_llm import embed_text
REPO = Path(__file__).resolve().parents[2]
FIXTURES = REPO / "tests" / "fixtures" / "docs"
#: The pre-phase-24 ``BOR_MAX_CONTEXT_CHARS`` default — the budget this
#: suite proves is gone from the document path.
OLD_CONTEXT_CAP = 24_000
QUESTION = "Show the end of your notes about the gitlab install playbook, please."
SMALL_QUESTION = "How is my Kubernetes cluster set up?"
MOCK_ANSWER_MARKER = "Deterministic mock answer for E2E"
# --- Content builders (deterministic, token-controlled) -------------------
def _repeated(line: str, min_chars: int) -> str:
""""line" (newline-terminated) repeated until at least min_chars chars."""
unit = line + "\n"
return unit * max(1, -(-min_chars // len(unit)))
def _doc_slices(content: str, n: int) -> list[str]:
"""Even slices of *content* (the last slice keeps the final line)."""
step = len(content) // n
return [content[i * step : (i + 1) * step] for i in range(n - 1)] + [
content[(n - 1) * step :]
]
def _gitlab_30k_doc(sentinel: str) -> str:
"""A ~30 000-char document (past the old 24k cap): a body of repeated
"gitlab install playbook" lines — the same tokens the question carries,
so hybrid retrieval genuinely hits — whose LAST line is a unique
sentinel only a tail echo can surface."""
body = _repeated(
"gitlab install playbook: run the gitlab install playbook on the homelab host.",
OLD_CONTEXT_CAP + 6_000,
)
return body + sentinel + "\n"
def _pair_doc(strong_line: str, filler_line: str, sentinel: str) -> tuple[str, list[str]]:
"""A ~16 000-char document: a ~2 000-char first chunk carrying the
question's key tokens, a ~14 000-char low-overlap remainder, and a
unique sentinel as the last line. Returns (content, chunk_texts)."""
chunk0 = _repeated(strong_line, 2_000)
chunk1 = _repeated(filler_line, 14_000)
return chunk0 + chunk1 + sentinel + "\n", [chunk0, chunk1]
# --- DB seeding (TRUNCATE-then-seed, cf. test_chat_rag.py) -----------------
def _seed_doc(
db: Session,
source: str,
path: str,
title: str,
content: str,
chunk_texts: Sequence[str],
) -> None:
"""One ``documents`` row + one ``chunks`` row per chunk text.
Each chunk's embedding is the mock's own ``embed_text`` vector, so the
app's live mock embedding of the question genuinely overlaps.
"""
doc = Document(
source=source,
path=path,
full_path=f"/tmp/{path}",
title=title,
content=content,
content_hash=hashlib.sha256(content.encode("utf-8")).hexdigest(),
indexed_at=datetime.now(UTC),
)
db.add(doc)
db.flush()
db.add_all(
Chunk(document_id=doc.id, position=i, content=chunk, embedding=embed_text(chunk))
for i, chunk in enumerate(chunk_texts)
)
def _reset_db(seed: Callable[[Session], None] | None = None) -> None:
"""Truncate the KB (and query log), then optionally run *seed*."""
with SessionLocal() as db:
db.execute(text("TRUNCATE chunks, documents, query_log"))
db.commit()
if seed is not None:
seed(db)
db.commit()
# --- Importer + thread helpers (test_chat_rag.py pattern) ------------------
async def _import_fixtures(mock_port: int) -> ImportSummary:
kwargs: dict[str, Any] = {
"_env_file": None,
"llm_base_url": f"http://127.0.0.1:{mock_port}/v1",
}
settings = Settings(**kwargs) # pyright: ignore[reportCallIssue]
return await import_sources([FIXTURES], LLMClient(settings))
def _run_in_thread(coro: Any) -> Any:
"""Run a coroutine on a worker thread.
Playwright's sync API keeps an asyncio loop running on the test thread,
so ``asyncio.run`` cannot be called directly from a test body.
"""
box: dict[str, Any] = {}
def runner() -> None:
try:
box["value"] = asyncio.run(coro)
except BaseException as e: # noqa: BLE001 — re-raised on the test thread
box["error"] = e
t = Thread(target=runner)
t.start()
t.join()
if "error" in box:
raise box["error"]
return box["value"]
def _ask(page: Page, app_url: str, question: str) -> Any:
"""Submit *question* and wait for the streamed brain bubble."""
page.set_default_timeout(30_000)
page.goto(app_url)
page.fill("#message-input", question)
page.click("#send-btn")
bubble = page.locator(".msg.brain .bubble")
bubble.first.wait_for(state="visible", timeout=30_000)
return bubble.first
def _last_query_log() -> QueryLog:
with SessionLocal() as db:
rows = db.scalars(select(QueryLog)).all()
assert len(rows) == 1, f"expected exactly one query_log row, got {len(rows)}"
return rows[0]
# --- Story tests -------------------------------------------------------------
def test_whole_document_over_old_cap_reaches_llm(
page: Page, app_url: str, mock_llm: int, db_ready: None
) -> None:
"""A 30k document (past the old 24k cap) reaches the LLM whole: its
tail sentinel — the last 160 chars of the context — is echoed back."""
sentinel = "WHOLE-DOC-TAIL-GITLAB-30K"
content = _gitlab_30k_doc(sentinel)
assert len(content) > OLD_CONTEXT_CAP # this is the point of the test
def seed(db: Session) -> None:
_seed_doc(
db,
"Homelab",
"gitlab-30k.md",
"GitLab Install Playbook (30k)",
content,
_doc_slices(content, 3),
)
_reset_db(seed)
bubble = _ask(page, app_url, QUESTION)
# The tail sentinel exists only on the document's last line — its
# presence proves the entire 30k document was in the LLM prompt.
expect(bubble).to_contain_text(sentinel, timeout=30_000)
expect(bubble).to_contain_text(MOCK_ANSWER_MARKER)
expect(bubble).not_to_contain_text(TRUNCATION_MARKER)
# Grounded: the document's source chip renders under the bubble.
chip = page.locator(".msg.brain .source-chip", has_text="gitlab-30k.md")
expect(chip).to_have_count(1)
expect(chip.first).to_contain_text("Homelab/gitlab-30k.md")
# Button recovers (never stale) and the turn was grounded.
expect(page.locator("#send-btn")).to_be_enabled()
expect(page.locator("#send-label")).to_have_text("Send")
row = _last_query_log()
assert row.question == QUESTION
assert row.deflected is False
assert "Homelab/gitlab-30k.md" in row.sources
def test_second_document_of_over_cap_pair_reaches_llm(
page: Page, app_url: str, mock_llm: int, db_ready: None
) -> None:
"""Two ~16k documents (32k combined — under the old budget the
lower-ranked one was truncated in place): the SECOND document, the
last block inside <documents>, reaches the LLM whole — its sentinel is
the one the tail echo surfaces."""
sentinel_a = "WHOLE-DOC-TAIL-PAIR-A"
sentinel_b = "WHOLE-DOC-TAIL-PAIR-B"
# Doc A's first chunk carries the question's key tokens → it ranks
# first in both candidate lists → it comes first in <documents>.
content_a, chunks_a = _pair_doc(
"gitlab install playbook: run the gitlab install playbook end to end.",
"the server room keeps a steady temperature and the racks are labelled.",
sentinel_a,
)
# Doc B's first chunk has only a weaker overlap ("playbook", "notes")
# → it ranks second → it is the LAST block inside <documents>.
content_b, chunks_b = _pair_doc(
"playbook notes: the playbook notes track what changed and where.",
"the rack elevation drawing shows cable trays and pdu positions.",
sentinel_b,
)
assert len(content_a) + len(content_b) > OLD_CONTEXT_CAP # 32k > 24k
def seed(db: Session) -> None:
_seed_doc(
db, "Homelab", "gitlab-install-playbook.md",
"GitLab Install Playbook", content_a, chunks_a,
)
_seed_doc(
db, "Homelab", "playbook-notes.md",
"Playbook Notes", content_b, chunks_b,
)
_reset_db(seed)
bubble = _ask(page, app_url, QUESTION)
# The tail echo quotes doc B's sentinel (the last block's tail) — doc B
# was in the prompt whole, past the old budget. Doc A's sentinel sits
# mid-prompt, so it must NOT be in the quoted tail: that is what pins
# the rank order (A first, B last inside <documents>).
expect(bubble).to_contain_text(sentinel_b, timeout=30_000)
expect(bubble).not_to_contain_text(sentinel_a)
expect(bubble).to_contain_text(MOCK_ANSWER_MARKER)
expect(bubble).not_to_contain_text(TRUNCATION_MARKER)
# Both documents are cited (top-2), in rank order.
chips = page.locator(".msg.brain .source-chip")
expect(chips).to_have_count(2)
expect(page.locator(".msg.brain .source-chip",
has_text="gitlab-install-playbook.md")).to_have_count(1)
expect(page.locator(".msg.brain .source-chip",
has_text="playbook-notes.md")).to_have_count(1)
row = _last_query_log()
assert row.question == QUESTION
assert row.deflected is False
assert "Homelab/gitlab-install-playbook.md" in row.sources
assert "Homelab/playbook-notes.md" in row.sources
def test_small_document_path_unchanged(
page: Page, app_url: str, mock_llm: int, db_ready: None
) -> None:
"""Regression: the standard (small) fixtures still take the grounded
path, byte-identical to before — no marker, kubernetes.md cited."""
_reset_db(None)
summary = _run_in_thread(_import_fixtures(mock_llm))
assert summary.added == 8 # A9 formats (fixture set unchanged)
bubble = _ask(page, app_url, SMALL_QUESTION)
expect(bubble).to_contain_text(SMALL_QUESTION, timeout=30_000)
expect(bubble).to_contain_text(MOCK_ANSWER_MARKER)
expect(bubble).not_to_contain_text(TRUNCATION_MARKER)
chip = page.locator(".msg.brain .source-chip", has_text="kubernetes.md")
expect(chip).to_have_count(1)
row = _last_query_log()
assert row.deflected is False
assert "docs/homelab/kubernetes.md" in row.sources