feat(rag): lite-model document summaries — non-markdown docs summarized at import, summary chunk retrieves and resolves to the full source doc
This commit is contained in:
@@ -11,6 +11,10 @@ Implements just enough of the aipi surface:
|
||||
off markers in the system prompt:
|
||||
- user message containing ``write a long answer`` -> a ~900-word
|
||||
deterministic numbered answer (long-answers story, phase 11)
|
||||
- ``SUMMARY_MODE`` -> the deterministic summary digest: the first 24
|
||||
tokens of the user message (the summarizer puts the capped document
|
||||
content there) — byte-stable for a given fixture (document summaries,
|
||||
phase 30)
|
||||
- ``DEFLECT_MODE`` -> honest "I haven't done anything like that" answer
|
||||
- otherwise -> upbeat answer quoting the provided document context
|
||||
- user message containing ``pretend to think slowly`` -> 3s warm-up delay
|
||||
@@ -156,6 +160,18 @@ def compose_answer(body: dict[str, Any]) -> str:
|
||||
user = _user(body)
|
||||
if LONG_ANSWER_TRIGGER in user.lower():
|
||||
answer = long_answer()
|
||||
elif "SUMMARY_MODE" in system:
|
||||
# Document summaries (phase 30): the ``lite`` stand-in returns a
|
||||
# deterministic digest — the first 24 tokens of the user message
|
||||
# (the summarizer puts the capped document content there). Byte-
|
||||
# stable for a given fixture, so the summary chunk's retrieval
|
||||
# rank is a pure function of the fixture text. Checked BEFORE the
|
||||
# DEFLECT_MODE branch (task 06) so a deflection prompt that ever
|
||||
# carries the marker cannot shadow the summary call.
|
||||
answer = (
|
||||
f"This document covers "
|
||||
f"{' '.join(TOKEN_RE.findall(user.lower())[:24])}."
|
||||
)
|
||||
elif "DEFLECT_MODE" in system:
|
||||
answer = (
|
||||
"Ah — I haven't done anything like that, so I don't want to make stuff up! "
|
||||
|
||||
@@ -0,0 +1,257 @@
|
||||
"""Phase 30 E2E (Playwright): a summary hit delivers the full source doc.
|
||||
|
||||
Story: ``.agent/user_stories/document-summaries.md``
|
||||
Run in isolation (DB must be up: ``podman compose up -d db``):
|
||||
|
||||
uv run pytest tests/e2e/test_document_summaries.py -v --no-cov
|
||||
|
||||
The fixture KB is a story-dedicated directory
|
||||
(``tests/fixtures/summary_kb/`` — the shared ``tests/fixtures/docs/``
|
||||
stays at its 8 pinned files) with two documents:
|
||||
|
||||
* ``quadlet/qwen-llamacpp.yaml`` — a non-markdown A9 doc. At import the
|
||||
mock ``lite`` model (``SUMMARY_MODE`` marker, ``tests/e2e/mock_llm.py``)
|
||||
reduces it to a deterministic 24-token digest, stored on
|
||||
``documents.summary`` and indexed as one ``is_summary`` chunk. The raw
|
||||
yaml body is deliberately token-diluted, so the document's best fused
|
||||
chunk is its summary chunk. The sentinel ``RESE-SUMMARY-SENTINEL-7f3a``
|
||||
sits on the document's LAST line — outside the 24-token digest,
|
||||
unreachable from the summary.
|
||||
* ``notes/qwen-llamacpp-notes.md`` — a markdown control doc (never
|
||||
summarized) that ranks first, which puts the yaml document LAST inside
|
||||
``<documents>``.
|
||||
|
||||
The mock LLM's tail-echo trigger (``END_OF_NOTES_TRIGGER``) makes the
|
||||
answer quote the last 160 chars of the document context — the tail of
|
||||
the LAST selected document. The sentinel therefore appears in the
|
||||
rendered answer **iff the entire yaml source document (not the summary
|
||||
digest) reached the LLM prompt** — the summary→parent-document resolution
|
||||
through the unchanged chunk→document mapping (A7 revised: never
|
||||
truncated), which is what this story is about.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from collections.abc import Callable, Sequence
|
||||
from pathlib import Path
|
||||
from threading import Thread
|
||||
from typing import Any
|
||||
|
||||
from playwright.sync_api import Page, expect
|
||||
from sqlalchemy import select, text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import Settings
|
||||
from app.db import SessionLocal
|
||||
from app.models import Document, QueryLog
|
||||
from app.rag.chunker import chunk_document
|
||||
from app.rag.importer import ImportSummary, import_sources
|
||||
from app.rag.llm import LLMClient
|
||||
from app.rag.retriever import RetrievedChunk, retrieve
|
||||
from tests.e2e.mock_llm import TOKEN_RE, embed_text
|
||||
|
||||
REPO = Path(__file__).resolve().parents[2]
|
||||
FIXTURES = REPO / "tests" / "fixtures" / "summary_kb"
|
||||
|
||||
QUESTION = (
|
||||
"What are the optimal parameters for qwen 3.8 on llama.cpp? "
|
||||
"show the end of your notes"
|
||||
)
|
||||
SENTINEL = "RESE-SUMMARY-SENTINEL-7f3a"
|
||||
SOURCE = "summary_kb"
|
||||
YAML_PATH = "quadlet/qwen-llamacpp.yaml"
|
||||
MD_PATH = "notes/qwen-llamacpp-notes.md"
|
||||
MOCK_ANSWER_MARKER = "Deterministic mock answer for E2E"
|
||||
|
||||
#: The importer's chunk policy (Settings defaults; the mock never trips
|
||||
#: the endpoint token-cap retry, so the target is never halved).
|
||||
CHUNK_TARGET = 2_000
|
||||
CHUNK_OVERLAP = 200
|
||||
|
||||
|
||||
# --- Importer + thread helpers (test_whole_document_context.py pattern) ---
|
||||
|
||||
|
||||
async def _import_fixtures(mock_port: int) -> ImportSummary:
|
||||
kwargs: dict[str, Any] = {
|
||||
"_env_file": None,
|
||||
"llm_base_url": f"http://127.0.0.1:{mock_port}/v1",
|
||||
}
|
||||
settings = Settings(**kwargs) # pyright: ignore[reportCallIssue]
|
||||
return await import_sources([FIXTURES], LLMClient(settings))
|
||||
|
||||
|
||||
def _run_in_thread(coro: Any) -> Any:
|
||||
"""Run a coroutine on a worker thread.
|
||||
|
||||
Playwright's sync API keeps an asyncio loop running on the test thread,
|
||||
so ``asyncio.run`` cannot be called directly from a test body.
|
||||
"""
|
||||
box: dict[str, Any] = {}
|
||||
|
||||
def runner() -> None:
|
||||
try:
|
||||
box["value"] = asyncio.run(coro)
|
||||
except BaseException as e: # noqa: BLE001 — re-raised on the test thread
|
||||
box["error"] = e
|
||||
|
||||
t = Thread(target=runner)
|
||||
t.start()
|
||||
t.join()
|
||||
if "error" in box:
|
||||
raise box["error"]
|
||||
return box["value"]
|
||||
|
||||
|
||||
def _reset_db(seed: Callable[[Session], None] | None = None) -> None:
|
||||
"""Truncate the KB (and query log + steering), then optionally seed."""
|
||||
with SessionLocal() as db:
|
||||
db.execute(text("TRUNCATE chunks, documents, query_log, steering_notes"))
|
||||
db.commit()
|
||||
if seed is not None:
|
||||
seed(db)
|
||||
db.commit()
|
||||
|
||||
|
||||
def _ask(page: Page, app_url: str, question: str) -> Any:
|
||||
"""Submit *question* and wait for the streamed brain bubble."""
|
||||
page.set_default_timeout(30_000)
|
||||
page.goto(app_url)
|
||||
page.fill("#message-input", question)
|
||||
page.click("#send-btn")
|
||||
bubble = page.locator(".msg.brain .bubble")
|
||||
bubble.first.wait_for(state="visible", timeout=30_000)
|
||||
return bubble.first
|
||||
|
||||
|
||||
def _last_query_log() -> QueryLog:
|
||||
with SessionLocal() as db:
|
||||
rows = db.scalars(select(QueryLog)).all()
|
||||
assert len(rows) == 1, f"expected exactly one query_log row, got {len(rows)}"
|
||||
return rows[0]
|
||||
|
||||
|
||||
def _doc(db: Session, path: str) -> Document:
|
||||
doc = db.scalar(select(Document).where(Document.path == path))
|
||||
assert doc is not None, f"fixture doc {path!r} was not imported"
|
||||
return doc
|
||||
|
||||
|
||||
def _expected_summary(content: str, source: str, path: str) -> str:
|
||||
"""The mock lite model's byte-stable digest + the code pointer line.
|
||||
|
||||
Mirrors ``mock_llm.compose_answer``'s ``SUMMARY_MODE`` branch (first
|
||||
24 tokens of the document content) plus the summarizer's deterministic
|
||||
``Source:`` line — no model output is ever trusted.
|
||||
"""
|
||||
digest = " ".join(TOKEN_RE.findall(content.lower())[:24])
|
||||
return f"This document covers {digest}.\nSource: {source}/{path}"
|
||||
|
||||
|
||||
def _chunks_by_path(chunks: Sequence[RetrievedChunk], path: str) -> list[RetrievedChunk]:
|
||||
return [c for c in chunks if c.document.path == path]
|
||||
|
||||
|
||||
# --- Story tests -------------------------------------------------------------
|
||||
|
||||
|
||||
def test_summary_hit_retrieves_full_source_document(
|
||||
page: Page, app_url: str, mock_llm: int, db_ready: None
|
||||
) -> None:
|
||||
"""A question whose best yaml match is the summary chunk yields an
|
||||
answer grounded in the FULL yaml source document: its tail sentinel —
|
||||
which the summary digest cannot contain — is echoed back, and the
|
||||
source chip cites the yaml path (deflected: false)."""
|
||||
_reset_db()
|
||||
summary = _run_in_thread(_import_fixtures(mock_llm))
|
||||
assert summary.added == 2 # yaml + md control
|
||||
assert summary.summaries == 1 and summary.summary_errors == 0
|
||||
assert summary.errors == 0
|
||||
|
||||
# Import state: exactly one embedded ``is_summary`` chunk (position
|
||||
# −1) whose text is the byte-stable mock digest + the deterministic
|
||||
# pointer line.
|
||||
yaml_content = (FIXTURES / YAML_PATH).read_text(encoding="utf-8")
|
||||
assert SENTINEL in yaml_content.splitlines()[-1] # last line, by design
|
||||
with SessionLocal() as db:
|
||||
yaml_doc = _doc(db, YAML_PATH)
|
||||
schunks = [c for c in yaml_doc.chunks if c.is_summary]
|
||||
assert len(schunks) == 1
|
||||
assert schunks[0].position == -1
|
||||
assert schunks[0].embedding is not None
|
||||
assert yaml_doc.summary == _expected_summary(yaml_content, SOURCE, YAML_PATH)
|
||||
|
||||
# Retrieval state: the summary chunk is the yaml document's best fused
|
||||
# chunk — the document enters the context through its summary, not
|
||||
# through the diluted raw yaml chunks.
|
||||
with SessionLocal() as db:
|
||||
chunks = retrieve(db, QUESTION, embed_text(QUESTION))
|
||||
yaml_chunks = _chunks_by_path(chunks, YAML_PATH)
|
||||
best_yaml = max(yaml_chunks, key=lambda c: c.score)
|
||||
assert best_yaml.is_summary
|
||||
assert len(yaml_chunks) >= 2 # summary + at least one raw candidate
|
||||
|
||||
bubble = _ask(page, app_url, QUESTION)
|
||||
|
||||
# The tail sentinel exists only on the document's last line and
|
||||
# cannot be in the summary digest — its presence proves the entire
|
||||
# source document was in the LLM prompt (summary→parent resolution).
|
||||
expect(bubble).to_contain_text(SENTINEL, timeout=30_000)
|
||||
expect(bubble).to_contain_text(MOCK_ANSWER_MARKER)
|
||||
|
||||
# Grounded: the yaml source chip renders (the md doc ranks first, so
|
||||
# both fixtures are cited).
|
||||
chip = page.locator(".msg.brain .source-chip", has_text=YAML_PATH)
|
||||
expect(chip).to_have_count(1)
|
||||
expect(chip.first).to_contain_text(f"{SOURCE}/{YAML_PATH}")
|
||||
expect(page.locator(".msg.brain .source-chip", has_text=MD_PATH)).to_have_count(1)
|
||||
|
||||
# Button recovers (never stale) and the turn was grounded, not
|
||||
# deflected.
|
||||
expect(page.locator("#send-btn")).to_be_enabled()
|
||||
expect(page.locator("#send-label")).to_have_text("Send")
|
||||
row = _last_query_log()
|
||||
assert row.question == QUESTION
|
||||
assert row.deflected is False
|
||||
assert f"{SOURCE}/{YAML_PATH}" in row.sources
|
||||
assert f"{SOURCE}/{MD_PATH}" in row.sources
|
||||
|
||||
|
||||
def test_markdown_control_doc_gets_no_summary_chunk(
|
||||
mock_llm: int, db_ready: None
|
||||
) -> None:
|
||||
"""Control: in the same KB the markdown doc gets no summary at all —
|
||||
its chunk count is exactly the raw chunks; the yaml doc has exactly
|
||||
one ``is_summary`` row and its raw chunk count is untouched by the
|
||||
summary."""
|
||||
_reset_db()
|
||||
summary = _run_in_thread(_import_fixtures(mock_llm))
|
||||
assert summary.added == 2
|
||||
|
||||
md_content = (FIXTURES / MD_PATH).read_text(encoding="utf-8")
|
||||
yaml_content = (FIXTURES / YAML_PATH).read_text(encoding="utf-8")
|
||||
with SessionLocal() as db:
|
||||
md_doc = _doc(db, MD_PATH)
|
||||
yaml_doc = _doc(db, YAML_PATH)
|
||||
md_chunks = [c for c in md_doc.chunks if not c.is_summary]
|
||||
yaml_raw = [c for c in yaml_doc.chunks if not c.is_summary]
|
||||
yaml_summary = [c for c in yaml_doc.chunks if c.is_summary]
|
||||
|
||||
# Markdown: never summarized (phase 30 scope — A9 non-markdown only).
|
||||
assert md_doc.summary is None
|
||||
assert len(md_chunks) == len(
|
||||
chunk_document(md_content, MD_PATH, CHUNK_TARGET, CHUNK_OVERLAP)
|
||||
)
|
||||
|
||||
# YAML: raw chunks exactly as chunked by the importer policy, plus
|
||||
# exactly one summary chunk (position −1, embedded, on the doc row).
|
||||
assert len(yaml_raw) == len(
|
||||
chunk_document(yaml_content, YAML_PATH, CHUNK_TARGET, CHUNK_OVERLAP)
|
||||
)
|
||||
assert sorted(c.position for c in yaml_raw) == list(range(len(yaml_raw)))
|
||||
assert len(yaml_summary) == 1
|
||||
assert yaml_summary[0].position == -1
|
||||
assert yaml_summary[0].embedding is not None
|
||||
assert yaml_doc.summary is not None
|
||||
assert yaml_doc.summary == yaml_summary[0].content
|
||||
assert yaml_doc.summary.endswith(f"\nSource: {SOURCE}/{YAML_PATH}")
|
||||
@@ -92,7 +92,11 @@ def test_sources_page_lists_indexed_docs(
|
||||
|
||||
login(page, app_url) # phase 16: the catalog is admin-only
|
||||
expect(page.locator("#stat-docs")).to_have_text("8")
|
||||
expect(page.locator("#stat-chunks")).to_have_text(str(summary.chunks))
|
||||
# Phase 30: non-markdown fixtures each gained one ``is_summary`` chunk,
|
||||
# so the Sources total is content chunks + summary chunks.
|
||||
expect(page.locator("#stat-chunks")).to_have_text(
|
||||
str(summary.chunks + summary.summaries)
|
||||
)
|
||||
expect(page.locator("#stat-last")).not_to_have_text("–")
|
||||
expect(page.locator("#sources-empty")).to_be_hidden()
|
||||
|
||||
|
||||
+17
-1
@@ -2,6 +2,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from app.config import Settings
|
||||
from app.rag.llm import LLMError
|
||||
|
||||
|
||||
class FakeEmbedder:
|
||||
@@ -9,7 +10,11 @@ class FakeEmbedder:
|
||||
``Embedder`` protocol in :mod:`app.rag.importer`).
|
||||
|
||||
Returns deterministic vectors of *dim* dimensions; records every call
|
||||
so tests can assert batching behaviour.
|
||||
so tests can assert batching behaviour. ``chat`` is the deterministic
|
||||
``lite``-model stand-in (phase 30): it returns
|
||||
``"Summary of <first token of the user content>"`` and raises
|
||||
:class:`LLMError` when the content contains the sentinel word
|
||||
``SUMMARY-BLOWUP`` (drives the importer's fail-soft summary path).
|
||||
"""
|
||||
|
||||
def __init__(self, dim: int = 768) -> None:
|
||||
@@ -17,8 +22,19 @@ class FakeEmbedder:
|
||||
self.settings = Settings(_env_file=None) # pyright: ignore[reportCallIssue]
|
||||
self.embed_batches = 0
|
||||
self.calls: list[list[str]] = []
|
||||
self.chat_calls: list[list[dict[str, str]]] = []
|
||||
|
||||
async def embed(self, texts: list[str]) -> list[list[float]]:
|
||||
self.calls.append(list(texts))
|
||||
self.embed_batches += 1
|
||||
return [[0.01 * (i % 97) for i in range(self.dim)] for _ in texts]
|
||||
|
||||
async def chat(
|
||||
self, messages: list[dict[str, str]], model: str | None = None
|
||||
) -> str:
|
||||
self.chat_calls.append(list(messages))
|
||||
user = next((m["content"] for m in messages if m.get("role") == "user"), "")
|
||||
if "SUMMARY-BLOWUP" in user:
|
||||
raise LLMError("simulated lite-model failure (SUMMARY-BLOWUP sentinel)")
|
||||
first = user.split()
|
||||
return "Summary of " + (first[0] if first else "<empty>")
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
# Qwen 3.8 on llama.cpp — deployment notes
|
||||
|
||||
## Optimal parameters
|
||||
|
||||
The optimal parameters for qwen 3.8 on llama.cpp came out of a week of
|
||||
benchmarks on the homelab GPU. Context length, flash attention and the
|
||||
batch size matter more than the mmap knob. The full flag set lives in
|
||||
`quadlet/qwen-llamacpp.yaml` — the yaml is the source of truth, these
|
||||
notes are the reasoning behind each picked value.
|
||||
|
||||
## What changed since last month
|
||||
|
||||
Switched the server image to the 0.1.43 release and moved the model
|
||||
files to the NVMe cache drive. Pinned the quant to q4_k_m; the repeat
|
||||
penalty is the knob that kept the rambles honest. The webui now fronts
|
||||
the raw server so chat sessions survive a container restart.
|
||||
@@ -0,0 +1,143 @@
|
||||
# qwen 3.8 llama.cpp optimal parameters deployment notes
|
||||
services:
|
||||
llamacpp-server:
|
||||
image: reg.local/ai/llamacpp-server:0.1.43
|
||||
restart: unless-stopped
|
||||
network_mode: host
|
||||
devices:
|
||||
- /dev/dri:/dev/dri
|
||||
volumes:
|
||||
- /srv/models:/models:ro
|
||||
- /srv/llamacpp/cache:/cache
|
||||
environment:
|
||||
- HOST=0.0.0.0
|
||||
- PORT=8081
|
||||
- MODEL=/models/qwen3-8b-instruct-q4_k_m.gguf
|
||||
- N_CTX=32768
|
||||
- N_BATCH=512
|
||||
- N_THREADS=12
|
||||
- FLASH_ATTN=1
|
||||
- MAIN_GPU=1
|
||||
- REPEAT_PENALTY=1.1
|
||||
- TEMPERATURE=0.7
|
||||
- TOP_K=40
|
||||
- TOP_P=0.9
|
||||
- MIN_P=0.05
|
||||
- SEED=42
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: gpu
|
||||
count: 1
|
||||
capabilities:
|
||||
- gpu
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-fsS", "http://127.0.0.1:8081/health"]
|
||||
interval: 30s
|
||||
start_period: 90s
|
||||
retries: 5
|
||||
webui:
|
||||
image: reg.local/ai/open-webui:0.6.12
|
||||
restart: unless-stopped
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /srv/webui/data:/app/data
|
||||
environment:
|
||||
- PORT=3000
|
||||
- UPSTREAM=http://127.0.0.1:8081
|
||||
- AUTO_UPGRADE=0
|
||||
|
||||
models:
|
||||
qwen3-8b-instruct-q4_k_m.gguf:
|
||||
sha256: "9f2c1e07b5a4d6c8f1a3e9b7d2c4a6f0e8b1d3c5a7f9e2b4c6d8f0a1b3c5d7e9"
|
||||
size_gb: 5.2
|
||||
quant: q4_k_m
|
||||
context: 32768
|
||||
mistral-7b-instruct-v0-q5_k_s.gguf:
|
||||
sha256: "b4d8f2c6a0e4f8b2d6c0a4e8f2b6d0c4a8e2f6b0d4c8a2e6f0b4d8c2a6e0f4b8"
|
||||
size_gb: 4.9
|
||||
quant: q5_k_s
|
||||
context: 16384
|
||||
gemma-2-2b-it-q4_k_m.gguf:
|
||||
sha256: "c7a2e4f8b0d6c1a3e5f7b9d2c4a6e8f0b2d4c6a8e0f2b4d6c8a0e2f4b6d8c0a2"
|
||||
size_gb: 1.6
|
||||
quant: q4_k_m
|
||||
context: 8192
|
||||
phi-2-2.7b-q5_k_s.gguf:
|
||||
sha256: "e1f3a5c7b9d1e3f5a7c9b1d3e5f7a9c1b3d5e7f9a1c3b5d7e9f1a3c5b7d9e1f3"
|
||||
size_gb: 2.1
|
||||
quant: q5_k_s
|
||||
context: 4096
|
||||
|
||||
benchmarks:
|
||||
gtx-1080-ti:
|
||||
q4_k_m:
|
||||
tok_per_s: 21.4
|
||||
p50_ms: 148
|
||||
p95_ms: 412
|
||||
q5_k_s:
|
||||
tok_per_s: 18.7
|
||||
p50_ms: 171
|
||||
p95_ms: 466
|
||||
rtx-4090:
|
||||
q4_k_m:
|
||||
tok_per_s: 88.2
|
||||
p50_ms: 34
|
||||
p95_ms: 96
|
||||
q5_k_s:
|
||||
tok_per_s: 74.6
|
||||
p50_ms: 41
|
||||
p95_ms: 118
|
||||
rtx-4060-ti:
|
||||
q4_k_m:
|
||||
tok_per_s: 46.9
|
||||
p50_ms: 62
|
||||
p95_ms: 178
|
||||
|
||||
tuning:
|
||||
repeat_penalty:
|
||||
tried: [1.0, 1.05, 1.1, 1.2]
|
||||
picked: 1.1
|
||||
why: 1.2 clipped mid-sentence twice
|
||||
temperature:
|
||||
tried: [0.5, 0.7, 0.9]
|
||||
picked: 0.7
|
||||
top_p:
|
||||
tried: [0.8, 0.9, 0.95]
|
||||
picked: 0.9
|
||||
min_p:
|
||||
tried: [0.0, 0.05, 0.1]
|
||||
picked: 0.05
|
||||
context:
|
||||
tried: [16384, 32768]
|
||||
picked: 32768
|
||||
why: 16384 evicted early turns in long chats
|
||||
|
||||
registry:
|
||||
mirror: reg.local/ai
|
||||
pull_policy: pinned
|
||||
scan_interval: 24h
|
||||
garbage_collect: weekly
|
||||
access: local-network-only
|
||||
|
||||
alerts:
|
||||
gpu_memory_high:
|
||||
threshold_pct: 92
|
||||
channel: ntfy
|
||||
repeat_after: 6h
|
||||
server_down:
|
||||
channel: ntfy
|
||||
repeat_after: 30m
|
||||
model_stale_days:
|
||||
value: 180
|
||||
channel: ntfy
|
||||
|
||||
maintenance:
|
||||
backup_cron: "0 4 * * *"
|
||||
log_rotate_days: 14
|
||||
update_policy: manual
|
||||
image_retention: 2
|
||||
model_download_mirror: reg.local/ai/models
|
||||
rollback: keep previous tags pinned in registry
|
||||
# RESE-SUMMARY-SENTINEL-7f3a
|
||||
@@ -72,6 +72,15 @@ class FakeRagLLM:
|
||||
self.embed_batches += 1
|
||||
return [_token_vec(t) for t in texts]
|
||||
|
||||
async def chat(
|
||||
self, messages: list[dict[str, str]], model: str | None = None
|
||||
) -> str:
|
||||
"""Deterministic ``lite`` stand-in for the import-time summaries
|
||||
(phase 30) — same convention as ``tests.fakes.FakeEmbedder.chat``."""
|
||||
user = next((m["content"] for m in messages if m.get("role") == "user"), "")
|
||||
first = user.split()
|
||||
return "Summary of " + (first[0] if first else "<empty>")
|
||||
|
||||
async def embed_one(self, text: str) -> list[float]:
|
||||
if self.embed_error is not None:
|
||||
raise self.embed_error
|
||||
|
||||
@@ -61,11 +61,31 @@ def test_import_fixtures_end_to_end(admin_client, db) -> None:
|
||||
k8s = next(d for d in docs if d.path == "homelab/kubernetes.md")
|
||||
assert "Talos Linux" in k8s.content and k8s.content_hash
|
||||
|
||||
# Phase 30: the four non-markdown fixtures each gained one embedded
|
||||
# ``is_summary`` chunk, so the DB holds content + summary chunks.
|
||||
n_chunks = db.scalar(select(func.count()).select_from(Chunk))
|
||||
assert n_chunks == summary.chunks
|
||||
assert n_chunks == summary.chunks + summary.summaries
|
||||
for c in db.scalars(select(Chunk)).all():
|
||||
assert c.embedding is not None and len(c.embedding) == 768
|
||||
|
||||
assert summary.summary_errors == 0
|
||||
for d in docs:
|
||||
non_md = Path(d.path).suffix.lower() not in (".md", ".markdown")
|
||||
schunks = [c for c in d.chunks if c.is_summary]
|
||||
if non_md:
|
||||
# Lite summary stored + exactly one embedded summary chunk (−1).
|
||||
assert d.summary is not None, f"{d.path} should have a summary"
|
||||
assert len(schunks) == 1
|
||||
assert schunks[0].position == -1
|
||||
assert schunks[0].content == d.summary
|
||||
assert schunks[0].embedding is not None
|
||||
else:
|
||||
# Markdown docs never get a summary (phase 30 scope).
|
||||
assert d.summary is None and not schunks
|
||||
assert summary.summaries == sum(
|
||||
1 for d in docs if Path(d.path).suffix.lower() not in (".md", ".markdown")
|
||||
)
|
||||
|
||||
# The Sources page consumes exactly this shape.
|
||||
r = admin_client.get("/api/docs") # phase 16: the catalog is admin-only
|
||||
assert r.status_code == 200
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
"""Integration: migration 0004 (document summaries) schema contract.
|
||||
|
||||
Drives the **real Alembic engine** against the live dev database
|
||||
(``podman compose up -d db``), mirroring the style of
|
||||
``test_migration_0002.py`` (information_schema assertions on the state the
|
||||
migration must leave):
|
||||
|
||||
* upgrade to head → ``documents.summary`` (TEXT, nullable) and
|
||||
``chunks.is_summary`` (BOOLEAN NOT NULL, default false) both exist, and a
|
||||
chunk inserted without the column gets ``is_summary = false`` (pre-0004
|
||||
insert paths stay valid);
|
||||
* downgrade to 0003 → both columns are gone;
|
||||
* upgrade to head again → both are back (round-trip).
|
||||
|
||||
The ``alembic`` fixture guarantees the DB ends at head even if a test
|
||||
fails or the process is interrupted.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterator
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from alembic.config import Config
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from alembic import command
|
||||
from app.db import db_available
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def alembic(db: Session) -> Iterator[Config]:
|
||||
"""Real Alembic config bound to the dev DB (URL from app settings).
|
||||
|
||||
Starts at head (repairs an interrupted earlier run); teardown upgrades
|
||||
to head no matter what happened, so the dev DB is never left below
|
||||
head.
|
||||
"""
|
||||
if not db_available():
|
||||
pytest.skip("Postgres not reachable — run `podman compose up -d db` first")
|
||||
cfg = Config() # no alembic.ini file — env.py gets the URL from app config
|
||||
cfg.set_main_option("script_location", "alembic")
|
||||
command.upgrade(cfg, "head")
|
||||
try:
|
||||
yield cfg
|
||||
finally:
|
||||
command.upgrade(cfg, "head")
|
||||
|
||||
|
||||
def _column(db: Session, table: str, column: str) -> tuple[Any, ...] | None:
|
||||
"""(data_type, is_nullable, column_default) for one column, or None."""
|
||||
row = db.execute(
|
||||
text(
|
||||
"SELECT data_type, is_nullable, column_default"
|
||||
" FROM information_schema.columns"
|
||||
" WHERE table_name = :t AND column_name = :c"
|
||||
),
|
||||
{"t": table, "c": column},
|
||||
).fetchone()
|
||||
return tuple(row) if row is not None else None
|
||||
|
||||
|
||||
def _version(db: Session) -> str | None:
|
||||
return db.execute(text("SELECT version_num FROM alembic_version")).scalar()
|
||||
|
||||
|
||||
def test_upgrade_to_head_adds_summary_columns(db: Session, alembic: Config) -> None:
|
||||
"""Upgrade to head: both columns exist with the locked types/defaults."""
|
||||
command.downgrade(alembic, "0003") # start from the pre-0004 state
|
||||
assert _version(db) == "0003"
|
||||
|
||||
command.upgrade(alembic, "head")
|
||||
assert _version(db) == "0004", "alembic_version must be at 0004 (head)"
|
||||
|
||||
summary = _column(db, "documents", "summary")
|
||||
assert summary is not None, "documents.summary is missing"
|
||||
assert summary[0] == "text", "documents.summary must be TEXT"
|
||||
assert summary[1] == "YES", "documents.summary must be NULLABLE"
|
||||
|
||||
is_summary = _column(db, "chunks", "is_summary")
|
||||
assert is_summary is not None, "chunks.is_summary is missing"
|
||||
assert is_summary[0] == "boolean", "chunks.is_summary must be BOOLEAN"
|
||||
assert is_summary[1] == "NO", "chunks.is_summary must be NOT NULL"
|
||||
assert is_summary[2] is not None and "false" in is_summary[2], (
|
||||
"chunks.is_summary must have server default false"
|
||||
)
|
||||
|
||||
|
||||
def test_is_summary_defaults_false_for_new_chunks(db: Session, alembic: Config) -> None:
|
||||
"""The default keeps old rows/insert paths valid: a chunk inserted
|
||||
without the column (the pre-0004 insert shape) lands as ``false``."""
|
||||
command.upgrade(alembic, "head")
|
||||
doc_id = db.execute(text("SELECT gen_random_uuid()")).scalar()
|
||||
try:
|
||||
db.execute(
|
||||
text(
|
||||
"INSERT INTO documents (id, source, path, full_path, title, content,"
|
||||
" content_hash, indexed_at) VALUES"
|
||||
" (:id, 'mig_test', 't.md', '/t.md', 'T', 'content here',"
|
||||
" repeat('0', 64), now())"
|
||||
),
|
||||
{"id": doc_id},
|
||||
)
|
||||
db.execute(
|
||||
text(
|
||||
"INSERT INTO chunks (id, document_id, position, content)"
|
||||
" VALUES (gen_random_uuid(), :id, 0, 'content here')"
|
||||
),
|
||||
{"id": doc_id},
|
||||
)
|
||||
db.commit()
|
||||
flag = db.execute(
|
||||
text("SELECT is_summary FROM chunks WHERE document_id = :id"), {"id": doc_id}
|
||||
).scalar()
|
||||
assert flag is False, "chunks.is_summary must default to false"
|
||||
finally:
|
||||
db.execute(text("DELETE FROM chunks WHERE document_id = :id"), {"id": doc_id})
|
||||
db.execute(text("DELETE FROM documents WHERE id = :id"), {"id": doc_id})
|
||||
db.commit()
|
||||
|
||||
|
||||
def test_downgrade_to_0003_removes_columns(db: Session, alembic: Config) -> None:
|
||||
"""Downgrade to 0003: both columns are dropped (A13 — reversible)."""
|
||||
command.downgrade(alembic, "0003")
|
||||
assert _version(db) == "0003"
|
||||
|
||||
assert _column(db, "documents", "summary") is None, "documents.summary must be dropped"
|
||||
assert _column(db, "chunks", "is_summary") is None, "chunks.is_summary must be dropped"
|
||||
|
||||
|
||||
def test_upgrade_round_trip_restores_columns(db: Session, alembic: Config) -> None:
|
||||
"""Upgrade back to head after the downgrade: both columns are back."""
|
||||
command.upgrade(alembic, "head")
|
||||
assert _version(db) == "0004", "round-trip upgrade must land at 0004 (head)"
|
||||
|
||||
summary = _column(db, "documents", "summary")
|
||||
assert summary is not None and summary[1] == "YES", "documents.summary must be back"
|
||||
|
||||
is_summary = _column(db, "chunks", "is_summary")
|
||||
assert is_summary is not None and is_summary[1] == "NO", "chunks.is_summary must be back"
|
||||
assert is_summary[2] is not None and "false" in is_summary[2], (
|
||||
"chunks.is_summary must keep its server default false after the round-trip"
|
||||
)
|
||||
@@ -47,18 +47,23 @@ def _doc(title: str, content: str) -> Document:
|
||||
|
||||
|
||||
def _chunk(
|
||||
doc: Document, score: float, cosine: float | None = None, fts_hit: bool = False
|
||||
doc: Document,
|
||||
score: float,
|
||||
cosine: float | None = None,
|
||||
fts_hit: bool = False,
|
||||
is_summary: bool = False,
|
||||
) -> RetrievedChunk:
|
||||
"""Fake candidate: *score* is the fused rank score; *cosine* (defaults to
|
||||
*score*) is the vector-similarity gate input."""
|
||||
return RetrievedChunk(
|
||||
chunk_id=uuid.uuid4(),
|
||||
position=0,
|
||||
position=-1 if is_summary else 0,
|
||||
content=doc.content[:32],
|
||||
score=score,
|
||||
document=doc,
|
||||
cosine=score if cosine is None else cosine,
|
||||
fts_hit=fts_hit,
|
||||
is_summary=is_summary,
|
||||
)
|
||||
|
||||
|
||||
@@ -167,6 +172,70 @@ def test_gate_zero_chunks_deflects_with_fallback_chips() -> None:
|
||||
assert 2 <= len(plan.suggestions) <= MAX_SUGGESTIONS
|
||||
|
||||
|
||||
# ---------- summary hits (phase 30: summary → full source document) ----------
|
||||
|
||||
|
||||
def test_summary_hit_on_selected_top_doc_counts() -> None:
|
||||
"""HIGH branch: the top document was hit via its summary chunk ⇒ 1.
|
||||
|
||||
Context assembly is unchanged (A7 revised): the *source* document's
|
||||
full content lands in the prompt, not the summary text alone.
|
||||
"""
|
||||
a = _doc("Alpha", "ALPHA_FULL_SOURCE_CONTENT")
|
||||
b = _doc("Beta", "BETA_FULL_SOURCE_CONTENT")
|
||||
chunks = [
|
||||
_chunk(a, 0.90, is_summary=True), # top doc reached through its summary
|
||||
_chunk(b, 0.50),
|
||||
]
|
||||
plan = chat_api.plan_turn(chunks, _settings(threshold=0.30))
|
||||
assert plan.deflected is False
|
||||
assert plan.summary_hits == 1
|
||||
# The full source document is what the LLM sees (phase 24 contract).
|
||||
assert "ALPHA_FULL_SOURCE_CONTENT" in plan.system_prompt
|
||||
|
||||
|
||||
def test_summary_hit_outside_top_n_selection_not_counted() -> None:
|
||||
"""A summary chunk on a document outside the top-N (default 2) selection
|
||||
does not count — only hits that landed in the selected context do."""
|
||||
a = _doc("Alpha", "ALPHA_CONTENT")
|
||||
b = _doc("Beta", "BETA_CONTENT")
|
||||
c = _doc("Gamma", "GAMMA_CONTENT")
|
||||
chunks = [
|
||||
_chunk(a, 0.90),
|
||||
_chunk(b, 0.80),
|
||||
_chunk(c, 0.70, is_summary=True), # 3rd-ranked doc — not selected
|
||||
]
|
||||
plan = chat_api.plan_turn(chunks, _settings(threshold=0.30))
|
||||
assert plan.deflected is False
|
||||
assert [d.title for d in plan.docs] == ["Alpha", "Beta"]
|
||||
assert plan.summary_hits == 0
|
||||
|
||||
|
||||
def test_low_branch_counts_summary_hit_on_selected_doc() -> None:
|
||||
"""LOW (deflected) branch records ``summary_hits`` too: the weak hit's
|
||||
parent is still the selected (weak-hit) document."""
|
||||
a = _doc("Gamma", "GAMMA_DOC_CONTENT")
|
||||
b = _doc("Delta", "DELTA_DOC_CONTENT")
|
||||
chunks = [
|
||||
_chunk(a, 0.05, cosine=0.05, is_summary=True), # weak cosine, no FTS
|
||||
_chunk(b, 0.03, cosine=0.03),
|
||||
]
|
||||
plan = chat_api.plan_turn(chunks, _settings(threshold=0.30))
|
||||
assert plan.deflected is True
|
||||
assert plan.summary_hits == 1
|
||||
|
||||
|
||||
def test_no_summary_chunks_yields_zero_summary_hits() -> None:
|
||||
"""Legacy chunks (``is_summary=false``) keep ``summary_hits == 0``."""
|
||||
a = _doc("Alpha", "ALPHA_CONTENT")
|
||||
b = _doc("Beta", "BETA_CONTENT")
|
||||
plan = chat_api.plan_turn([_chunk(a, 0.90), _chunk(b, 0.40)], _settings(threshold=0.30))
|
||||
assert plan.summary_hits == 0
|
||||
plan_low = chat_api.plan_turn([_chunk(a, 0.05, cosine=0.05)], _settings(threshold=0.30))
|
||||
assert plan_low.deflected is True
|
||||
assert plan_low.summary_hits == 0
|
||||
|
||||
|
||||
# ---------- prompt content (LOW vs HIGH) ----------
|
||||
|
||||
|
||||
|
||||
@@ -25,6 +25,9 @@ def test_defaults_match_locked_decisions(monkeypatch: pytest.MonkeyPatch) -> Non
|
||||
s = _settings()
|
||||
assert s.llm_chat_model == "turbo"
|
||||
assert s.llm_embed_model == "embed"
|
||||
# A5 extended (phase 30): one-shot completions default to the ``lite``
|
||||
# model on the same endpoint.
|
||||
assert s.llm_summary_model == "lite"
|
||||
assert s.embedding_dim == 768
|
||||
assert s.llm_base_url.endswith("/v1")
|
||||
# A8 (revised): the honesty gate input is the best cosine, default 0.62.
|
||||
@@ -53,6 +56,22 @@ def test_env_override(monkeypatch) -> None:
|
||||
assert s.llm_chat_model == "juggernaut"
|
||||
|
||||
|
||||
def test_llm_summary_model_env_override(monkeypatch) -> None:
|
||||
"""Phase 30: ``BOR_LLM_SUMMARY_MODEL`` overrides the ``lite`` default"""
|
||||
monkeypatch.setenv("BOR_LLM_SUMMARY_MODEL", "mini")
|
||||
s = _settings()
|
||||
assert s.llm_summary_model == "mini"
|
||||
|
||||
|
||||
def test_summary_max_chars_default_and_env_override(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Phase 30: document content sent to the ``lite`` model is capped at
|
||||
``BOR_SUMMARY_MAX_CHARS`` (default 12 000 chars per call)."""
|
||||
monkeypatch.delenv("BOR_SUMMARY_MAX_CHARS", raising=False)
|
||||
assert _settings().summary_max_chars == 12_000
|
||||
monkeypatch.setenv("BOR_SUMMARY_MAX_CHARS", "5000")
|
||||
assert _settings().summary_max_chars == 5000
|
||||
|
||||
|
||||
def test_max_output_tokens_env_override(monkeypatch) -> None:
|
||||
monkeypatch.setenv("BOR_MAX_OUTPUT_TOKENS", "1234")
|
||||
s = _settings()
|
||||
|
||||
+211
-4
@@ -1,12 +1,18 @@
|
||||
"""Unit tests: importer directory walk + sha256 delta logic.
|
||||
"""Unit tests: importer directory walk + sha256 delta logic + summaries.
|
||||
|
||||
The walk tests are pure filesystem (``tmp_path``); the delta tests run
|
||||
against the local compose Postgres (preferred — a real vector table),
|
||||
skipping with clear instructions when the stack is not up.
|
||||
The walk tests are pure filesystem (``tmp_path``); the delta and summary
|
||||
tests run against the local compose Postgres (preferred — a real vector
|
||||
table), skipping with clear instructions when the stack is not up.
|
||||
|
||||
Summaries (phase 30): non-markdown files get a ``lite``-model summary via
|
||||
the fake's deterministic ``chat`` (``"Summary of <first token>"``); the
|
||||
sentinel word ``SUMMARY-BLOWUP`` makes ``chat`` raise :class:`LLMError`
|
||||
for the fail-soft path.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
@@ -15,6 +21,8 @@ from sqlalchemy import func, select
|
||||
from app.models import Chunk, Document
|
||||
from app.rag.importer import (
|
||||
EXCLUDED_DIRS,
|
||||
ImportSummary,
|
||||
_store_summary,
|
||||
import_sources,
|
||||
iter_importable_files,
|
||||
)
|
||||
@@ -350,6 +358,205 @@ def test_multi_format_import_counts_per_format_and_titles_stem(db, tmp_path: Pat
|
||||
_cleanup_source(db, root.name)
|
||||
|
||||
|
||||
# ---------- phase 30: lite-model summaries for non-markdown files ----------
|
||||
|
||||
|
||||
def test_non_markdown_file_gets_stored_and_indexed_summary(db, tmp_path: Path) -> None:
|
||||
"""A ``.yaml`` file is summarized: ``documents.summary`` is set and one
|
||||
``is_summary`` chunk (position −1, embedded) is indexed alongside the
|
||||
content chunks."""
|
||||
root = tmp_path / "sumsrc"
|
||||
root.mkdir()
|
||||
(root / "svc.yaml").write_text("alpha services:\n gitlab:\n port: 8929\n")
|
||||
llm = FakeEmbedder()
|
||||
try:
|
||||
summary = asyncio.run(import_sources([root], llm, session=db))
|
||||
assert summary.added == 1
|
||||
assert summary.summaries == 1
|
||||
assert summary.summary_errors == 0
|
||||
doc = db.scalar(
|
||||
select(Document).where(Document.source == root.name, Document.path == "svc.yaml")
|
||||
)
|
||||
assert doc is not None
|
||||
assert doc.summary is not None
|
||||
# Deterministic fake reply + the code-appended pointer line.
|
||||
assert doc.summary.startswith("Summary of alpha")
|
||||
assert doc.summary.endswith(f"Source: {root.name}/svc.yaml")
|
||||
schunks = [c for c in doc.chunks if c.is_summary]
|
||||
assert len(schunks) == 1
|
||||
assert schunks[0].position == -1
|
||||
assert schunks[0].content == doc.summary
|
||||
assert schunks[0].embedding is not None and len(schunks[0].embedding) == 768
|
||||
# Content chunks stay 0-based and are never flagged as summaries.
|
||||
content = [c for c in doc.chunks if not c.is_summary]
|
||||
assert sorted(c.position for c in content) == list(range(len(content)))
|
||||
finally:
|
||||
_cleanup_source(db, root.name)
|
||||
|
||||
|
||||
def test_markdown_file_never_gets_summary(db, tmp_path: Path) -> None:
|
||||
"""Markdown is already natural language: no summary, no ``is_summary``
|
||||
chunk, and the ``lite`` model is never called."""
|
||||
root = tmp_path / "mdsrc"
|
||||
root.mkdir()
|
||||
(root / "note.md").write_text("# Note\n\nmarkdown body\n")
|
||||
llm = FakeEmbedder()
|
||||
try:
|
||||
summary = asyncio.run(import_sources([root], llm, session=db))
|
||||
assert summary.added == 1
|
||||
assert summary.summaries == 0 and summary.summary_errors == 0
|
||||
assert llm.chat_calls == [] # the model was never asked
|
||||
doc = db.scalar(
|
||||
select(Document).where(Document.source == root.name, Document.path == "note.md")
|
||||
)
|
||||
assert doc is not None
|
||||
assert doc.summary is None
|
||||
assert doc.chunks and all(not c.is_summary for c in doc.chunks)
|
||||
finally:
|
||||
_cleanup_source(db, root.name)
|
||||
|
||||
|
||||
def test_summary_failure_is_fail_soft(db, tmp_path: Path) -> None:
|
||||
"""A ``lite``-model failure must never lose the document: the file is
|
||||
fully indexed (content chunks + embeddings), ``documents.summary`` stays
|
||||
NULL, and the failure is counted in ``summary_errors``."""
|
||||
root = tmp_path / "blowup"
|
||||
root.mkdir()
|
||||
(root / "bad.txt").write_text("SUMMARY-BLOWUP the lite model chokes on this\n")
|
||||
llm = FakeEmbedder()
|
||||
try:
|
||||
summary = asyncio.run(import_sources([root], llm, session=db))
|
||||
assert summary.errors == 0 # the document itself imported fine
|
||||
assert summary.added == 1
|
||||
assert summary.summaries == 0
|
||||
assert summary.summary_errors == 1
|
||||
doc = db.scalar(
|
||||
select(Document).where(Document.source == root.name, Document.path == "bad.txt")
|
||||
)
|
||||
assert doc is not None
|
||||
assert doc.summary is None
|
||||
assert len(doc.chunks) == 1
|
||||
assert doc.chunks[0].embedding is not None # content chunk embedded
|
||||
assert all(not c.is_summary for c in doc.chunks)
|
||||
finally:
|
||||
_cleanup_source(db, root.name)
|
||||
|
||||
|
||||
def test_summary_chunk_is_replaced_on_reimport(db, tmp_path: Path) -> None:
|
||||
"""Re-importing a changed non-markdown file keeps exactly one
|
||||
``is_summary`` chunk — the old one is gone, the new summary is stored
|
||||
and embedded, and the content chunks stay 0-based."""
|
||||
root = tmp_path / "repl"
|
||||
root.mkdir()
|
||||
path = root / "cfg.yaml"
|
||||
path.write_text("alpha settings:\n host: one\n")
|
||||
llm = FakeEmbedder()
|
||||
try:
|
||||
asyncio.run(import_sources([root], llm, session=db))
|
||||
path.write_text("bravo settings:\n host: two\n")
|
||||
summary = asyncio.run(import_sources([root], llm, session=db))
|
||||
assert summary.updated == 1
|
||||
assert summary.summaries == 1 and summary.summary_errors == 0
|
||||
doc = db.scalar(
|
||||
select(Document).where(Document.source == root.name, Document.path == "cfg.yaml")
|
||||
)
|
||||
assert doc is not None
|
||||
assert doc.summary is not None and doc.summary.startswith("Summary of bravo")
|
||||
schunks = [c for c in doc.chunks if c.is_summary]
|
||||
assert len(schunks) == 1 # the old one was deleted
|
||||
assert schunks[0].position == -1
|
||||
assert schunks[0].content == doc.summary
|
||||
assert schunks[0].embedding is not None
|
||||
assert "alpha" not in schunks[0].content # no stale summary text
|
||||
assert sorted(c.position for c in doc.chunks if not c.is_summary) == [0]
|
||||
finally:
|
||||
_cleanup_source(db, root.name)
|
||||
|
||||
|
||||
def test_store_summary_replaces_an_existing_summary_chunk(db, tmp_path: Path) -> None:
|
||||
"""Replacement unit, driven directly: with a pre-existing
|
||||
``is_summary`` chunk in place, ``_store_summary`` deletes the old one
|
||||
and leaves exactly one (new) summary chunk + updated
|
||||
``documents.summary`` — the at-most-one-summary invariant."""
|
||||
root = tmp_path / "direct"
|
||||
root.mkdir()
|
||||
(root / "a.yaml").write_text("alpha x\n")
|
||||
llm = FakeEmbedder()
|
||||
try:
|
||||
asyncio.run(import_sources([root], llm, session=db))
|
||||
doc = db.scalar(
|
||||
select(Document).where(Document.source == root.name, Document.path == "a.yaml")
|
||||
)
|
||||
assert doc is not None and doc.summary is not None
|
||||
assert any(c.is_summary for c in doc.chunks) # the first import's summary
|
||||
counters = ImportSummary()
|
||||
asyncio.run(
|
||||
_store_summary(
|
||||
session=db, doc=doc, source=root.name, rel="a.yaml",
|
||||
content=doc.content, llm=llm, summary=counters,
|
||||
)
|
||||
)
|
||||
assert counters.summaries == 1 and counters.summary_errors == 0
|
||||
schunks = [c for c in doc.chunks if c.is_summary]
|
||||
assert len(schunks) == 1 # the old one was deleted
|
||||
assert schunks[0].position == -1
|
||||
assert schunks[0].content == doc.summary
|
||||
assert schunks[0].embedding is not None
|
||||
finally:
|
||||
_cleanup_source(db, root.name)
|
||||
|
||||
|
||||
def test_store_summary_fail_soft_leaves_document_untouched(db, tmp_path: Path) -> None:
|
||||
"""A ``lite`` failure inside ``_store_summary`` rolls back only the
|
||||
summary rows: the previous summary (if any) and the document survive,
|
||||
and the failure is counted."""
|
||||
root = tmp_path / "directfail"
|
||||
root.mkdir()
|
||||
(root / "a.yaml").write_text("alpha x\n")
|
||||
llm = FakeEmbedder()
|
||||
try:
|
||||
asyncio.run(import_sources([root], llm, session=db))
|
||||
doc = db.scalar(
|
||||
select(Document).where(Document.source == root.name, Document.path == "a.yaml")
|
||||
)
|
||||
assert doc is not None and doc.summary is not None
|
||||
previous_summary = doc.summary
|
||||
(root / "a.yaml").write_text("SUMMARY-BLOWUP now the lite model fails\n")
|
||||
counters = ImportSummary()
|
||||
asyncio.run(
|
||||
_store_summary(
|
||||
session=db, doc=doc, source=root.name, rel="a.yaml",
|
||||
content="SUMMARY-BLOWUP now the lite model fails\n",
|
||||
llm=llm, summary=counters,
|
||||
)
|
||||
)
|
||||
assert counters.summaries == 0 and counters.summary_errors == 1
|
||||
assert doc.summary == previous_summary # rolled back, not nulled
|
||||
schunks = [c for c in doc.chunks if c.is_summary]
|
||||
assert len(schunks) == 1 # the old one survived the rollback
|
||||
assert schunks[0].content == previous_summary
|
||||
finally:
|
||||
_cleanup_source(db, root.name)
|
||||
|
||||
|
||||
def test_import_summary_log_line_includes_summary_counters(
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
"""PLAN §9 summary line: the phase-30 counters sit between
|
||||
``embed_batches`` and ``formats``."""
|
||||
s = ImportSummary()
|
||||
s.files, s.added, s.chunks, s.embed_batches = 3, 3, 5, 4
|
||||
s.summaries, s.summary_errors = 2, 1
|
||||
s.formats = {"md": 1, "yaml": 2}
|
||||
with caplog.at_level(logging.INFO, logger="app.importer"):
|
||||
s.log()
|
||||
line = caplog.records[-1].getMessage()
|
||||
assert line == (
|
||||
"import: summary files=3 added=3 updated=0 unchanged=0 pruned=0 errors=0 "
|
||||
"chunks=5 embed_batches=4 summaries=2 summary_errors=1 formats=yaml:2,md:1"
|
||||
)
|
||||
|
||||
|
||||
def test_prune_removes_files_now_excluded_by_format_filter(db, tmp_path: Path) -> None:
|
||||
"""Previously-imported junk leaves the index: a file that no longer
|
||||
matches the A9 extension filter is pruned on the next ``prune=True`` run.
|
||||
|
||||
@@ -275,17 +275,42 @@ class _FakeChatStream:
|
||||
return chunk
|
||||
|
||||
|
||||
class _FakeCompletion:
|
||||
"""One fake non-streaming ChatCompletion (``choices[].message`` shape).
|
||||
|
||||
``content=None`` mirrors the real wire where the field can be absent or
|
||||
empty (reasoning-only replies, provider quirks).
|
||||
"""
|
||||
|
||||
def __init__(self, content: str | None, empty_choices: bool = False) -> None:
|
||||
if empty_choices:
|
||||
self.choices = []
|
||||
else:
|
||||
self.choices = [SimpleNamespace(message=SimpleNamespace(content=content))]
|
||||
|
||||
|
||||
class _FakeCompletions:
|
||||
def __init__(self, chunks: list | None = None, fail: Exception | None = None) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
chunks: list | None = None,
|
||||
fail: Exception | None = None,
|
||||
completion: _FakeCompletion | None = None,
|
||||
) -> None:
|
||||
self.chunks = chunks or []
|
||||
self.fail = fail
|
||||
self.completion = completion
|
||||
self.kwargs: dict | None = None
|
||||
self.chat_kwargs: dict | None = None
|
||||
|
||||
async def create(self, **kwargs) -> _FakeChatStream:
|
||||
async def create(self, **kwargs) -> _FakeChatStream | _FakeCompletion:
|
||||
self.kwargs = kwargs
|
||||
if self.fail is not None:
|
||||
raise self.fail
|
||||
return _FakeChatStream(self.chunks)
|
||||
if kwargs.get("stream"):
|
||||
return _FakeChatStream(self.chunks)
|
||||
self.chat_kwargs = kwargs
|
||||
assert self.completion is not None
|
||||
return self.completion
|
||||
|
||||
|
||||
def _make_stream_client(
|
||||
@@ -428,3 +453,85 @@ def test_chat_stream_llm_error_passes_through_unwrapped() -> None:
|
||||
llm, _ = _make_stream_client(fail=LLMError("already wrapped"))
|
||||
with pytest.raises(LLMError, match="already wrapped"):
|
||||
asyncio.run(_collect(llm, [{"role": "user", "content": "q"}]))
|
||||
|
||||
|
||||
# ---------- one-shot chat: LLMClient.chat (phase 30, task 01) ----------
|
||||
|
||||
|
||||
def _make_chat_client(
|
||||
completion: _FakeCompletion | None = None,
|
||||
fail: Exception | None = None,
|
||||
**settings_kwargs: Any,
|
||||
) -> tuple[LLMClient, _FakeCompletions]:
|
||||
completions = _FakeCompletions(fail=fail, completion=completion)
|
||||
fake_openai = SimpleNamespace(chat=SimpleNamespace(completions=completions))
|
||||
llm = LLMClient(_settings(**settings_kwargs))
|
||||
llm._client = fake_openai # pyright: ignore[reportAttributeAccessIssue]
|
||||
return llm, completions
|
||||
|
||||
|
||||
def test_chat_returns_trimmed_content_with_locked_params() -> None:
|
||||
"""Default model is ``lite`` (BOR_LLM_SUMMARY_MODEL), non-streaming,
|
||||
low temperature, fixed 2048-token budget — summaries are short."""
|
||||
llm, completions = _make_chat_client(_FakeCompletion(" Summary text.\n"))
|
||||
messages = [{"role": "system", "content": "s"}, {"role": "user", "content": "u"}]
|
||||
out = asyncio.run(llm.chat(messages))
|
||||
assert out == "Summary text."
|
||||
assert completions.chat_kwargs is not None
|
||||
assert completions.chat_kwargs["model"] == "lite"
|
||||
assert completions.chat_kwargs["stream"] is False
|
||||
assert completions.chat_kwargs["temperature"] == 0.2
|
||||
assert completions.chat_kwargs["max_tokens"] == 2048
|
||||
assert completions.chat_kwargs["messages"] == messages
|
||||
|
||||
|
||||
def test_chat_default_model_comes_from_llm_summary_model_setting() -> None:
|
||||
llm, completions = _make_chat_client(
|
||||
_FakeCompletion("x"), llm_summary_model="tiny"
|
||||
)
|
||||
asyncio.run(llm.chat([{"role": "user", "content": "q"}]))
|
||||
assert completions.chat_kwargs is not None
|
||||
assert completions.chat_kwargs["model"] == "tiny"
|
||||
|
||||
|
||||
def test_chat_explicit_model_overrides_the_default() -> None:
|
||||
llm, completions = _make_chat_client(
|
||||
_FakeCompletion("x"), llm_summary_model="tiny"
|
||||
)
|
||||
asyncio.run(llm.chat([{"role": "user", "content": "q"}], model="special"))
|
||||
assert completions.chat_kwargs is not None
|
||||
assert completions.chat_kwargs["model"] == "special"
|
||||
|
||||
|
||||
def test_chat_transport_failure_wrapped_as_llm_error_with_base_url() -> None:
|
||||
"""HTTP/transport failures (incl. >=400 surfaced by the SDK) are wrapped
|
||||
with the base URL in the message — same style as chat_stream."""
|
||||
llm, _ = _make_chat_client(fail=RuntimeError("HTTP 502 Bad Gateway"))
|
||||
with pytest.raises(LLMError, match="HTTP 502") as exc:
|
||||
asyncio.run(llm.chat([{"role": "user", "content": "q"}]))
|
||||
assert "aipi.reeseapps.com" in str(exc.value)
|
||||
|
||||
|
||||
def test_chat_llm_error_passes_through_unwrapped() -> None:
|
||||
llm, _ = _make_chat_client(fail=LLMError("already wrapped"))
|
||||
with pytest.raises(LLMError, match="already wrapped"):
|
||||
asyncio.run(llm.chat([{"role": "user", "content": "q"}]))
|
||||
|
||||
|
||||
def test_chat_empty_choices_raises_llm_error() -> None:
|
||||
llm, _ = _make_chat_client(_FakeCompletion(None, empty_choices=True))
|
||||
with pytest.raises(LLMError, match="no choices"):
|
||||
asyncio.run(llm.chat([{"role": "user", "content": "q"}]))
|
||||
|
||||
|
||||
def test_chat_missing_content_raises_llm_error() -> None:
|
||||
"""A silent empty summary must never be stored — None content fails."""
|
||||
llm, _ = _make_chat_client(_FakeCompletion(None))
|
||||
with pytest.raises(LLMError, match="empty content"):
|
||||
asyncio.run(llm.chat([{"role": "user", "content": "q"}]))
|
||||
|
||||
|
||||
def test_chat_whitespace_only_content_raises_llm_error() -> None:
|
||||
llm, _ = _make_chat_client(_FakeCompletion(" \n\t "))
|
||||
with pytest.raises(LLMError, match="empty content"):
|
||||
asyncio.run(llm.chat([{"role": "user", "content": "q"}]))
|
||||
|
||||
@@ -7,6 +7,7 @@ chat integration tests against real Postgres; the pure mapping logic in
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -97,7 +98,8 @@ from app.rag.retriever import fuse, lexical_tsquery # noqa: E402
|
||||
|
||||
|
||||
def _rc(
|
||||
doc_path: str, cosine: float = 0.0, fts_hit: bool = False, position: int = 0
|
||||
doc_path: str, cosine: float = 0.0, fts_hit: bool = False, position: int = 0,
|
||||
is_summary: bool = False,
|
||||
) -> RetrievedChunk:
|
||||
return RetrievedChunk(
|
||||
chunk_id=uuid.uuid4(),
|
||||
@@ -107,6 +109,7 @@ def _rc(
|
||||
document=_doc(doc_path, "x" * 20),
|
||||
cosine=cosine,
|
||||
fts_hit=fts_hit,
|
||||
is_summary=is_summary,
|
||||
)
|
||||
|
||||
|
||||
@@ -186,3 +189,137 @@ def test_fuse_rejects_nonpositive_k() -> None:
|
||||
|
||||
def test_fuse_empty_lists() -> None:
|
||||
assert fuse([], [], k=60) == []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 30: is_summary survives both candidate lists and the fusion
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
from app.models import Chunk # noqa: E402
|
||||
from app.rag.retriever import _lexical_candidates, _vector_candidates # noqa: E402
|
||||
|
||||
|
||||
class _FakeResult:
|
||||
"""Stands in for SQLAlchemy's RowMapping result (``.all()`` only)."""
|
||||
|
||||
def __init__(self, rows: list) -> None:
|
||||
self._rows = rows
|
||||
|
||||
def all(self) -> list:
|
||||
return self._rows
|
||||
|
||||
|
||||
class _FakeSession:
|
||||
"""Returns canned rows from ``execute`` without touching Postgres."""
|
||||
|
||||
def __init__(self, rows: list) -> None:
|
||||
self._rows = rows
|
||||
self.statements: list = []
|
||||
|
||||
def execute(self, stmt, params: dict | None = None) -> _FakeResult:
|
||||
self.statements.append((stmt, params))
|
||||
return _FakeResult(self._rows)
|
||||
|
||||
|
||||
def _chunk_row(is_summary: bool) -> Chunk:
|
||||
doc = _doc("summary-src.yaml", "RAW_YAML_CONTENT")
|
||||
return Chunk(
|
||||
id=uuid.uuid4(),
|
||||
document_id=doc.id,
|
||||
position=-1, # the summary chunk's position (phase 30)
|
||||
content="Summary text",
|
||||
is_summary=is_summary,
|
||||
)
|
||||
|
||||
|
||||
def test_vector_candidates_carry_is_summary_flag() -> None:
|
||||
"""The vector list copies ``Chunk.is_summary`` onto each candidate."""
|
||||
doc = _doc("summary-src.yaml", "RAW_YAML_CONTENT")
|
||||
summary = _chunk_row(is_summary=True)
|
||||
ordinary = _chunk_row(is_summary=False)
|
||||
ordinary.position = 0
|
||||
ordinary.content = "ordinary content"
|
||||
rows = [
|
||||
(summary, 0.123456, doc),
|
||||
(ordinary, 0.2, doc),
|
||||
]
|
||||
out = _vector_candidates(_FakeSession(rows), [0.0] * 768, limit=5) # pyright: ignore[reportArgumentType]
|
||||
assert len(out) == 2
|
||||
by_pos = {rc.position: rc for rc in out}
|
||||
assert by_pos[-1].is_summary is True # the summary chunk (position −1)
|
||||
assert by_pos[0].is_summary is False # ordinary content chunk
|
||||
assert by_pos[-1].cosine == pytest.approx(0.876544) # 1 − distance, still rounded
|
||||
|
||||
|
||||
def test_vector_candidates_default_is_summary_false_for_legacy_chunks() -> None:
|
||||
"""Pre-phase-30 rows have ``is_summary=false`` — candidates stay False."""
|
||||
doc = _doc("legacy.md", "LEGACY")
|
||||
legacy = Chunk(
|
||||
id=uuid.uuid4(),
|
||||
document_id=doc.id,
|
||||
position=0,
|
||||
content="legacy content",
|
||||
is_summary=False,
|
||||
)
|
||||
out = _vector_candidates(_FakeSession([(legacy, 0.5, doc)]), [0.0] * 768, limit=5) # pyright: ignore[reportArgumentType]
|
||||
assert out[0].is_summary is False
|
||||
|
||||
|
||||
def _lexical_row(is_summary: bool, doc_path: str) -> object:
|
||||
"""One row of ``_LEXICAL_SQL`` (attribute access, as SQLAlchemy returns)."""
|
||||
doc = _doc(doc_path, "DOC_BODY")
|
||||
return SimpleNamespace(
|
||||
chunk_id=uuid.uuid4(),
|
||||
position=-1 if is_summary else 0,
|
||||
content="summary chunk text" if is_summary else "content chunk text",
|
||||
doc_id=doc.id,
|
||||
source=doc.source,
|
||||
path=doc.path,
|
||||
full_path=doc.full_path,
|
||||
title=doc.title,
|
||||
doc_content=doc.content,
|
||||
content_hash=doc.content_hash,
|
||||
indexed_at=None,
|
||||
is_summary=is_summary,
|
||||
rank=0.33,
|
||||
)
|
||||
|
||||
|
||||
def test_lexical_candidates_carry_is_summary_flag() -> None:
|
||||
"""The lexical list reads ``c.is_summary`` from the raw row."""
|
||||
rows = [_lexical_row(True, "summary-src.yaml"), _lexical_row(False, "other.md")]
|
||||
out = _lexical_candidates(_FakeSession(rows), "how do i configure the thing", limit=10) # pyright: ignore[reportArgumentType]
|
||||
assert len(out) == 2
|
||||
by_path = {rc.document.path: rc for rc in out}
|
||||
assert by_path["summary-src.yaml"].is_summary is True
|
||||
assert by_path["summary-src.yaml"].position == -1
|
||||
assert by_path["other.md"].is_summary is False
|
||||
assert all(rc.fts_hit is True for rc in out)
|
||||
|
||||
|
||||
def test_fuse_keeps_is_summary_on_double_hit() -> None:
|
||||
"""A summary chunk in both lists keeps the flag after fusion."""
|
||||
v1 = _rc("s.yaml", cosine=0.9, is_summary=True)
|
||||
l1 = _rc("s.yaml", cosine=0.9, is_summary=True) # lexical copy of the same chunk
|
||||
l1.chunk_id = v1.chunk_id
|
||||
out = fuse([v1], [l1], k=60)
|
||||
assert len(out) == 1
|
||||
assert out[0].is_summary is True
|
||||
assert out[0].fts_hit is True
|
||||
assert out[0].score == pytest.approx(2 / 61)
|
||||
|
||||
|
||||
def test_fuse_keeps_is_summary_on_lexical_only_hit() -> None:
|
||||
"""A summary-only lexical hit (no vector rank) keeps the flag."""
|
||||
out = fuse([], [_rc("s.yaml", is_summary=True)], k=60)
|
||||
assert len(out) == 1
|
||||
assert out[0].is_summary is True
|
||||
assert out[0].fts_hit is True
|
||||
assert out[0].cosine == 0.0
|
||||
|
||||
|
||||
def test_fuse_default_is_summary_stays_false_for_legacy_chunks() -> None:
|
||||
"""Neither list flagged ⇒ fusion never invents a summary flag."""
|
||||
out = fuse([_rc("a.md", cosine=0.8)], [_rc("b.md")], k=60)
|
||||
assert len(out) == 2
|
||||
assert all(rc.is_summary is False for rc in out)
|
||||
|
||||
@@ -0,0 +1,173 @@
|
||||
"""Unit: document summarizer (phase 30, task 03).
|
||||
|
||||
Covers the ``SUMMARY_MODE`` prompt (marker + instruction, capped user
|
||||
content), the code-deterministic ``Source: <source>/<path>`` pointer,
|
||||
and the rejection of empty/whitespace model output.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
from app.config import Settings, get_settings
|
||||
from app.rag.llm import LLMError
|
||||
from app.rag.retriever import TRUNCATION_MARKER
|
||||
from app.rag.summarizer import (
|
||||
SUMMARY_INSTRUCTION,
|
||||
SUMMARY_MODE,
|
||||
SYSTEM_PROMPT,
|
||||
build_summary_prompt,
|
||||
generate_summary,
|
||||
)
|
||||
|
||||
|
||||
class _FakeLLM:
|
||||
"""Duck-typed stand-in for ``LLMClient`` (``chat`` + ``settings``).
|
||||
|
||||
Records the messages and the ``model`` kwarg it was called with; can
|
||||
return a canned reply or raise (e.g. :class:`LLMError`).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
reply: str | None = "Backups run nightly at 02:00 via the borg schedule.",
|
||||
fail: Exception | None = None,
|
||||
) -> None:
|
||||
self._reply = reply
|
||||
self._fail = fail
|
||||
self.settings = Settings(_env_file=None) # pyright: ignore[reportCallIssue]
|
||||
self.messages: list[dict[str, str]] = []
|
||||
self.model: str | None = None
|
||||
|
||||
async def chat(
|
||||
self, messages: list[dict[str, str]], model: str | None = None
|
||||
) -> str:
|
||||
self.messages = list(messages)
|
||||
self.model = model
|
||||
if self._fail is not None:
|
||||
raise self._fail
|
||||
assert self._reply is not None
|
||||
return self._reply
|
||||
|
||||
|
||||
# ---------- build_summary_prompt: system ----------
|
||||
|
||||
|
||||
def test_system_prompt_has_marker_and_locked_instruction() -> None:
|
||||
assert SYSTEM_PROMPT.startswith(SUMMARY_MODE)
|
||||
assert SUMMARY_INSTRUCTION in SYSTEM_PROMPT
|
||||
for fragment in (
|
||||
"plain-text summary of this document in natural",
|
||||
"what it configures/defines",
|
||||
"Do not use markdown",
|
||||
"Do not invent anything that is not in the document",
|
||||
):
|
||||
assert fragment in SYSTEM_PROMPT
|
||||
system, _ = build_summary_prompt("Homelab", "a.yaml", "content")
|
||||
assert system == SYSTEM_PROMPT
|
||||
assert SUMMARY_MODE in system # the marker the E2E mock keys on
|
||||
|
||||
|
||||
# ---------- build_summary_prompt: user (capped content) ----------
|
||||
|
||||
|
||||
def test_user_prompt_is_full_content_when_under_cap() -> None:
|
||||
content = "services:\n borg:\n port: 9999"
|
||||
_, user = build_summary_prompt("Homelab", "a.yaml", content, max_chars=12_000)
|
||||
assert user == content
|
||||
assert TRUNCATION_MARKER not in user
|
||||
|
||||
|
||||
def test_user_prompt_at_exact_cap_is_not_truncated() -> None:
|
||||
content = "z" * 64
|
||||
_, user = build_summary_prompt("Homelab", "a.yaml", content, max_chars=64)
|
||||
assert user == content
|
||||
assert TRUNCATION_MARKER not in user
|
||||
|
||||
|
||||
def test_user_prompt_truncated_with_marker_when_over_custom_cap() -> None:
|
||||
content = "x" * 100 + "TAIL"
|
||||
_, user = build_summary_prompt("Homelab", "a.yaml", content, max_chars=100)
|
||||
assert user == "x" * 100 + "\n" + TRUNCATION_MARKER
|
||||
assert "TAIL" not in user # overflow is gone, not squeezed in
|
||||
assert user.endswith(TRUNCATION_MARKER)
|
||||
|
||||
|
||||
def test_user_prompt_truncated_at_default_cap() -> None:
|
||||
"""No explicit cap → ``BOR_SUMMARY_MAX_CHARS`` (read from the live
|
||||
settings, so the test holds for any configured value)."""
|
||||
cap = get_settings().summary_max_chars
|
||||
content = "y" * (cap + 50)
|
||||
_, user = build_summary_prompt("Homelab", "a.yaml", content)
|
||||
assert user == "y" * cap + "\n" + TRUNCATION_MARKER
|
||||
|
||||
|
||||
# ---------- generate_summary: pointer + validation ----------
|
||||
|
||||
|
||||
def test_generate_summary_returns_model_text_plus_deterministic_pointer() -> None:
|
||||
llm = _FakeLLM(reply="Backups run nightly at 02:00 via the borg schedule.")
|
||||
out = asyncio.run(
|
||||
generate_summary(llm, source="Homelab", path="backups/borg.yaml", content="c")
|
||||
)
|
||||
expected = (
|
||||
"Backups run nightly at 02:00 via the borg schedule.\n"
|
||||
"Source: Homelab/backups/borg.yaml"
|
||||
)
|
||||
assert out == expected
|
||||
assert out.splitlines()[-1] == "Source: Homelab/backups/borg.yaml"
|
||||
|
||||
|
||||
def test_generate_summary_calls_the_configured_summary_model() -> None:
|
||||
llm = _FakeLLM(reply="s")
|
||||
asyncio.run(generate_summary(llm, source="Homelab", path="a.yaml", content="c"))
|
||||
assert llm.model == llm.settings.llm_summary_model # the ``lite`` default
|
||||
assert llm.model == "lite"
|
||||
assert [m["role"] for m in llm.messages] == ["system", "user"]
|
||||
assert SUMMARY_MODE in llm.messages[0]["content"]
|
||||
assert llm.messages[1] == {"role": "user", "content": "c"}
|
||||
|
||||
|
||||
def test_generate_summary_strips_model_text_before_appending_pointer() -> None:
|
||||
llm = _FakeLLM(reply=" padded summary. \n")
|
||||
out = asyncio.run(generate_summary(llm, source="Deployments", path="f.txt", content="c"))
|
||||
assert out == "padded summary.\nSource: Deployments/f.txt"
|
||||
|
||||
|
||||
def test_pointer_is_code_deterministic_even_if_model_writes_its_own() -> None:
|
||||
"""The pointer must never be model-generated: even a model reply that
|
||||
contains a bogus 'Source:' line ends with the code-appended one."""
|
||||
llm = _FakeLLM(reply="The document itself says Source: fake/other.yaml inside.")
|
||||
out = asyncio.run(generate_summary(llm, source="Homelab", path="real.yaml", content="c"))
|
||||
assert out.splitlines()[-1] == "Source: Homelab/real.yaml"
|
||||
|
||||
|
||||
def test_generate_summary_sends_capped_content_to_the_model() -> None:
|
||||
"""The cap applies to what the model actually receives (overflow cut
|
||||
at the cap + marker) — read from the live settings for any value."""
|
||||
llm = _FakeLLM(reply="s")
|
||||
content = "w" * (get_settings().summary_max_chars + 50)
|
||||
asyncio.run(generate_summary(llm, source="Homelab", path="a.yaml", content=content))
|
||||
cap = get_settings().summary_max_chars
|
||||
assert llm.messages[1]["content"] == "w" * cap + "\n" + TRUNCATION_MARKER
|
||||
|
||||
|
||||
def test_generate_summary_rejects_whitespace_only_reply() -> None:
|
||||
llm = _FakeLLM(reply=" \n\t ")
|
||||
with pytest.raises(LLMError, match="empty content"):
|
||||
asyncio.run(generate_summary(llm, source="Homelab", path="a.yaml", content="c"))
|
||||
|
||||
|
||||
def test_generate_summary_rejects_empty_reply() -> None:
|
||||
llm = _FakeLLM(reply="")
|
||||
with pytest.raises(LLMError, match="empty content"):
|
||||
asyncio.run(generate_summary(llm, source="Homelab", path="a.yaml", content="c"))
|
||||
|
||||
|
||||
def test_generate_summary_propagates_llm_error_from_client() -> None:
|
||||
llm = _FakeLLM(
|
||||
fail=LLMError("chat completion from https://aipi.reeseapps.com/v1 failed: boom")
|
||||
)
|
||||
with pytest.raises(LLMError, match="boom"):
|
||||
asyncio.run(generate_summary(llm, source="Homelab", path="a.yaml", content="c"))
|
||||
Reference in New Issue
Block a user