feat(rag): lite-model document summaries — non-markdown docs summarized at import, summary chunk retrieves and resolves to the full source doc
This commit is contained in:
+211
-4
@@ -1,12 +1,18 @@
|
||||
"""Unit tests: importer directory walk + sha256 delta logic.
|
||||
"""Unit tests: importer directory walk + sha256 delta logic + summaries.
|
||||
|
||||
The walk tests are pure filesystem (``tmp_path``); the delta tests run
|
||||
against the local compose Postgres (preferred — a real vector table),
|
||||
skipping with clear instructions when the stack is not up.
|
||||
The walk tests are pure filesystem (``tmp_path``); the delta and summary
|
||||
tests run against the local compose Postgres (preferred — a real vector
|
||||
table), skipping with clear instructions when the stack is not up.
|
||||
|
||||
Summaries (phase 30): non-markdown files get a ``lite``-model summary via
|
||||
the fake's deterministic ``chat`` (``"Summary of <first token>"``); the
|
||||
sentinel word ``SUMMARY-BLOWUP`` makes ``chat`` raise :class:`LLMError`
|
||||
for the fail-soft path.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
@@ -15,6 +21,8 @@ from sqlalchemy import func, select
|
||||
from app.models import Chunk, Document
|
||||
from app.rag.importer import (
|
||||
EXCLUDED_DIRS,
|
||||
ImportSummary,
|
||||
_store_summary,
|
||||
import_sources,
|
||||
iter_importable_files,
|
||||
)
|
||||
@@ -350,6 +358,205 @@ def test_multi_format_import_counts_per_format_and_titles_stem(db, tmp_path: Pat
|
||||
_cleanup_source(db, root.name)
|
||||
|
||||
|
||||
# ---------- phase 30: lite-model summaries for non-markdown files ----------
|
||||
|
||||
|
||||
def test_non_markdown_file_gets_stored_and_indexed_summary(db, tmp_path: Path) -> None:
|
||||
"""A ``.yaml`` file is summarized: ``documents.summary`` is set and one
|
||||
``is_summary`` chunk (position −1, embedded) is indexed alongside the
|
||||
content chunks."""
|
||||
root = tmp_path / "sumsrc"
|
||||
root.mkdir()
|
||||
(root / "svc.yaml").write_text("alpha services:\n gitlab:\n port: 8929\n")
|
||||
llm = FakeEmbedder()
|
||||
try:
|
||||
summary = asyncio.run(import_sources([root], llm, session=db))
|
||||
assert summary.added == 1
|
||||
assert summary.summaries == 1
|
||||
assert summary.summary_errors == 0
|
||||
doc = db.scalar(
|
||||
select(Document).where(Document.source == root.name, Document.path == "svc.yaml")
|
||||
)
|
||||
assert doc is not None
|
||||
assert doc.summary is not None
|
||||
# Deterministic fake reply + the code-appended pointer line.
|
||||
assert doc.summary.startswith("Summary of alpha")
|
||||
assert doc.summary.endswith(f"Source: {root.name}/svc.yaml")
|
||||
schunks = [c for c in doc.chunks if c.is_summary]
|
||||
assert len(schunks) == 1
|
||||
assert schunks[0].position == -1
|
||||
assert schunks[0].content == doc.summary
|
||||
assert schunks[0].embedding is not None and len(schunks[0].embedding) == 768
|
||||
# Content chunks stay 0-based and are never flagged as summaries.
|
||||
content = [c for c in doc.chunks if not c.is_summary]
|
||||
assert sorted(c.position for c in content) == list(range(len(content)))
|
||||
finally:
|
||||
_cleanup_source(db, root.name)
|
||||
|
||||
|
||||
def test_markdown_file_never_gets_summary(db, tmp_path: Path) -> None:
|
||||
"""Markdown is already natural language: no summary, no ``is_summary``
|
||||
chunk, and the ``lite`` model is never called."""
|
||||
root = tmp_path / "mdsrc"
|
||||
root.mkdir()
|
||||
(root / "note.md").write_text("# Note\n\nmarkdown body\n")
|
||||
llm = FakeEmbedder()
|
||||
try:
|
||||
summary = asyncio.run(import_sources([root], llm, session=db))
|
||||
assert summary.added == 1
|
||||
assert summary.summaries == 0 and summary.summary_errors == 0
|
||||
assert llm.chat_calls == [] # the model was never asked
|
||||
doc = db.scalar(
|
||||
select(Document).where(Document.source == root.name, Document.path == "note.md")
|
||||
)
|
||||
assert doc is not None
|
||||
assert doc.summary is None
|
||||
assert doc.chunks and all(not c.is_summary for c in doc.chunks)
|
||||
finally:
|
||||
_cleanup_source(db, root.name)
|
||||
|
||||
|
||||
def test_summary_failure_is_fail_soft(db, tmp_path: Path) -> None:
|
||||
"""A ``lite``-model failure must never lose the document: the file is
|
||||
fully indexed (content chunks + embeddings), ``documents.summary`` stays
|
||||
NULL, and the failure is counted in ``summary_errors``."""
|
||||
root = tmp_path / "blowup"
|
||||
root.mkdir()
|
||||
(root / "bad.txt").write_text("SUMMARY-BLOWUP the lite model chokes on this\n")
|
||||
llm = FakeEmbedder()
|
||||
try:
|
||||
summary = asyncio.run(import_sources([root], llm, session=db))
|
||||
assert summary.errors == 0 # the document itself imported fine
|
||||
assert summary.added == 1
|
||||
assert summary.summaries == 0
|
||||
assert summary.summary_errors == 1
|
||||
doc = db.scalar(
|
||||
select(Document).where(Document.source == root.name, Document.path == "bad.txt")
|
||||
)
|
||||
assert doc is not None
|
||||
assert doc.summary is None
|
||||
assert len(doc.chunks) == 1
|
||||
assert doc.chunks[0].embedding is not None # content chunk embedded
|
||||
assert all(not c.is_summary for c in doc.chunks)
|
||||
finally:
|
||||
_cleanup_source(db, root.name)
|
||||
|
||||
|
||||
def test_summary_chunk_is_replaced_on_reimport(db, tmp_path: Path) -> None:
|
||||
"""Re-importing a changed non-markdown file keeps exactly one
|
||||
``is_summary`` chunk — the old one is gone, the new summary is stored
|
||||
and embedded, and the content chunks stay 0-based."""
|
||||
root = tmp_path / "repl"
|
||||
root.mkdir()
|
||||
path = root / "cfg.yaml"
|
||||
path.write_text("alpha settings:\n host: one\n")
|
||||
llm = FakeEmbedder()
|
||||
try:
|
||||
asyncio.run(import_sources([root], llm, session=db))
|
||||
path.write_text("bravo settings:\n host: two\n")
|
||||
summary = asyncio.run(import_sources([root], llm, session=db))
|
||||
assert summary.updated == 1
|
||||
assert summary.summaries == 1 and summary.summary_errors == 0
|
||||
doc = db.scalar(
|
||||
select(Document).where(Document.source == root.name, Document.path == "cfg.yaml")
|
||||
)
|
||||
assert doc is not None
|
||||
assert doc.summary is not None and doc.summary.startswith("Summary of bravo")
|
||||
schunks = [c for c in doc.chunks if c.is_summary]
|
||||
assert len(schunks) == 1 # the old one was deleted
|
||||
assert schunks[0].position == -1
|
||||
assert schunks[0].content == doc.summary
|
||||
assert schunks[0].embedding is not None
|
||||
assert "alpha" not in schunks[0].content # no stale summary text
|
||||
assert sorted(c.position for c in doc.chunks if not c.is_summary) == [0]
|
||||
finally:
|
||||
_cleanup_source(db, root.name)
|
||||
|
||||
|
||||
def test_store_summary_replaces_an_existing_summary_chunk(db, tmp_path: Path) -> None:
|
||||
"""Replacement unit, driven directly: with a pre-existing
|
||||
``is_summary`` chunk in place, ``_store_summary`` deletes the old one
|
||||
and leaves exactly one (new) summary chunk + updated
|
||||
``documents.summary`` — the at-most-one-summary invariant."""
|
||||
root = tmp_path / "direct"
|
||||
root.mkdir()
|
||||
(root / "a.yaml").write_text("alpha x\n")
|
||||
llm = FakeEmbedder()
|
||||
try:
|
||||
asyncio.run(import_sources([root], llm, session=db))
|
||||
doc = db.scalar(
|
||||
select(Document).where(Document.source == root.name, Document.path == "a.yaml")
|
||||
)
|
||||
assert doc is not None and doc.summary is not None
|
||||
assert any(c.is_summary for c in doc.chunks) # the first import's summary
|
||||
counters = ImportSummary()
|
||||
asyncio.run(
|
||||
_store_summary(
|
||||
session=db, doc=doc, source=root.name, rel="a.yaml",
|
||||
content=doc.content, llm=llm, summary=counters,
|
||||
)
|
||||
)
|
||||
assert counters.summaries == 1 and counters.summary_errors == 0
|
||||
schunks = [c for c in doc.chunks if c.is_summary]
|
||||
assert len(schunks) == 1 # the old one was deleted
|
||||
assert schunks[0].position == -1
|
||||
assert schunks[0].content == doc.summary
|
||||
assert schunks[0].embedding is not None
|
||||
finally:
|
||||
_cleanup_source(db, root.name)
|
||||
|
||||
|
||||
def test_store_summary_fail_soft_leaves_document_untouched(db, tmp_path: Path) -> None:
|
||||
"""A ``lite`` failure inside ``_store_summary`` rolls back only the
|
||||
summary rows: the previous summary (if any) and the document survive,
|
||||
and the failure is counted."""
|
||||
root = tmp_path / "directfail"
|
||||
root.mkdir()
|
||||
(root / "a.yaml").write_text("alpha x\n")
|
||||
llm = FakeEmbedder()
|
||||
try:
|
||||
asyncio.run(import_sources([root], llm, session=db))
|
||||
doc = db.scalar(
|
||||
select(Document).where(Document.source == root.name, Document.path == "a.yaml")
|
||||
)
|
||||
assert doc is not None and doc.summary is not None
|
||||
previous_summary = doc.summary
|
||||
(root / "a.yaml").write_text("SUMMARY-BLOWUP now the lite model fails\n")
|
||||
counters = ImportSummary()
|
||||
asyncio.run(
|
||||
_store_summary(
|
||||
session=db, doc=doc, source=root.name, rel="a.yaml",
|
||||
content="SUMMARY-BLOWUP now the lite model fails\n",
|
||||
llm=llm, summary=counters,
|
||||
)
|
||||
)
|
||||
assert counters.summaries == 0 and counters.summary_errors == 1
|
||||
assert doc.summary == previous_summary # rolled back, not nulled
|
||||
schunks = [c for c in doc.chunks if c.is_summary]
|
||||
assert len(schunks) == 1 # the old one survived the rollback
|
||||
assert schunks[0].content == previous_summary
|
||||
finally:
|
||||
_cleanup_source(db, root.name)
|
||||
|
||||
|
||||
def test_import_summary_log_line_includes_summary_counters(
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
"""PLAN §9 summary line: the phase-30 counters sit between
|
||||
``embed_batches`` and ``formats``."""
|
||||
s = ImportSummary()
|
||||
s.files, s.added, s.chunks, s.embed_batches = 3, 3, 5, 4
|
||||
s.summaries, s.summary_errors = 2, 1
|
||||
s.formats = {"md": 1, "yaml": 2}
|
||||
with caplog.at_level(logging.INFO, logger="app.importer"):
|
||||
s.log()
|
||||
line = caplog.records[-1].getMessage()
|
||||
assert line == (
|
||||
"import: summary files=3 added=3 updated=0 unchanged=0 pruned=0 errors=0 "
|
||||
"chunks=5 embed_batches=4 summaries=2 summary_errors=1 formats=yaml:2,md:1"
|
||||
)
|
||||
|
||||
|
||||
def test_prune_removes_files_now_excluded_by_format_filter(db, tmp_path: Path) -> None:
|
||||
"""Previously-imported junk leaves the index: a file that no longer
|
||||
matches the A9 extension filter is pruned on the next ``prune=True`` run.
|
||||
|
||||
Reference in New Issue
Block a user