feat(rag): lite-model document summaries — non-markdown docs summarized at import, summary chunk retrieves and resolves to the full source doc

This commit is contained in:
2026-08-25 17:48:37 -04:00
parent 9809482a4b
commit 572a4190a6
32 changed files with 1806 additions and 26 deletions
+45
View File
@@ -0,0 +1,45 @@
"""document summaries: documents.summary + chunks.is_summary
Revision ID: 0004
Revises: 0003
Create Date: 2026-08-25
Phase 30 (document-summaries story, A13 — two additive, reversible
columns, no table rework):
* ``documents.summary`` — TEXT, nullable. Natural-language summary of the
document produced at import time by the aipi ``lite`` model (non-markdown
A9 documents only). NULL for markdown docs and for documents imported
before summaries existed (or whose summary generation failed — the
fail-soft path still indexes the document without a summary).
* ``chunks.is_summary`` — BOOLEAN NOT NULL DEFAULT false. Marks the single
extra summary chunk (position -1) that mirrors ``documents.summary`` into
the embedding space, so hybrid search has a well-embedding
natural-language target for badly-formatted raw text. The default keeps
every pre-0004 row valid; the summary chunk flows through the unchanged
A7 retrieval path (cosine ∪ FTS ∪ RRF) and the existing chunk→document
mapping resolves a summary hit to its full source document.
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision = "0004"
down_revision = "0003"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column("documents", sa.Column("summary", sa.Text(), nullable=True))
op.add_column(
"chunks",
sa.Column("is_summary", sa.Boolean(), nullable=False, server_default=sa.text("false")),
)
def downgrade() -> None:
op.drop_column("chunks", "is_summary")
op.drop_column("documents", "summary")