feat(rag): lite-model document summaries — non-markdown docs summarized at import, summary chunk retrieves and resolves to the full source doc

tests/integration/test_chat_api.py

@@ -72,6 +72,15 @@ class FakeRagLLM:
         self.embed_batches += 1
         return [_token_vec(t) for t in texts]
 
+    async def chat(
+        self, messages: list[dict[str, str]], model: str | None = None
+    ) -> str:
+        """Deterministic ``lite`` stand-in for the import-time summaries
+        (phase 30) — same convention as ``tests.fakes.FakeEmbedder.chat``."""
+        user = next((m["content"] for m in messages if m.get("role") == "user"), "")
+        first = user.split()
+        return "Summary of " + (first[0] if first else "<empty>")
+
     async def embed_one(self, text: str) -> list[float]:
         if self.embed_error is not None:
             raise self.embed_error

tests/integration/test_importer_e2e.py

@@ -61,11 +61,31 @@ def test_import_fixtures_end_to_end(admin_client, db) -> None:
     k8s = next(d for d in docs if d.path == "homelab/kubernetes.md")
     assert "Talos Linux" in k8s.content and k8s.content_hash
 
+    # Phase 30: the four non-markdown fixtures each gained one embedded
+    # ``is_summary`` chunk, so the DB holds content + summary chunks.
     n_chunks = db.scalar(select(func.count()).select_from(Chunk))
-    assert n_chunks == summary.chunks
+    assert n_chunks == summary.chunks + summary.summaries
     for c in db.scalars(select(Chunk)).all():
         assert c.embedding is not None and len(c.embedding) == 768
 
+    assert summary.summary_errors == 0
+    for d in docs:
+        non_md = Path(d.path).suffix.lower() not in (".md", ".markdown")
+        schunks = [c for c in d.chunks if c.is_summary]
+        if non_md:
+            # Lite summary stored + exactly one embedded summary chunk (−1).
+            assert d.summary is not None, f"{d.path} should have a summary"
+            assert len(schunks) == 1
+            assert schunks[0].position == -1
+            assert schunks[0].content == d.summary
+            assert schunks[0].embedding is not None
+        else:
+            # Markdown docs never get a summary (phase 30 scope).
+            assert d.summary is None and not schunks
+    assert summary.summaries == sum(
+        1 for d in docs if Path(d.path).suffix.lower() not in (".md", ".markdown")
+    )
+
     # The Sources page consumes exactly this shape.
     r = admin_client.get("/api/docs")  # phase 16: the catalog is admin-only
     assert r.status_code == 200

tests/integration/test_migration_0004.py

@@ -0,0 +1,144 @@
+"""Integration: migration 0004 (document summaries) schema contract.
+
+Drives the **real Alembic engine** against the live dev database
+(``podman compose up -d db``), mirroring the style of
+``test_migration_0002.py`` (information_schema assertions on the state the
+migration must leave):
+
+* upgrade to head → ``documents.summary`` (TEXT, nullable) and
+  ``chunks.is_summary`` (BOOLEAN NOT NULL, default false) both exist, and a
+  chunk inserted without the column gets ``is_summary = false`` (pre-0004
+  insert paths stay valid);
+* downgrade to 0003 → both columns are gone;
+* upgrade to head again → both are back (round-trip).
+
+The ``alembic`` fixture guarantees the DB ends at head even if a test
+fails or the process is interrupted.
+"""
+from __future__ import annotations
+
+from collections.abc import Iterator
+from typing import Any
+
+import pytest
+from alembic.config import Config
+from sqlalchemy import text
+from sqlalchemy.orm import Session
+
+from alembic import command
+from app.db import db_available
+
+
+@pytest.fixture()
+def alembic(db: Session) -> Iterator[Config]:
+    """Real Alembic config bound to the dev DB (URL from app settings).
+
+    Starts at head (repairs an interrupted earlier run); teardown upgrades
+    to head no matter what happened, so the dev DB is never left below
+    head.
+    """
+    if not db_available():
+        pytest.skip("Postgres not reachable — run `podman compose up -d db` first")
+    cfg = Config()  # no alembic.ini file — env.py gets the URL from app config
+    cfg.set_main_option("script_location", "alembic")
+    command.upgrade(cfg, "head")
+    try:
+        yield cfg
+    finally:
+        command.upgrade(cfg, "head")
+
+
+def _column(db: Session, table: str, column: str) -> tuple[Any, ...] | None:
+    """(data_type, is_nullable, column_default) for one column, or None."""
+    row = db.execute(
+        text(
+            "SELECT data_type, is_nullable, column_default"
+            " FROM information_schema.columns"
+            " WHERE table_name = :t AND column_name = :c"
+        ),
+        {"t": table, "c": column},
+    ).fetchone()
+    return tuple(row) if row is not None else None
+
+
+def _version(db: Session) -> str | None:
+    return db.execute(text("SELECT version_num FROM alembic_version")).scalar()
+
+
+def test_upgrade_to_head_adds_summary_columns(db: Session, alembic: Config) -> None:
+    """Upgrade to head: both columns exist with the locked types/defaults."""
+    command.downgrade(alembic, "0003")  # start from the pre-0004 state
+    assert _version(db) == "0003"
+
+    command.upgrade(alembic, "head")
+    assert _version(db) == "0004", "alembic_version must be at 0004 (head)"
+
+    summary = _column(db, "documents", "summary")
+    assert summary is not None, "documents.summary is missing"
+    assert summary[0] == "text", "documents.summary must be TEXT"
+    assert summary[1] == "YES", "documents.summary must be NULLABLE"
+
+    is_summary = _column(db, "chunks", "is_summary")
+    assert is_summary is not None, "chunks.is_summary is missing"
+    assert is_summary[0] == "boolean", "chunks.is_summary must be BOOLEAN"
+    assert is_summary[1] == "NO", "chunks.is_summary must be NOT NULL"
+    assert is_summary[2] is not None and "false" in is_summary[2], (
+        "chunks.is_summary must have server default false"
+    )
+
+
+def test_is_summary_defaults_false_for_new_chunks(db: Session, alembic: Config) -> None:
+    """The default keeps old rows/insert paths valid: a chunk inserted
+    without the column (the pre-0004 insert shape) lands as ``false``."""
+    command.upgrade(alembic, "head")
+    doc_id = db.execute(text("SELECT gen_random_uuid()")).scalar()
+    try:
+        db.execute(
+            text(
+                "INSERT INTO documents (id, source, path, full_path, title, content,"
+                " content_hash, indexed_at) VALUES"
+                " (:id, 'mig_test', 't.md', '/t.md', 'T', 'content here',"
+                " repeat('0', 64), now())"
+            ),
+            {"id": doc_id},
+        )
+        db.execute(
+            text(
+                "INSERT INTO chunks (id, document_id, position, content)"
+                " VALUES (gen_random_uuid(), :id, 0, 'content here')"
+            ),
+            {"id": doc_id},
+        )
+        db.commit()
+        flag = db.execute(
+            text("SELECT is_summary FROM chunks WHERE document_id = :id"), {"id": doc_id}
+        ).scalar()
+        assert flag is False, "chunks.is_summary must default to false"
+    finally:
+        db.execute(text("DELETE FROM chunks WHERE document_id = :id"), {"id": doc_id})
+        db.execute(text("DELETE FROM documents WHERE id = :id"), {"id": doc_id})
+        db.commit()
+
+
+def test_downgrade_to_0003_removes_columns(db: Session, alembic: Config) -> None:
+    """Downgrade to 0003: both columns are dropped (A13 — reversible)."""
+    command.downgrade(alembic, "0003")
+    assert _version(db) == "0003"
+
+    assert _column(db, "documents", "summary") is None, "documents.summary must be dropped"
+    assert _column(db, "chunks", "is_summary") is None, "chunks.is_summary must be dropped"
+
+
+def test_upgrade_round_trip_restores_columns(db: Session, alembic: Config) -> None:
+    """Upgrade back to head after the downgrade: both columns are back."""
+    command.upgrade(alembic, "head")
+    assert _version(db) == "0004", "round-trip upgrade must land at 0004 (head)"
+
+    summary = _column(db, "documents", "summary")
+    assert summary is not None and summary[1] == "YES", "documents.summary must be back"
+
+    is_summary = _column(db, "chunks", "is_summary")
+    assert is_summary is not None and is_summary[1] == "NO", "chunks.is_summary must be back"
+    assert is_summary[2] is not None and "false" in is_summary[2], (
+        "chunks.is_summary must keep its server default false after the round-trip"
+    )