phase: 163_pdf_strip_additive_selfheal

All gates verified. Final report:

**Phase 163 — final verification pass (all 5 tasks already complete; verification-only run)**

- Verified D1–D4 in code: `page_strip_mode` column + migration 0027, `DocContent` absent-never-null rule keyed off `page_count`; `_persist_pdf_pages` render → atomic tmp+`os.replace` writes → stale-tail-last + `strip_class` stamp; `_regate_pdf_strip` on both unchanged paths (mirror gate, heal, repair, fail-soft, one INFO line only when acting); viewer mode branch + Text/Pages toggle (Text default, `aria-pressed`, no innerHTML, lazy imgs).
- `uv run pytest --cov=app --cov-report=term-missing` → **3796 tests, exit 0, coverage 99%** (>90%)
- `uv run pytest tests/e2e/test_pdf_strip_additive_selfheal.py -v --no-cov` → **5 passed** in isolation
- `test_pdf_pages_in_sources + test_pictorial_pdf + test_garbage_pdf` → 10 passed; `test_document_viewer(_context) + test_image_documents` → 20 passed
- `uv run ruff check .` clean; `uv run pyright` → 0 errors; alembic **0027 downgrade→upgrade CLI round-trip clean** (+ integration pins)
- Completion criteria: additive view w/ toggle ✅ · primary DOM byte-identity ✅ · write-first order/mid-write-failure/atomic pins ✅ · heal + repair E2E ✅ · zero vision/LLM on fast path (mock-captured) ✅ · non-qualifying payload byte-identity ✅ · docs pass ✅
- **Documentation**: README.md + AGENTS.md reviewed in full — the phase's updates are accurate; all config-table defaults cross-checked against `app/config.py` (no drift), every page route, script, and E2E command referenced exists, every `.env.example` var documented — **verified consistent, no changes needed**.
- Notable: live `Letumdei` repair is owner-runtime post-deploy per D5 (next production Sync self-heals; no live changes made); the modified `.agents/screenshots/doc_modal_scrollbar_1280.png` is an E2E-regenerated artifact left for the pipeline commit.
- **Next pending phase:** `164_chat_error_honesty`

tests/integration/test_docs_api.py

@@ -10,7 +10,10 @@ per phase 124 A4) and the phase-148 (task 02) PDF page route
 (``GET /api/documents/{id}/pages/{n}`` — the same trust model and 404
 posture for a page-qualifying document's rendered page PNGs) + the
 content endpoint's ``image_url`` / ``page_urls`` keys (present for the
-qualifying row, ABSENT otherwise — never ``null``).
+qualifying row, ABSENT otherwise — never ``null``), plus the phase-163
+(task 01) ``page_strip_mode`` class signal (``"primary"`` /``"additive"``
+for a stamped strip, ABSENT for a pre-phase row and for every
+non-qualifying document).
 
 Uses the real compose Postgres (``db`` fixture) and FastAPI's TestClient.
 """
@@ -1381,7 +1384,8 @@ def test_content_endpoint_exposes_the_image_affordance(
 
 # ---------------------------------------------------------------------------
 # Phase 148 (task 02) — the PDF page BYTES route + the content payload's
-# ``page_urls`` affordance
+# ``page_urls`` affordance (phase 163 task 01 adds the ``page_strip_mode``
+# class signal beside it)
 # ---------------------------------------------------------------------------
 
 
@@ -1404,6 +1408,7 @@ def _seed_page_doc(
     path: str = "besta.pdf",
     page_count: int | None = 2,
     write_copies: bool = True,
+    page_strip_mode: str | None = None,
 ) -> Document:
     """One PDF document row with ``page_count`` + its rendered page
     copies (the importer's ``pdf_page_dir`` layout —
@@ -1411,7 +1416,8 @@ def _seed_page_doc(
     sentinel bytes, so a page mix-up between the two copies is caught.
     ``page_count=None`` is a non-qualifying PDF (no copies);
     ``write_copies=False`` leaves a qualifying row without its files
-    (the lost-copy corner)."""
+    (the lost-copy corner); ``page_strip_mode`` (phase 163) stamps the
+    strip CLASS — ``None`` is the pre-phase row."""
     doc = Document(
         id=uuid.uuid4(),
         source=source,
@@ -1424,6 +1430,7 @@ def _seed_page_doc(
         created_at=datetime(2026, 1, 1, tzinfo=UTC),
         summary="IKEA BESTA frame assembly manual (page descriptions).",
         page_count=page_count,
+        page_strip_mode=page_strip_mode,
     )
     db.add(doc)
     db.flush()
@@ -1610,6 +1617,79 @@ def test_content_endpoint_exposes_the_page_urls_affordance(
         )
         assert r.status_code == 200
         assert "page_urls" not in r.json()  # absent — never null (non-qualifying PDF)
+        # Phase 163 (task 01): the class key follows the same rule — a
+        # pre-phase row (mode NULL) omits it while keeping page_urls.
+        assert "page_strip_mode" not in r.json()
+    finally:
+        _cleanup_kb(db)
+
+
+@pytest.mark.parametrize("mode", ["primary", "additive"])
+def test_content_endpoint_carries_the_strip_class(
+    admin_client: TestClient, db, tmp_path: Path, mode: str
+) -> None:
+    """Phase 163 (task 01, LOCKED A2/A3) — the real-DB round-trip of the
+    strip CLASS: a stamped row (``page_strip_mode`` = ``"primary"`` — the
+    phase-133 fallback class, the strip IS the view — or ``"additive"`` —
+    the phase-140 pictorial class, text + a Text/Pages toggle) serves the
+    class VERBATIM beside the untouched ``page_urls`` list (the viewer's
+    branch signal, task 04); every other key stands where phase 148 left
+    it."""
+    _cleanup_kb(db)
+    try:
+        pages_dir = tmp_path / "pages"
+        _seed_page_doc(db, pages_dir, page_count=3, page_strip_mode=mode)
+        r = admin_client.get(
+            "/api/documents/content", params={"source": "PdfSrc", "path": "besta.pdf"}
+        )
+        assert r.status_code == 200
+        body = r.json()
+        assert body["page_strip_mode"] == mode
+        assert len(body["page_urls"]) == 3  # the strip rides along, unchanged
+    finally:
+        _cleanup_kb(db)
+
+
+def test_content_strip_class_never_travels_without_the_strip(
+    admin_client: TestClient, db, tmp_path: Path
+) -> None:
+    """Phase 163 (task 01) — the two strip keys are ONE block keyed off
+    ``page_count``: a row with a stray ``page_strip_mode`` and no strip
+    (``page_count`` NULL — the corner a partial repair could leave)
+    serializes NEITHER key, byte-identically to a plain non-qualifying
+    document (the viewer must never be handed a class for pages that do
+    not exist)."""
+    _cleanup_kb(db)
+    try:
+        pages_dir = tmp_path / "pages"
+        _seed_page_doc(
+            db,
+            pages_dir,
+            path="stray.pdf",
+            page_count=None,
+            write_copies=False,
+            page_strip_mode="additive",
+        )
+        r = admin_client.get(
+            "/api/documents/content", params={"source": "PdfSrc", "path": "stray.pdf"}
+        )
+        assert r.status_code == 200
+        body = r.json()
+        assert "page_strip_mode" not in body
+        assert "page_urls" not in body
+        # The exact pre-phase key set (the byte-identity pin).
+        assert set(body) == {
+            "source",
+            "path",
+            "title",
+            "format",
+            "summary",
+            "created_at",
+            "content",
+            "indexed_at",
+            "chunks",
+            "is_image",
+        }
     finally:
         _cleanup_kb(db)
 

tests/integration/test_migration_0027.py

@@ -0,0 +1,370 @@
+"""Integration: migration 0027 (documents.page_strip_mode) schema contract
+(phase 163, task 01).
+
+Drives the **real Alembic engine** against the live dev database
+(``podman compose up -d db``), mirroring the house pattern of
+``test_migration_0026.py`` (information_schema assertions on the state
+the migration must leave). The tests target the 0026 → 0027 step
+explicitly so later migrations cannot break the pins:
+
+* upgrade 0026 → 0027 → ``page_strip_mode`` exists with the full
+  contract — VARCHAR(16), NULLABLE, NO server default (NULL = "no strip"
+  for every pre-phase row, every non-qualifying document and every
+  fail-soft miss) and UNINDEXED (a display fact, never a query surface —
+  the content endpoint reads it from the ROW, the ``file_hash`` /
+  ``page_count`` convention) — while the 0026 ``documents`` schema
+  survives, in particular ``page_count`` keeping its 0026 contract and
+  ``content_hash`` staying NOT NULL and INDEXED;
+* rows inserted while the DB is at 0026 backfill ``page_strip_mode`` to
+  NULL — including a row whose ``page_count`` IS set (the pre-phase strip
+  with no stamped class: the wire omits the key and the viewer keeps
+  today's "primary" DOM until the task-03 re-gate stamps it);
+* the ORM contract agrees: a freshly inserted ``Document`` with no mode
+  reads ``page_strip_mode is None`` through a fresh session, and each of
+  the two classes (``"primary"`` / ``"additive"``) round-trips verbatim;
+* downgrade to 0026 → the column is GONE (A13) while the row + its
+  content + its ``page_count`` survive; upgrade back to 0027 → the column
+  is back (round-trip).
+
+The ``alembic`` fixture guarantees the DB ends at head even if a test
+fails or the process is interrupted.
+"""
+from __future__ import annotations
+
+import hashlib
+import uuid
+from collections.abc import Iterator
+from typing import Any
+
+import pytest
+from alembic.config import Config
+from sqlalchemy import text
+from sqlalchemy.orm import Session
+
+from alembic import command
+from app.db import SessionLocal, db_available
+from app.models import Document
+
+SOURCE = "mig0027"
+CONTENT = "content here"
+CONTENT_HASH = hashlib.sha256(CONTENT.encode()).hexdigest()
+
+
+@pytest.fixture()
+def alembic(db: Session) -> Iterator[Config]:
+    """Real Alembic config bound to the dev DB (URL from app settings).
+
+    Starts at head (repairs an interrupted earlier run); teardown
+    upgrades to head no matter what happened, so the dev DB is never
+    left below head.
+    """
+    if not db_available():
+        pytest.skip("Postgres not reachable — run `podman compose up -d db` first")
+    cfg = Config()  # no alembic.ini file — env.py gets the URL from app config
+    cfg.set_main_option("script_location", "alembic")
+    command.upgrade(cfg, "head")
+    try:
+        yield cfg
+    finally:
+        # Release the test session's open transaction BEFORE the repair
+        # DDL: an idle-in-transaction SELECT holds an ACCESS SHARE lock
+        # on ``documents``, which would deadlock the repair's
+        # ``ALTER TABLE`` forever.
+        db.rollback()
+        command.upgrade(cfg, "head")
+
+
+def _version(db: Session) -> str | None:
+    return db.execute(text("SELECT version_num FROM alembic_version")).scalar()
+
+
+def _column(db: Session, column: str) -> tuple[Any, ...] | None:
+    """(data_type, is_nullable, column_default, character_maximum_length)
+    for one documents column."""
+    row = db.execute(
+        text(
+            "SELECT data_type, is_nullable, column_default, character_maximum_length"
+            " FROM information_schema.columns"
+            " WHERE table_name = 'documents' AND column_name = :c"
+        ),
+        {"c": column},
+    ).fetchone()
+    return tuple(row) if row is not None else None
+
+
+def _index_covering(db: Session, column: str) -> str | None:
+    """The name of any documents index whose definition mentions the
+    column (None = UNINDEXED)."""
+    return db.execute(
+        text(
+            "SELECT indexname FROM pg_indexes"
+            " WHERE tablename = 'documents' AND indexdef ILIKE :like"
+        ),
+        {"like": f"%{column}%"},
+    ).scalar()
+
+
+def _insert_sql(db: Session, path: str, page_count: int | None = None) -> uuid.UUID:
+    """Insert one documents row with the PRE-0027 column set (the 0026
+    shape — the mode column, when present, is omitted so a NULL backfill
+    is what the row reads; ``page_count`` is 0026's own column and may be
+    set to model a pre-phase strip)."""
+    doc_id = uuid.uuid4()
+    db.execute(
+        text(
+            "INSERT INTO documents (id, source, path, full_path, title, content,"
+            " content_hash, page_count)"
+            " VALUES (:id, :s, :p, :fp, :t, :c, :h, :pc)"
+        ),
+        {
+            "id": doc_id,
+            "s": SOURCE,
+            "p": path,
+            "fp": f"/tmp/{path}",
+            "t": f"Doc {path}",
+            "c": CONTENT,
+            "h": CONTENT_HASH,
+            "pc": page_count,
+        },
+    )
+    db.commit()
+    return doc_id
+
+
+def _delete(db: Session, doc_id: uuid.UUID) -> None:
+    db.execute(text("DELETE FROM documents WHERE id = :id"), {"id": doc_id})
+    db.commit()
+
+
+def _assert_0026_schema_survives(db: Session) -> None:
+    """The 0026 documents contract must survive the additive upgrade —
+    ``page_count`` keeps its INTEGER / NULLABLE / NO-default / UNINDEXED
+    shape (the sibling column this one rides) and ``content_hash`` stays
+    NOT NULL and indexed (the retriever's identity)."""
+    page_count = _column(db, "page_count")
+    assert page_count is not None, "documents.page_count (0026) must survive the upgrade"
+    assert page_count[0] == "integer", "page_count must stay INTEGER"
+    assert page_count[1] == "YES", "page_count must stay NULLABLE"
+    assert page_count[2] is None, "page_count must keep NO server default"
+    assert _index_covering(db, "page_count") is None, "page_count must stay UNINDEXED"
+
+    content = _column(db, "content_hash")
+    assert content is not None and content[0] == "character varying" and content[1] == "NO", (
+        "documents.content_hash (0001) must keep its contract"
+    )
+    assert (
+        _index_covering(db, "content_hash") is not None
+    ), "the ix_documents_content_hash index must survive the upgrade"
+
+    indexed = _column(db, "indexed_at")
+    assert indexed is not None and indexed[0] == "timestamp with time zone", (
+        "documents.indexed_at (0001) must survive the upgrade"
+    )
+    is_image = _column(db, "is_image")
+    assert is_image is not None and is_image[0] == "boolean" and is_image[1] == "NO", (
+        "documents.is_image (0022) must survive the upgrade"
+    )
+
+
+def test_upgrade_to_0027_adds_page_strip_mode(db: Session, alembic: Config) -> None:
+    """Upgrade 0026 → 0027: ``page_strip_mode`` exists with the full
+    contract (VARCHAR(16), NULLABLE, NO server default — NULL = "no
+    strip" — and UNINDEXED: a display fact, never a query surface), is
+    ABSENT at 0026, and every pre-0027 row backfills it to NULL —
+    including the row whose ``page_count`` IS set (the pre-phase strip
+    with no stamped class), while the 0026 table contract (the
+    ``page_count`` sibling, the indexed ``content_hash``) survives.
+    """
+    command.downgrade(alembic, "0026")  # start from the pre-0027 state
+    assert _version(db) == "0026"
+    assert _column(db, "page_strip_mode") is None, "page_strip_mode must be absent at 0026"
+
+    plain_id = _insert_sql(db, "plain.pdf")  # the 0026 column set, no strip
+    pre_strip_id = _insert_sql(db, "prephase.pdf", page_count=36)  # a live pre-phase strip
+
+    try:
+        command.upgrade(alembic, "0027")
+        assert _version(db) == "0027", "alembic_version must be at 0027"
+
+        col = _column(db, "page_strip_mode")
+        assert col is not None, "documents.page_strip_mode is missing"
+        assert col[0] == "character varying", "page_strip_mode must be VARCHAR"
+        assert col[3] == 16, "page_strip_mode must be VARCHAR(16) — the two class names"
+        assert col[1] == "YES", (
+            "page_strip_mode must be NULLABLE (NULL = no strip, and the pre-phase shape)"
+        )
+        assert col[2] is None, (
+            "page_strip_mode must carry NO server default — NULL is the meaningful"
+            " pre-phase state"
+        )
+        assert _index_covering(db, "page_strip_mode") is None, (
+            "page_strip_mode must be UNINDEXED — a display fact, never a query"
+            " surface (the content endpoint reads it from the row)"
+        )
+
+        # Every pre-0027 row backfills to NULL — no backfill, no default.
+        assert (
+            db.execute(
+                text("SELECT page_count, page_strip_mode FROM documents WHERE id = :id"),
+                {"id": plain_id},
+            ).fetchone()
+            == (None, None)
+        ), "a pre-0027 row without a strip must read both columns NULL"
+        assert (
+            db.execute(
+                text("SELECT page_count, page_strip_mode FROM documents WHERE id = :id"),
+                {"id": pre_strip_id},
+            ).fetchone()
+            == (36, None)
+        ), (
+            "a pre-phase strip (page_count set) must backfill page_strip_mode to"
+            " NULL — the wire omits the key and the viewer keeps today's DOM"
+        )
+
+        _assert_0026_schema_survives(db)
+    finally:
+        _delete(db, plain_id)
+        _delete(db, pre_strip_id)
+
+
+@pytest.mark.parametrize("mode", ["primary", "additive"])
+def test_orm_page_strip_mode_round_trips(db: Session, alembic: Config, mode: str) -> None:
+    """The ORM contract agrees with the column contract: each class
+    (``"primary"`` = the fallback class, ``"additive"`` = the pictorial
+    class) round-trips VERBATIM through a fresh session, and a row
+    created with no mode reads ``page_strip_mode is None`` (the
+    pre-phase shape — NULL until the importer stamps it)."""
+    command.upgrade(alembic, "head")
+    stamped = Document(
+        source=SOURCE,
+        path="stamped.pdf",
+        full_path="/tmp/stamped.pdf",
+        title="stamped",
+        content=CONTENT,
+        content_hash=CONTENT_HASH,
+        page_count=36,
+        page_strip_mode=mode,
+    )
+    unstamped = Document(
+        source=SOURCE,
+        path="unstamped.md",
+        full_path="/tmp/unstamped.md",
+        title="unstamped",
+        content=CONTENT,
+        content_hash=CONTENT_HASH,
+    )
+    db.add(stamped)
+    db.add(unstamped)
+    db.commit()
+    try:
+        with SessionLocal() as fresh:
+            reloaded = fresh.get(Document, stamped.id)
+            assert reloaded is not None, "the stamped row must be readable"
+            assert reloaded.page_strip_mode == mode, (
+                f"the {mode!r} class must round-trip through the DB verbatim"
+            )
+            assert reloaded.page_count == 36, "the sibling page_count must round-trip too"
+            reloaded_unstamped = fresh.get(Document, unstamped.id)
+            assert reloaded_unstamped is not None, "the unstamped row must be readable"
+            assert reloaded_unstamped.page_strip_mode is None, (
+                "a row created without a mode must read NULL (the pre-phase shape)"
+            )
+            assert reloaded_unstamped.content_hash == CONTENT_HASH, (
+                "content_hash must be untouched by the page_strip_mode addition"
+            )
+    finally:
+        _delete(db, stamped.id)
+        _delete(db, unstamped.id)
+
+
+def test_downgrade_to_0026_drops_the_column(db: Session, alembic: Config) -> None:
+    """Downgrade 0027 → 0026: the page_strip_mode column is gone (A13 —
+    fully reversible, the class is re-derivable from the source file by
+    the deterministic re-gate) while the row + its content + its 0026
+    ``page_count`` survive, and the rest of the table contract is intact."""
+    command.upgrade(alembic, "head")
+    row = Document(
+        source=SOURCE,
+        path="down.pdf",
+        full_path="/tmp/down.pdf",
+        title="down",
+        content=CONTENT,
+        content_hash=CONTENT_HASH,
+        page_count=36,
+        page_strip_mode="additive",
+    )
+    db.add(row)
+    db.commit()
+    try:
+        command.downgrade(alembic, "0026")
+        assert _version(db) == "0026"
+        assert _column(db, "page_strip_mode") is None, "page_strip_mode must be dropped"
+
+        surviving = db.execute(
+            text(
+                "SELECT source, path, content, content_hash, page_count, is_image"
+                " FROM documents WHERE id = :id"
+            ),
+            {"id": row.id},
+        ).fetchone()
+        assert surviving is not None and surviving[0] == SOURCE, (
+            "the row must survive the column drop"
+        )
+        assert surviving[1] == "down.pdf" and surviving[2] == CONTENT, (
+            "path + content must survive the drop"
+        )
+        assert surviving[3] == CONTENT_HASH, (
+            "content_hash (the retriever's identity) must survive the drop"
+        )
+        assert surviving[4] == 36, (
+            "page_count (the 0026 sibling) must survive the drop — the strip"
+            " itself is untouched by this migration's reversal"
+        )
+        assert surviving[5] is False, "is_image must survive the drop"
+        assert _index_covering(db, "content_hash") is not None, (
+            "the content_hash index must survive the downgrade"
+        )
+    finally:
+        _delete(db, row.id)
+    # Repair: the fixture teardown re-upgrades to head.
+
+
+def test_upgrade_round_trip_restores_the_column(db: Session, alembic: Config) -> None:
+    """Downgrade to 0026, then upgrade back to 0027: ``page_strip_mode``
+    is back with the full contract (VARCHAR(16), NULLABLE, NO server
+    default, UNINDEXED), a row seeded at 0026 — strip included — reads
+    the mode as NULL (the pre-phase shape on the round-trip), and the
+    0026 schema (the ``page_count`` sibling, the indexed
+    ``content_hash``) is intact."""
+    command.downgrade(alembic, "0026")
+    pre_id = _insert_sql(db, "round.pdf", page_count=36)
+    try:
+        command.upgrade(alembic, "0027")
+        assert _version(db) == "0027", "round-trip upgrade must land at 0027"
+
+        col = _column(db, "page_strip_mode")
+        assert col is not None, "documents.page_strip_mode must be back"
+        assert col[0] == "character varying", "page_strip_mode must be VARCHAR after the round-trip"
+        assert col[3] == 16, "page_strip_mode must be VARCHAR(16) after the round-trip"
+        assert col[1] == "YES", "page_strip_mode must be NULLABLE after the round-trip"
+        assert col[2] is None, (
+            "page_strip_mode must still carry NO server default after the round-trip"
+        )
+        assert _index_covering(db, "page_strip_mode") is None, (
+            "page_strip_mode must stay UNINDEXED after the round-trip"
+        )
+
+        assert (
+            db.execute(
+                text("SELECT page_count, page_strip_mode FROM documents WHERE id = :id"),
+                {"id": pre_id},
+            ).fetchone()
+            == (36, None)
+        ), (
+            "a row seeded at 0026 must read page_strip_mode NULL after the"
+            " round-trip while its page_count stands"
+        )
+
+        _assert_0026_schema_survives(db)
+    finally:
+        _delete(db, pre_id)