phase: 148_pdf_pages_in_sources
**Phase 148 — final verification pass: complete, 100% green.**
- **Verified (tasks 01–04 already implemented):** page render/persist at index (`_persist_page_copies`, fail-soft, stale-replace, qualifier-loss, prune hook via `_delete_page_copies`); public `GET /api/documents/{id}/pages/{n}` (uuid4-credential, 404-all-non-servable, `image/png` + cache header); `DocContent.page_urls` additive-omitted; shared `renderDocument` strip (lazy imgs, "Page N" captions, text + Summary suppressed, onerror note, createElement-only); migration 0026; `pdf_page_dir` setting; `.env.example` entry
- **Completion criteria:** (1) pictorial + fallback classes get `p{N}.png` + `page_count`, text-heavy gets neither — pinned, pass; (2) unchanged re-sync renders zero (fast path) — pass; (3) changed re-index replaces copies, qualifier-loss/prune delete — pass; (4) route 404 matrix + payload key absent-never-null — pass; (5) modal AND /document.html strip with suppression, text-heavy byte-identical — E2E pass; (6) see below; (7) no completed-phase drift — full suite green
- **Tests:** `uv run pytest --cov=app --cov-report=term-missing` → **3461 passed, 0 failed**, `app/` coverage **99%** (>90%); `uv run pytest tests/e2e/test_pdf_pages_in_sources.py -v --no-cov` → **4 passed** in isolation; `uv run ruff check .` clean; `uv run pyright` → 0 errors, 0 warnings
- **Documentation:** README had **1 pre-existing inconsistency** — the Production Deployment section's standalone `podman run` example (and compose note) omitted the **required** `BOR_ADMIN_PASSWORD`/`BOR_SESSION_SECRET` (the fail-loud boot gate the README's own Admin section documents — the container would refuse to start); fixed with the two `-e` flags + a note for the compose path. All else verified consistent: 70/70 `.env.example` vars documented, all defaults match `config.py`, pages table matches routes, Containerfile/esbuild/uid-10001/healthcheck/entrypoint claims accurate, QA commands match pytest config. **AGENTS.md: verified consistent — no changes needed**
- **Notable:** repo uses `Containerfile` (Podman) — README's `podman build` example is valid as-is; no live infrastructure touched
- **Next pending phase:** `149_sync_progress_box`
This commit is contained in:
@@ -3,6 +3,15 @@ and GET /api/docs/tree (phase 97, task 02 — the full recursive KB tree
|
||||
the RAG view renders: shape, ordering, counts, summaries, the 403 gate,
|
||||
the superset rule, and the stat-walk equivalence with ``GET /api/docs``).
|
||||
|
||||
Also the BYTES routes and their content-payload affordances: the
|
||||
phase-122 (task 04) image route (``GET /api/documents/{id}/image``
|
||||
— exact bytes, the extension's Content-Type, the 404 matrix, public
|
||||
per phase 124 A4) and the phase-148 (task 02) PDF page route
|
||||
(``GET /api/documents/{id}/pages/{n}`` — the same trust model and 404
|
||||
posture for a page-qualifying document's rendered page PNGs) + the
|
||||
content endpoint's ``image_url`` / ``page_urls`` keys (present for the
|
||||
qualifying row, ABSENT otherwise — never ``null``).
|
||||
|
||||
Uses the real compose Postgres (``db`` fixture) and FastAPI's TestClient.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
@@ -1370,6 +1379,241 @@ def test_content_endpoint_exposes_the_image_affordance(
|
||||
_cleanup_kb(db)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 148 (task 02) — the PDF page BYTES route + the content payload's
|
||||
# ``page_urls`` affordance
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _page_settings(pages_dir: Path) -> Settings:
|
||||
"""Fresh settings (the ``.env`` file ignored) with the phase-148
|
||||
``pdf_page_dir`` pinned to *pages_dir* — the page route reads the
|
||||
setting at request time, and the copies under test never land in
|
||||
the real home (the ``get_settings`` monkeypatch pattern)."""
|
||||
return Settings(
|
||||
_env_file=None, # pyright: ignore[reportCallIssue]
|
||||
pdf_page_dir=str(pages_dir),
|
||||
)
|
||||
|
||||
|
||||
def _seed_page_doc(
|
||||
db,
|
||||
pages_dir: Path,
|
||||
*,
|
||||
source: str = "PdfSrc",
|
||||
path: str = "besta.pdf",
|
||||
page_count: int | None = 2,
|
||||
write_copies: bool = True,
|
||||
) -> Document:
|
||||
"""One PDF document row with ``page_count`` + its rendered page
|
||||
copies (the importer's ``pdf_page_dir`` layout —
|
||||
``<doc-id>-p{N}.png``, N 1-based); page N is the ``PNG_1X1 + N``
|
||||
sentinel bytes, so a page mix-up between the two copies is caught.
|
||||
``page_count=None`` is a non-qualifying PDF (no copies);
|
||||
``write_copies=False`` leaves a qualifying row without its files
|
||||
(the lost-copy corner)."""
|
||||
doc = Document(
|
||||
id=uuid.uuid4(),
|
||||
source=source,
|
||||
path=path,
|
||||
full_path=f"/tmp/{path}",
|
||||
title=path.rsplit(".", 1)[0],
|
||||
content="[Page 1 illustration: the parts tray] [Page 2 illustration: step 1]",
|
||||
content_hash=hashlib.sha256(path.encode("utf-8")).hexdigest(),
|
||||
indexed_at=datetime.now(UTC),
|
||||
created_at=datetime(2026, 1, 1, tzinfo=UTC),
|
||||
summary="IKEA BESTA frame assembly manual (page descriptions).",
|
||||
page_count=page_count,
|
||||
)
|
||||
db.add(doc)
|
||||
db.flush()
|
||||
if write_copies and page_count:
|
||||
pages_dir.mkdir(parents=True, exist_ok=True)
|
||||
for n in range(1, page_count + 1):
|
||||
(pages_dir / f"{doc.id}-p{n}.png").write_bytes(PNG_1X1 + str(n).encode("ascii"))
|
||||
db.add_all(
|
||||
[
|
||||
Chunk(
|
||||
document_id=doc.id, position=0, content=doc.content, embedding=[0.01] * 768
|
||||
),
|
||||
Chunk(
|
||||
document_id=doc.id,
|
||||
position=-1,
|
||||
content=doc.summary,
|
||||
embedding=[0.01] * 768,
|
||||
is_summary=True,
|
||||
),
|
||||
]
|
||||
)
|
||||
db.commit()
|
||||
return doc
|
||||
|
||||
|
||||
def test_page_route_serves_exact_bytes_with_png_type(
|
||||
admin_client: TestClient, db, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""The serve contract: each page of a qualifying doc streams the
|
||||
EXACT stored PNG bytes (a per-page sentinel — a page mix-up is
|
||||
caught) as ``image/png`` (a page is always a PNG —
|
||||
``render_pdf_pages``'s ``tobytes("png")``) with
|
||||
``Cache-Control: private, max-age=3600`` (the image route's
|
||||
header — content-addressed by the document id + page number).
|
||||
The path is built ONLY from the row's id + ``n`` (the row
|
||||
derives it; ``n`` is the int path parameter) — the copies
|
||||
written under the tmp ``pdf_page_dir`` are what come back."""
|
||||
_cleanup_kb(db)
|
||||
try:
|
||||
pages_dir = tmp_path / "pages"
|
||||
monkeypatch.setattr(docs_api, "get_settings", lambda: _page_settings(pages_dir))
|
||||
doc = _seed_page_doc(db, pages_dir, page_count=2)
|
||||
for n in (1, 2):
|
||||
r = admin_client.get(f"/api/documents/{doc.id}/pages/{n}")
|
||||
assert r.status_code == 200, n
|
||||
assert r.headers["content-type"] == "image/png"
|
||||
assert r.headers["cache-control"] == "private, max-age=3600"
|
||||
assert r.content == PNG_1X1 + str(n).encode("ascii") # the exact stored bytes
|
||||
finally:
|
||||
_cleanup_kb(db)
|
||||
|
||||
|
||||
def test_page_route_404_matrix(
|
||||
admin_client: TestClient, db, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""Every non-servable case is 404 ``document not found`` (the
|
||||
router's unknown-document shape — the same detail string the
|
||||
content endpoint uses): a missing id, a MALFORMED id (an
|
||||
unparseable string maps here, not to a 422 — the image route's
|
||||
mapping: a guessed id is an unknown document), a text doc, an
|
||||
image doc, a PDF row whose ``page_count`` is NULL (a
|
||||
non-qualifying doc — the route knows nothing of its pages),
|
||||
``n`` outside ``1..page_count`` (``n=0``/``n=-1`` below the range
|
||||
— the int parameter makes fractions impossible — and ``n=3``
|
||||
above a 2-page doc), and a qualifying row whose copy was lost on
|
||||
disk (defensive — the row exists, the bytes don't; the viewer's
|
||||
onerror note covers it). The int parameter is the no-traversal
|
||||
pin: a NON-int ``n`` 422s at the router, and no request value
|
||||
ever reaches a filesystem path (the out-of-range 404s show the
|
||||
path is bounded by the ROW's extent)."""
|
||||
_cleanup_kb(db)
|
||||
try:
|
||||
pages_dir = tmp_path / "pages"
|
||||
monkeypatch.setattr(docs_api, "get_settings", lambda: _page_settings(pages_dir))
|
||||
_seed_doc(db, "TextSrc", "note.md", "Note", 1, datetime.now(UTC))
|
||||
db.commit() # the module's _seed_doc leaves the row uncommitted
|
||||
text_doc_id = db.scalar(
|
||||
select(Document.id).where(Document.source == "TextSrc", Document.path == "note.md")
|
||||
)
|
||||
img = _seed_image_doc(db, tmp_path, path="pic.png")
|
||||
plain = _seed_page_doc(db, pages_dir, path="texty.pdf", page_count=None)
|
||||
lost = _seed_page_doc(db, pages_dir, path="lost.pdf")
|
||||
assert (pages_dir / f"{lost.id}-p1.png").is_file() # the copies were written
|
||||
(pages_dir / f"{lost.id}-p1.png").unlink() # the copy is lost (the row remains)
|
||||
qualifying = _seed_page_doc(db, pages_dir, path="besta.pdf")
|
||||
|
||||
for doc_id, n in (
|
||||
(str(uuid.uuid4()), 1), # missing id
|
||||
("abc", 1), # malformed id → 404, not 422
|
||||
(str(text_doc_id), 1), # text doc
|
||||
(str(img.id), 1), # image doc
|
||||
(str(plain.id), 1), # PDF, page_count NULL (non-qualifying)
|
||||
(str(lost.id), 1), # qualifying doc, copy lost on disk
|
||||
(str(qualifying.id), 0), # below range
|
||||
(str(qualifying.id), -1), # negative — the int parse allows it, the range rejects it
|
||||
(str(qualifying.id), 3), # above range (page_count 2)
|
||||
):
|
||||
r = admin_client.get(f"/api/documents/{doc_id}/pages/{n}")
|
||||
assert r.status_code == 404, (doc_id, n)
|
||||
assert r.json() == {"detail": "document not found"}, (doc_id, n)
|
||||
|
||||
# The int parameter (the no-traversal pin): a NON-int n is a
|
||||
# router-level 422 — no other ``n`` shape reaches the handler.
|
||||
r = admin_client.get(f"/api/documents/{qualifying.id}/pages/abc")
|
||||
assert r.status_code == 422
|
||||
finally:
|
||||
_cleanup_kb(db)
|
||||
|
||||
|
||||
def test_page_route_is_public_like_the_shared_chat_content(
|
||||
admin_client: TestClient, db, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""The image route's phase-124 A4 trust model verbatim: a shared
|
||||
chat can cite a page FIGURE (the viewer's strip, task 03), so the
|
||||
BYTES serve ANONYMOUSLY — the unguessable ``uuid4`` document id IS
|
||||
the credential, no enumeration value (PLAN A10: the shared chats
|
||||
are the app's ONLY anonymous surface). An ANONYMOUS caller (a
|
||||
FRESH client — no session) gets the page PNG, and a signed-in
|
||||
caller gets the same. The document's TEXT (the content endpoint)
|
||||
stays user-gated — only the page bytes are public."""
|
||||
_cleanup_kb(db)
|
||||
try:
|
||||
pages_dir = tmp_path / "pages"
|
||||
monkeypatch.setattr(docs_api, "get_settings", lambda: _page_settings(pages_dir))
|
||||
doc = _seed_page_doc(db, pages_dir)
|
||||
anonymous = TestClient(fastapi_app)
|
||||
r = anonymous.get(f"/api/documents/{doc.id}/pages/1")
|
||||
assert r.status_code == 200
|
||||
assert r.headers["content-type"] == "image/png"
|
||||
assert r.content == PNG_1X1 + b"1"
|
||||
# The signed-in client (admin) gets the same bytes.
|
||||
r_admin = admin_client.get(f"/api/documents/{doc.id}/pages/1")
|
||||
assert r_admin.status_code == 200
|
||||
assert r_admin.content == PNG_1X1 + b"1"
|
||||
finally:
|
||||
_cleanup_kb(db)
|
||||
|
||||
|
||||
def test_content_endpoint_exposes_the_page_urls_affordance(
|
||||
admin_client: TestClient, db, tmp_path: Path
|
||||
) -> None:
|
||||
"""The content endpoint (the viewer's data source — task 03's
|
||||
strip reads from it): a page-qualifying PDF (``page_count`` set)
|
||||
carries ``page_urls`` = the EXACT 1-based route list over
|
||||
``1..page_count`` (the count == ``page_count`` — a per-page render
|
||||
gap is still listed; the route 404s that N); text docs, image
|
||||
docs, and non-qualifying PDFs (``page_count`` NULL) keep the key
|
||||
ABSENT from the JSON — never ``null`` (the ``image_url``
|
||||
omission rule — a non-qualifying doc's payload is byte-identical
|
||||
to pre-phase)."""
|
||||
_cleanup_kb(db)
|
||||
try:
|
||||
pages_dir = tmp_path / "pages"
|
||||
doc = _seed_page_doc(db, pages_dir, page_count=3)
|
||||
r = admin_client.get(
|
||||
"/api/documents/content", params={"source": "PdfSrc", "path": "besta.pdf"}
|
||||
)
|
||||
assert r.status_code == 200
|
||||
body = r.json()
|
||||
assert body["page_urls"] == [
|
||||
f"/api/documents/{doc.id}/pages/1",
|
||||
f"/api/documents/{doc.id}/pages/2",
|
||||
f"/api/documents/{doc.id}/pages/3",
|
||||
] # the exact 1-based list, count == page_count
|
||||
|
||||
_seed_doc(db, "TextSrc", "note.md", "Note", 1, datetime.now(UTC))
|
||||
db.commit() # the app's endpoint session reads committed data only
|
||||
r = admin_client.get(
|
||||
"/api/documents/content", params={"source": "TextSrc", "path": "note.md"}
|
||||
)
|
||||
assert r.status_code == 200
|
||||
assert "page_urls" not in r.json() # absent — never null (text doc)
|
||||
|
||||
_seed_image_doc(db, tmp_path, source="ImgSrc", path="pic.png")
|
||||
r = admin_client.get(
|
||||
"/api/documents/content", params={"source": "ImgSrc", "path": "pic.png"}
|
||||
)
|
||||
assert r.status_code == 200
|
||||
assert "page_urls" not in r.json() # absent — never null (image doc)
|
||||
|
||||
_seed_page_doc(db, pages_dir, path="texty.pdf", page_count=None)
|
||||
r = admin_client.get(
|
||||
"/api/documents/content", params={"source": "PdfSrc", "path": "texty.pdf"}
|
||||
)
|
||||
assert r.status_code == 200
|
||||
assert "page_urls" not in r.json() # absent — never null (non-qualifying PDF)
|
||||
finally:
|
||||
_cleanup_kb(db)
|
||||
|
||||
|
||||
def _tree_file_nodes_all(sources) -> list[dict]:
|
||||
"""Every file node of a tree response, walked recursively (all
|
||||
sources — env-registered 0-document sources may join the response
|
||||
|
||||
@@ -0,0 +1,351 @@
|
||||
"""Integration: migration 0026 (documents.page_count) schema contract
|
||||
(phase 148, task 01).
|
||||
|
||||
Drives the **real Alembic engine** against the live dev database
|
||||
(``podman compose up -d db``), mirroring the house pattern of
|
||||
``test_migration_0025.py`` (information_schema assertions on the state
|
||||
the migration must leave). The tests target the 0025 → 0026 step
|
||||
explicitly so later migrations cannot break the pins:
|
||||
|
||||
* upgrade 0025 → 0026 → ``page_count`` exists with the full contract —
|
||||
INTEGER, NULLABLE, NO server default (NULL = "not page-qualifying"
|
||||
for every pre-phase row and every fail-soft miss) — and is UNINDEXED
|
||||
(a display fact, never a query surface — the page route of task 02
|
||||
reads it from the ROW after the uuid4 id lookup, the ``file_hash``
|
||||
convention of 0025) while the 0025 ``documents`` schema survives —
|
||||
in particular ``content_hash`` stays NOT NULL and INDEXED (the
|
||||
retriever's dedupe/caching identity, untouched) and ``file_hash``
|
||||
keeps its 0025 contract;
|
||||
* rows inserted while the DB is at 0025 backfill ``page_count`` to
|
||||
NULL — the pre-phase shape, no backfill, no default;
|
||||
* the ORM contract agrees: a freshly inserted ``Document`` with no
|
||||
``page_count`` reads ``page_count is None`` through a fresh session
|
||||
(the pre-phase shape), and an explicit count round-trips;
|
||||
* downgrade to 0025 → the column is GONE (A13) while the row + its
|
||||
content survive; upgrade back to 0026 → the column is back
|
||||
(round-trip).
|
||||
|
||||
The ``alembic`` fixture guarantees the DB ends at head even if a test
|
||||
fails or the process is interrupted.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import uuid
|
||||
from collections.abc import Iterator
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from alembic.config import Config
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from alembic import command
|
||||
from app.db import SessionLocal, db_available
|
||||
from app.models import Document
|
||||
|
||||
SOURCE = "mig0026"
|
||||
CONTENT = "content here"
|
||||
CONTENT_HASH = hashlib.sha256(CONTENT.encode()).hexdigest()
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def alembic(db: Session) -> Iterator[Config]:
|
||||
"""Real Alembic config bound to the dev DB (URL from app settings).
|
||||
|
||||
Starts at head (repairs an interrupted earlier run); teardown
|
||||
upgrades to head no matter what happened, so the dev DB is never
|
||||
left below head.
|
||||
"""
|
||||
if not db_available():
|
||||
pytest.skip("Postgres not reachable — run `podman compose up -d db` first")
|
||||
cfg = Config() # no alembic.ini file — env.py gets the URL from app config
|
||||
cfg.set_main_option("script_location", "alembic")
|
||||
command.upgrade(cfg, "head")
|
||||
try:
|
||||
yield cfg
|
||||
finally:
|
||||
# Release the test session's open transaction BEFORE the repair
|
||||
# DDL: an idle-in-transaction SELECT holds an ACCESS SHARE lock
|
||||
# on ``documents``, which would deadlock the repair's
|
||||
# ``ALTER TABLE`` forever.
|
||||
db.rollback()
|
||||
command.upgrade(cfg, "head")
|
||||
|
||||
|
||||
def _version(db: Session) -> str | None:
|
||||
return db.execute(text("SELECT version_num FROM alembic_version")).scalar()
|
||||
|
||||
|
||||
def _column(db: Session, column: str) -> tuple[Any, ...] | None:
|
||||
"""(data_type, is_nullable, column_default) for one documents
|
||||
column."""
|
||||
row = db.execute(
|
||||
text(
|
||||
"SELECT data_type, is_nullable, column_default"
|
||||
" FROM information_schema.columns"
|
||||
" WHERE table_name = 'documents' AND column_name = :c"
|
||||
),
|
||||
{"c": column},
|
||||
).fetchone()
|
||||
return tuple(row) if row is not None else None
|
||||
|
||||
|
||||
def _index_covering(db: Session, column: str) -> str | None:
|
||||
"""The name of any documents index whose definition mentions the
|
||||
column (None = UNINDEXED)."""
|
||||
return db.execute(
|
||||
text(
|
||||
"SELECT indexname FROM pg_indexes"
|
||||
" WHERE tablename = 'documents' AND indexdef ILIKE :like"
|
||||
),
|
||||
{"like": f"%{column}%"},
|
||||
).scalar()
|
||||
|
||||
|
||||
def _insert_sql(db: Session, path: str) -> uuid.UUID:
|
||||
"""Insert one documents row with the PRE-0026 column set (the 0025
|
||||
shape — the page_count column, when present, is omitted so a NULL
|
||||
backfill is what the row reads)."""
|
||||
doc_id = uuid.uuid4()
|
||||
db.execute(
|
||||
text(
|
||||
"INSERT INTO documents (id, source, path, full_path, title, content,"
|
||||
" content_hash) VALUES (:id, :s, :p, :fp, :t, :c, :h)"
|
||||
),
|
||||
{
|
||||
"id": doc_id,
|
||||
"s": SOURCE,
|
||||
"p": path,
|
||||
"fp": f"/tmp/{path}",
|
||||
"t": f"Doc {path}",
|
||||
"c": CONTENT,
|
||||
"h": CONTENT_HASH,
|
||||
},
|
||||
)
|
||||
db.commit()
|
||||
return doc_id
|
||||
|
||||
|
||||
def _delete(db: Session, doc_id: uuid.UUID) -> None:
|
||||
db.execute(text("DELETE FROM documents WHERE id = :id"), {"id": doc_id})
|
||||
db.commit()
|
||||
|
||||
|
||||
def _assert_0025_schema_survives(db: Session) -> None:
|
||||
"""The 0025 documents contract must survive the additive upgrade —
|
||||
in particular ``content_hash`` stays NOT NULL and indexed (the
|
||||
retriever's identity — untouched by phase 148) and ``file_hash``
|
||||
keeps its 0025 contract (VARCHAR(64), nullable, unindexed)."""
|
||||
content = _column(db, "content_hash")
|
||||
assert content is not None and content[0] == "character varying" and content[1] == "NO", (
|
||||
"documents.content_hash (0001) must keep its 0025 contract"
|
||||
)
|
||||
assert (
|
||||
_index_covering(db, "content_hash") is not None
|
||||
), "the ix_documents_content_hash index must survive the upgrade"
|
||||
|
||||
file_hash = _column(db, "file_hash")
|
||||
assert file_hash is not None and file_hash[0] == "character varying", (
|
||||
"documents.file_hash (0025) must survive the upgrade"
|
||||
)
|
||||
assert file_hash[1] == "YES", "file_hash must stay NULLABLE"
|
||||
assert file_hash[2] is None, "file_hash must keep NO server default"
|
||||
assert _index_covering(db, "file_hash") is None, (
|
||||
"file_hash must stay UNINDEXED"
|
||||
)
|
||||
|
||||
indexed = _column(db, "indexed_at")
|
||||
assert indexed is not None and indexed[0] == "timestamp with time zone", (
|
||||
"documents.indexed_at (0001) must survive the upgrade"
|
||||
)
|
||||
is_image = _column(db, "is_image")
|
||||
assert is_image is not None and is_image[0] == "boolean" and is_image[1] == "NO", (
|
||||
"documents.is_image (0022) must survive the upgrade"
|
||||
)
|
||||
image_path = _column(db, "image_path")
|
||||
assert image_path is not None and image_path[1] == "YES", (
|
||||
"documents.image_path (0022) must survive the upgrade"
|
||||
)
|
||||
|
||||
|
||||
def test_upgrade_to_0026_adds_page_count(db: Session, alembic: Config) -> None:
|
||||
"""Upgrade 0025 → 0026: ``page_count`` exists with the full
|
||||
contract (INTEGER, NULLABLE, NO server default — NULL = "not
|
||||
page-qualifying" for every pre-phase row — and UNINDEXED: a
|
||||
display fact, never a query surface), is ABSENT at 0025, and every
|
||||
pre-0026 row backfills ``page_count`` to NULL (no backfill
|
||||
possible — a migration cannot know which documents qualified) —
|
||||
while the 0025 table contract (incl. the INDEXED ``content_hash``
|
||||
and the unindexed ``file_hash``) survives."""
|
||||
command.downgrade(alembic, "0025") # start from the pre-0026 state
|
||||
assert _version(db) == "0025"
|
||||
assert _column(db, "page_count") is None, "page_count must be absent at 0025"
|
||||
|
||||
pre_id = _insert_sql(db, "pre.md") # the 0025 column set
|
||||
try:
|
||||
command.upgrade(alembic, "0026")
|
||||
assert _version(db) == "0026", "alembic_version must be at 0026"
|
||||
|
||||
col = _column(db, "page_count")
|
||||
assert col is not None, "documents.page_count is missing"
|
||||
assert col[0] == "integer", "page_count must be INTEGER"
|
||||
assert col[1] == "YES", (
|
||||
"page_count must be NULLABLE (NULL = not page-qualifying)"
|
||||
)
|
||||
assert col[2] is None, (
|
||||
"page_count must carry NO server default — NULL is the meaningful"
|
||||
" pre-phase state"
|
||||
)
|
||||
assert _index_covering(db, "page_count") is None, (
|
||||
"page_count must be UNINDEXED — a display fact, never a query"
|
||||
" surface (the page route reads it from the row after the id lookup)"
|
||||
)
|
||||
|
||||
# The pre-0026 row backfilled to NULL (the pre-phase shape).
|
||||
assert db.execute(
|
||||
text(
|
||||
"SELECT content_hash, page_count FROM documents WHERE id = :id"
|
||||
),
|
||||
{"id": pre_id},
|
||||
).fetchone() == (CONTENT_HASH, None), (
|
||||
"every pre-0026 row must backfill page_count to NULL — no"
|
||||
" backfill, no default — while content_hash survives"
|
||||
)
|
||||
|
||||
_assert_0025_schema_survives(db)
|
||||
finally:
|
||||
_delete(db, pre_id)
|
||||
|
||||
|
||||
def test_orm_page_count_round_trips(db: Session, alembic: Config) -> None:
|
||||
"""The ORM contract agrees with the column contract: a freshly
|
||||
inserted ``Document`` with NO ``page_count`` reads
|
||||
``page_count is None`` through a FRESH session (the pre-phase
|
||||
shape — NULL until the importer's PDF branch sets it), and an
|
||||
explicit count round-trips the value."""
|
||||
command.upgrade(alembic, "head")
|
||||
row_counted = Document(
|
||||
source=SOURCE,
|
||||
path="counted.pdf",
|
||||
full_path="/tmp/counted.pdf",
|
||||
title="counted",
|
||||
content=CONTENT,
|
||||
content_hash=CONTENT_HASH,
|
||||
page_count=24,
|
||||
)
|
||||
row_uncounted = Document(
|
||||
source=SOURCE,
|
||||
path="uncounted.md",
|
||||
full_path="/tmp/uncounted.md",
|
||||
title="uncounted",
|
||||
content=CONTENT,
|
||||
content_hash=CONTENT_HASH,
|
||||
)
|
||||
db.add(row_counted)
|
||||
db.add(row_uncounted)
|
||||
db.commit()
|
||||
try:
|
||||
with SessionLocal() as fresh:
|
||||
reloaded_counted = fresh.get(Document, row_counted.id)
|
||||
assert reloaded_counted is not None, "the counted row must be readable"
|
||||
assert reloaded_counted.page_count == 24, (
|
||||
"the explicit page_count must round-trip through the DB"
|
||||
)
|
||||
reloaded_uncounted = fresh.get(Document, row_uncounted.id)
|
||||
assert reloaded_uncounted is not None, "the uncounted row must be readable"
|
||||
assert reloaded_uncounted.page_count is None, (
|
||||
"a row created without page_count must read NULL (the pre-phase"
|
||||
" shape)"
|
||||
)
|
||||
assert reloaded_uncounted.content_hash == CONTENT_HASH, (
|
||||
"content_hash must be untouched by the page_count addition"
|
||||
)
|
||||
finally:
|
||||
_delete(db, row_counted.id)
|
||||
_delete(db, row_uncounted.id)
|
||||
|
||||
|
||||
def test_downgrade_to_0025_drops_the_column(db: Session, alembic: Config) -> None:
|
||||
"""Downgrade 0026 → 0025: the page_count column is gone (A13 —
|
||||
fully reversible, the copies are re-derivable from the source
|
||||
files on the next sync) while the row + its content (incl. the
|
||||
0025 ``file_hash`` sibling) survive, and the rest of the 0025
|
||||
table contract is intact."""
|
||||
command.upgrade(alembic, "head")
|
||||
row = Document(
|
||||
source=SOURCE,
|
||||
path="down.pdf",
|
||||
full_path="/tmp/down.pdf",
|
||||
title="down",
|
||||
content=CONTENT,
|
||||
content_hash=CONTENT_HASH,
|
||||
page_count=24,
|
||||
)
|
||||
db.add(row)
|
||||
db.commit()
|
||||
try:
|
||||
command.downgrade(alembic, "0025")
|
||||
assert _version(db) == "0025"
|
||||
assert _column(db, "page_count") is None, "page_count must be dropped"
|
||||
|
||||
surviving = db.execute(
|
||||
text(
|
||||
"SELECT source, path, content, content_hash, file_hash, is_image,"
|
||||
" image_path FROM documents WHERE id = :id"
|
||||
),
|
||||
{"id": row.id},
|
||||
).fetchone()
|
||||
assert surviving is not None and surviving[0] == SOURCE, (
|
||||
"the row must survive the column drop"
|
||||
)
|
||||
assert surviving[1] == "down.pdf" and surviving[2] == CONTENT, (
|
||||
"path + content must survive the drop"
|
||||
)
|
||||
assert surviving[3] == CONTENT_HASH, (
|
||||
"content_hash (the retriever's identity) must survive the drop"
|
||||
)
|
||||
assert surviving[4] is None and surviving[5] is False and surviving[6] is None, (
|
||||
"file_hash + is_image + image_path must survive the drop"
|
||||
)
|
||||
assert _index_covering(db, "content_hash") is not None, (
|
||||
"the content_hash index must survive the downgrade"
|
||||
)
|
||||
finally:
|
||||
_delete(db, row.id)
|
||||
# Repair: the fixture teardown re-upgrades to head.
|
||||
|
||||
|
||||
def test_upgrade_round_trip_restores_the_column(db: Session, alembic: Config) -> None:
|
||||
"""Downgrade to 0025, then upgrade back to 0026: ``page_count`` is
|
||||
back with the full contract (INTEGER, NULLABLE, NO server default,
|
||||
UNINDEXED), a row seeded at 0025 reads ``page_count is None`` (the
|
||||
pre-phase shape on the round-trip), and the 0025 schema (incl. the
|
||||
indexed ``content_hash``) is intact."""
|
||||
command.downgrade(alembic, "0025")
|
||||
pre_id = _insert_sql(db, "round.pdf")
|
||||
try:
|
||||
command.upgrade(alembic, "0026")
|
||||
assert _version(db) == "0026", "round-trip upgrade must land at 0026"
|
||||
|
||||
col = _column(db, "page_count")
|
||||
assert col is not None, "documents.page_count must be back"
|
||||
assert col[0] == "integer", "page_count must be INTEGER after the round-trip"
|
||||
assert col[1] == "YES", "page_count must be NULLABLE after the round-trip"
|
||||
assert col[2] is None, (
|
||||
"page_count must still carry NO server default after the round-trip"
|
||||
)
|
||||
assert _index_covering(db, "page_count") is None, (
|
||||
"page_count must stay UNINDEXED after the round-trip"
|
||||
)
|
||||
|
||||
assert db.execute(
|
||||
text("SELECT page_count FROM documents WHERE id = :id"),
|
||||
{"id": pre_id},
|
||||
).scalar() is None, (
|
||||
"a row seeded at 0025 must read page_count NULL after the round-trip"
|
||||
)
|
||||
|
||||
_assert_0025_schema_survives(db)
|
||||
finally:
|
||||
_delete(db, pre_id)
|
||||
Reference in New Issue
Block a user