phase: 148_pdf_pages_in_sources
**Phase 148 — final verification pass: complete, 100% green.**
- **Verified (tasks 01–04 already implemented):** page render/persist at index (`_persist_page_copies`, fail-soft, stale-replace, qualifier-loss, prune hook via `_delete_page_copies`); public `GET /api/documents/{id}/pages/{n}` (uuid4-credential, 404-all-non-servable, `image/png` + cache header); `DocContent.page_urls` additive-omitted; shared `renderDocument` strip (lazy imgs, "Page N" captions, text + Summary suppressed, onerror note, createElement-only); migration 0026; `pdf_page_dir` setting; `.env.example` entry
- **Completion criteria:** (1) pictorial + fallback classes get `p{N}.png` + `page_count`, text-heavy gets neither — pinned, pass; (2) unchanged re-sync renders zero (fast path) — pass; (3) changed re-index replaces copies, qualifier-loss/prune delete — pass; (4) route 404 matrix + payload key absent-never-null — pass; (5) modal AND /document.html strip with suppression, text-heavy byte-identical — E2E pass; (6) see below; (7) no completed-phase drift — full suite green
- **Tests:** `uv run pytest --cov=app --cov-report=term-missing` → **3461 passed, 0 failed**, `app/` coverage **99%** (>90%); `uv run pytest tests/e2e/test_pdf_pages_in_sources.py -v --no-cov` → **4 passed** in isolation; `uv run ruff check .` clean; `uv run pyright` → 0 errors, 0 warnings
- **Documentation:** README had **1 pre-existing inconsistency** — the Production Deployment section's standalone `podman run` example (and compose note) omitted the **required** `BOR_ADMIN_PASSWORD`/`BOR_SESSION_SECRET` (the fail-loud boot gate the README's own Admin section documents — the container would refuse to start); fixed with the two `-e` flags + a note for the compose path. All else verified consistent: 70/70 `.env.example` vars documented, all defaults match `config.py`, pages table matches routes, Containerfile/esbuild/uid-10001/healthcheck/entrypoint claims accurate, QA commands match pytest config. **AGENTS.md: verified consistent — no changes needed**
- **Notable:** repo uses `Containerfile` (Podman) — README's `podman build` example is valid as-is; no live infrastructure touched
- **Next pending phase:** `149_sync_progress_box`
This commit is contained in:
@@ -3,6 +3,15 @@ and GET /api/docs/tree (phase 97, task 02 — the full recursive KB tree
|
||||
the RAG view renders: shape, ordering, counts, summaries, the 403 gate,
|
||||
the superset rule, and the stat-walk equivalence with ``GET /api/docs``).
|
||||
|
||||
Also the BYTES routes and their content-payload affordances: the
|
||||
phase-122 (task 04) image route (``GET /api/documents/{id}/image``
|
||||
— exact bytes, the extension's Content-Type, the 404 matrix, public
|
||||
per phase 124 A4) and the phase-148 (task 02) PDF page route
|
||||
(``GET /api/documents/{id}/pages/{n}`` — the same trust model and 404
|
||||
posture for a page-qualifying document's rendered page PNGs) + the
|
||||
content endpoint's ``image_url`` / ``page_urls`` keys (present for the
|
||||
qualifying row, ABSENT otherwise — never ``null``).
|
||||
|
||||
Uses the real compose Postgres (``db`` fixture) and FastAPI's TestClient.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
@@ -1370,6 +1379,241 @@ def test_content_endpoint_exposes_the_image_affordance(
|
||||
_cleanup_kb(db)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 148 (task 02) — the PDF page BYTES route + the content payload's
|
||||
# ``page_urls`` affordance
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _page_settings(pages_dir: Path) -> Settings:
|
||||
"""Fresh settings (the ``.env`` file ignored) with the phase-148
|
||||
``pdf_page_dir`` pinned to *pages_dir* — the page route reads the
|
||||
setting at request time, and the copies under test never land in
|
||||
the real home (the ``get_settings`` monkeypatch pattern)."""
|
||||
return Settings(
|
||||
_env_file=None, # pyright: ignore[reportCallIssue]
|
||||
pdf_page_dir=str(pages_dir),
|
||||
)
|
||||
|
||||
|
||||
def _seed_page_doc(
|
||||
db,
|
||||
pages_dir: Path,
|
||||
*,
|
||||
source: str = "PdfSrc",
|
||||
path: str = "besta.pdf",
|
||||
page_count: int | None = 2,
|
||||
write_copies: bool = True,
|
||||
) -> Document:
|
||||
"""One PDF document row with ``page_count`` + its rendered page
|
||||
copies (the importer's ``pdf_page_dir`` layout —
|
||||
``<doc-id>-p{N}.png``, N 1-based); page N is the ``PNG_1X1 + N``
|
||||
sentinel bytes, so a page mix-up between the two copies is caught.
|
||||
``page_count=None`` is a non-qualifying PDF (no copies);
|
||||
``write_copies=False`` leaves a qualifying row without its files
|
||||
(the lost-copy corner)."""
|
||||
doc = Document(
|
||||
id=uuid.uuid4(),
|
||||
source=source,
|
||||
path=path,
|
||||
full_path=f"/tmp/{path}",
|
||||
title=path.rsplit(".", 1)[0],
|
||||
content="[Page 1 illustration: the parts tray] [Page 2 illustration: step 1]",
|
||||
content_hash=hashlib.sha256(path.encode("utf-8")).hexdigest(),
|
||||
indexed_at=datetime.now(UTC),
|
||||
created_at=datetime(2026, 1, 1, tzinfo=UTC),
|
||||
summary="IKEA BESTA frame assembly manual (page descriptions).",
|
||||
page_count=page_count,
|
||||
)
|
||||
db.add(doc)
|
||||
db.flush()
|
||||
if write_copies and page_count:
|
||||
pages_dir.mkdir(parents=True, exist_ok=True)
|
||||
for n in range(1, page_count + 1):
|
||||
(pages_dir / f"{doc.id}-p{n}.png").write_bytes(PNG_1X1 + str(n).encode("ascii"))
|
||||
db.add_all(
|
||||
[
|
||||
Chunk(
|
||||
document_id=doc.id, position=0, content=doc.content, embedding=[0.01] * 768
|
||||
),
|
||||
Chunk(
|
||||
document_id=doc.id,
|
||||
position=-1,
|
||||
content=doc.summary,
|
||||
embedding=[0.01] * 768,
|
||||
is_summary=True,
|
||||
),
|
||||
]
|
||||
)
|
||||
db.commit()
|
||||
return doc
|
||||
|
||||
|
||||
def test_page_route_serves_exact_bytes_with_png_type(
|
||||
admin_client: TestClient, db, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""The serve contract: each page of a qualifying doc streams the
|
||||
EXACT stored PNG bytes (a per-page sentinel — a page mix-up is
|
||||
caught) as ``image/png`` (a page is always a PNG —
|
||||
``render_pdf_pages``'s ``tobytes("png")``) with
|
||||
``Cache-Control: private, max-age=3600`` (the image route's
|
||||
header — content-addressed by the document id + page number).
|
||||
The path is built ONLY from the row's id + ``n`` (the row
|
||||
derives it; ``n`` is the int path parameter) — the copies
|
||||
written under the tmp ``pdf_page_dir`` are what come back."""
|
||||
_cleanup_kb(db)
|
||||
try:
|
||||
pages_dir = tmp_path / "pages"
|
||||
monkeypatch.setattr(docs_api, "get_settings", lambda: _page_settings(pages_dir))
|
||||
doc = _seed_page_doc(db, pages_dir, page_count=2)
|
||||
for n in (1, 2):
|
||||
r = admin_client.get(f"/api/documents/{doc.id}/pages/{n}")
|
||||
assert r.status_code == 200, n
|
||||
assert r.headers["content-type"] == "image/png"
|
||||
assert r.headers["cache-control"] == "private, max-age=3600"
|
||||
assert r.content == PNG_1X1 + str(n).encode("ascii") # the exact stored bytes
|
||||
finally:
|
||||
_cleanup_kb(db)
|
||||
|
||||
|
||||
def test_page_route_404_matrix(
|
||||
admin_client: TestClient, db, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""Every non-servable case is 404 ``document not found`` (the
|
||||
router's unknown-document shape — the same detail string the
|
||||
content endpoint uses): a missing id, a MALFORMED id (an
|
||||
unparseable string maps here, not to a 422 — the image route's
|
||||
mapping: a guessed id is an unknown document), a text doc, an
|
||||
image doc, a PDF row whose ``page_count`` is NULL (a
|
||||
non-qualifying doc — the route knows nothing of its pages),
|
||||
``n`` outside ``1..page_count`` (``n=0``/``n=-1`` below the range
|
||||
— the int parameter makes fractions impossible — and ``n=3``
|
||||
above a 2-page doc), and a qualifying row whose copy was lost on
|
||||
disk (defensive — the row exists, the bytes don't; the viewer's
|
||||
onerror note covers it). The int parameter is the no-traversal
|
||||
pin: a NON-int ``n`` 422s at the router, and no request value
|
||||
ever reaches a filesystem path (the out-of-range 404s show the
|
||||
path is bounded by the ROW's extent)."""
|
||||
_cleanup_kb(db)
|
||||
try:
|
||||
pages_dir = tmp_path / "pages"
|
||||
monkeypatch.setattr(docs_api, "get_settings", lambda: _page_settings(pages_dir))
|
||||
_seed_doc(db, "TextSrc", "note.md", "Note", 1, datetime.now(UTC))
|
||||
db.commit() # the module's _seed_doc leaves the row uncommitted
|
||||
text_doc_id = db.scalar(
|
||||
select(Document.id).where(Document.source == "TextSrc", Document.path == "note.md")
|
||||
)
|
||||
img = _seed_image_doc(db, tmp_path, path="pic.png")
|
||||
plain = _seed_page_doc(db, pages_dir, path="texty.pdf", page_count=None)
|
||||
lost = _seed_page_doc(db, pages_dir, path="lost.pdf")
|
||||
assert (pages_dir / f"{lost.id}-p1.png").is_file() # the copies were written
|
||||
(pages_dir / f"{lost.id}-p1.png").unlink() # the copy is lost (the row remains)
|
||||
qualifying = _seed_page_doc(db, pages_dir, path="besta.pdf")
|
||||
|
||||
for doc_id, n in (
|
||||
(str(uuid.uuid4()), 1), # missing id
|
||||
("abc", 1), # malformed id → 404, not 422
|
||||
(str(text_doc_id), 1), # text doc
|
||||
(str(img.id), 1), # image doc
|
||||
(str(plain.id), 1), # PDF, page_count NULL (non-qualifying)
|
||||
(str(lost.id), 1), # qualifying doc, copy lost on disk
|
||||
(str(qualifying.id), 0), # below range
|
||||
(str(qualifying.id), -1), # negative — the int parse allows it, the range rejects it
|
||||
(str(qualifying.id), 3), # above range (page_count 2)
|
||||
):
|
||||
r = admin_client.get(f"/api/documents/{doc_id}/pages/{n}")
|
||||
assert r.status_code == 404, (doc_id, n)
|
||||
assert r.json() == {"detail": "document not found"}, (doc_id, n)
|
||||
|
||||
# The int parameter (the no-traversal pin): a NON-int n is a
|
||||
# router-level 422 — no other ``n`` shape reaches the handler.
|
||||
r = admin_client.get(f"/api/documents/{qualifying.id}/pages/abc")
|
||||
assert r.status_code == 422
|
||||
finally:
|
||||
_cleanup_kb(db)
|
||||
|
||||
|
||||
def test_page_route_is_public_like_the_shared_chat_content(
|
||||
admin_client: TestClient, db, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""The image route's phase-124 A4 trust model verbatim: a shared
|
||||
chat can cite a page FIGURE (the viewer's strip, task 03), so the
|
||||
BYTES serve ANONYMOUSLY — the unguessable ``uuid4`` document id IS
|
||||
the credential, no enumeration value (PLAN A10: the shared chats
|
||||
are the app's ONLY anonymous surface). An ANONYMOUS caller (a
|
||||
FRESH client — no session) gets the page PNG, and a signed-in
|
||||
caller gets the same. The document's TEXT (the content endpoint)
|
||||
stays user-gated — only the page bytes are public."""
|
||||
_cleanup_kb(db)
|
||||
try:
|
||||
pages_dir = tmp_path / "pages"
|
||||
monkeypatch.setattr(docs_api, "get_settings", lambda: _page_settings(pages_dir))
|
||||
doc = _seed_page_doc(db, pages_dir)
|
||||
anonymous = TestClient(fastapi_app)
|
||||
r = anonymous.get(f"/api/documents/{doc.id}/pages/1")
|
||||
assert r.status_code == 200
|
||||
assert r.headers["content-type"] == "image/png"
|
||||
assert r.content == PNG_1X1 + b"1"
|
||||
# The signed-in client (admin) gets the same bytes.
|
||||
r_admin = admin_client.get(f"/api/documents/{doc.id}/pages/1")
|
||||
assert r_admin.status_code == 200
|
||||
assert r_admin.content == PNG_1X1 + b"1"
|
||||
finally:
|
||||
_cleanup_kb(db)
|
||||
|
||||
|
||||
def test_content_endpoint_exposes_the_page_urls_affordance(
|
||||
admin_client: TestClient, db, tmp_path: Path
|
||||
) -> None:
|
||||
"""The content endpoint (the viewer's data source — task 03's
|
||||
strip reads from it): a page-qualifying PDF (``page_count`` set)
|
||||
carries ``page_urls`` = the EXACT 1-based route list over
|
||||
``1..page_count`` (the count == ``page_count`` — a per-page render
|
||||
gap is still listed; the route 404s that N); text docs, image
|
||||
docs, and non-qualifying PDFs (``page_count`` NULL) keep the key
|
||||
ABSENT from the JSON — never ``null`` (the ``image_url``
|
||||
omission rule — a non-qualifying doc's payload is byte-identical
|
||||
to pre-phase)."""
|
||||
_cleanup_kb(db)
|
||||
try:
|
||||
pages_dir = tmp_path / "pages"
|
||||
doc = _seed_page_doc(db, pages_dir, page_count=3)
|
||||
r = admin_client.get(
|
||||
"/api/documents/content", params={"source": "PdfSrc", "path": "besta.pdf"}
|
||||
)
|
||||
assert r.status_code == 200
|
||||
body = r.json()
|
||||
assert body["page_urls"] == [
|
||||
f"/api/documents/{doc.id}/pages/1",
|
||||
f"/api/documents/{doc.id}/pages/2",
|
||||
f"/api/documents/{doc.id}/pages/3",
|
||||
] # the exact 1-based list, count == page_count
|
||||
|
||||
_seed_doc(db, "TextSrc", "note.md", "Note", 1, datetime.now(UTC))
|
||||
db.commit() # the app's endpoint session reads committed data only
|
||||
r = admin_client.get(
|
||||
"/api/documents/content", params={"source": "TextSrc", "path": "note.md"}
|
||||
)
|
||||
assert r.status_code == 200
|
||||
assert "page_urls" not in r.json() # absent — never null (text doc)
|
||||
|
||||
_seed_image_doc(db, tmp_path, source="ImgSrc", path="pic.png")
|
||||
r = admin_client.get(
|
||||
"/api/documents/content", params={"source": "ImgSrc", "path": "pic.png"}
|
||||
)
|
||||
assert r.status_code == 200
|
||||
assert "page_urls" not in r.json() # absent — never null (image doc)
|
||||
|
||||
_seed_page_doc(db, pages_dir, path="texty.pdf", page_count=None)
|
||||
r = admin_client.get(
|
||||
"/api/documents/content", params={"source": "PdfSrc", "path": "texty.pdf"}
|
||||
)
|
||||
assert r.status_code == 200
|
||||
assert "page_urls" not in r.json() # absent — never null (non-qualifying PDF)
|
||||
finally:
|
||||
_cleanup_kb(db)
|
||||
|
||||
|
||||
def _tree_file_nodes_all(sources) -> list[dict]:
|
||||
"""Every file node of a tree response, walked recursively (all
|
||||
sources — env-registered 0-document sources may join the response
|
||||
|
||||
Reference in New Issue
Block a user