phase: 122_image_documents
Build and Push Containers / build-and-push-app (push) Successful in 1m57s
Build and Push Containers / build-and-push-db (push) Failing after 13s

**Phase 122 (image documents) — final verification pass: all green. No code changes were needed; defects found: none.**

**Verified (implementation already complete in working tree, reviewed end-to-end):**
- Toggle (`BOR_IMAGES`/`BOR_IMAGE_EXTENSIONS`/`BOR_IMAGE_DIR`, off by default) + `GET /api/config` `images` flag
- Ingest: bytes digest, `image_dir` persistent copy, `content = summary = vision description` (chat-model call; only text embedded), fail-soft skip + `images_failed` counter
- Serve/display: `/api/documents/{id}/image` route (404 matrix), viewer `<img>` + description, Sources 48px lazy thumbnails, chat inline source figure (alt = summary), agent `read` marker
- Prune guard: images-off syncs never prune `is_image` docs

**Test / lint / coverage (exact commands & outcomes):**
- `uv run pytest` → exit 0 (green; note: pytest 9.1.1 `-q` omits the final count line in output — exit code authoritative)
- `uv run pytest --cov=app --cov-report=term-missing` → **2715 passed, exit 0, TOTAL 99%** (>90% gate)
- `uv run ruff check . && uv run pyright` → "All checks passed!" / "0 errors, 0 warnings, 0 informations"
- `uv run pytest tests/e2e/test_image_documents.py -v --no-cov` → **4 passed, exit 0** (isolation)

**Completion criteria:** (1) images=true → described/embedded/displayed docs: ✅ (E2E + integration) · (2) images=false byte-identical + image docs survive sync: ✅ (E2E negative app + unit/integration) · (3) viewer + chat rendering with alt text; failed description skips + logs, sync completes: ✅ · (4) test/lint/coverage gates: ✅ · (5) commit + phase move: deferred to harness per this pass's rules (working tree left uncommitted).

**Notable deviation (pre-existing, documented in code):** image route uses `require_user` (phase-79 posture, same gate as the document content endpoint) rather than the phase text's "public" parenthetical — matches the endpoint it mirrors.

**Next pending phase:** `123_chat_image_questions`.
This commit is contained in:
2026-09-25 01:54:23 -04:00
parent 0f77e9a876
commit a19d78d284
63 changed files with 5484 additions and 111 deletions
+37 -9
View File
@@ -48,25 +48,30 @@ def test_health_reports_ok(client) -> None:
def test_config_returns_default_app_metadata(client, db: Session) -> None:
"""GET /api/config is public (anonymous) and returns exactly five
"""GET /api/config is public (anonymous) and returns exactly six
keys — the phase-39 app metadata, the phase-59 docs flag (inert
false while BOR_DOCS_REPO is empty — the "Save as doc" gating),
and the phase-62 UI customization strings (composer placeholder,
footer line). Phase 91: with an empty ui_settings table the
effective strings are the env defaults (B1 — DB-over-env, the row
absent here); the retired CSS-file theming's ``theme`` key is gone
(task 03 — the five keys are the entire contract)."""
the phase-122 images flag (default false in the test env —
LOCKED A3: off by default), and the phase-62 UI customization
strings (composer placeholder, footer line). Phase 91: with an
empty ui_settings table the effective strings are the env defaults
(B1 — DB-over-env, the row absent here); the retired CSS-file
theming's ``theme`` key is gone (task 03 — the six keys are the
entire contract)."""
_clear_ui_settings(db)
r = client.get("/api/config")
assert r.status_code == 200
body = r.json()
assert set(body) == {
"app_name", "version", "docs_repo_configured",
"input_placeholder", "footer_text",
"images", "input_placeholder", "footer_text",
}
assert body["app_name"] == "Brain of Reese"
assert body["version"] == get_settings().app_version
assert body["docs_repo_configured"] is False
# Phase 122 (task 01): the images flag is the default off (the
# test env sets no BOR_IMAGES) — a real bool, not a truthy string.
assert body["images"] is False
# Phase 62: UNSET => the phase-61 neutral copy stands (the
# byte-identical contract).
assert body["input_placeholder"] == "Ask me anything…"
@@ -91,7 +96,7 @@ def test_config_follows_overridden_app_name(client, db: Session) -> None:
body = r.json()
assert set(body) == {
"app_name", "version", "docs_repo_configured",
"input_placeholder", "footer_text",
"images", "input_placeholder", "footer_text",
}
assert body["app_name"] == "Brain of Testy"
assert body["version"] == "0.1.0"
@@ -122,7 +127,7 @@ def test_config_serves_ui_customization_overrides(client, db: Session) -> None:
body = r.json()
assert set(body) == {
"app_name", "version", "docs_repo_configured",
"input_placeholder", "footer_text",
"images", "input_placeholder", "footer_text",
}
assert body["input_placeholder"] == "Ask the vault…"
assert body["footer_text"] == "Powered by my own models"
@@ -153,6 +158,29 @@ def test_config_docs_flag_tracks_settings(client, db: Session) -> None:
fastapi_app.dependency_overrides.clear()
def test_config_images_flag_tracks_settings(client, db: Session) -> None:
"""Phase 122 (task 01): ``images`` mirrors ``settings.images``
(``BOR_IMAGES``) — a real bool (never a truthy string) that flips
true the moment the toggle is on: that flag is the entire frontend
gating of the image affordances (the phase-123 attach control,
optionally the Sources page hint)."""
from app.config import Settings
from app.main import app as fastapi_app
_clear_ui_settings(db)
fastapi_app.dependency_overrides[get_settings] = lambda: Settings(
images=True,
)
try:
r = client.get("/api/config")
assert r.status_code == 200
body = r.json()
assert isinstance(body["images"], bool)
assert body["images"] is True
finally:
fastapi_app.dependency_overrides.clear()
def test_suggestions_returns_list(client) -> None:
# Phase 79: the chips are user-gated — sign in as the admin first
# (the test's purpose is the list shape, not the auth contract).
+3
View File
@@ -431,8 +431,11 @@ def test_document_content_admin_contract(client: TestClient, db) -> None:
"content",
"indexed_at",
"chunks",
"is_image", # added in phase 122 (task 04) — always present
}
assert body["summary"] is None
assert body["is_image"] is False # text doc — and no image_url key (never null)
assert "image_url" not in body
# Unknown docs still 404 (same shape as the phase-16 pin).
r = client.get("/api/documents/content", params={"source": "docs", "path": "nope.md"})
+121
View File
@@ -2076,3 +2076,124 @@ def test_done_event_related_defaults_empty_and_old_payload_parses() -> None:
dumped = ChatDoneEvent(**new_payload).model_dump()
assert [r["path"] for r in dumped["related"]] == ["b.md"]
assert [r["path"] for r in dumped["sources"]] == ["a.md"]
# ---------------------------------------------------------------------------
# Phase 122, task 05 — the SSE source frame's OPTIONAL image_url (the
# shared ``source_ref_with_image`` builder: present on an image doc's
# ref only, omitted — never null — on every text ref). Task 06
# finalizes the phase-122 suite here with the story-level pins.
# ---------------------------------------------------------------------------
def _seed_image_doc(db) -> Document:
"""An ``is_image`` documents row the agent can ``read`` (no chunks
needed — the read tool resolves the row by ``(source, path)`` and
serves its ``content``; the frame needs ``is_image`` + ``id``
only). The fixture's teardown truncates the tables."""
doc = Document(
id=uuid.uuid4(),
source="docs",
path="pic.png",
full_path="/tmp/pic.png",
title="pic",
content="A red square on a white background.",
summary="A red square on a white background.",
content_hash="0" * 64,
created_at=_FIXTURE_CREATED_AT,
is_image=True,
image_path="/tmp/pic.png",
)
db.add(doc)
db.commit()
return doc
def test_grounded_turn_reading_image_doc_frame_carries_image_url_only_for_it(
client, db, seeded_kb: FakeRagLLM
) -> None:
"""Task 05 wire contract: a mocked grounded answer whose agent
READS an image doc → the done frame's ref for THAT doc alone
carries ``image_url`` (the bytes route the frontend's sources
block renders from); the text-doc related refs carry NO
``image_url`` key at all (the omission rule — the key is absent,
never null)."""
doc = _seed_image_doc(db)
scripted = FakeRagLLM(
tool_script=[
[
ToolCallPiece(
id="call_1",
name="read",
arguments={"path": "docs/pic.png"},
)
]
]
)
fastapi_app.dependency_overrides[chat_api.get_llm] = lambda: scripted
try:
_, _, frames = _stream_chat(client, QUESTION)
finally:
fastapi_app.dependency_overrides.clear()
done = frames[-1]
assert done["deflected"] is False
# The read image doc is the citation surface (phase 119, A1) — and
# its ref is the frame's ONLY image_url carrier.
sources = done["sources"]
assert [(s["source"], s["path"]) for s in sources] == [("docs", "pic.png")]
assert sources[0]["image_url"] == f"/api/documents/{doc.id}/image"
# The related tier (ranks 6–7 for the Kubernetes question) is text
# docs — the key is ABSENT on every one of them, not null.
related = done["related"]
assert related
for ref in related:
assert "image_url" not in ref
def test_text_only_grounded_turn_frame_has_no_image_url_key_anywhere(
client, db, seeded_kb: FakeRagLLM
) -> None:
"""The omission rule at the BYTE level (the phase's byte-identity
criterion): a grounded turn whose cited + related docs are ALL
text docs serializes a done frame with no ``image_url`` key
anywhere — checked on the raw wire text (not a re-serialized
dict), and every ref keeps exactly the pre-phase-122 key set."""
scripted = FakeRagLLM(
tool_script=[
[
ToolCallPiece(
id="call_1",
name="read",
arguments={"path": "docs/homelab/kubernetes.md"},
)
]
]
)
fastapi_app.dependency_overrides[chat_api.get_llm] = lambda: scripted
try:
with client.stream("POST", "/api/chat", json={"message": QUESTION}) as r:
assert r.status_code == 200
buf = ""
raw_frames: list[str] = []
for part in r.iter_text():
buf += part
while "\n\n" in buf:
frame, buf = buf.split("\n\n", 1)
frame = frame.strip()
if frame.startswith("data:"):
raw_frames.append(frame)
finally:
fastapi_app.dependency_overrides.clear()
done_raw = [f for f in raw_frames if '"type": "done"' in f]
assert len(done_raw) == 1
# The BYTE check: the key is absent from the wire text itself.
assert "image_url" not in done_raw[0]
done = json.loads(done_raw[0].removeprefix("data:").strip())
assert done["deflected"] is False
assert [(s["source"], s["path"]) for s in done["sources"]] == [
("docs", "homelab/kubernetes.md")
]
for ref in done["sources"] + done["related"]:
assert set(ref) == {"source", "path", "title"} # the pre-122 key set
+611
View File
@@ -7,22 +7,33 @@ Uses the real compose Postgres (``db`` fixture) and FastAPI's TestClient.
"""
from __future__ import annotations
import asyncio
import base64
import hashlib
import inspect
import itertools
import logging
import uuid
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any
import pytest
from fastapi.testclient import TestClient
from sqlalchemy import delete, func, select, text
import app.api.docs as docs_api
import app.rag.importer as rag_importer
from app.config import Settings
from app.core import tokens as token_service
from app.main import app as fastapi_app
from app.models import Chunk, Document, FolderSummary, GitSource
from app.rag import git_sources as rag_git_sources
from app.rag.folder_summaries import missing_folder_summaries
from app.rag.importer import import_sources
from app.rag.llm import LLMError
from app.rag.summarizer import DESCRIBE_PROMPT
from tests.fakes import FakeEmbedder
_TREE_TABLES = "chunks, documents, folder_summaries, git_sources"
@@ -823,3 +834,603 @@ def test_docs_tree_stat_walk_equivalence_with_flat_list(admin_client, db) -> Non
)
_truncate_tree_tables(db)
# ---------------------------------------------------------------------------
# Phase 122, task 02 — image ingest end-to-end (the full ``import_sources``
# pipeline against the real DB; task 06 finalizes the phase-122 suite here
# with the image route + content-endpoint shapes).
# ---------------------------------------------------------------------------
#: A real 1×1 transparent PNG — the importer is content-agnostic (it
#: never parses the image), but a well-formed fixture keeps the tests
#: honest about what a real upload looks like.
PNG_1X1 = base64.b64decode(
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAC0lEQVR4nGP4DwQACfsD/fteaysAAAAASUVORK5CYII="
)
def _image_llm[
ImageLLM: FakeEmbedder
](tmp_path, llm_cls: type[ImageLLM] = FakeEmbedder, **kwargs) -> ImageLLM:
"""A fake LLM with the phase-122 image knobs (toggle ON by default;
the image dir defaults under *tmp_path* unless overridden).
*llm_cls* (task 03) may be the mock vision client (``_MockVisionLLM``
below) for the no-seam-patch end-to-end path — the PEP 695 type
parameter keeps the helper's return type honest (``chat_models``).
"""
kwargs.setdefault("_env_file", None)
kwargs.setdefault("images", True)
kwargs.setdefault("image_dir", str(tmp_path / "images"))
llm = llm_cls()
llm.settings = Settings(**kwargs) # pyright: ignore[reportCallIssue]
return llm
def _patch_description(monkeypatch, description) -> None:
"""Pin the task-02 seam (``rag_importer._describe_or_skip``) to
*description*. Task 03 fills the seam with the CHAT model's vision
call — these end-to-end mechanics do not change with it."""
async def _fake(llm, *, data, source, rel, full_path):
return description
monkeypatch.setattr(rag_importer, "_describe_or_skip", _fake)
def _cleanup_source(db, source: str) -> None:
for doc in db.scalars(select(Document).where(Document.source == source)).all():
db.delete(doc)
db.commit()
def test_import_sources_images_on_indexes_image_docs(
db, tmp_path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""``images=True`` end-to-end: a standalone image in a source becomes
a Document — bytes digested, the persistent copy in ``image_dir``
(``<doc-id>.png``), ``content`` = the (mock) description, and ONLY
that text is embedded (the content chunks + the phase-30 summary
chunk, 768 dims) — while a text doc in the same source stays a
plain text doc."""
source_root = tmp_path / "imgsource"
source_root.mkdir()
(source_root / "diagram.png").write_bytes(PNG_1X1)
(source_root / "notes.md").write_text("# Notes\n\nBody text.\n", encoding="utf-8")
llm = _image_llm(tmp_path)
_patch_description(monkeypatch, "A network diagram of the homelab VLANs.")
try:
summary = asyncio.run(
import_sources([source_root], llm, session=db, prune=True)
)
assert (summary.files, summary.added, summary.images_failed) == (2, 2, 0)
assert summary.formats == {"md": 1, "png": 1}
doc = db.scalar(
select(Document).where(
Document.source == source_root.name, Document.path == "diagram.png"
)
)
assert doc is not None, "the image file must become a document"
assert doc.is_image is True and doc.image_path is not None
assert doc.content == "A network diagram of the homelab VLANs."
assert doc.title == "diagram" # the non-markdown stem rule
copy = Path(doc.image_path)
assert copy.parent == Path(llm.settings.image_dir)
assert copy.name == f"{doc.id}.png"
assert copy.read_bytes() == PNG_1X1
# The ONLY embedded text is the description (the embedding model
# never sees pixels): every content chunk carries it, the
# phase-30 summary chunk exists + is embedded, 768 dims. Task
# 03: the description IS the summary (stored verbatim — no
# ``lite`` call, no pointer line), so the summary chunk mirrors
# ``doc.content`` exactly.
chunks = db.scalars(select(Chunk).where(Chunk.document_id == doc.id)).all()
content_chunks = [c for c in chunks if not c.is_summary]
assert [c.content for c in content_chunks] == [doc.content]
assert doc.summary == doc.content
summary_chunks = [c for c in chunks if c.is_summary]
assert len(summary_chunks) == 1 and summary_chunks[0].position == -1
assert summary_chunks[0].content == doc.content
for c in chunks:
assert c.embedding is not None and len(c.embedding) == 768
# The text doc is untouched by the image machinery.
md = db.scalar(
select(Document).where(
Document.source == source_root.name, Document.path == "notes.md"
)
)
assert md is not None and md.is_image is False and md.image_path is None
finally:
_cleanup_source(db, source_root.name)
class _MockVisionLLM(FakeEmbedder):
"""The phase-122 mock VISION client (task 03): the CHAT model
answers the multimodal describe call (the image bytes' data URL)
with a fixed, retrieval-oriented description; text (``lite``) calls
keep the ``FakeEmbedder`` behaviour. The REAL
``rag_importer._describe_or_skip`` → ``summarizer.describe_image``
chain runs end-to-end against it (no seam patch); every chat
call's model is recorded (``chat_models``)."""
DESCRIPTION = (
"A network diagram of the homelab VLANs: the core switch, the "
"router, and three labeled subnets."
)
def __init__(self) -> None:
super().__init__()
self.chat_models: list[str | None] = []
async def chat(self, messages, model=None):
self.chat_calls.append(list(messages))
self.chat_models.append(model)
user = next((m["content"] for m in messages if m.get("role") == "user"), "")
if isinstance(user, list):
# The phase-122 describe call — the multimodal message.
return self.DESCRIPTION
first = user.split()
return "Summary of " + (first[0] if first else "<empty>")
def test_import_sources_mock_vision_client_end_to_end(db, tmp_path) -> None:
"""Task 03 end-to-end (NO seam patch): a fixture PNG through the
mock vision client — the real ``_describe_or_skip`` →
``describe_image`` → CHAT-model call — yields a doc whose
``content`` == ``summary`` == the description, with its
``is_summary`` chunk embedded, and whose ONLY embedded text is that
description (the embedding model never sees pixels)."""
source_root = tmp_path / "visione2e"
source_root.mkdir()
(source_root / "diagram.png").write_bytes(PNG_1X1)
llm = _image_llm(tmp_path, _MockVisionLLM)
try:
summary = asyncio.run(
import_sources([source_root], llm, session=db)
)
assert (summary.files, summary.added, summary.images_failed) == (1, 1, 0)
# The describe call went to the CHAT model (LOCKED A3), once.
assert llm.chat_models == [llm.settings.llm_chat_model]
# The wire shape: the multimodal user message — the fixed
# prompt's text part + the image's data-URL part.
(message,) = llm.chat_calls[0]
assert message["role"] == "user"
content: Any = message["content"] # the multimodal part list
assert content[0] == {"type": "text", "text": DESCRIBE_PROMPT}
assert content[1]["type"] == "image_url"
assert content[1]["image_url"]["url"].startswith("data:image/png;base64,")
doc = db.scalar(
select(Document).where(
Document.source == source_root.name, Document.path == "diagram.png"
)
)
assert doc is not None, "the image file must become a document"
assert doc.is_image is True and doc.image_path is not None
assert doc.content == _MockVisionLLM.DESCRIPTION
assert doc.summary == _MockVisionLLM.DESCRIPTION # task 03: verbatim
# The ONLY embedded text of the doc is the description — twice
# (the one content chunk + the is_summary chunk), 768 dims.
chunks = db.scalars(select(Chunk).where(Chunk.document_id == doc.id)).all()
summary_chunks = [c for c in chunks if c.is_summary]
assert len(summary_chunks) == 1 and summary_chunks[0].position == -1
assert all(c.content == _MockVisionLLM.DESCRIPTION for c in chunks)
for c in chunks:
assert c.embedding is not None and len(c.embedding) == 768
embedded = [t for batch in llm.calls for t in batch]
assert embedded == [_MockVisionLLM.DESCRIPTION] * 2
finally:
_cleanup_source(db, source_root.name)
class _MockVisionFailsLLM(FakeEmbedder):
"""A NON-VISION chat model (LOCKED A3's honest failure): the
multimodal describe call raises (the SDK errors — a chat model
without vision rejects the ``image_url`` part), text (``lite``)
calls keep the ``FakeEmbedder`` behaviour."""
async def chat(self, messages, model=None):
self.chat_calls.append(list(messages))
user = next((m["content"] for m in messages if m.get("role") == "user"), "")
if isinstance(user, list):
raise LLMError("simulated non-vision chat model (test sentinel)")
first = user.split()
return "Summary of " + (first[0] if first else "<empty>")
def test_import_sources_failing_vision_skips_image_keeps_sync_green(
db, tmp_path, caplog: pytest.LogCaptureFixture
) -> None:
"""LOCKED A3 fail-soft end-to-end (the task-06 integration pin):
a fixture PNG through a NON-VISION chat model — the real seam, no
patch — skips the image doc (``images_failed == 1``, NO row, NO
orphan copy — not even the image dir) while the TEXT doc in the
same source is indexed as usual (the sync completes, no row
mutation anywhere for the failed image)."""
source_root = tmp_path / "visionfail"
source_root.mkdir()
(source_root / "diagram.png").write_bytes(PNG_1X1)
(source_root / "notes.md").write_text("# Notes\n\nBody.\n", encoding="utf-8")
llm = _image_llm(tmp_path, _MockVisionFailsLLM)
try:
with caplog.at_level(logging.INFO, logger="app.importer"):
summary = asyncio.run(
import_sources([source_root], llm, session=db)
)
assert (summary.files, summary.added, summary.images_failed) == (2, 1, 1)
# The image: no row, no copy (the dir itself was never created).
assert (
db.scalar(
select(Document).where(
Document.source == source_root.name, Document.path == "diagram.png"
)
)
is None
)
assert not Path(llm.settings.image_dir).expanduser().exists()
# The text doc indexed as usual (the sync stayed green).
md = db.scalar(
select(Document).where(
Document.source == source_root.name, Document.path == "notes.md"
)
)
assert md is not None and md.is_image is False
# The importer's warning names the document (the PLAN §9 signal).
warnings = [
r
for r in caplog.records
if r.name == "app.importer" and "image description failed" in r.getMessage()
]
assert len(warnings) == 1 and warnings[0].levelno == logging.WARNING
assert f"source={source_root.name} path=diagram.png" in warnings[0].getMessage()
finally:
_cleanup_source(db, source_root.name)
def test_import_sources_images_off_ignores_and_prune_guard_protects(
db, tmp_path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""``images=False`` (the default) end-to-end: the walk ignores the
image file entirely (not counted, no row, no copy), and a
``prune=True`` run MUST NOT delete a pre-existing image doc — the
LOCKED prune guard (invisible to the walk ≠ deleted)."""
source_root = tmp_path / "imgsrc_off"
source_root.mkdir()
(source_root / "diagram.png").write_bytes(PNG_1X1)
_patch_description(monkeypatch, "A network diagram.")
try:
llm_on = _image_llm(tmp_path)
s_on = asyncio.run(import_sources([source_root], llm_on, session=db))
assert s_on.added == 1
doc = db.scalar(
select(Document).where(
Document.source == source_root.name, Document.path == "diagram.png"
)
)
assert doc is not None
copy = Path(doc.image_path)
assert copy.exists()
# Toggle OFF (a fresh fake on the code defaults): the walk is
# blind to the file, and prune protects the pre-existing image
# doc + its copy.
llm_off = FakeEmbedder() # Settings(_env_file=None) → images False
s_off = asyncio.run(
import_sources([source_root], llm_off, session=db, prune=True)
)
assert (s_off.files, s_off.added, s_off.pruned) == (0, 0, 0)
assert db.scalar(select(Document).where(Document.id == doc.id)) is not None
assert copy.exists(), "the copy survives with the doc"
finally:
_cleanup_source(db, source_root.name)
def test_import_sources_toggle_on_prunes_deleted_image_with_copy(
db, tmp_path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Toggle ON, the image file deleted: the normal prune runs — the
doc row AND its ``image_dir`` copy are removed (the copy's
lifecycle is tied to the row)."""
source_root = tmp_path / "imgsrc_prune"
source_root.mkdir()
(source_root / "diagram.png").write_bytes(PNG_1X1)
llm_on = _image_llm(tmp_path)
_patch_description(monkeypatch, "A network diagram.")
try:
asyncio.run(import_sources([source_root], llm_on, session=db))
doc = db.scalar(
select(Document).where(
Document.source == source_root.name, Document.path == "diagram.png"
)
)
assert doc is not None
copy = Path(doc.image_path)
assert copy.exists()
(source_root / "diagram.png").unlink()
s = asyncio.run(
import_sources([source_root], llm_on, session=db, prune=True)
)
assert (s.pruned, s.added, s.unchanged) == (1, 0, 0)
assert (
db.scalar(select(Document).where(Document.source == source_root.name))
is None
)
assert not copy.exists(), "the image_dir copy is deleted with the doc"
finally:
_cleanup_source(db, source_root.name)
# ---------------------------------------------------------------------------
# Phase 122 (task 04) — the image BYTES route + the content/tree wire
# affordance (the serve side of the image-document contract).
# ---------------------------------------------------------------------------
def _seed_image_doc(
db,
tmp_path: Path,
*,
source: str = "ImgSrc",
path: str = "pic.png",
data: bytes = PNG_1X1,
content: str = "A red square on a white background.",
image_path: str | None = "auto",
doc_id: str | None = None,
) -> Document:
"""One ``is_image`` document row with its persistent copy (the
importer's ``image_dir`` layout) under *tmp_path*; ``image_path``
``"auto"`` writes the copy, ``None`` leaves the row without a copy
(the lost-copy corner)."""
doc = Document(
id=uuid.UUID(doc_id) if doc_id else uuid.uuid4(),
source=source,
path=path,
full_path=f"/tmp/{path}",
title=path.rsplit(".", 1)[0],
content=content,
content_hash=hashlib.sha256(data).hexdigest(),
indexed_at=datetime.now(UTC),
created_at=datetime(2026, 1, 1, tzinfo=UTC),
summary=content,
is_image=True,
)
if image_path == "auto":
copy = tmp_path / f"{doc.id}{Path(path).suffix}"
copy.write_bytes(data)
doc.image_path = str(copy)
elif image_path is not None:
doc.image_path = image_path
db.add(doc)
db.flush()
db.add_all(
[
Chunk(
document_id=doc.id, position=0, content=content, embedding=[0.01] * 768
),
Chunk(
document_id=doc.id,
position=-1,
content=content,
embedding=[0.01] * 768,
is_summary=True,
),
]
)
db.commit()
return doc
def _cleanup_kb(db) -> None:
"""Truncate the KB tables. The settle commit matters: SQLAlchemy
does NOT autoflush pending ORM objects before a raw ``text()``
statement — a pending chunks INSERT flushed *after* the TRUNCATE
would FK-violate (the document row is already gone), so any
pending state is committed (then truncated) first."""
db.commit()
db.execute(text("TRUNCATE chunks, documents"))
db.commit()
@pytest.mark.parametrize(
("ext", "mime"),
[
(".png", "image/png"),
(".jpg", "image/jpeg"),
(".jpeg", "image/jpeg"),
(".webp", "image/webp"),
(".gif", "image/gif"),
(".bmp", "image/bmp"),
],
)
def test_image_route_serves_exact_bytes_with_content_type(
admin_client: TestClient, db, tmp_path: Path, ext: str, mime: str
) -> None:
"""The serve contract: the route streams the EXACT stored bytes
(a per-extension sentinel — a mix-up between the six formats is
caught) with the extension's ``Content-Type`` (the
``IMAGE_MIMES`` map — one map, one truth) and
``Cache-Control: private, max-age=3600`` (content-hashed bytes —
long enough, bustable by re-upload)."""
_cleanup_kb(db)
try:
data = PNG_1X1 + ext.encode("ascii") # per-extension sentinel bytes
doc = _seed_image_doc(db, tmp_path, path=f"pic{ext}", data=data)
r = admin_client.get(f"/api/documents/{doc.id}/image")
assert r.status_code == 200
assert r.headers["content-type"] == mime # exact type per extension
assert r.headers["cache-control"] == "private, max-age=3600"
assert r.content == data # the exact uploaded bytes, nothing else
finally:
_cleanup_kb(db)
def test_image_route_404_matrix(admin_client: TestClient, db, tmp_path: Path) -> None:
"""Every non-servable case is 404 ``document not found`` (the
router's unknown-document shape — the same detail string the
content endpoint uses): a missing id, a MALFORMED id (an unparseable
string maps here, not to a 422 — a guessed id is an unknown
document), a text doc, an image doc whose ``image_path`` is NULL,
and a row whose copy was lost on disk (defensive — the row exists,
the bytes don't)."""
_cleanup_kb(db)
try:
_seed_doc(db, "TextSrc", "note.md", "Note", 1, datetime.now(UTC))
db.commit() # the module's _seed_doc leaves the row uncommitted
text_doc_id = db.scalar(
select(Document.id).where(
Document.source == "TextSrc", Document.path == "note.md"
)
)
no_copy = _seed_image_doc(db, tmp_path, path="nopy.png", image_path=None)
lost = _seed_image_doc(db, tmp_path, path="lost.png")
assert lost.image_path is not None # the "auto" copy was written
Path(lost.image_path).unlink() # the copy is lost (the row remains)
for doc_id in (
str(uuid.uuid4()), # missing id
"not-a-uuid", # malformed id → 404, not 422
str(text_doc_id), # text doc
str(no_copy.id), # image doc, image_path NULL
str(lost.id), # image doc, copy lost
):
r = admin_client.get(f"/api/documents/{doc_id}/image")
assert r.status_code == 404, doc_id
assert r.json() == {"detail": "document not found"}, doc_id
finally:
_cleanup_kb(db)
def test_image_route_requires_user_like_the_content_endpoint(
admin_client: TestClient, db, tmp_path: Path
) -> None:
"""Phase 79 posture (the task's "PUBLIC, like the document content
endpoint" — the content endpoint has been user-gated since phase
79; the ONLY anonymous surface is the shared chats, PLAN A10): an
anonymous caller gets 401 ``authentication required`` before any
row is read (a FRESH client — the module's fixture ``client``
stays unsigned here), a signed-in caller gets the bytes."""
_cleanup_kb(db)
try:
doc = _seed_image_doc(db, tmp_path)
anonymous = TestClient(fastapi_app)
r = anonymous.get(f"/api/documents/{doc.id}/image")
assert r.status_code == 401
assert r.json() == {"detail": "authentication required"}
# The signed-in client (admin) passes.
assert admin_client.get(f"/api/documents/{doc.id}/image").status_code == 200
finally:
_cleanup_kb(db)
def test_content_endpoint_exposes_the_image_affordance(
admin_client: TestClient, db, tmp_path: Path
) -> None:
"""The content endpoint (the viewer's data source): ``is_image`` is
ALWAYS present (text doc: false — the one new key; the wire shape
gains nothing else), and ``image_url`` — the bytes route's path —
is present for an image doc and ABSENT for a text doc (never null,
the ``DocContent`` omission rule)."""
_cleanup_kb(db)
try:
_seed_doc(db, "TextSrc", "note.md", "Note", 1, datetime.now(UTC))
db.commit() # the app's endpoint session reads committed data only
r = admin_client.get(
"/api/documents/content", params={"source": "TextSrc", "path": "note.md"}
)
assert r.status_code == 200
body = r.json()
assert body["is_image"] is False
assert "image_url" not in body # absent — never null (text doc)
doc = _seed_image_doc(db, tmp_path, source="ImgSrc", path="pic.png")
r = admin_client.get(
"/api/documents/content", params={"source": "ImgSrc", "path": "pic.png"}
)
assert r.status_code == 200
body = r.json()
assert body["is_image"] is True
assert body["image_url"] == f"/api/documents/{doc.id}/image"
assert body["content"] == body["summary"] # the description (task 03)
finally:
_cleanup_kb(db)
def _tree_file_nodes_all(sources) -> list[dict]:
"""Every file node of a tree response, walked recursively (all
sources — env-registered 0-document sources may join the response
when the ``git_sources`` table is truncated, and they carry no
file nodes; the assertions below hold over whatever files exist).
"""
files: list[dict] = []
def _walk(node: dict) -> None:
for child in node.get("children", ()):
if child["kind"] == "file":
files.append(child)
else:
_walk(child)
for source in sources:
_walk(source)
return files
def test_tree_image_file_node_affordance_and_text_node_byte_identical(
admin_client: TestClient, db, tmp_path: Path
) -> None:
"""The tree (the RAG view's single fetch): an image doc's file node
carries the thumbnail affordance (``is_image`` true,
``image_url`` = the bytes route's path, ``summary`` verbatim — the
RAG view's thumbnail ``alt``); EVERY text file node keeps the
pre-phase wire shape byte-identically (the six keys — no
``is_image``/``image_url``/``summary`` — the phase's
byte-identical criterion: the fields are row-driven, so a KB with
no image rows serializes exactly as pre-phase)."""
_truncate_tree_tables(db)
try:
base = datetime.now(UTC)
_seed_doc(db, "MixedSrc", "a.md", "A", 1, base)
img = _seed_image_doc(db, tmp_path, source="MixedSrc", path="pic.png")
r = admin_client.get("/api/docs/tree") # both seeds committed (_seed_image_doc)
assert r.status_code == 200
files = {f["path"]: f for f in _tree_file_nodes_all(r.json()["sources"])}
# Text node: byte-identical pre-phase wire shape (no image keys).
assert set(files["a.md"]) == {
"kind", "path", "title", "chunks", "created_at", "indexed_at"
}
# Image node: the affordance rides the node.
pic = files["pic.png"]
assert pic["is_image"] is True
assert pic["image_url"] == f"/api/documents/{img.id}/image"
assert pic["summary"] == "A red square on a white background."
finally:
_truncate_tree_tables(db)
def test_tree_with_no_image_rows_is_byte_identical(admin_client: TestClient, db) -> None:
"""The pre-phase KB (no ``is_image`` rows): the image-docs map is
empty and EVERY file node serializes in the pre-phase shape — the
row-driven fields introduce no wire change at all (the phase's
byte-identical criterion, the toggle irrelevant)."""
_truncate_tree_tables(db)
try:
base = datetime.now(UTC)
_seed_doc(db, "PlainSrc", "x.md", "X", 2, base)
db.commit() # the app's endpoint session reads committed data only
r = admin_client.get("/api/docs/tree")
assert r.status_code == 200
files = _tree_file_nodes_all(r.json()["sources"])
assert len(files) == 1 # the seeded doc (env sources carry no files)
assert set(files[0]) == {
"kind", "path", "title", "chunks", "created_at", "indexed_at"
}
finally:
_truncate_tree_tables(db)
+7 -2
View File
@@ -422,11 +422,16 @@ def test_content_200_all_fields(client, db) -> None:
assert r.status_code == 200
body = r.json()
# Wire-additive (phase 106, task 05): ``created_at`` joins the
# content shape (after ``summary``, before ``content``).
# content shape (after ``summary``, before ``content``) — and
# (phase 122, task 04) ``is_image`` joins it ALWAYS present
# (text docs: false); ``image_url`` is ABSENT for a text doc
# (never null — the ``DocContent`` omission rule).
assert set(body) == {
"source", "path", "title", "format", "summary", "created_at",
"content", "indexed_at", "chunks",
"content", "indexed_at", "chunks", "is_image",
}
assert body["is_image"] is False
assert "image_url" not in body # absent — never null (text doc)
datetime.fromisoformat(body["created_at"]) # raises if not ISO-8601
assert body["source"] == "Homelab"
assert body["path"] == "kubernetes.md"
+344
View File
@@ -0,0 +1,344 @@
"""Integration: migration 0022 (documents.is_image + image_path) schema
contract (phase 122, task 02).
Drives the **real Alembic engine** against the live dev database
(``podman compose up -d db``), mirroring the house pattern of
``test_migration_0021.py`` (information_schema assertions on the state
the migration must leave). The tests target the 0021 → 0022 step
explicitly so later migrations cannot break the pins:
* upgrade 0021 → 0022 → ``is_image`` exists with the full contract —
BOOLEAN, NOT NULL, server default ``false`` — and ``image_path`` —
TEXT, NULLABLE, no server default — while the 0021 ``documents``
schema (``content``/``content_hash`` NOT NULL, ``summary`` NULLABLE,
``created_at`` + ``created_at_manual``, the (source, path) unique
constraint — asserted column-based, since the suite's table
self-heal renames copied constraints) survives;
* a ``documents`` row inserted while the DB is at 0021 backfills
``is_image`` to ``false`` and ``image_path`` to NULL (every
pre-phase-122 row is a text doc — the LOCKED A3 default);
* the ORM contract agrees: a freshly inserted ``Document`` without the
image fields reads ``is_image is False`` / ``image_path is None``,
and one with them round-trips through a fresh session;
* downgrade to 0021 → both columns are GONE (A13) while the row
survives; upgrade back to 0022 → the columns are back (round-trip).
The ``alembic`` fixture guarantees the DB ends at head even if a test
fails or the process is interrupted.
"""
from __future__ import annotations
import uuid
from collections.abc import Iterator
from typing import Any
import pytest
from alembic.config import Config
from sqlalchemy import text
from sqlalchemy.orm import Session
from alembic import command
from app.db import SessionLocal, db_available
from app.models import Document
SOURCE = "mig0022"
PATH_TEXT = "notes/readme.md"
PATH_IMAGE = "notes/diagram.png"
@pytest.fixture()
def alembic(db: Session) -> Iterator[Config]:
"""Real Alembic config bound to the dev DB (URL from app settings).
Starts at head (repairs an interrupted earlier run); teardown
upgrades to head no matter what happened, so the dev DB is never
left below head.
"""
if not db_available():
pytest.skip("Postgres not reachable — run `podman compose up -d db` first")
cfg = Config() # no alembic.ini file — env.py gets the URL from app config
cfg.set_main_option("script_location", "alembic")
command.upgrade(cfg, "head")
try:
yield cfg
finally:
# Release the test session's open transaction BEFORE the repair
# DDL: an idle-in-transaction SELECT holds an ACCESS SHARE lock
# on ``documents``, which would deadlock the repair's
# ``ALTER TABLE`` (0022) forever.
db.rollback()
command.upgrade(cfg, "head")
def _version(db: Session) -> str | None:
return db.execute(text("SELECT version_num FROM alembic_version")).scalar()
def _column(db: Session, column: str) -> tuple[Any, ...] | None:
"""(data_type, is_nullable, column_default) for one documents
column."""
row = db.execute(
text(
"SELECT data_type, is_nullable, column_default"
" FROM information_schema.columns"
" WHERE table_name = 'documents' AND column_name = :c"
),
{"c": column},
).fetchone()
return tuple(row) if row is not None else None
def _insert_sql(db: Session, path: str) -> uuid.UUID:
"""Insert one documents row with the PRE-0022 column set (the 0021
shape — the image columns, when present, are omitted so their
backfill is what the row reads)."""
row_id = uuid.uuid4()
db.execute(
text(
"INSERT INTO documents (id, source, path, full_path, title,"
" content, content_hash)"
" VALUES (:id, :s, :p, :f, :t, :c, :h)"
),
{
"id": row_id,
"s": SOURCE,
"p": path,
"f": f"/tmp/{SOURCE}/{path}",
"t": path.rsplit("/", 1)[-1],
"c": "content",
"h": "0" * 64,
},
)
db.commit()
return row_id
def _delete(db: Session, row_id: uuid.UUID) -> None:
db.execute(text("DELETE FROM documents WHERE id = :id"), {"id": row_id})
db.commit()
def _unique_column_sets(db: Session) -> set[tuple[str, ...]]:
"""The column tuples of every UNIQUE constraint on ``documents``.
Column-based (not name-based): the integration suite's table
self-heal (``tests/integration/conftest.py``) rewrites bloated
tables via ``CREATE TABLE (LIKE …)``, which renames copied
constraints (PG auto-names them) — the (source, path) uniqueness
contract is what must hold, not the original name.
"""
rows = db.execute(
text(
"SELECT (SELECT string_agg(a.attname, ',' ORDER BY k.ord)"
" FROM unnest(c.conkey) WITH ORDINALITY k(attnum, ord)"
" JOIN pg_attribute a"
" ON a.attrelid = c.conrelid AND a.attnum = k.attnum)"
" FROM pg_constraint c"
" WHERE c.contype = 'u' AND c.conrelid = 'documents'::regclass"
)
).fetchall()
return {tuple(r[0].split(",")) for r in rows}
def test_upgrade_to_0022_adds_image_columns(db: Session, alembic: Config) -> None:
"""Upgrade 0021 → 0022: ``is_image`` exists with the full contract
(BOOLEAN, NOT NULL, server default ``false`` — every pre-phase-122
row is a text doc) and ``image_path`` (TEXT, NULLABLE, no server
default — NULL for text docs), both ABSENT at 0021; a pre-0022 row
backfills ``is_image`` to ``false`` + ``image_path`` to NULL; and
the 0021 table contract survives the additive upgrade."""
command.downgrade(alembic, "0021") # start from the pre-0022 state
assert _version(db) == "0021"
assert _column(db, "is_image") is None, "is_image must be absent at 0021"
assert _column(db, "image_path") is None, "image_path must be absent at 0021"
pre_id = _insert_sql(db, PATH_TEXT) # the 0021 column set
try:
command.upgrade(alembic, "0022")
assert _version(db) == "0022", "alembic_version must be at 0022"
is_image = _column(db, "is_image")
assert is_image is not None, "documents.is_image is missing"
assert is_image[0] == "boolean", "is_image must be BOOLEAN"
assert is_image[1] == "NO", "is_image must be NOT NULL"
assert is_image[2] == "false", (
"is_image must carry the `false` server default — every"
" pre-phase-122 row is a text doc"
)
image_path = _column(db, "image_path")
assert image_path is not None, "documents.image_path is missing"
assert image_path[0] == "text", "image_path must be TEXT"
assert image_path[1] == "YES", "image_path must be NULLABLE"
assert image_path[2] is None, (
"image_path must carry NO server default — NULL is the"
" text-doc value"
)
# The pre-0022 row backfilled to (false, NULL) — a text doc.
row = db.execute(
text("SELECT is_image, image_path FROM documents WHERE id = :id"),
{"id": pre_id},
).fetchone()
assert row is not None, "the pre-0022 row must survive the upgrade"
assert row[0] is False, "the backfilled is_image must be false"
assert row[1] is None, "the backfilled image_path must be NULL"
# A row written without the image columns reads the same
# (the Python-side defaults are False/None — same values).
new_id = _insert_sql(db, PATH_IMAGE)
try:
backfilled = db.execute(
text("SELECT is_image, image_path FROM documents WHERE id = :id"),
{"id": new_id},
).fetchone()
assert backfilled == (False, None), (
"an omitted image state must read (false, NULL)"
)
finally:
_delete(db, new_id)
# The 0021 schema survives the additive upgrade.
content = _column(db, "content")
assert content is not None and content[0] == "text" and content[1] == "NO", (
"documents.content (0001) must keep its 0021 contract"
)
hash_col = _column(db, "content_hash")
assert (
hash_col is not None
and hash_col[0] == "character varying"
and hash_col[1] == "NO"
), "documents.content_hash (0001) must survive the upgrade"
summary = _column(db, "summary")
assert summary is not None and summary[0] == "text" and summary[1] == "YES", (
"documents.summary (phase 30) must survive the upgrade"
)
created = _column(db, "created_at")
assert created is not None and created[0] == "timestamp with time zone"
assert created[1] == "NO" and "now()" in str(created[2]), (
"documents.created_at (0020) must keep its `now()` server default"
)
manual = _column(db, "created_at_manual")
assert manual is not None and manual[0] == "boolean" and manual[1] == "NO"
assert manual[2] == "false", (
"documents.created_at_manual (0020) must keep its `false` default"
)
assert ("source", "path") in _unique_column_sets(db), (
"the (source, path) unique constraint must survive the upgrade"
)
finally:
_delete(db, pre_id)
def test_orm_image_fields_round_trip(db: Session, alembic: Config) -> None:
"""The ORM contract agrees with the column contract: a freshly
inserted ``Document`` WITHOUT the image fields reads ``is_image is
False`` / ``image_path is None`` (the text-doc default state), and
one WITH them round-trips the pair through a FRESH session."""
command.upgrade(alembic, "head")
text_doc = Document(
source=SOURCE,
path=PATH_TEXT,
full_path=f"/tmp/{SOURCE}/{PATH_TEXT}",
title="readme",
content="# readme\n",
content_hash="1" * 64,
)
image_doc = Document(
source=SOURCE,
path=PATH_IMAGE,
full_path=f"/tmp/{SOURCE}/{PATH_IMAGE}",
title="diagram",
content="A description of the diagram.",
content_hash="2" * 64,
is_image=True,
image_path="/srv/bor-images/diagram.png",
)
db.add(text_doc)
db.add(image_doc)
db.commit()
try:
with SessionLocal() as fresh:
reloaded_text = fresh.get(Document, text_doc.id)
assert reloaded_text is not None, "the text row must be readable"
assert reloaded_text.is_image is False, (
"an omitted is_image must read the False default"
)
assert reloaded_text.image_path is None, (
"an omitted image_path must read NULL"
)
reloaded_image = fresh.get(Document, image_doc.id)
assert reloaded_image is not None, "the image row must be readable"
assert reloaded_image.is_image is True
assert reloaded_image.image_path == "/srv/bor-images/diagram.png"
finally:
_delete(db, text_doc.id)
_delete(db, image_doc.id)
def test_downgrade_to_0021_drops_the_columns(db: Session, alembic: Config) -> None:
"""Downgrade 0022 → 0021: both image columns are gone (A13 — fully
reversible) while the row + its 0021 columns survive, and the rest
of the 0021 table contract (``content``, ``content_hash``,
``created_at``) is intact."""
command.upgrade(alembic, "head")
row = Document(
source=SOURCE,
path=PATH_IMAGE,
full_path=f"/tmp/{SOURCE}/{PATH_IMAGE}",
title="diagram",
content="A description of the diagram.",
content_hash="3" * 64,
is_image=True,
image_path="/srv/bor-images/diagram.png",
)
db.add(row)
db.commit()
try:
command.downgrade(alembic, "0021")
assert _version(db) == "0021"
assert _column(db, "is_image") is None, "is_image must be dropped"
assert _column(db, "image_path") is None, "image_path must be dropped"
surviving = db.execute(
text(
"SELECT source, path, title, content, content_hash, created_at"
" FROM documents WHERE id = :id"
),
{"id": row.id},
).fetchone()
assert surviving is not None, "the row must survive the column drops"
assert surviving[0] == SOURCE and surviving[1] == PATH_IMAGE
assert surviving[3] == "A description of the diagram."
assert surviving[4] == "3" * 64
assert surviving[5] is not None, "created_at must survive the drops"
assert ("source", "path") in _unique_column_sets(db), (
"the (source, path) unique constraint must survive the downgrade"
)
finally:
_delete(db, row.id)
# Repair: the fixture teardown re-upgrades to head.
def test_upgrade_round_trip_restores_the_columns(db: Session, alembic: Config) -> None:
"""Downgrade to 0021, then upgrade back to 0022: both columns are
back with the full contract (``is_image`` BOOLEAN NOT NULL default
``false``; ``image_path`` TEXT NULLABLE no default)."""
command.downgrade(alembic, "0021")
command.upgrade(alembic, "0022")
assert _version(db) == "0022", "round-trip upgrade must land at 0022"
is_image = _column(db, "is_image")
assert is_image is not None, "documents.is_image must be back"
assert is_image[0] == "boolean", "is_image must be BOOLEAN after the round-trip"
assert is_image[1] == "NO", "is_image must be NOT NULL after the round-trip"
assert is_image[2] == "false", (
"is_image must still carry the `false` server default"
)
image_path = _column(db, "image_path")
assert image_path is not None, "documents.image_path must be back"
assert image_path[0] == "text"
assert image_path[1] == "YES"
assert image_path[2] is None, "image_path must still carry NO server default"
+4 -3
View File
@@ -216,10 +216,11 @@ def test_admin_semantic_fields_round_trip(client: TestClient, db: Session) -> No
def _config_keys() -> set[str]:
"""The /api/config key set after task 03: the five phase-39/59/62
keys — the retired CSS-file theming's ``theme`` key is gone."""
"""The /api/config key set after task 03 (phase 91): the retired
CSS-file theming's ``theme`` key is gone; phase 122 (task 01) added
the ``images`` flag — the six keys below are the contract."""
return {"app_name", "version", "docs_repo_configured",
"input_placeholder", "footer_text"}
"images", "input_placeholder", "footer_text"}
def test_api_config_env_only_deployment_returns_env_strings(client: TestClient) -> None: