phase: 122_image_documents
**Phase 122 (image documents) — final verification pass: all green. No code changes were needed; defects found: none.**
**Verified (implementation already complete in working tree, reviewed end-to-end):**
- Toggle (`BOR_IMAGES`/`BOR_IMAGE_EXTENSIONS`/`BOR_IMAGE_DIR`, off by default) + `GET /api/config` `images` flag
- Ingest: bytes digest, `image_dir` persistent copy, `content = summary = vision description` (chat-model call; only text embedded), fail-soft skip + `images_failed` counter
- Serve/display: `/api/documents/{id}/image` route (404 matrix), viewer `<img>` + description, Sources 48px lazy thumbnails, chat inline source figure (alt = summary), agent `read` marker
- Prune guard: images-off syncs never prune `is_image` docs
**Test / lint / coverage (exact commands & outcomes):**
- `uv run pytest` → exit 0 (green; note: pytest 9.1.1 `-q` omits the final count line in output — exit code authoritative)
- `uv run pytest --cov=app --cov-report=term-missing` → **2715 passed, exit 0, TOTAL 99%** (>90% gate)
- `uv run ruff check . && uv run pyright` → "All checks passed!" / "0 errors, 0 warnings, 0 informations"
- `uv run pytest tests/e2e/test_image_documents.py -v --no-cov` → **4 passed, exit 0** (isolation)
**Completion criteria:** (1) images=true → described/embedded/displayed docs: ✅ (E2E + integration) · (2) images=false byte-identical + image docs survive sync: ✅ (E2E negative app + unit/integration) · (3) viewer + chat rendering with alt text; failed description skips + logs, sync completes: ✅ · (4) test/lint/coverage gates: ✅ · (5) commit + phase move: deferred to harness per this pass's rules (working tree left uncommitted).
**Notable deviation (pre-existing, documented in code):** image route uses `require_user` (phase-79 posture, same gate as the document content endpoint) rather than the phase text's "public" parenthetical — matches the endpoint it mirrors.
**Next pending phase:** `123_chat_image_questions`.
This commit is contained in:
@@ -48,25 +48,30 @@ def test_health_reports_ok(client) -> None:
|
||||
|
||||
|
||||
def test_config_returns_default_app_metadata(client, db: Session) -> None:
|
||||
"""GET /api/config is public (anonymous) and returns exactly five
|
||||
"""GET /api/config is public (anonymous) and returns exactly six
|
||||
keys — the phase-39 app metadata, the phase-59 docs flag (inert
|
||||
false while BOR_DOCS_REPO is empty — the "Save as doc" gating),
|
||||
and the phase-62 UI customization strings (composer placeholder,
|
||||
footer line). Phase 91: with an empty ui_settings table the
|
||||
effective strings are the env defaults (B1 — DB-over-env, the row
|
||||
absent here); the retired CSS-file theming's ``theme`` key is gone
|
||||
(task 03 — the five keys are the entire contract)."""
|
||||
the phase-122 images flag (default false in the test env —
|
||||
LOCKED A3: off by default), and the phase-62 UI customization
|
||||
strings (composer placeholder, footer line). Phase 91: with an
|
||||
empty ui_settings table the effective strings are the env defaults
|
||||
(B1 — DB-over-env, the row absent here); the retired CSS-file
|
||||
theming's ``theme`` key is gone (task 03 — the six keys are the
|
||||
entire contract)."""
|
||||
_clear_ui_settings(db)
|
||||
r = client.get("/api/config")
|
||||
assert r.status_code == 200
|
||||
body = r.json()
|
||||
assert set(body) == {
|
||||
"app_name", "version", "docs_repo_configured",
|
||||
"input_placeholder", "footer_text",
|
||||
"images", "input_placeholder", "footer_text",
|
||||
}
|
||||
assert body["app_name"] == "Brain of Reese"
|
||||
assert body["version"] == get_settings().app_version
|
||||
assert body["docs_repo_configured"] is False
|
||||
# Phase 122 (task 01): the images flag is the default off (the
|
||||
# test env sets no BOR_IMAGES) — a real bool, not a truthy string.
|
||||
assert body["images"] is False
|
||||
# Phase 62: UNSET => the phase-61 neutral copy stands (the
|
||||
# byte-identical contract).
|
||||
assert body["input_placeholder"] == "Ask me anything…"
|
||||
@@ -91,7 +96,7 @@ def test_config_follows_overridden_app_name(client, db: Session) -> None:
|
||||
body = r.json()
|
||||
assert set(body) == {
|
||||
"app_name", "version", "docs_repo_configured",
|
||||
"input_placeholder", "footer_text",
|
||||
"images", "input_placeholder", "footer_text",
|
||||
}
|
||||
assert body["app_name"] == "Brain of Testy"
|
||||
assert body["version"] == "0.1.0"
|
||||
@@ -122,7 +127,7 @@ def test_config_serves_ui_customization_overrides(client, db: Session) -> None:
|
||||
body = r.json()
|
||||
assert set(body) == {
|
||||
"app_name", "version", "docs_repo_configured",
|
||||
"input_placeholder", "footer_text",
|
||||
"images", "input_placeholder", "footer_text",
|
||||
}
|
||||
assert body["input_placeholder"] == "Ask the vault…"
|
||||
assert body["footer_text"] == "Powered by my own models"
|
||||
@@ -153,6 +158,29 @@ def test_config_docs_flag_tracks_settings(client, db: Session) -> None:
|
||||
fastapi_app.dependency_overrides.clear()
|
||||
|
||||
|
||||
def test_config_images_flag_tracks_settings(client, db: Session) -> None:
|
||||
"""Phase 122 (task 01): ``images`` mirrors ``settings.images``
|
||||
(``BOR_IMAGES``) — a real bool (never a truthy string) that flips
|
||||
true the moment the toggle is on: that flag is the entire frontend
|
||||
gating of the image affordances (the phase-123 attach control,
|
||||
optionally the Sources page hint)."""
|
||||
from app.config import Settings
|
||||
from app.main import app as fastapi_app
|
||||
|
||||
_clear_ui_settings(db)
|
||||
fastapi_app.dependency_overrides[get_settings] = lambda: Settings(
|
||||
images=True,
|
||||
)
|
||||
try:
|
||||
r = client.get("/api/config")
|
||||
assert r.status_code == 200
|
||||
body = r.json()
|
||||
assert isinstance(body["images"], bool)
|
||||
assert body["images"] is True
|
||||
finally:
|
||||
fastapi_app.dependency_overrides.clear()
|
||||
|
||||
|
||||
def test_suggestions_returns_list(client) -> None:
|
||||
# Phase 79: the chips are user-gated — sign in as the admin first
|
||||
# (the test's purpose is the list shape, not the auth contract).
|
||||
|
||||
@@ -431,8 +431,11 @@ def test_document_content_admin_contract(client: TestClient, db) -> None:
|
||||
"content",
|
||||
"indexed_at",
|
||||
"chunks",
|
||||
"is_image", # added in phase 122 (task 04) — always present
|
||||
}
|
||||
assert body["summary"] is None
|
||||
assert body["is_image"] is False # text doc — and no image_url key (never null)
|
||||
assert "image_url" not in body
|
||||
|
||||
# Unknown docs still 404 (same shape as the phase-16 pin).
|
||||
r = client.get("/api/documents/content", params={"source": "docs", "path": "nope.md"})
|
||||
|
||||
@@ -2076,3 +2076,124 @@ def test_done_event_related_defaults_empty_and_old_payload_parses() -> None:
|
||||
dumped = ChatDoneEvent(**new_payload).model_dump()
|
||||
assert [r["path"] for r in dumped["related"]] == ["b.md"]
|
||||
assert [r["path"] for r in dumped["sources"]] == ["a.md"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 122, task 05 — the SSE source frame's OPTIONAL image_url (the
|
||||
# shared ``source_ref_with_image`` builder: present on an image doc's
|
||||
# ref only, omitted — never null — on every text ref). Task 06
|
||||
# finalizes the phase-122 suite here with the story-level pins.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _seed_image_doc(db) -> Document:
|
||||
"""An ``is_image`` documents row the agent can ``read`` (no chunks
|
||||
needed — the read tool resolves the row by ``(source, path)`` and
|
||||
serves its ``content``; the frame needs ``is_image`` + ``id``
|
||||
only). The fixture's teardown truncates the tables."""
|
||||
doc = Document(
|
||||
id=uuid.uuid4(),
|
||||
source="docs",
|
||||
path="pic.png",
|
||||
full_path="/tmp/pic.png",
|
||||
title="pic",
|
||||
content="A red square on a white background.",
|
||||
summary="A red square on a white background.",
|
||||
content_hash="0" * 64,
|
||||
created_at=_FIXTURE_CREATED_AT,
|
||||
is_image=True,
|
||||
image_path="/tmp/pic.png",
|
||||
)
|
||||
db.add(doc)
|
||||
db.commit()
|
||||
return doc
|
||||
|
||||
|
||||
def test_grounded_turn_reading_image_doc_frame_carries_image_url_only_for_it(
|
||||
client, db, seeded_kb: FakeRagLLM
|
||||
) -> None:
|
||||
"""Task 05 wire contract: a mocked grounded answer whose agent
|
||||
READS an image doc → the done frame's ref for THAT doc alone
|
||||
carries ``image_url`` (the bytes route the frontend's sources
|
||||
block renders from); the text-doc related refs carry NO
|
||||
``image_url`` key at all (the omission rule — the key is absent,
|
||||
never null)."""
|
||||
doc = _seed_image_doc(db)
|
||||
scripted = FakeRagLLM(
|
||||
tool_script=[
|
||||
[
|
||||
ToolCallPiece(
|
||||
id="call_1",
|
||||
name="read",
|
||||
arguments={"path": "docs/pic.png"},
|
||||
)
|
||||
]
|
||||
]
|
||||
)
|
||||
fastapi_app.dependency_overrides[chat_api.get_llm] = lambda: scripted
|
||||
try:
|
||||
_, _, frames = _stream_chat(client, QUESTION)
|
||||
finally:
|
||||
fastapi_app.dependency_overrides.clear()
|
||||
|
||||
done = frames[-1]
|
||||
assert done["deflected"] is False
|
||||
# The read image doc is the citation surface (phase 119, A1) — and
|
||||
# its ref is the frame's ONLY image_url carrier.
|
||||
sources = done["sources"]
|
||||
assert [(s["source"], s["path"]) for s in sources] == [("docs", "pic.png")]
|
||||
assert sources[0]["image_url"] == f"/api/documents/{doc.id}/image"
|
||||
# The related tier (ranks 6–7 for the Kubernetes question) is text
|
||||
# docs — the key is ABSENT on every one of them, not null.
|
||||
related = done["related"]
|
||||
assert related
|
||||
for ref in related:
|
||||
assert "image_url" not in ref
|
||||
|
||||
|
||||
def test_text_only_grounded_turn_frame_has_no_image_url_key_anywhere(
|
||||
client, db, seeded_kb: FakeRagLLM
|
||||
) -> None:
|
||||
"""The omission rule at the BYTE level (the phase's byte-identity
|
||||
criterion): a grounded turn whose cited + related docs are ALL
|
||||
text docs serializes a done frame with no ``image_url`` key
|
||||
anywhere — checked on the raw wire text (not a re-serialized
|
||||
dict), and every ref keeps exactly the pre-phase-122 key set."""
|
||||
scripted = FakeRagLLM(
|
||||
tool_script=[
|
||||
[
|
||||
ToolCallPiece(
|
||||
id="call_1",
|
||||
name="read",
|
||||
arguments={"path": "docs/homelab/kubernetes.md"},
|
||||
)
|
||||
]
|
||||
]
|
||||
)
|
||||
fastapi_app.dependency_overrides[chat_api.get_llm] = lambda: scripted
|
||||
try:
|
||||
with client.stream("POST", "/api/chat", json={"message": QUESTION}) as r:
|
||||
assert r.status_code == 200
|
||||
buf = ""
|
||||
raw_frames: list[str] = []
|
||||
for part in r.iter_text():
|
||||
buf += part
|
||||
while "\n\n" in buf:
|
||||
frame, buf = buf.split("\n\n", 1)
|
||||
frame = frame.strip()
|
||||
if frame.startswith("data:"):
|
||||
raw_frames.append(frame)
|
||||
finally:
|
||||
fastapi_app.dependency_overrides.clear()
|
||||
|
||||
done_raw = [f for f in raw_frames if '"type": "done"' in f]
|
||||
assert len(done_raw) == 1
|
||||
# The BYTE check: the key is absent from the wire text itself.
|
||||
assert "image_url" not in done_raw[0]
|
||||
done = json.loads(done_raw[0].removeprefix("data:").strip())
|
||||
assert done["deflected"] is False
|
||||
assert [(s["source"], s["path"]) for s in done["sources"]] == [
|
||||
("docs", "homelab/kubernetes.md")
|
||||
]
|
||||
for ref in done["sources"] + done["related"]:
|
||||
assert set(ref) == {"source", "path", "title"} # the pre-122 key set
|
||||
|
||||
@@ -7,22 +7,33 @@ Uses the real compose Postgres (``db`` fixture) and FastAPI's TestClient.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import hashlib
|
||||
import inspect
|
||||
import itertools
|
||||
import logging
|
||||
import uuid
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlalchemy import delete, func, select, text
|
||||
|
||||
import app.api.docs as docs_api
|
||||
import app.rag.importer as rag_importer
|
||||
from app.config import Settings
|
||||
from app.core import tokens as token_service
|
||||
from app.main import app as fastapi_app
|
||||
from app.models import Chunk, Document, FolderSummary, GitSource
|
||||
from app.rag import git_sources as rag_git_sources
|
||||
from app.rag.folder_summaries import missing_folder_summaries
|
||||
from app.rag.importer import import_sources
|
||||
from app.rag.llm import LLMError
|
||||
from app.rag.summarizer import DESCRIBE_PROMPT
|
||||
from tests.fakes import FakeEmbedder
|
||||
|
||||
_TREE_TABLES = "chunks, documents, folder_summaries, git_sources"
|
||||
|
||||
@@ -823,3 +834,603 @@ def test_docs_tree_stat_walk_equivalence_with_flat_list(admin_client, db) -> Non
|
||||
)
|
||||
|
||||
_truncate_tree_tables(db)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 122, task 02 — image ingest end-to-end (the full ``import_sources``
|
||||
# pipeline against the real DB; task 06 finalizes the phase-122 suite here
|
||||
# with the image route + content-endpoint shapes).
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
#: A real 1×1 transparent PNG — the importer is content-agnostic (it
|
||||
#: never parses the image), but a well-formed fixture keeps the tests
|
||||
#: honest about what a real upload looks like.
|
||||
PNG_1X1 = base64.b64decode(
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAC0lEQVR4nGP4DwQACfsD/fteaysAAAAASUVORK5CYII="
|
||||
)
|
||||
|
||||
|
||||
def _image_llm[
|
||||
ImageLLM: FakeEmbedder
|
||||
](tmp_path, llm_cls: type[ImageLLM] = FakeEmbedder, **kwargs) -> ImageLLM:
|
||||
"""A fake LLM with the phase-122 image knobs (toggle ON by default;
|
||||
the image dir defaults under *tmp_path* unless overridden).
|
||||
*llm_cls* (task 03) may be the mock vision client (``_MockVisionLLM``
|
||||
below) for the no-seam-patch end-to-end path — the PEP 695 type
|
||||
parameter keeps the helper's return type honest (``chat_models``).
|
||||
"""
|
||||
kwargs.setdefault("_env_file", None)
|
||||
kwargs.setdefault("images", True)
|
||||
kwargs.setdefault("image_dir", str(tmp_path / "images"))
|
||||
llm = llm_cls()
|
||||
llm.settings = Settings(**kwargs) # pyright: ignore[reportCallIssue]
|
||||
return llm
|
||||
|
||||
|
||||
def _patch_description(monkeypatch, description) -> None:
|
||||
"""Pin the task-02 seam (``rag_importer._describe_or_skip``) to
|
||||
*description*. Task 03 fills the seam with the CHAT model's vision
|
||||
call — these end-to-end mechanics do not change with it."""
|
||||
|
||||
async def _fake(llm, *, data, source, rel, full_path):
|
||||
return description
|
||||
|
||||
monkeypatch.setattr(rag_importer, "_describe_or_skip", _fake)
|
||||
|
||||
|
||||
def _cleanup_source(db, source: str) -> None:
|
||||
for doc in db.scalars(select(Document).where(Document.source == source)).all():
|
||||
db.delete(doc)
|
||||
db.commit()
|
||||
|
||||
|
||||
def test_import_sources_images_on_indexes_image_docs(
|
||||
db, tmp_path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""``images=True`` end-to-end: a standalone image in a source becomes
|
||||
a Document — bytes digested, the persistent copy in ``image_dir``
|
||||
(``<doc-id>.png``), ``content`` = the (mock) description, and ONLY
|
||||
that text is embedded (the content chunks + the phase-30 summary
|
||||
chunk, 768 dims) — while a text doc in the same source stays a
|
||||
plain text doc."""
|
||||
source_root = tmp_path / "imgsource"
|
||||
source_root.mkdir()
|
||||
(source_root / "diagram.png").write_bytes(PNG_1X1)
|
||||
(source_root / "notes.md").write_text("# Notes\n\nBody text.\n", encoding="utf-8")
|
||||
llm = _image_llm(tmp_path)
|
||||
_patch_description(monkeypatch, "A network diagram of the homelab VLANs.")
|
||||
try:
|
||||
summary = asyncio.run(
|
||||
import_sources([source_root], llm, session=db, prune=True)
|
||||
)
|
||||
assert (summary.files, summary.added, summary.images_failed) == (2, 2, 0)
|
||||
assert summary.formats == {"md": 1, "png": 1}
|
||||
|
||||
doc = db.scalar(
|
||||
select(Document).where(
|
||||
Document.source == source_root.name, Document.path == "diagram.png"
|
||||
)
|
||||
)
|
||||
assert doc is not None, "the image file must become a document"
|
||||
assert doc.is_image is True and doc.image_path is not None
|
||||
assert doc.content == "A network diagram of the homelab VLANs."
|
||||
assert doc.title == "diagram" # the non-markdown stem rule
|
||||
copy = Path(doc.image_path)
|
||||
assert copy.parent == Path(llm.settings.image_dir)
|
||||
assert copy.name == f"{doc.id}.png"
|
||||
assert copy.read_bytes() == PNG_1X1
|
||||
|
||||
# The ONLY embedded text is the description (the embedding model
|
||||
# never sees pixels): every content chunk carries it, the
|
||||
# phase-30 summary chunk exists + is embedded, 768 dims. Task
|
||||
# 03: the description IS the summary (stored verbatim — no
|
||||
# ``lite`` call, no pointer line), so the summary chunk mirrors
|
||||
# ``doc.content`` exactly.
|
||||
chunks = db.scalars(select(Chunk).where(Chunk.document_id == doc.id)).all()
|
||||
content_chunks = [c for c in chunks if not c.is_summary]
|
||||
assert [c.content for c in content_chunks] == [doc.content]
|
||||
assert doc.summary == doc.content
|
||||
summary_chunks = [c for c in chunks if c.is_summary]
|
||||
assert len(summary_chunks) == 1 and summary_chunks[0].position == -1
|
||||
assert summary_chunks[0].content == doc.content
|
||||
for c in chunks:
|
||||
assert c.embedding is not None and len(c.embedding) == 768
|
||||
|
||||
# The text doc is untouched by the image machinery.
|
||||
md = db.scalar(
|
||||
select(Document).where(
|
||||
Document.source == source_root.name, Document.path == "notes.md"
|
||||
)
|
||||
)
|
||||
assert md is not None and md.is_image is False and md.image_path is None
|
||||
finally:
|
||||
_cleanup_source(db, source_root.name)
|
||||
|
||||
|
||||
class _MockVisionLLM(FakeEmbedder):
|
||||
"""The phase-122 mock VISION client (task 03): the CHAT model
|
||||
answers the multimodal describe call (the image bytes' data URL)
|
||||
with a fixed, retrieval-oriented description; text (``lite``) calls
|
||||
keep the ``FakeEmbedder`` behaviour. The REAL
|
||||
``rag_importer._describe_or_skip`` → ``summarizer.describe_image``
|
||||
chain runs end-to-end against it (no seam patch); every chat
|
||||
call's model is recorded (``chat_models``)."""
|
||||
|
||||
DESCRIPTION = (
|
||||
"A network diagram of the homelab VLANs: the core switch, the "
|
||||
"router, and three labeled subnets."
|
||||
)
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.chat_models: list[str | None] = []
|
||||
|
||||
async def chat(self, messages, model=None):
|
||||
self.chat_calls.append(list(messages))
|
||||
self.chat_models.append(model)
|
||||
user = next((m["content"] for m in messages if m.get("role") == "user"), "")
|
||||
if isinstance(user, list):
|
||||
# The phase-122 describe call — the multimodal message.
|
||||
return self.DESCRIPTION
|
||||
first = user.split()
|
||||
return "Summary of " + (first[0] if first else "<empty>")
|
||||
|
||||
|
||||
def test_import_sources_mock_vision_client_end_to_end(db, tmp_path) -> None:
|
||||
"""Task 03 end-to-end (NO seam patch): a fixture PNG through the
|
||||
mock vision client — the real ``_describe_or_skip`` →
|
||||
``describe_image`` → CHAT-model call — yields a doc whose
|
||||
``content`` == ``summary`` == the description, with its
|
||||
``is_summary`` chunk embedded, and whose ONLY embedded text is that
|
||||
description (the embedding model never sees pixels)."""
|
||||
source_root = tmp_path / "visione2e"
|
||||
source_root.mkdir()
|
||||
(source_root / "diagram.png").write_bytes(PNG_1X1)
|
||||
llm = _image_llm(tmp_path, _MockVisionLLM)
|
||||
try:
|
||||
summary = asyncio.run(
|
||||
import_sources([source_root], llm, session=db)
|
||||
)
|
||||
assert (summary.files, summary.added, summary.images_failed) == (1, 1, 0)
|
||||
# The describe call went to the CHAT model (LOCKED A3), once.
|
||||
assert llm.chat_models == [llm.settings.llm_chat_model]
|
||||
|
||||
# The wire shape: the multimodal user message — the fixed
|
||||
# prompt's text part + the image's data-URL part.
|
||||
(message,) = llm.chat_calls[0]
|
||||
assert message["role"] == "user"
|
||||
content: Any = message["content"] # the multimodal part list
|
||||
assert content[0] == {"type": "text", "text": DESCRIBE_PROMPT}
|
||||
assert content[1]["type"] == "image_url"
|
||||
assert content[1]["image_url"]["url"].startswith("data:image/png;base64,")
|
||||
|
||||
doc = db.scalar(
|
||||
select(Document).where(
|
||||
Document.source == source_root.name, Document.path == "diagram.png"
|
||||
)
|
||||
)
|
||||
assert doc is not None, "the image file must become a document"
|
||||
assert doc.is_image is True and doc.image_path is not None
|
||||
assert doc.content == _MockVisionLLM.DESCRIPTION
|
||||
assert doc.summary == _MockVisionLLM.DESCRIPTION # task 03: verbatim
|
||||
|
||||
# The ONLY embedded text of the doc is the description — twice
|
||||
# (the one content chunk + the is_summary chunk), 768 dims.
|
||||
chunks = db.scalars(select(Chunk).where(Chunk.document_id == doc.id)).all()
|
||||
summary_chunks = [c for c in chunks if c.is_summary]
|
||||
assert len(summary_chunks) == 1 and summary_chunks[0].position == -1
|
||||
assert all(c.content == _MockVisionLLM.DESCRIPTION for c in chunks)
|
||||
for c in chunks:
|
||||
assert c.embedding is not None and len(c.embedding) == 768
|
||||
embedded = [t for batch in llm.calls for t in batch]
|
||||
assert embedded == [_MockVisionLLM.DESCRIPTION] * 2
|
||||
finally:
|
||||
_cleanup_source(db, source_root.name)
|
||||
|
||||
|
||||
class _MockVisionFailsLLM(FakeEmbedder):
|
||||
"""A NON-VISION chat model (LOCKED A3's honest failure): the
|
||||
multimodal describe call raises (the SDK errors — a chat model
|
||||
without vision rejects the ``image_url`` part), text (``lite``)
|
||||
calls keep the ``FakeEmbedder`` behaviour."""
|
||||
|
||||
async def chat(self, messages, model=None):
|
||||
self.chat_calls.append(list(messages))
|
||||
user = next((m["content"] for m in messages if m.get("role") == "user"), "")
|
||||
if isinstance(user, list):
|
||||
raise LLMError("simulated non-vision chat model (test sentinel)")
|
||||
first = user.split()
|
||||
return "Summary of " + (first[0] if first else "<empty>")
|
||||
|
||||
|
||||
def test_import_sources_failing_vision_skips_image_keeps_sync_green(
|
||||
db, tmp_path, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
"""LOCKED A3 fail-soft end-to-end (the task-06 integration pin):
|
||||
a fixture PNG through a NON-VISION chat model — the real seam, no
|
||||
patch — skips the image doc (``images_failed == 1``, NO row, NO
|
||||
orphan copy — not even the image dir) while the TEXT doc in the
|
||||
same source is indexed as usual (the sync completes, no row
|
||||
mutation anywhere for the failed image)."""
|
||||
source_root = tmp_path / "visionfail"
|
||||
source_root.mkdir()
|
||||
(source_root / "diagram.png").write_bytes(PNG_1X1)
|
||||
(source_root / "notes.md").write_text("# Notes\n\nBody.\n", encoding="utf-8")
|
||||
llm = _image_llm(tmp_path, _MockVisionFailsLLM)
|
||||
try:
|
||||
with caplog.at_level(logging.INFO, logger="app.importer"):
|
||||
summary = asyncio.run(
|
||||
import_sources([source_root], llm, session=db)
|
||||
)
|
||||
assert (summary.files, summary.added, summary.images_failed) == (2, 1, 1)
|
||||
# The image: no row, no copy (the dir itself was never created).
|
||||
assert (
|
||||
db.scalar(
|
||||
select(Document).where(
|
||||
Document.source == source_root.name, Document.path == "diagram.png"
|
||||
)
|
||||
)
|
||||
is None
|
||||
)
|
||||
assert not Path(llm.settings.image_dir).expanduser().exists()
|
||||
# The text doc indexed as usual (the sync stayed green).
|
||||
md = db.scalar(
|
||||
select(Document).where(
|
||||
Document.source == source_root.name, Document.path == "notes.md"
|
||||
)
|
||||
)
|
||||
assert md is not None and md.is_image is False
|
||||
# The importer's warning names the document (the PLAN §9 signal).
|
||||
warnings = [
|
||||
r
|
||||
for r in caplog.records
|
||||
if r.name == "app.importer" and "image description failed" in r.getMessage()
|
||||
]
|
||||
assert len(warnings) == 1 and warnings[0].levelno == logging.WARNING
|
||||
assert f"source={source_root.name} path=diagram.png" in warnings[0].getMessage()
|
||||
finally:
|
||||
_cleanup_source(db, source_root.name)
|
||||
|
||||
|
||||
def test_import_sources_images_off_ignores_and_prune_guard_protects(
|
||||
db, tmp_path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""``images=False`` (the default) end-to-end: the walk ignores the
|
||||
image file entirely (not counted, no row, no copy), and a
|
||||
``prune=True`` run MUST NOT delete a pre-existing image doc — the
|
||||
LOCKED prune guard (invisible to the walk ≠ deleted)."""
|
||||
source_root = tmp_path / "imgsrc_off"
|
||||
source_root.mkdir()
|
||||
(source_root / "diagram.png").write_bytes(PNG_1X1)
|
||||
_patch_description(monkeypatch, "A network diagram.")
|
||||
try:
|
||||
llm_on = _image_llm(tmp_path)
|
||||
s_on = asyncio.run(import_sources([source_root], llm_on, session=db))
|
||||
assert s_on.added == 1
|
||||
doc = db.scalar(
|
||||
select(Document).where(
|
||||
Document.source == source_root.name, Document.path == "diagram.png"
|
||||
)
|
||||
)
|
||||
assert doc is not None
|
||||
copy = Path(doc.image_path)
|
||||
assert copy.exists()
|
||||
|
||||
# Toggle OFF (a fresh fake on the code defaults): the walk is
|
||||
# blind to the file, and prune protects the pre-existing image
|
||||
# doc + its copy.
|
||||
llm_off = FakeEmbedder() # Settings(_env_file=None) → images False
|
||||
s_off = asyncio.run(
|
||||
import_sources([source_root], llm_off, session=db, prune=True)
|
||||
)
|
||||
assert (s_off.files, s_off.added, s_off.pruned) == (0, 0, 0)
|
||||
assert db.scalar(select(Document).where(Document.id == doc.id)) is not None
|
||||
assert copy.exists(), "the copy survives with the doc"
|
||||
finally:
|
||||
_cleanup_source(db, source_root.name)
|
||||
|
||||
|
||||
def test_import_sources_toggle_on_prunes_deleted_image_with_copy(
|
||||
db, tmp_path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""Toggle ON, the image file deleted: the normal prune runs — the
|
||||
doc row AND its ``image_dir`` copy are removed (the copy's
|
||||
lifecycle is tied to the row)."""
|
||||
source_root = tmp_path / "imgsrc_prune"
|
||||
source_root.mkdir()
|
||||
(source_root / "diagram.png").write_bytes(PNG_1X1)
|
||||
llm_on = _image_llm(tmp_path)
|
||||
_patch_description(monkeypatch, "A network diagram.")
|
||||
try:
|
||||
asyncio.run(import_sources([source_root], llm_on, session=db))
|
||||
doc = db.scalar(
|
||||
select(Document).where(
|
||||
Document.source == source_root.name, Document.path == "diagram.png"
|
||||
)
|
||||
)
|
||||
assert doc is not None
|
||||
copy = Path(doc.image_path)
|
||||
assert copy.exists()
|
||||
|
||||
(source_root / "diagram.png").unlink()
|
||||
s = asyncio.run(
|
||||
import_sources([source_root], llm_on, session=db, prune=True)
|
||||
)
|
||||
assert (s.pruned, s.added, s.unchanged) == (1, 0, 0)
|
||||
assert (
|
||||
db.scalar(select(Document).where(Document.source == source_root.name))
|
||||
is None
|
||||
)
|
||||
assert not copy.exists(), "the image_dir copy is deleted with the doc"
|
||||
finally:
|
||||
_cleanup_source(db, source_root.name)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 122 (task 04) — the image BYTES route + the content/tree wire
|
||||
# affordance (the serve side of the image-document contract).
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _seed_image_doc(
|
||||
db,
|
||||
tmp_path: Path,
|
||||
*,
|
||||
source: str = "ImgSrc",
|
||||
path: str = "pic.png",
|
||||
data: bytes = PNG_1X1,
|
||||
content: str = "A red square on a white background.",
|
||||
image_path: str | None = "auto",
|
||||
doc_id: str | None = None,
|
||||
) -> Document:
|
||||
"""One ``is_image`` document row with its persistent copy (the
|
||||
importer's ``image_dir`` layout) under *tmp_path*; ``image_path``
|
||||
``"auto"`` writes the copy, ``None`` leaves the row without a copy
|
||||
(the lost-copy corner)."""
|
||||
doc = Document(
|
||||
id=uuid.UUID(doc_id) if doc_id else uuid.uuid4(),
|
||||
source=source,
|
||||
path=path,
|
||||
full_path=f"/tmp/{path}",
|
||||
title=path.rsplit(".", 1)[0],
|
||||
content=content,
|
||||
content_hash=hashlib.sha256(data).hexdigest(),
|
||||
indexed_at=datetime.now(UTC),
|
||||
created_at=datetime(2026, 1, 1, tzinfo=UTC),
|
||||
summary=content,
|
||||
is_image=True,
|
||||
)
|
||||
if image_path == "auto":
|
||||
copy = tmp_path / f"{doc.id}{Path(path).suffix}"
|
||||
copy.write_bytes(data)
|
||||
doc.image_path = str(copy)
|
||||
elif image_path is not None:
|
||||
doc.image_path = image_path
|
||||
db.add(doc)
|
||||
db.flush()
|
||||
db.add_all(
|
||||
[
|
||||
Chunk(
|
||||
document_id=doc.id, position=0, content=content, embedding=[0.01] * 768
|
||||
),
|
||||
Chunk(
|
||||
document_id=doc.id,
|
||||
position=-1,
|
||||
content=content,
|
||||
embedding=[0.01] * 768,
|
||||
is_summary=True,
|
||||
),
|
||||
]
|
||||
)
|
||||
db.commit()
|
||||
return doc
|
||||
|
||||
|
||||
def _cleanup_kb(db) -> None:
|
||||
"""Truncate the KB tables. The settle commit matters: SQLAlchemy
|
||||
does NOT autoflush pending ORM objects before a raw ``text()``
|
||||
statement — a pending chunks INSERT flushed *after* the TRUNCATE
|
||||
would FK-violate (the document row is already gone), so any
|
||||
pending state is committed (then truncated) first."""
|
||||
db.commit()
|
||||
db.execute(text("TRUNCATE chunks, documents"))
|
||||
db.commit()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("ext", "mime"),
|
||||
[
|
||||
(".png", "image/png"),
|
||||
(".jpg", "image/jpeg"),
|
||||
(".jpeg", "image/jpeg"),
|
||||
(".webp", "image/webp"),
|
||||
(".gif", "image/gif"),
|
||||
(".bmp", "image/bmp"),
|
||||
],
|
||||
)
|
||||
def test_image_route_serves_exact_bytes_with_content_type(
|
||||
admin_client: TestClient, db, tmp_path: Path, ext: str, mime: str
|
||||
) -> None:
|
||||
"""The serve contract: the route streams the EXACT stored bytes
|
||||
(a per-extension sentinel — a mix-up between the six formats is
|
||||
caught) with the extension's ``Content-Type`` (the
|
||||
``IMAGE_MIMES`` map — one map, one truth) and
|
||||
``Cache-Control: private, max-age=3600`` (content-hashed bytes —
|
||||
long enough, bustable by re-upload)."""
|
||||
_cleanup_kb(db)
|
||||
try:
|
||||
data = PNG_1X1 + ext.encode("ascii") # per-extension sentinel bytes
|
||||
doc = _seed_image_doc(db, tmp_path, path=f"pic{ext}", data=data)
|
||||
r = admin_client.get(f"/api/documents/{doc.id}/image")
|
||||
assert r.status_code == 200
|
||||
assert r.headers["content-type"] == mime # exact type per extension
|
||||
assert r.headers["cache-control"] == "private, max-age=3600"
|
||||
assert r.content == data # the exact uploaded bytes, nothing else
|
||||
finally:
|
||||
_cleanup_kb(db)
|
||||
|
||||
|
||||
def test_image_route_404_matrix(admin_client: TestClient, db, tmp_path: Path) -> None:
|
||||
"""Every non-servable case is 404 ``document not found`` (the
|
||||
router's unknown-document shape — the same detail string the
|
||||
content endpoint uses): a missing id, a MALFORMED id (an unparseable
|
||||
string maps here, not to a 422 — a guessed id is an unknown
|
||||
document), a text doc, an image doc whose ``image_path`` is NULL,
|
||||
and a row whose copy was lost on disk (defensive — the row exists,
|
||||
the bytes don't)."""
|
||||
_cleanup_kb(db)
|
||||
try:
|
||||
_seed_doc(db, "TextSrc", "note.md", "Note", 1, datetime.now(UTC))
|
||||
db.commit() # the module's _seed_doc leaves the row uncommitted
|
||||
text_doc_id = db.scalar(
|
||||
select(Document.id).where(
|
||||
Document.source == "TextSrc", Document.path == "note.md"
|
||||
)
|
||||
)
|
||||
no_copy = _seed_image_doc(db, tmp_path, path="nopy.png", image_path=None)
|
||||
lost = _seed_image_doc(db, tmp_path, path="lost.png")
|
||||
assert lost.image_path is not None # the "auto" copy was written
|
||||
Path(lost.image_path).unlink() # the copy is lost (the row remains)
|
||||
|
||||
for doc_id in (
|
||||
str(uuid.uuid4()), # missing id
|
||||
"not-a-uuid", # malformed id → 404, not 422
|
||||
str(text_doc_id), # text doc
|
||||
str(no_copy.id), # image doc, image_path NULL
|
||||
str(lost.id), # image doc, copy lost
|
||||
):
|
||||
r = admin_client.get(f"/api/documents/{doc_id}/image")
|
||||
assert r.status_code == 404, doc_id
|
||||
assert r.json() == {"detail": "document not found"}, doc_id
|
||||
finally:
|
||||
_cleanup_kb(db)
|
||||
|
||||
|
||||
def test_image_route_requires_user_like_the_content_endpoint(
|
||||
admin_client: TestClient, db, tmp_path: Path
|
||||
) -> None:
|
||||
"""Phase 79 posture (the task's "PUBLIC, like the document content
|
||||
endpoint" — the content endpoint has been user-gated since phase
|
||||
79; the ONLY anonymous surface is the shared chats, PLAN A10): an
|
||||
anonymous caller gets 401 ``authentication required`` before any
|
||||
row is read (a FRESH client — the module's fixture ``client``
|
||||
stays unsigned here), a signed-in caller gets the bytes."""
|
||||
_cleanup_kb(db)
|
||||
try:
|
||||
doc = _seed_image_doc(db, tmp_path)
|
||||
anonymous = TestClient(fastapi_app)
|
||||
r = anonymous.get(f"/api/documents/{doc.id}/image")
|
||||
assert r.status_code == 401
|
||||
assert r.json() == {"detail": "authentication required"}
|
||||
# The signed-in client (admin) passes.
|
||||
assert admin_client.get(f"/api/documents/{doc.id}/image").status_code == 200
|
||||
finally:
|
||||
_cleanup_kb(db)
|
||||
|
||||
|
||||
def test_content_endpoint_exposes_the_image_affordance(
|
||||
admin_client: TestClient, db, tmp_path: Path
|
||||
) -> None:
|
||||
"""The content endpoint (the viewer's data source): ``is_image`` is
|
||||
ALWAYS present (text doc: false — the one new key; the wire shape
|
||||
gains nothing else), and ``image_url`` — the bytes route's path —
|
||||
is present for an image doc and ABSENT for a text doc (never null,
|
||||
the ``DocContent`` omission rule)."""
|
||||
_cleanup_kb(db)
|
||||
try:
|
||||
_seed_doc(db, "TextSrc", "note.md", "Note", 1, datetime.now(UTC))
|
||||
db.commit() # the app's endpoint session reads committed data only
|
||||
r = admin_client.get(
|
||||
"/api/documents/content", params={"source": "TextSrc", "path": "note.md"}
|
||||
)
|
||||
assert r.status_code == 200
|
||||
body = r.json()
|
||||
assert body["is_image"] is False
|
||||
assert "image_url" not in body # absent — never null (text doc)
|
||||
|
||||
doc = _seed_image_doc(db, tmp_path, source="ImgSrc", path="pic.png")
|
||||
r = admin_client.get(
|
||||
"/api/documents/content", params={"source": "ImgSrc", "path": "pic.png"}
|
||||
)
|
||||
assert r.status_code == 200
|
||||
body = r.json()
|
||||
assert body["is_image"] is True
|
||||
assert body["image_url"] == f"/api/documents/{doc.id}/image"
|
||||
assert body["content"] == body["summary"] # the description (task 03)
|
||||
finally:
|
||||
_cleanup_kb(db)
|
||||
|
||||
|
||||
def _tree_file_nodes_all(sources) -> list[dict]:
|
||||
"""Every file node of a tree response, walked recursively (all
|
||||
sources — env-registered 0-document sources may join the response
|
||||
when the ``git_sources`` table is truncated, and they carry no
|
||||
file nodes; the assertions below hold over whatever files exist).
|
||||
"""
|
||||
files: list[dict] = []
|
||||
|
||||
def _walk(node: dict) -> None:
|
||||
for child in node.get("children", ()):
|
||||
if child["kind"] == "file":
|
||||
files.append(child)
|
||||
else:
|
||||
_walk(child)
|
||||
|
||||
for source in sources:
|
||||
_walk(source)
|
||||
return files
|
||||
|
||||
|
||||
def test_tree_image_file_node_affordance_and_text_node_byte_identical(
|
||||
admin_client: TestClient, db, tmp_path: Path
|
||||
) -> None:
|
||||
"""The tree (the RAG view's single fetch): an image doc's file node
|
||||
carries the thumbnail affordance (``is_image`` true,
|
||||
``image_url`` = the bytes route's path, ``summary`` verbatim — the
|
||||
RAG view's thumbnail ``alt``); EVERY text file node keeps the
|
||||
pre-phase wire shape byte-identically (the six keys — no
|
||||
``is_image``/``image_url``/``summary`` — the phase's
|
||||
byte-identical criterion: the fields are row-driven, so a KB with
|
||||
no image rows serializes exactly as pre-phase)."""
|
||||
_truncate_tree_tables(db)
|
||||
try:
|
||||
base = datetime.now(UTC)
|
||||
_seed_doc(db, "MixedSrc", "a.md", "A", 1, base)
|
||||
img = _seed_image_doc(db, tmp_path, source="MixedSrc", path="pic.png")
|
||||
|
||||
r = admin_client.get("/api/docs/tree") # both seeds committed (_seed_image_doc)
|
||||
assert r.status_code == 200
|
||||
files = {f["path"]: f for f in _tree_file_nodes_all(r.json()["sources"])}
|
||||
|
||||
# Text node: byte-identical pre-phase wire shape (no image keys).
|
||||
assert set(files["a.md"]) == {
|
||||
"kind", "path", "title", "chunks", "created_at", "indexed_at"
|
||||
}
|
||||
|
||||
# Image node: the affordance rides the node.
|
||||
pic = files["pic.png"]
|
||||
assert pic["is_image"] is True
|
||||
assert pic["image_url"] == f"/api/documents/{img.id}/image"
|
||||
assert pic["summary"] == "A red square on a white background."
|
||||
finally:
|
||||
_truncate_tree_tables(db)
|
||||
|
||||
|
||||
def test_tree_with_no_image_rows_is_byte_identical(admin_client: TestClient, db) -> None:
|
||||
"""The pre-phase KB (no ``is_image`` rows): the image-docs map is
|
||||
empty and EVERY file node serializes in the pre-phase shape — the
|
||||
row-driven fields introduce no wire change at all (the phase's
|
||||
byte-identical criterion, the toggle irrelevant)."""
|
||||
_truncate_tree_tables(db)
|
||||
try:
|
||||
base = datetime.now(UTC)
|
||||
_seed_doc(db, "PlainSrc", "x.md", "X", 2, base)
|
||||
db.commit() # the app's endpoint session reads committed data only
|
||||
r = admin_client.get("/api/docs/tree")
|
||||
assert r.status_code == 200
|
||||
files = _tree_file_nodes_all(r.json()["sources"])
|
||||
assert len(files) == 1 # the seeded doc (env sources carry no files)
|
||||
assert set(files[0]) == {
|
||||
"kind", "path", "title", "chunks", "created_at", "indexed_at"
|
||||
}
|
||||
finally:
|
||||
_truncate_tree_tables(db)
|
||||
|
||||
@@ -422,11 +422,16 @@ def test_content_200_all_fields(client, db) -> None:
|
||||
assert r.status_code == 200
|
||||
body = r.json()
|
||||
# Wire-additive (phase 106, task 05): ``created_at`` joins the
|
||||
# content shape (after ``summary``, before ``content``).
|
||||
# content shape (after ``summary``, before ``content``) — and
|
||||
# (phase 122, task 04) ``is_image`` joins it ALWAYS present
|
||||
# (text docs: false); ``image_url`` is ABSENT for a text doc
|
||||
# (never null — the ``DocContent`` omission rule).
|
||||
assert set(body) == {
|
||||
"source", "path", "title", "format", "summary", "created_at",
|
||||
"content", "indexed_at", "chunks",
|
||||
"content", "indexed_at", "chunks", "is_image",
|
||||
}
|
||||
assert body["is_image"] is False
|
||||
assert "image_url" not in body # absent — never null (text doc)
|
||||
datetime.fromisoformat(body["created_at"]) # raises if not ISO-8601
|
||||
assert body["source"] == "Homelab"
|
||||
assert body["path"] == "kubernetes.md"
|
||||
|
||||
@@ -0,0 +1,344 @@
|
||||
"""Integration: migration 0022 (documents.is_image + image_path) schema
|
||||
contract (phase 122, task 02).
|
||||
|
||||
Drives the **real Alembic engine** against the live dev database
|
||||
(``podman compose up -d db``), mirroring the house pattern of
|
||||
``test_migration_0021.py`` (information_schema assertions on the state
|
||||
the migration must leave). The tests target the 0021 → 0022 step
|
||||
explicitly so later migrations cannot break the pins:
|
||||
|
||||
* upgrade 0021 → 0022 → ``is_image`` exists with the full contract —
|
||||
BOOLEAN, NOT NULL, server default ``false`` — and ``image_path`` —
|
||||
TEXT, NULLABLE, no server default — while the 0021 ``documents``
|
||||
schema (``content``/``content_hash`` NOT NULL, ``summary`` NULLABLE,
|
||||
``created_at`` + ``created_at_manual``, the (source, path) unique
|
||||
constraint — asserted column-based, since the suite's table
|
||||
self-heal renames copied constraints) survives;
|
||||
* a ``documents`` row inserted while the DB is at 0021 backfills
|
||||
``is_image`` to ``false`` and ``image_path`` to NULL (every
|
||||
pre-phase-122 row is a text doc — the LOCKED A3 default);
|
||||
* the ORM contract agrees: a freshly inserted ``Document`` without the
|
||||
image fields reads ``is_image is False`` / ``image_path is None``,
|
||||
and one with them round-trips through a fresh session;
|
||||
* downgrade to 0021 → both columns are GONE (A13) while the row
|
||||
survives; upgrade back to 0022 → the columns are back (round-trip).
|
||||
|
||||
The ``alembic`` fixture guarantees the DB ends at head even if a test
|
||||
fails or the process is interrupted.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from collections.abc import Iterator
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from alembic.config import Config
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from alembic import command
|
||||
from app.db import SessionLocal, db_available
|
||||
from app.models import Document
|
||||
|
||||
SOURCE = "mig0022"
|
||||
PATH_TEXT = "notes/readme.md"
|
||||
PATH_IMAGE = "notes/diagram.png"
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def alembic(db: Session) -> Iterator[Config]:
|
||||
"""Real Alembic config bound to the dev DB (URL from app settings).
|
||||
|
||||
Starts at head (repairs an interrupted earlier run); teardown
|
||||
upgrades to head no matter what happened, so the dev DB is never
|
||||
left below head.
|
||||
"""
|
||||
if not db_available():
|
||||
pytest.skip("Postgres not reachable — run `podman compose up -d db` first")
|
||||
cfg = Config() # no alembic.ini file — env.py gets the URL from app config
|
||||
cfg.set_main_option("script_location", "alembic")
|
||||
command.upgrade(cfg, "head")
|
||||
try:
|
||||
yield cfg
|
||||
finally:
|
||||
# Release the test session's open transaction BEFORE the repair
|
||||
# DDL: an idle-in-transaction SELECT holds an ACCESS SHARE lock
|
||||
# on ``documents``, which would deadlock the repair's
|
||||
# ``ALTER TABLE`` (0022) forever.
|
||||
db.rollback()
|
||||
command.upgrade(cfg, "head")
|
||||
|
||||
|
||||
def _version(db: Session) -> str | None:
|
||||
return db.execute(text("SELECT version_num FROM alembic_version")).scalar()
|
||||
|
||||
|
||||
def _column(db: Session, column: str) -> tuple[Any, ...] | None:
|
||||
"""(data_type, is_nullable, column_default) for one documents
|
||||
column."""
|
||||
row = db.execute(
|
||||
text(
|
||||
"SELECT data_type, is_nullable, column_default"
|
||||
" FROM information_schema.columns"
|
||||
" WHERE table_name = 'documents' AND column_name = :c"
|
||||
),
|
||||
{"c": column},
|
||||
).fetchone()
|
||||
return tuple(row) if row is not None else None
|
||||
|
||||
|
||||
def _insert_sql(db: Session, path: str) -> uuid.UUID:
|
||||
"""Insert one documents row with the PRE-0022 column set (the 0021
|
||||
shape — the image columns, when present, are omitted so their
|
||||
backfill is what the row reads)."""
|
||||
row_id = uuid.uuid4()
|
||||
db.execute(
|
||||
text(
|
||||
"INSERT INTO documents (id, source, path, full_path, title,"
|
||||
" content, content_hash)"
|
||||
" VALUES (:id, :s, :p, :f, :t, :c, :h)"
|
||||
),
|
||||
{
|
||||
"id": row_id,
|
||||
"s": SOURCE,
|
||||
"p": path,
|
||||
"f": f"/tmp/{SOURCE}/{path}",
|
||||
"t": path.rsplit("/", 1)[-1],
|
||||
"c": "content",
|
||||
"h": "0" * 64,
|
||||
},
|
||||
)
|
||||
db.commit()
|
||||
return row_id
|
||||
|
||||
|
||||
def _delete(db: Session, row_id: uuid.UUID) -> None:
|
||||
db.execute(text("DELETE FROM documents WHERE id = :id"), {"id": row_id})
|
||||
db.commit()
|
||||
|
||||
|
||||
def _unique_column_sets(db: Session) -> set[tuple[str, ...]]:
|
||||
"""The column tuples of every UNIQUE constraint on ``documents``.
|
||||
|
||||
Column-based (not name-based): the integration suite's table
|
||||
self-heal (``tests/integration/conftest.py``) rewrites bloated
|
||||
tables via ``CREATE TABLE (LIKE …)``, which renames copied
|
||||
constraints (PG auto-names them) — the (source, path) uniqueness
|
||||
contract is what must hold, not the original name.
|
||||
"""
|
||||
rows = db.execute(
|
||||
text(
|
||||
"SELECT (SELECT string_agg(a.attname, ',' ORDER BY k.ord)"
|
||||
" FROM unnest(c.conkey) WITH ORDINALITY k(attnum, ord)"
|
||||
" JOIN pg_attribute a"
|
||||
" ON a.attrelid = c.conrelid AND a.attnum = k.attnum)"
|
||||
" FROM pg_constraint c"
|
||||
" WHERE c.contype = 'u' AND c.conrelid = 'documents'::regclass"
|
||||
)
|
||||
).fetchall()
|
||||
return {tuple(r[0].split(",")) for r in rows}
|
||||
|
||||
|
||||
def test_upgrade_to_0022_adds_image_columns(db: Session, alembic: Config) -> None:
|
||||
"""Upgrade 0021 → 0022: ``is_image`` exists with the full contract
|
||||
(BOOLEAN, NOT NULL, server default ``false`` — every pre-phase-122
|
||||
row is a text doc) and ``image_path`` (TEXT, NULLABLE, no server
|
||||
default — NULL for text docs), both ABSENT at 0021; a pre-0022 row
|
||||
backfills ``is_image`` to ``false`` + ``image_path`` to NULL; and
|
||||
the 0021 table contract survives the additive upgrade."""
|
||||
command.downgrade(alembic, "0021") # start from the pre-0022 state
|
||||
assert _version(db) == "0021"
|
||||
assert _column(db, "is_image") is None, "is_image must be absent at 0021"
|
||||
assert _column(db, "image_path") is None, "image_path must be absent at 0021"
|
||||
|
||||
pre_id = _insert_sql(db, PATH_TEXT) # the 0021 column set
|
||||
try:
|
||||
command.upgrade(alembic, "0022")
|
||||
assert _version(db) == "0022", "alembic_version must be at 0022"
|
||||
|
||||
is_image = _column(db, "is_image")
|
||||
assert is_image is not None, "documents.is_image is missing"
|
||||
assert is_image[0] == "boolean", "is_image must be BOOLEAN"
|
||||
assert is_image[1] == "NO", "is_image must be NOT NULL"
|
||||
assert is_image[2] == "false", (
|
||||
"is_image must carry the `false` server default — every"
|
||||
" pre-phase-122 row is a text doc"
|
||||
)
|
||||
|
||||
image_path = _column(db, "image_path")
|
||||
assert image_path is not None, "documents.image_path is missing"
|
||||
assert image_path[0] == "text", "image_path must be TEXT"
|
||||
assert image_path[1] == "YES", "image_path must be NULLABLE"
|
||||
assert image_path[2] is None, (
|
||||
"image_path must carry NO server default — NULL is the"
|
||||
" text-doc value"
|
||||
)
|
||||
|
||||
# The pre-0022 row backfilled to (false, NULL) — a text doc.
|
||||
row = db.execute(
|
||||
text("SELECT is_image, image_path FROM documents WHERE id = :id"),
|
||||
{"id": pre_id},
|
||||
).fetchone()
|
||||
assert row is not None, "the pre-0022 row must survive the upgrade"
|
||||
assert row[0] is False, "the backfilled is_image must be false"
|
||||
assert row[1] is None, "the backfilled image_path must be NULL"
|
||||
|
||||
# A row written without the image columns reads the same
|
||||
# (the Python-side defaults are False/None — same values).
|
||||
new_id = _insert_sql(db, PATH_IMAGE)
|
||||
try:
|
||||
backfilled = db.execute(
|
||||
text("SELECT is_image, image_path FROM documents WHERE id = :id"),
|
||||
{"id": new_id},
|
||||
).fetchone()
|
||||
assert backfilled == (False, None), (
|
||||
"an omitted image state must read (false, NULL)"
|
||||
)
|
||||
finally:
|
||||
_delete(db, new_id)
|
||||
|
||||
# The 0021 schema survives the additive upgrade.
|
||||
content = _column(db, "content")
|
||||
assert content is not None and content[0] == "text" and content[1] == "NO", (
|
||||
"documents.content (0001) must keep its 0021 contract"
|
||||
)
|
||||
hash_col = _column(db, "content_hash")
|
||||
assert (
|
||||
hash_col is not None
|
||||
and hash_col[0] == "character varying"
|
||||
and hash_col[1] == "NO"
|
||||
), "documents.content_hash (0001) must survive the upgrade"
|
||||
summary = _column(db, "summary")
|
||||
assert summary is not None and summary[0] == "text" and summary[1] == "YES", (
|
||||
"documents.summary (phase 30) must survive the upgrade"
|
||||
)
|
||||
created = _column(db, "created_at")
|
||||
assert created is not None and created[0] == "timestamp with time zone"
|
||||
assert created[1] == "NO" and "now()" in str(created[2]), (
|
||||
"documents.created_at (0020) must keep its `now()` server default"
|
||||
)
|
||||
manual = _column(db, "created_at_manual")
|
||||
assert manual is not None and manual[0] == "boolean" and manual[1] == "NO"
|
||||
assert manual[2] == "false", (
|
||||
"documents.created_at_manual (0020) must keep its `false` default"
|
||||
)
|
||||
assert ("source", "path") in _unique_column_sets(db), (
|
||||
"the (source, path) unique constraint must survive the upgrade"
|
||||
)
|
||||
finally:
|
||||
_delete(db, pre_id)
|
||||
|
||||
|
||||
def test_orm_image_fields_round_trip(db: Session, alembic: Config) -> None:
|
||||
"""The ORM contract agrees with the column contract: a freshly
|
||||
inserted ``Document`` WITHOUT the image fields reads ``is_image is
|
||||
False`` / ``image_path is None`` (the text-doc default state), and
|
||||
one WITH them round-trips the pair through a FRESH session."""
|
||||
command.upgrade(alembic, "head")
|
||||
text_doc = Document(
|
||||
source=SOURCE,
|
||||
path=PATH_TEXT,
|
||||
full_path=f"/tmp/{SOURCE}/{PATH_TEXT}",
|
||||
title="readme",
|
||||
content="# readme\n",
|
||||
content_hash="1" * 64,
|
||||
)
|
||||
image_doc = Document(
|
||||
source=SOURCE,
|
||||
path=PATH_IMAGE,
|
||||
full_path=f"/tmp/{SOURCE}/{PATH_IMAGE}",
|
||||
title="diagram",
|
||||
content="A description of the diagram.",
|
||||
content_hash="2" * 64,
|
||||
is_image=True,
|
||||
image_path="/srv/bor-images/diagram.png",
|
||||
)
|
||||
db.add(text_doc)
|
||||
db.add(image_doc)
|
||||
db.commit()
|
||||
try:
|
||||
with SessionLocal() as fresh:
|
||||
reloaded_text = fresh.get(Document, text_doc.id)
|
||||
assert reloaded_text is not None, "the text row must be readable"
|
||||
assert reloaded_text.is_image is False, (
|
||||
"an omitted is_image must read the False default"
|
||||
)
|
||||
assert reloaded_text.image_path is None, (
|
||||
"an omitted image_path must read NULL"
|
||||
)
|
||||
reloaded_image = fresh.get(Document, image_doc.id)
|
||||
assert reloaded_image is not None, "the image row must be readable"
|
||||
assert reloaded_image.is_image is True
|
||||
assert reloaded_image.image_path == "/srv/bor-images/diagram.png"
|
||||
finally:
|
||||
_delete(db, text_doc.id)
|
||||
_delete(db, image_doc.id)
|
||||
|
||||
|
||||
def test_downgrade_to_0021_drops_the_columns(db: Session, alembic: Config) -> None:
|
||||
"""Downgrade 0022 → 0021: both image columns are gone (A13 — fully
|
||||
reversible) while the row + its 0021 columns survive, and the rest
|
||||
of the 0021 table contract (``content``, ``content_hash``,
|
||||
``created_at``) is intact."""
|
||||
command.upgrade(alembic, "head")
|
||||
row = Document(
|
||||
source=SOURCE,
|
||||
path=PATH_IMAGE,
|
||||
full_path=f"/tmp/{SOURCE}/{PATH_IMAGE}",
|
||||
title="diagram",
|
||||
content="A description of the diagram.",
|
||||
content_hash="3" * 64,
|
||||
is_image=True,
|
||||
image_path="/srv/bor-images/diagram.png",
|
||||
)
|
||||
db.add(row)
|
||||
db.commit()
|
||||
try:
|
||||
command.downgrade(alembic, "0021")
|
||||
assert _version(db) == "0021"
|
||||
assert _column(db, "is_image") is None, "is_image must be dropped"
|
||||
assert _column(db, "image_path") is None, "image_path must be dropped"
|
||||
|
||||
surviving = db.execute(
|
||||
text(
|
||||
"SELECT source, path, title, content, content_hash, created_at"
|
||||
" FROM documents WHERE id = :id"
|
||||
),
|
||||
{"id": row.id},
|
||||
).fetchone()
|
||||
assert surviving is not None, "the row must survive the column drops"
|
||||
assert surviving[0] == SOURCE and surviving[1] == PATH_IMAGE
|
||||
assert surviving[3] == "A description of the diagram."
|
||||
assert surviving[4] == "3" * 64
|
||||
assert surviving[5] is not None, "created_at must survive the drops"
|
||||
|
||||
assert ("source", "path") in _unique_column_sets(db), (
|
||||
"the (source, path) unique constraint must survive the downgrade"
|
||||
)
|
||||
finally:
|
||||
_delete(db, row.id)
|
||||
# Repair: the fixture teardown re-upgrades to head.
|
||||
|
||||
|
||||
def test_upgrade_round_trip_restores_the_columns(db: Session, alembic: Config) -> None:
|
||||
"""Downgrade to 0021, then upgrade back to 0022: both columns are
|
||||
back with the full contract (``is_image`` BOOLEAN NOT NULL default
|
||||
``false``; ``image_path`` TEXT NULLABLE no default)."""
|
||||
command.downgrade(alembic, "0021")
|
||||
command.upgrade(alembic, "0022")
|
||||
assert _version(db) == "0022", "round-trip upgrade must land at 0022"
|
||||
|
||||
is_image = _column(db, "is_image")
|
||||
assert is_image is not None, "documents.is_image must be back"
|
||||
assert is_image[0] == "boolean", "is_image must be BOOLEAN after the round-trip"
|
||||
assert is_image[1] == "NO", "is_image must be NOT NULL after the round-trip"
|
||||
assert is_image[2] == "false", (
|
||||
"is_image must still carry the `false` server default"
|
||||
)
|
||||
|
||||
image_path = _column(db, "image_path")
|
||||
assert image_path is not None, "documents.image_path must be back"
|
||||
assert image_path[0] == "text"
|
||||
assert image_path[1] == "YES"
|
||||
assert image_path[2] is None, "image_path must still carry NO server default"
|
||||
@@ -216,10 +216,11 @@ def test_admin_semantic_fields_round_trip(client: TestClient, db: Session) -> No
|
||||
|
||||
|
||||
def _config_keys() -> set[str]:
|
||||
"""The /api/config key set after task 03: the five phase-39/59/62
|
||||
keys — the retired CSS-file theming's ``theme`` key is gone."""
|
||||
"""The /api/config key set after task 03 (phase 91): the retired
|
||||
CSS-file theming's ``theme`` key is gone; phase 122 (task 01) added
|
||||
the ``images`` flag — the six keys below are the contract."""
|
||||
return {"app_name", "version", "docs_repo_configured",
|
||||
"input_placeholder", "footer_text"}
|
||||
"images", "input_placeholder", "footer_text"}
|
||||
|
||||
|
||||
def test_api_config_env_only_deployment_returns_env_strings(client: TestClient) -> None:
|
||||
|
||||
Reference in New Issue
Block a user