phase: 152_sync_progress_image_pass
All criteria verified. The CLI path is hook-less (only `app/api/sync.py` passes the hook), and the byte-identity is pinned by `test_on_image_progress_none_run_is_byte_identical`. ## Final Report — Phase 152 final verification pass - **Verified (no new code needed):** `SyncStatus.images_done/images_total` + reset/clear rules in `app/api/sync.py`; `on_image_progress` threaded through the importer's three PDF vision spans with the continuous per-file counter; the `_image_hook` closure; `sources.js` `· image n/m` label part (box + title + aria). - **C1** status climbs 1→N + UI shows `· image n/m`: ✅ unit 3/15→15/15, integration 1/15→7/15, E2E 1/4→4/4 (title/aria ride the full string) - **C2** never frozen: ✅ E2E asserts ≥3 distinct positions, starting at 1, non-decreasing, ≤ total, reaching it - **C3** cleared at overview/summaries + all terminals; per-file reset via `_hook`; no stale position for image-less files: ✅ unit pins + E2E settle (image part gone in box/title/aria, endpoint 0/0) - **C4** `/api/sync/status` additive-only: ✅ full-dict pin passes (all pre-existing keys unchanged + the two new 0/0 keys) - **C5** hook-less caller byte-identical: ✅ CLI passes no hook; `test_on_image_progress_none_run_is_byte_identical` green - **C6** `uv run pytest`: **3526 passed, exit 0** (95s); dedicated E2E in isolation: **2 passed** (95s); `uv run ruff check . && uv run pyright`: **clean**; coverage: **99% on `app/` (>90% gate)** - Regression: all five neighbouring E2E suites green in isolation (progress_box 8, chunk_progress 2, label_mobile 4, pdf_support 6, image_documents 4) - **C7** atomic commit: left to the harness (no git index/history touched per rules) - **Docs:** README.md — sync-label paragraph already updated this phase, accurately; all `.env.example` vars documented; routes/commands verified — **no changes needed**. AGENTS.md — all references (PLAN, user_stories, test_smoke.py, .env.example, gpg config, .gitignore) verified — **no changes needed** - **Defects found in prior work:** none (one prior env-leak defect was already fixed inside this phase's task 04, conftest) - No live-infrastructure touched; no ntfy used. **Next pending phase:** none — `todo/` held only this phase
This commit is contained in:
+508
-1
@@ -18,7 +18,8 @@ import base64
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
from collections.abc import Callable
|
||||
import textwrap
|
||||
from collections.abc import Callable, Sequence
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
@@ -41,6 +42,11 @@ from app.rag.importer import (
|
||||
match_extension,
|
||||
)
|
||||
from app.rag.llm import EmbeddingError, LLMError
|
||||
from app.rag.pdf import (
|
||||
extract_pdf_images_with_stats,
|
||||
extract_pdf_page_texts,
|
||||
pictorial_pages,
|
||||
)
|
||||
from tests.fakes import FakeEmbedder
|
||||
|
||||
#: The original seven A9 formats as dotted suffixes (pre-phase-47 default
|
||||
@@ -2088,3 +2094,504 @@ def test_fallback_embedded_images_still_join_the_pipeline(
|
||||
assert image_doc.content != d1 and image_doc.content != d2
|
||||
finally:
|
||||
_cleanup_source(db, "both")
|
||||
|
||||
|
||||
# ---------- phase 152 (task 02): on_image_progress plumbing ----------
|
||||
|
||||
#: A text block comfortably over the phase-140 100-char sparseness bar
|
||||
#: (a USABLE layer, zero pictorial candidates even with the toggle on
|
||||
#: — the "no vision span" class). Inserted wrapped at 60 chars: a
|
||||
#: single long line would be CLIPPED at the page edge on extraction,
|
||||
#: while the wrapped multi-line read round-trips (the phase-140
|
||||
#: fixture's class property).
|
||||
_P152_DENSE_TEXT = (
|
||||
"The quadlet unit file defines the container interface, the image, "
|
||||
"the network, and the restart policy. Every key maps to a docker "
|
||||
"flag in the generated service file, and comments use the usual "
|
||||
"semicolon form throughout the whole document."
|
||||
)
|
||||
assert len(_P152_DENSE_TEXT.strip()) > 100 # the class property, not a typo
|
||||
|
||||
|
||||
def _p152_solid_png(value: int) -> bytes:
|
||||
"""A 40x30 solid-color PNG (channel *value*): every value used
|
||||
below stays under the phase-151 blank gate's 235 mean floor (kept,
|
||||
never blank-skipped), and distinct values are distinct images (the
|
||||
dedupe keeps every one)."""
|
||||
pix = fitz.Pixmap(fitz.csRGB, fitz.IRect(0, 0, 40, 30), 0)
|
||||
pix.clear_with(value)
|
||||
return pix.tobytes("png")
|
||||
|
||||
|
||||
def _build_image_pdf(
|
||||
path: Path,
|
||||
values: Sequence[int],
|
||||
dense: bool = True,
|
||||
repeat_first: bool = False,
|
||||
) -> None:
|
||||
"""A REAL one-page PDF: *dense* a text block over the 100-char bar
|
||||
(a USABLE layer — the fallback never fires; zero pictorial
|
||||
candidates even with the toggle on) carrying *values* the
|
||||
solid-color embedded images (one placement each, distinct).
|
||||
``dense=False`` inserts the sparse ``BESTA`` marker instead (a
|
||||
pictorial candidate under the default 100-char bar).
|
||||
``repeat_first`` adds a SECOND placement of the first image (the
|
||||
same stream — one xref, the parser's repeated-placement class,
|
||||
phase 151: the dedupe drops it, the position count is
|
||||
post-dedupe)."""
|
||||
doc = fitz.open()
|
||||
page = doc.new_page()
|
||||
text = _P152_DENSE_TEXT if dense else "BESTA"
|
||||
for i, line in enumerate(textwrap.wrap(text, 60)):
|
||||
page.insert_text((72, 72 + i * 14), line)
|
||||
y = 300
|
||||
for value in values:
|
||||
page.insert_image(fitz.Rect(72, y, 112, y + 30), stream=_p152_solid_png(value))
|
||||
y += 40
|
||||
if repeat_first:
|
||||
page.insert_image(
|
||||
fitz.Rect(150, 300, 190, 330), stream=_p152_solid_png(values[0])
|
||||
)
|
||||
doc.save(path)
|
||||
doc.close()
|
||||
|
||||
|
||||
def _build_pictorial_image_pdf(path: Path, values: Sequence[int]) -> None:
|
||||
"""A REAL two-page PDF — the ONE continuous counter's two-span
|
||||
case: page 1 is SPARSE (``BESTA`` — a pictorial candidate under
|
||||
the default 100-char bar), page 2 is dense (never a candidate) and
|
||||
carries *values* the solid-color embedded images. Both the
|
||||
phase-140 pictorial pass (before the document commit) and the
|
||||
phase-126 embedded-image pass (after it) fire on it."""
|
||||
doc = fitz.open()
|
||||
doc.new_page().insert_text((72, 72), "BESTA")
|
||||
page = doc.new_page()
|
||||
for i, line in enumerate(textwrap.wrap(_P152_DENSE_TEXT, 60)):
|
||||
page.insert_text((72, 72 + i * 14), line)
|
||||
y = 300
|
||||
for value in values:
|
||||
page.insert_image(fitz.Rect(72, y, 112, y + 30), stream=_p152_solid_png(value))
|
||||
y += 40
|
||||
doc.save(path)
|
||||
doc.close()
|
||||
|
||||
|
||||
def _build_garbage_image_pdf(path: Path, values: Sequence[int]) -> None:
|
||||
"""A REAL two-page garbage-text PDF (every page XML-ish — the
|
||||
phase-133 gate fires, the page-screenshot fallback runs) with
|
||||
*values* the solid-color embedded images on page 2 — the fallback
|
||||
span + the embedded span, ONE continuous counter."""
|
||||
doc = fitz.open()
|
||||
for p in range(1, 3):
|
||||
page = doc.new_page()
|
||||
for i in range(10):
|
||||
page.insert_text((72, 72 + i * 14), f"<foo:p{p}>{i}</foo:p{p}>")
|
||||
y = 400
|
||||
for value in values:
|
||||
doc[1].insert_image(
|
||||
fitz.Rect(72, y, 112, y + 30), stream=_p152_solid_png(value)
|
||||
)
|
||||
y += 40
|
||||
doc.save(path)
|
||||
doc.close()
|
||||
|
||||
|
||||
def _p152_summary_fields(s: ImportSummary) -> tuple:
|
||||
"""One run's observable outcome (the byte-identity pin's left
|
||||
side): every count a hook could plausibly move + the changed
|
||||
paths' RELS (the source names are equal by construction — the two
|
||||
roots carry the same directory name)."""
|
||||
return (
|
||||
s.files, s.added, s.updated, s.unchanged, s.pruned, s.errors,
|
||||
s.chunks, s.embed_batches, s.summaries, s.summary_errors,
|
||||
s.images_failed, s.pdfs_screenshotted, s.pdfs_pictorial,
|
||||
s.pdf_pictorial_pages, s.pdf_images_deduped, s.pdf_images_blank_skipped,
|
||||
tuple(rel for _, rel in s.changed_paths),
|
||||
)
|
||||
|
||||
|
||||
def _p152_rows(db, source: str) -> list[tuple]:
|
||||
"""A source's document rows (the byte-identity pin's right side),
|
||||
identity-blind: the fields a hook could plausibly move."""
|
||||
docs = db.scalars(select(Document).where(Document.source == source)).all()
|
||||
return sorted(
|
||||
(d.path, d.title, d.content, d.content_hash, d.is_image, d.summary,
|
||||
d.image_context)
|
||||
for d in docs
|
||||
)
|
||||
|
||||
|
||||
|
||||
def test_on_image_progress_embedded_pass_climbs_one_per_image(
|
||||
db, tmp_path: Path
|
||||
) -> None:
|
||||
"""The phase-152 core pin: a PDF with N DISTINCT embedded images
|
||||
(one placement each — post-dedupe, post-blank-gate: the work
|
||||
actually about to happen) fires the hook ONCE per image, 1-based,
|
||||
with the span's total, at the START of each image's pipeline step:
|
||||
``(1, N) … (N, N)``. A repeated placement of the first image (the
|
||||
parser's dedupe, phase 151) adds NO position (the total is the
|
||||
post-dedupe count), a clean usable text layer burns no fallback or
|
||||
pictorial span (the embedded pass's totals stand alone), and the
|
||||
phase-147 fast path on a re-sync (raw bytes unchanged) burns no
|
||||
vision call and fires the hook never."""
|
||||
root = tmp_path / "imgprog"
|
||||
root.mkdir()
|
||||
values = (90, 130, 170, 210) # 4 distinct images, all under the blank floor
|
||||
_build_image_pdf(root / "report.pdf", values, repeat_first=True)
|
||||
n = len(values)
|
||||
llm = _PageVisionLLM()
|
||||
llm.settings = _pdf_settings(tmp_path, images=True)
|
||||
events: list[tuple[int, int]] = []
|
||||
try:
|
||||
summary = asyncio.run(
|
||||
import_sources(
|
||||
[root], llm, session=db,
|
||||
on_image_progress=lambda pos, total: events.append((pos, total)),
|
||||
)
|
||||
)
|
||||
# The PDF + its N image docs; the repeat placement deduped.
|
||||
assert (summary.files, summary.added) == (1, 1 + n)
|
||||
assert summary.pdf_images_deduped == 1
|
||||
assert (summary.images_failed, summary.pdfs_screenshotted,
|
||||
summary.pdfs_pictorial) == (0, 0, 0)
|
||||
# The embedded pass is the ONLY vision span: N slow calls.
|
||||
assert llm.vision_calls == n
|
||||
# The core pin: once per image, 1-based, the span's total.
|
||||
assert events == [(i, n) for i in range(1, n + 1)]
|
||||
# The phase-147 fast path: a re-sync (bytes unchanged) burns no
|
||||
# vision call and fires the hook never — the events list grows
|
||||
# not at all.
|
||||
summary2 = asyncio.run(
|
||||
import_sources(
|
||||
[root], llm, session=db,
|
||||
on_image_progress=lambda pos, total: events.append((pos, total)),
|
||||
)
|
||||
)
|
||||
assert summary2.unchanged == 1
|
||||
assert llm.vision_calls == n
|
||||
assert len(events) == n
|
||||
finally:
|
||||
_cleanup_source(db, root.name)
|
||||
|
||||
|
||||
def test_on_image_progress_pictorial_then_embedded_is_one_counter(
|
||||
db, tmp_path: Path
|
||||
) -> None:
|
||||
"""ASSUMPTION A3: the spans of ONE file are ONE continuous counter
|
||||
— the embedded pass starts where the pictorial pass ended (``base
|
||||
= images_done_so_far``): the positions climb ``1…C+N`` with NO
|
||||
restart and never above the combined total (the pictorial span's
|
||||
total is its candidate-page count, the embedded span's total the
|
||||
combined count)."""
|
||||
root = tmp_path / "imgprog-two"
|
||||
root.mkdir()
|
||||
values = (90, 130, 170)
|
||||
_build_pictorial_image_pdf(root / "ikea.pdf", values)
|
||||
# The span inputs, computed IN the test (no hardcoded lists — the
|
||||
# phase-140 idiom): the deterministic gate's candidates and the
|
||||
# post-dedupe/post-gate image count.
|
||||
pdf = root / "ikea.pdf"
|
||||
page_texts = extract_pdf_page_texts(pdf)
|
||||
n_cand = len(pictorial_pages(page_texts, min_chars=100))
|
||||
images, _ = extract_pdf_images_with_stats(pdf)
|
||||
n_img = len(images)
|
||||
assert (n_cand, n_img) == (1, len(values))
|
||||
llm = _PageVisionLLM()
|
||||
llm.settings = Settings(
|
||||
_env_file=None, # pyright: ignore[reportCallIssue]
|
||||
import_extensions="md,txt,pdf",
|
||||
images=True,
|
||||
image_dir=str(tmp_path / "images"),
|
||||
pdf_pictorial=True, # default on — the pass runs on the sparse page
|
||||
)
|
||||
events: list[tuple[int, int]] = []
|
||||
try:
|
||||
summary = asyncio.run(
|
||||
import_sources(
|
||||
[root], llm, session=db,
|
||||
on_image_progress=lambda pos, total: events.append((pos, total)),
|
||||
)
|
||||
)
|
||||
assert (summary.files, summary.added) == (1, 1 + n_img)
|
||||
assert (summary.pdfs_pictorial, summary.pdf_pictorial_pages) == (1, 1)
|
||||
# The continuous counter: 1…C (total C) then C+1…C+N (total
|
||||
# C+N) — no restart, never above the combined total.
|
||||
assert events == (
|
||||
[(i, n_cand) for i in range(1, n_cand + 1)]
|
||||
+ [(n_cand + i, n_cand + n_img) for i in range(1, n_img + 1)]
|
||||
)
|
||||
finally:
|
||||
_cleanup_source(db, root.name)
|
||||
|
||||
|
||||
def test_on_image_progress_fallback_span_climbs_into_embedded(
|
||||
db, tmp_path: Path
|
||||
) -> None:
|
||||
"""The owner's stuck-label class: a garbage-text PDF's per-page
|
||||
screenshot describes (the phase-133 fallback span — the span that
|
||||
froze the label for minutes) report the rendered-page count
|
||||
(``1…M`` with total ``M``) and hand their reported count to the
|
||||
embedded pass, which continues the SAME counter (``M+1`` with
|
||||
total ``M+N``) — every slow call has a position before it."""
|
||||
root = tmp_path / "imgprog-fb"
|
||||
root.mkdir()
|
||||
_build_garbage_image_pdf(root / "garbage.pdf", (90,))
|
||||
llm = _PageVisionLLM()
|
||||
llm.settings = _pdf_settings(tmp_path, images=True)
|
||||
events: list[tuple[int, int]] = []
|
||||
try:
|
||||
summary = asyncio.run(
|
||||
import_sources(
|
||||
[root], llm, session=db,
|
||||
on_image_progress=lambda pos, total: events.append((pos, total)),
|
||||
)
|
||||
)
|
||||
assert summary.pdfs_screenshotted == 1
|
||||
assert (summary.files, summary.added, summary.images_failed) == (1, 2, 0)
|
||||
# 2 fallback pages (total 2) + 1 embedded image continuing at
|
||||
# 3 (total 3) — ONE counter across the two spans.
|
||||
assert events == [(1, 2), (2, 2), (3, 3)]
|
||||
# Every slow call (2 pages + 1 image) was preceded by a
|
||||
# position — the label never sits frozen on a busy run.
|
||||
assert llm.vision_calls == 3
|
||||
finally:
|
||||
_cleanup_source(db, root.name)
|
||||
|
||||
|
||||
def test_on_image_progress_text_only_pdf_never_fires(
|
||||
db, tmp_path: Path
|
||||
) -> None:
|
||||
"""A clean-text PDF with NO embedded images fires the hook NEVER,
|
||||
with BOTH default-on toggles: the garbage gate stays closed (no
|
||||
fallback span), the pictorial pass's deterministic gate finds zero
|
||||
candidates (a no-op — the clean-text byte-identity class), and the
|
||||
embedded pass's list is empty. The skipped spans report nothing
|
||||
and advance the counter not at all (there is no vision call at
|
||||
all)."""
|
||||
root = tmp_path / "imgprog-none"
|
||||
root.mkdir()
|
||||
_build_image_pdf(root / "clean.pdf", ()) # dense text, zero images
|
||||
llm = _PageVisionLLM()
|
||||
llm.settings = Settings(
|
||||
_env_file=None, # pyright: ignore[reportCallIssue]
|
||||
import_extensions="md,txt,pdf",
|
||||
images=True,
|
||||
image_dir=str(tmp_path / "images"),
|
||||
pdf_pictorial=True, # default on — the pass runs and no-ops
|
||||
)
|
||||
events: list[tuple[int, int]] = []
|
||||
try:
|
||||
summary = asyncio.run(
|
||||
import_sources(
|
||||
[root], llm, session=db,
|
||||
on_image_progress=lambda pos, total: events.append((pos, total)),
|
||||
)
|
||||
)
|
||||
assert (summary.files, summary.added) == (1, 1)
|
||||
assert (summary.pdfs_screenshotted, summary.pdfs_pictorial) == (0, 0)
|
||||
assert llm.vision_calls == 0 # no span ran — zero slow calls
|
||||
assert events == []
|
||||
finally:
|
||||
_cleanup_source(db, root.name)
|
||||
|
||||
|
||||
def test_on_image_progress_toggle_off_reports_nothing(
|
||||
db, tmp_path: Path
|
||||
) -> None:
|
||||
"""The ``images`` toggle OFF skips the embedded span entirely (the
|
||||
prune guard keeps any existing image docs) — a PDF that WOULD
|
||||
carry N embedded images reports nothing (a skipped span advances
|
||||
the counter not at all) and indexes as text only."""
|
||||
root = tmp_path / "imgprog-off"
|
||||
root.mkdir()
|
||||
_build_image_pdf(root / "off.pdf", (90, 130))
|
||||
llm = _PageVisionLLM()
|
||||
llm.settings = _pdf_settings(tmp_path, images=False)
|
||||
events: list[tuple[int, int]] = []
|
||||
try:
|
||||
summary = asyncio.run(
|
||||
import_sources(
|
||||
[root], llm, session=db,
|
||||
on_image_progress=lambda pos, total: events.append((pos, total)),
|
||||
)
|
||||
)
|
||||
assert (summary.files, summary.added) == (1, 1)
|
||||
assert llm.vision_calls == 0
|
||||
assert events == []
|
||||
finally:
|
||||
_cleanup_source(db, root.name)
|
||||
|
||||
|
||||
def test_on_image_progress_none_run_is_byte_identical(
|
||||
db, tmp_path: Path
|
||||
) -> None:
|
||||
"""The regression pin: a hook-less run (the CLI/chat default —
|
||||
``on_image_progress=None``, every call site a guarded no-op) lands
|
||||
byte-identical to a run WITH the hook: the same documents (path,
|
||||
title, content, digest, image flag, summary, context) and the same
|
||||
summary counters. The hook observes; it never changes the run."""
|
||||
# Both roots carry the SAME directory name — the source (the
|
||||
# walk's root.name) is the document's provenance and part of the
|
||||
# stored image context, so the comparison needs it equal.
|
||||
root_a = tmp_path / "a" / "same_src"
|
||||
root_a.mkdir(parents=True)
|
||||
root_b = tmp_path / "b" / "same_src"
|
||||
root_b.mkdir(parents=True)
|
||||
values = (90, 130, 170, 210)
|
||||
_build_image_pdf(root_a / "same.pdf", values)
|
||||
_build_image_pdf(root_b / "same.pdf", values)
|
||||
llm_a, llm_b = _PageVisionLLM(), _PageVisionLLM()
|
||||
llm_a.settings = _pdf_settings(tmp_path, images=True)
|
||||
llm_b.settings = _pdf_settings(tmp_path, images=True)
|
||||
events: list[tuple[int, int]] = []
|
||||
try:
|
||||
# Run A (hook-less) first, snapshot its rows, clear the shared
|
||||
# source name, then run B (hooked) — the rows line up 1:1.
|
||||
summary_a = asyncio.run(import_sources([root_a], llm_a, session=db))
|
||||
rows_a = _p152_rows(db, "same_src")
|
||||
_cleanup_source(db, "same_src")
|
||||
summary_b = asyncio.run(
|
||||
import_sources(
|
||||
[root_b], llm_b, session=db,
|
||||
on_image_progress=lambda pos, total: events.append((pos, total)),
|
||||
)
|
||||
)
|
||||
rows_b = _p152_rows(db, "same_src")
|
||||
# The hook ran in run B (N image positions, the span's total)
|
||||
# — and changed nothing observable.
|
||||
assert events == [(i, len(values)) for i in range(1, len(values) + 1)]
|
||||
assert _p152_summary_fields(summary_a) == _p152_summary_fields(summary_b)
|
||||
assert rows_a == rows_b
|
||||
finally:
|
||||
_cleanup_source(db, "same_src")
|
||||
|
||||
|
||||
class _InterleavingVisionLLM(_PageVisionLLM):
|
||||
"""A ``_PageVisionLLM`` variant that records each vision call in
|
||||
a SHARED event sequence (phase 152, task 04): the importer fires
|
||||
the ``on_image_progress`` hook BEFORE the
|
||||
``_index_one_embedded_image`` describe seam, so the sequence reads
|
||||
hook, vision, hook, vision, … — the interleaving pin (the label
|
||||
moves at the START of each slow call, never after it)."""
|
||||
|
||||
def __init__(self, events: list[tuple]) -> None:
|
||||
super().__init__()
|
||||
self.events = events
|
||||
|
||||
async def chat(self, messages, model=None): # pyright: ignore[reportIncompatibleMethodOverride]
|
||||
reply = await super().chat(messages, model)
|
||||
user = next((m["content"] for m in messages if m.get("role") == "user"), "")
|
||||
if isinstance(user, list):
|
||||
self.events.append(("vision", self.vision_calls))
|
||||
return reply
|
||||
|
||||
|
||||
def test_on_image_progress_hook_fires_before_each_describe_call(
|
||||
db, tmp_path: Path
|
||||
) -> None:
|
||||
"""Phase 152 (task 04): the hook fires BEFORE the corresponding
|
||||
describe call (the contract: at the START of each slow call, never
|
||||
after it) — the interleaving pinned by recording events in the
|
||||
fake: for a PDF with N embedded images the sequence reads
|
||||
hook(1,N), vision(1), hook(2,N), vision(2), …, hook(N,N),
|
||||
vision(N) (the fake's vision call IS the describe seam the hook
|
||||
precedes — the label moves while the describe is about to start,
|
||||
so a busy run never reads frozen)."""
|
||||
root = tmp_path / "imgprog-order"
|
||||
root.mkdir()
|
||||
values = (90, 130, 170)
|
||||
_build_image_pdf(root / "order.pdf", values)
|
||||
n = len(values)
|
||||
events: list[tuple] = []
|
||||
llm = _InterleavingVisionLLM(events)
|
||||
llm.settings = _pdf_settings(tmp_path, images=True)
|
||||
try:
|
||||
summary = asyncio.run(
|
||||
import_sources(
|
||||
[root], llm, session=db,
|
||||
on_image_progress=lambda pos, total: events.append(("image", pos, total)),
|
||||
)
|
||||
)
|
||||
assert (summary.files, summary.added) == (1, 1 + n)
|
||||
assert llm.vision_calls == n # one describe per image
|
||||
# The interleaving: hook BEFORE each describe call, in order.
|
||||
expected: list[tuple] = []
|
||||
for i in range(1, n + 1):
|
||||
expected.append(("image", i, n))
|
||||
expected.append(("vision", i))
|
||||
assert events == expected
|
||||
finally:
|
||||
_cleanup_source(db, root.name)
|
||||
|
||||
|
||||
def test_on_image_progress_blank_gate_excludes_skipped_images_from_the_total(
|
||||
db, tmp_path: Path
|
||||
) -> None:
|
||||
"""Phase 152 (task 04, the phase-151 interaction): images skipped
|
||||
by the dedupe/blank gate are NOT counted in the hook's total —
|
||||
the total is the work actually about to happen (the pass's
|
||||
post-dedupe, post-gate list). A PDF with 2 real + 1 near-white
|
||||
image (250 — the blank gate's class, mean ≥ 235): the gate
|
||||
(default ON) drops the near-white one BEFORE the pass — the hook
|
||||
fires for the 2 kept with total 2 (the skipped image burns no
|
||||
vision call, no position); with the kill switch OFF
|
||||
(``pdf_image_skip_blank=False`` — the pre-phase-151 behavior) the
|
||||
near-white image is indexed too — the hook fires for all 3 with
|
||||
total 3 (the total is the work about to happen under the knob's
|
||||
state)."""
|
||||
values_a = (90, 130, 250) # 250 = near-white (the blank gate's class)
|
||||
values_b = (110, 150, 250) # distinct bytes — a different document set
|
||||
# Both roots carry the SAME directory name — the source is part of
|
||||
# the stored image context, so the runs line up 1:1 (the house
|
||||
# cleanup idiom separates them).
|
||||
root_a = tmp_path / "a" / "same_src"
|
||||
root_a.mkdir(parents=True)
|
||||
root_b = tmp_path / "b" / "same_src"
|
||||
root_b.mkdir(parents=True)
|
||||
_build_image_pdf(root_a / "blank.pdf", values_a)
|
||||
_build_image_pdf(root_b / "blank.pdf", values_b)
|
||||
llm_a = _PageVisionLLM()
|
||||
llm_a.settings = _pdf_settings(tmp_path, images=True) # skip_blank default ON
|
||||
events_a: list[tuple[int, int]] = []
|
||||
try:
|
||||
summary_a = asyncio.run(
|
||||
import_sources(
|
||||
[root_a], llm_a, session=db,
|
||||
on_image_progress=lambda pos, total: events_a.append((pos, total)),
|
||||
)
|
||||
)
|
||||
# The blank gate dropped the near-white image BEFORE the
|
||||
# pass — the total is the 2 actually about to happen.
|
||||
assert summary_a.pdf_images_blank_skipped == 1
|
||||
assert (summary_a.files, summary_a.added) == (1, 3)
|
||||
assert llm_a.vision_calls == 2
|
||||
assert events_a == [(1, 2), (2, 2)]
|
||||
# The knob OFF: the near-white image is indexed too — the
|
||||
# total is all 3 (the pre-phase behavior, the work about to
|
||||
# happen under the knob's state). Fresh source (the house
|
||||
# cleanup idiom — run B's rows would otherwise be updates of
|
||||
# run A's rows, same synthetic image paths).
|
||||
_cleanup_source(db, "same_src")
|
||||
llm_b = _PageVisionLLM()
|
||||
llm_b.settings = Settings(
|
||||
_env_file=None, # pyright: ignore[reportCallIssue]
|
||||
import_extensions="md,txt,pdf",
|
||||
images=True,
|
||||
image_dir=str(tmp_path / "images"),
|
||||
pdf_pictorial=False,
|
||||
pdf_image_skip_blank=False,
|
||||
)
|
||||
events_b: list[tuple[int, int]] = []
|
||||
summary_b = asyncio.run(
|
||||
import_sources(
|
||||
[root_b], llm_b, session=db,
|
||||
on_image_progress=lambda pos, total: events_b.append((pos, total)),
|
||||
)
|
||||
)
|
||||
assert summary_b.pdf_images_blank_skipped == 0
|
||||
assert (summary_b.files, summary_b.added) == (1, 4)
|
||||
assert llm_b.vision_calls == 3
|
||||
assert events_b == [(1, 3), (2, 3), (3, 3)]
|
||||
finally:
|
||||
_cleanup_source(db, "same_src")
|
||||
|
||||
Reference in New Issue
Block a user