phase: 129_sync_chunk_progress
Build and Push Containers / build-and-push-app (push) Successful in 8m23s
Build and Push Containers / build-and-push-db (push) Successful in 40s

All green. Final report:

**Phase 129 final verification pass — 100% green** (previous work intact; this pass re-verified everything and fixed two pre-existing README inconsistencies)

- Verified on-spec: `LLMClient.embed` per-batch `on_progress` hook; importer `on_chunk_progress` plumbing (text/image/PDF branches; summary embed excluded); `SyncStatus.current_chunk/chunks_total` reset+clear contract; UI `.sync-label-chunk` pinned part with untruncated `title`/`#sync-result`
- `uv run pytest --cov=app -q`: exit 0, ~3041 passed, 0 failed, `app/` coverage **99%** (>90% gate)
- `uv run pytest tests/e2e/test_sync_chunk_progress.py -v --no-cov` (isolated, DB up): **2 passed** (endpoint chunk climb with true `chunk_document` total + UI `· chunk n/m` part)
- `uv run ruff check .`: all checks passed · `uv run pyright`: 0 errors, 0 warnings
- Completion criteria: all 5 ✅ (incl. no behavior change in completed phases — full suite green, CLI/chat no-op paths byte-identical)
- Docs: README.md fixed 2 pre-existing inconsistencies (upload section claimed "unpacks + scans immediately" — phase 90 made it unpack+register only, scan deferred to Sync; troubleshooting claimed "120s client timeout" — actual guard is the 300 s pre-token `TURN_TIMEOUT_MS`). Phase-129 label behavior already documented. AGENTS.md re-verified consistent — no changes
- Removed stray untracked `NOTES.md` (8-byte artifact); no live-infra changes; no commits — all changes left in working tree for the harness
- Next pending phase: `130_chat_answer_clusters`
This commit is contained in:
2026-09-28 03:38:14 -04:00
parent 1139f007cf
commit 2d0f608246
53 changed files with 3590 additions and 157 deletions
+115 -1
View File
@@ -54,12 +54,14 @@ class _FakeEmbeddingsService:
self,
dim: int = 768,
fail: Exception | None = None,
fail_from: int = 1,
drop_index: int = -1,
http_error: int | None = None,
too_large_min: int | None = None,
) -> None:
self.dim = dim
self.fail = fail
self.fail_from = fail_from # 1-based: fail on this call number onward
self.drop_index = drop_index
self.http_error = http_error
self.too_large_min = too_large_min
@@ -67,7 +69,7 @@ class _FakeEmbeddingsService:
async def create(self, *, model: str, input: list[str]) -> _Response:
self.calls.append(list(input))
if self.fail is not None:
if self.fail is not None and len(self.calls) >= self.fail_from:
raise self.fail
return _Response(
[
@@ -275,6 +277,118 @@ def test_single_oversized_text_fails_actionably() -> None:
assert llm.embed_batches == 0
# ---------- embed on_progress hook (phase 129, task 01) ----------
def test_embed_on_progress_reports_batch_starts_against_batch_texts() -> None:
"""The hook fires once per TOP-LEVEL batch, before the request, with
``(first_chunk_1based, total)`` — asserted against ``_batch_texts``'s
actual partition: each first index = the previous first + the
previous batch's size, total always ``len(texts)``."""
service = _FakeEmbeddingsService()
llm, _ = _make_client(service, embed_batch_size=4) # count cap binds
texts = [f"t{i}" for i in range(11)] # 4 + 4 + 3 ⇒ three top-level batches
events: list[tuple[int, int]] = []
vecs = asyncio.run(
llm.embed(texts, on_progress=lambda first, total: events.append((first, total)))
)
batches = llm._batch_texts(texts)
assert [len(b) for b in batches] == [4, 4, 3]
# Walk the partition: each first index = the previous first + the
# previous batch's size (offset counts chunks of earlier batches).
expected: list[tuple[int, int]] = []
offset = 0
for batch in batches:
expected.append((offset + 1, len(texts)))
offset += len(batch)
assert events == expected == [(1, 11), (5, 11), (9, 11)]
# first-indices strictly increasing, total always the full input count
firsts = [first for first, _ in events]
assert firsts == sorted(firsts) and len(set(firsts)) == len(firsts)
assert all(total == 11 for _, total in events)
assert len(vecs) == 11
assert llm.embed_batches == 3
def test_embed_on_progress_token_budget_partition() -> None:
"""The same contract holds when the TOKEN budget (not the count cap)
binds the partition — one oversized text per request."""
service = _FakeEmbeddingsService()
llm, _ = _make_client(service, embed_batch_size=16) # token budget binds
texts = ["x" * 2000 for _ in range(5)] # 2×2000 > 2800-char budget
events: list[tuple[int, int]] = []
asyncio.run(
llm.embed(texts, on_progress=lambda first, total: events.append((first, total)))
)
batches = llm._batch_texts(texts)
assert [len(b) for b in batches] == [1, 1, 1, 1, 1]
assert events == [(1, 5), (2, 5), (3, 5), (4, 5), (5, 5)]
def test_embed_on_progress_fires_for_a_failing_batch() -> None:
"""The hook precedes the request's try/except: a failing batch still
announced its start — the position is visible at the failure point."""
service = _FakeEmbeddingsService(
fail=RuntimeError("connection refused"), fail_from=2
) # first batch succeeds, the second fails
llm, _ = _make_client(service) # embed_batch_size=2 ⇒ [a,b] + [c]
events: list[tuple[int, int]] = []
def hook(first: int, total: int) -> None:
events.append((first, total))
with pytest.raises(EmbeddingError, match="connection refused"):
asyncio.run(llm.embed(["a", "b", "c"], on_progress=hook))
assert events == [(1, 3), (3, 3)]
assert llm.embed_batches == 1 # the failing batch did not count
def test_embed_on_progress_halved_batches_do_not_subreport() -> None:
"""The halving backstop is rare by construction and does NOT sub-report:
a halved batch leaves the announced position approximate (documented in
the ``embed`` docstring) — one top-level batch ⇒ exactly one hook call."""
service = _FakeEmbeddingsService(too_large_min=3)
llm, _ = _make_client(service, embed_batch_size=16) # all 8 fit one request
texts = [f"t{i}" for i in range(8)]
events: list[tuple[int, int]] = []
vecs = asyncio.run(
llm.embed(texts, on_progress=lambda first, total: events.append((first, total)))
)
assert events == [(1, 8)]
assert len(vecs) == 8
def test_embed_on_progress_none_is_a_noop_identical_to_default() -> None:
"""Explicit ``None`` (and the default) keep the pre-hook client: the
same request sequence, the same vectors, the same counter."""
service_default = _FakeEmbeddingsService()
llm_default, _ = _make_client(service_default)
service_none = _FakeEmbeddingsService()
llm_none, _ = _make_client(service_none)
texts = [f"t{i}" for i in range(5)]
vecs_default = asyncio.run(llm_default.embed(texts))
vecs_none = asyncio.run(llm_none.embed(texts, on_progress=None))
assert vecs_default == vecs_none
assert service_default.calls == service_none.calls
assert [len(c) for c in service_default.calls] == [2, 2, 1]
assert llm_default.embed_batches == llm_none.embed_batches == 3
def test_embed_empty_texts_never_calls_on_progress() -> None:
"""The early return for an empty input list precedes the batch loop —
no hook calls, no requests."""
service = _FakeEmbeddingsService()
llm, http = _make_client(service)
events: list[tuple[int, int]] = []
vecs = asyncio.run(
llm.embed([], on_progress=lambda first, total: events.append((first, total)))
)
assert vecs == []
assert events == []
assert http.bodies == []
# ---------- chat streaming (phase 03) ----------