phase: 129_sync_chunk_progress
All green. Final report:
**Phase 129 final verification pass — 100% green** (previous work intact; this pass re-verified everything and fixed two pre-existing README inconsistencies)
- Verified on-spec: `LLMClient.embed` per-batch `on_progress` hook; importer `on_chunk_progress` plumbing (text/image/PDF branches; summary embed excluded); `SyncStatus.current_chunk/chunks_total` reset+clear contract; UI `.sync-label-chunk` pinned part with untruncated `title`/`#sync-result`
- `uv run pytest --cov=app -q`: exit 0, ~3041 passed, 0 failed, `app/` coverage **99%** (>90% gate)
- `uv run pytest tests/e2e/test_sync_chunk_progress.py -v --no-cov` (isolated, DB up): **2 passed** (endpoint chunk climb with true `chunk_document` total + UI `· chunk n/m` part)
- `uv run ruff check .`: all checks passed · `uv run pyright`: 0 errors, 0 warnings
- Completion criteria: all 5 ✅ (incl. no behavior change in completed phases — full suite green, CLI/chat no-op paths byte-identical)
- Docs: README.md fixed 2 pre-existing inconsistencies (upload section claimed "unpacks + scans immediately" — phase 90 made it unpack+register only, scan deferred to Sync; troubleshooting claimed "120s client timeout" — actual guard is the 300 s pre-token `TURN_TIMEOUT_MS`). Phase-129 label behavior already documented. AGENTS.md re-verified consistent — no changes
- Removed stray untracked `NOTES.md` (8-byte artifact); no live-infra changes; no commits — all changes left in working tree for the harness
- Next pending phase: `130_chat_answer_clusters`
This commit is contained in:
@@ -63,12 +63,39 @@ visible span carries main + count, the space is the CSS gap). Every
|
||||
other label state (idle "Sync sources", settle "Synced HH:MM", the
|
||||
202/409 click, the re-attach) goes through setSyncLabel(main) —
|
||||
count hidden.
|
||||
|
||||
Phase 129 (task 04) adds the label's THIRD pinned part — the chunk
|
||||
position: #sync-label ships a hidden .sync-label-chunk span after the
|
||||
count span (the pinned "· chunk n/m" — the in-flight file's
|
||||
embed-batch position, batch granularity: n the 1-based first chunk of
|
||||
the in-flight batch, m the file's total content chunks). The ONLY
|
||||
builder that reads the status's chunk fields is fmtSyncPhaseLabel's
|
||||
import fall-through — "· chunk n/m" renders while chunks_total > 0
|
||||
and hides in every other state (the prelude, the 202/409 click, the
|
||||
upload run — its status has no chunk fields, overview, summaries —
|
||||
its own (n/m) is its position, idle / settle). setSyncLabel(mainText,
|
||||
countText, chunkText) lands the part under the same present/absent
|
||||
contract as the count part, and the UNTRUNCATED full label — main +
|
||||
" " + count + " " + chunk — rides the button title + #sync-result
|
||||
(A4 — the in-flight import reads "Syncing… <file> (2/3) · chunk
|
||||
14/38"). The part keeps the pinned mono CSS (flex 0 0 auto, nowrap,
|
||||
--mono, the rule right after .sync-label-count). A node harness (the
|
||||
test-document-viewer pattern) behaviorally pins the rendered parts +
|
||||
the full title across every label state.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
#: The node harness needs node (the test-document-viewer pattern).
|
||||
HAVE_NODE = shutil.which("node") is not None
|
||||
|
||||
FRONTEND = Path(__file__).resolve().parents[2] / "frontend"
|
||||
SOURCES_JS = FRONTEND / "assets" / "sources.js"
|
||||
STYLES_CSS = FRONTEND / "assets" / "styles.css"
|
||||
@@ -219,21 +246,26 @@ def test_fmt_sync_label_returns_the_parts_contract() -> None:
|
||||
|
||||
|
||||
def test_set_sync_label_is_the_single_parts_writer() -> None:
|
||||
"""setSyncLabel(mainText, countText = "") — Phase 128 (task 01):
|
||||
the SINGLE point that writes #sync-label. The MAIN part gets
|
||||
mainText (textContent only — the house rule, no HTML strings);
|
||||
the COUNT part gets countText when present (textContent +
|
||||
hidden=false — the "(n/m)" position is visible, NEVER clipped) and
|
||||
is emptied + hidden=true when absent (the count span is hidden
|
||||
whenever there is no "(n/m)" to show). Then the UNTRUNCATED full
|
||||
label — mainText + (countText ? " " + countText : "") — rides the
|
||||
button title + #sync-result (the aria-live announcer) exactly as
|
||||
before (A4 — the screen reader hears the whole thing)."""
|
||||
"""setSyncLabel(mainText, countText = "", chunkText = "") — Phase
|
||||
128 (task 01) + Phase 129 (task 04): the SINGLE point that writes
|
||||
#sync-label. The MAIN part gets mainText (textContent only — the
|
||||
house rule, no HTML strings); the COUNT part gets countText when
|
||||
present (textContent + hidden=false — the "(n/m)" position is
|
||||
visible, NEVER clipped) and is emptied + hidden=true when absent;
|
||||
the CHUNK part (phase 129) gets chunkText under the SAME contract
|
||||
(present → the pinned "· chunk n/m" is visible, absent → emptied
|
||||
+ hidden=true — the part renders only while a file's embed batch
|
||||
is in flight). Then the UNTRUNCATED full label — mainText +
|
||||
(countText ? " " + countText : "") + (chunkText ? " " + chunkText
|
||||
: "") — rides the button title + #sync-result (the aria-live
|
||||
announcer) (A4 — the screen reader hears the whole "Syncing…
|
||||
<file> (2/3) · chunk 14/38")."""
|
||||
body = _fn(_js(), "setSyncLabel")
|
||||
assert 'function setSyncLabel(mainText, countText = "")' in body
|
||||
assert 'function setSyncLabel(mainText, countText = "", chunkText = "")' in body
|
||||
# the parts: textContent only (the house rule), null-guarded refs
|
||||
assert "syncLabelMain.textContent = mainText" in body
|
||||
assert "syncLabelCount.textContent = countText" in body
|
||||
assert "syncLabelChunk.textContent = chunkText" in body
|
||||
# present count → visible; absent count → hidden, emptied
|
||||
assert "syncLabelCount.hidden = false" in body
|
||||
assert 'syncLabelCount.textContent = ""' in body
|
||||
@@ -241,11 +273,22 @@ def test_set_sync_label_is_the_single_parts_writer() -> None:
|
||||
i_vis = body.find("syncLabelCount.hidden = false")
|
||||
i_hid = body.find("syncLabelCount.hidden = true")
|
||||
assert -1 < i_vis < i_hid, "the present-count branch precedes the absent one"
|
||||
# present chunk → visible; absent chunk → hidden, emptied (same contract)
|
||||
assert "syncLabelChunk.hidden = false" in body
|
||||
assert 'syncLabelChunk.textContent = ""' in body
|
||||
assert "syncLabelChunk.hidden = true" in body
|
||||
i_cvis = body.find("syncLabelChunk.hidden = false")
|
||||
i_chid = body.find("syncLabelChunk.hidden = true")
|
||||
assert -1 < i_cvis < i_chid, "the present-chunk branch precedes the absent one"
|
||||
# A4: the untruncated full label rides the title + the announcer —
|
||||
# main + " " + count (the join space), bare main when no count.
|
||||
# main + " " + count + " " + chunk (the join spaces = the CSS
|
||||
# gaps), bare main when both are absent.
|
||||
assert (
|
||||
'const full = countText ? `${mainText} ${countText}` : mainText'
|
||||
) in body, "the full label is main + ' ' + count (bare main when absent)"
|
||||
"const full =\n"
|
||||
" mainText +\n"
|
||||
" (countText ? ` ${countText}` : \"\") +\n"
|
||||
" (chunkText ? ` ${chunkText}` : \"\");"
|
||||
) in body, ("the full label is main + ' ' + count + ' ' + chunk (spaces = the CSS gaps)")
|
||||
i_full = body.find("const full")
|
||||
i_title = body.find("syncBtn.title = full")
|
||||
i_result = body.find("syncResult.textContent = full")
|
||||
@@ -281,8 +324,11 @@ def test_fmt_sync_phase_label_is_the_exact_d2_parts_contract() -> None:
|
||||
assert "current_summary" not in ov and "summaries_done" not in ov, (
|
||||
"the overview label carries no folder and no counts"
|
||||
)
|
||||
# "summaries": the exact template — optional folder, counts always.
|
||||
i_fall = body.find("return fmtSyncLabel")
|
||||
# "summaries": the exact template — optional folder, counts always,
|
||||
# NO chunk part (the summaries phase's own (n/m) is its position —
|
||||
# the status's chunk fields are cleared at this transition, so the
|
||||
# branch never reads them).
|
||||
i_fall = body.find("fmtSyncLabel(")
|
||||
assert i_sum < i_fall, "the summaries branch precedes the fall-through"
|
||||
sum_ = body[i_sum:i_fall]
|
||||
assert 'status.current_summary ? ` ${status.current_summary}` : ""' in sum_, (
|
||||
@@ -295,12 +341,40 @@ def test_fmt_sync_phase_label_is_the_exact_d2_parts_contract() -> None:
|
||||
"count: `(${status.summaries_done}/${status.summaries_total})`"
|
||||
) in sum_, "the count part is ALWAYS emitted in the summaries phase"
|
||||
assert "total > 0" not in sum_, ("no conditional — the hook always carries a total")
|
||||
assert 'chunk: ""' in ov, "the overview label carries no chunk part"
|
||||
assert 'chunk: ""' in sum_, (
|
||||
"the summaries label carries no chunk part (its (n/m) is its position)"
|
||||
)
|
||||
assert "chunks_total" not in ov, "the overview branch never reads the chunk fields"
|
||||
assert "chunks_total" not in sum_, "the summaries branch never reads the chunk fields"
|
||||
# anything else (null prelude, "import"): byte-identical to today's
|
||||
# sync label — same builder, same fields, same order.
|
||||
# sync label — same builder, same fields, same order — plus the
|
||||
# phase-129 chunk part: "· chunk n/m" ONLY while chunks_total > 0
|
||||
# (a file's embed batch is in flight — n the 1-based first chunk
|
||||
# of the batch, m the file's total content chunks, batch
|
||||
# granularity), "" otherwise (0/0 before the first batch — the
|
||||
# part hides). The fall-through is the ONLY branch that reads the
|
||||
# status's chunk fields.
|
||||
i_destr = body.find("const { main, count } = fmtSyncLabel(")
|
||||
assert -1 < i_destr < i_fall, "the fall-through destructures the phase-64 parts"
|
||||
fall = body[i_destr:]
|
||||
assert (
|
||||
'return fmtSyncLabel("sync", status.current_file, '
|
||||
"status.files_done, status.files_total);"
|
||||
) in body[i_fall:]
|
||||
'const { main, count } = fmtSyncLabel(\n'
|
||||
' "sync", status.current_file, status.files_done, status.files_total\n'
|
||||
" );"
|
||||
) in fall, "the phase-64 parts builder, unchanged"
|
||||
assert "status.chunks_total > 0" in fall, (
|
||||
"the chunk part renders only while a batch is in flight"
|
||||
)
|
||||
assert "`· chunk ${status.current_chunk}/${status.chunks_total}`" in fall
|
||||
i_guard = fall.find("status.chunks_total > 0")
|
||||
i_else = fall.find(": \"\";", i_guard)
|
||||
assert -1 < i_guard < i_else, "0/0 (before the first batch) → the empty chunk part"
|
||||
assert "return { main, count, chunk }" in fall
|
||||
# the bare builder (the upload job + the 202/409 click) returns NO
|
||||
# chunk part — those labels stay bare.
|
||||
bare = _fn(_js(), "fmtSyncLabel")
|
||||
assert "chunk" not in bare, "fmtSyncLabel stays chunk-free (upload + the click stay bare)"
|
||||
|
||||
|
||||
def test_enter_running_state_is_phase_aware_and_writes_the_parts() -> None:
|
||||
@@ -312,8 +386,10 @@ def test_enter_running_state_is_phase_aware_and_writes_the_parts() -> None:
|
||||
endpoint has no phase) and the 202/409 click keep the bare
|
||||
fmtSyncLabel parts, byte-identical. Phase 128 (task 01): the entry
|
||||
itself no longer writes the label — it hands the parts to the
|
||||
SINGLE writer (setSyncLabel(main, count)), which lands the main
|
||||
part, the pinned count part (hidden when absent), and the FULL
|
||||
SINGLE writer (setSyncLabel(main, count, chunk) — phase 129:
|
||||
the chunk part destructures with a "" default), which lands the
|
||||
main part, the pinned count part (hidden when absent), the pinned
|
||||
chunk part (phase 129 — hidden when absent), and the FULL
|
||||
untruncated label on the button title + #sync-result (A4 — set
|
||||
for EVERY running label; CSS ellipsizes the main part only)."""
|
||||
body = _fn(_js(), "enterSyncRunningState")
|
||||
@@ -332,8 +408,14 @@ def test_enter_running_state_is_phase_aware_and_writes_the_parts() -> None:
|
||||
i_fall = body.find("fmtSyncLabel(kind, currentFile, done, total)")
|
||||
assert 'kind === "sync" && status' in body
|
||||
assert -1 < i_phase < i_fall, "the phase label precedes the fall-through"
|
||||
# the parts are handed to the single writer (A4 lives in it).
|
||||
i_set = body.find("setSyncLabel(main, count)")
|
||||
# the parts are handed to the single writer (A4 lives in it) —
|
||||
# phase 129: the chunk part destructures with a "" default (the
|
||||
# bare builders — the upload job, the 202/409 click — return no
|
||||
# chunk part, so the part stays hidden there).
|
||||
assert "const { main, count, chunk = \"\" } =" in body, (
|
||||
"the chunk part destructures with the '' default (bare builders return none)"
|
||||
)
|
||||
i_set = body.find("setSyncLabel(main, count, chunk)")
|
||||
assert -1 < i_fall < i_set, "the parts are written after the split"
|
||||
assert body.count("setSyncLabel(") == 1, "one writer call per entry"
|
||||
assert "syncBtn.title =" not in body, ("the title write lives in setSyncLabel (A4)")
|
||||
@@ -369,6 +451,211 @@ def test_settle_and_idle_labels_go_through_the_parts_writer() -> None:
|
||||
assert "syncLabel.textContent" not in js, ("the parts writer is the single entry point")
|
||||
|
||||
|
||||
# =====================================================================
|
||||
# Phase 129 (task 04) — the label's third part: the chunk position
|
||||
# =====================================================================
|
||||
|
||||
|
||||
def test_sync_label_markup_ships_the_hidden_chunk_part() -> None:
|
||||
"""#sync-label (the shell's RAG view) ships the phase-129 THIRD
|
||||
part after the count span — the pinned "· chunk n/m" span
|
||||
(.sync-label-chunk), hidden + empty by default (the same ship
|
||||
contract as the count span: no chunk position at idle — the part
|
||||
renders only while a file's embed batch is in flight)."""
|
||||
html = _html()
|
||||
btn = html[html.find('id="sync-btn"') : html.find("</button>", html.find('id="sync-btn"'))]
|
||||
label = re.search(
|
||||
r'<span\b[^>]*id="sync-label"[^>]*>'
|
||||
r'<span class="sync-label-main">Sync sources</span>'
|
||||
r'<span class="sync-label-count" hidden></span>'
|
||||
r'<span class="sync-label-chunk" hidden></span>'
|
||||
r"</span>",
|
||||
btn,
|
||||
)
|
||||
assert label, (
|
||||
"the #sync-label parts markup (main + hidden count + hidden chunk) is missing"
|
||||
)
|
||||
|
||||
|
||||
def test_sync_label_chunk_css_is_the_pinned_part() -> None:
|
||||
""".sync-label-chunk (phase 129 task 04): the same pinned-part
|
||||
language as .sync-label-count — flex 0 0 auto (NEVER clipped),
|
||||
white-space nowrap, the --mono font (a position reads as data) —
|
||||
and it sits right AFTER the count rule (both mono parts read as
|
||||
data)."""
|
||||
css = _css()
|
||||
count_block = re.search(r"\.sync-label-count\s*\{", css)
|
||||
chunk_block = re.search(r"\.sync-label-chunk\s*\{([^}]*)\}", css)
|
||||
assert chunk_block, "styles.css must style .sync-label-chunk"
|
||||
for prop in ("flex: 0 0 auto", "white-space: nowrap", "font-family: var(--mono)"):
|
||||
assert prop in chunk_block.group(1), f".sync-label-chunk must carry {prop!r}"
|
||||
assert count_block and count_block.start() < chunk_block.start(), (
|
||||
"the chunk rule sits right after the count rule"
|
||||
)
|
||||
|
||||
|
||||
#: The node harness script (the test-document-viewer pattern): the
|
||||
#: REAL extracted label functions run against stub element refs, and
|
||||
#: every label state is snapshotted as a JSON row —
|
||||
#: [tag, title, announcer, main, count, countHidden, chunk, chunkHidden].
|
||||
#: No JS template literals in the script body (plain strings only —
|
||||
#: the script is assembled by concatenation, no placeholders).
|
||||
_CHUNK_NODE_HEAD = """
|
||||
const syncLabelMain = { textContent: "" };
|
||||
const syncLabelCount = { textContent: "", hidden: true };
|
||||
const syncLabelChunk = { textContent: "", hidden: true };
|
||||
const syncResult = { textContent: "" };
|
||||
const syncBtn = {
|
||||
disabled: false,
|
||||
title: "",
|
||||
attrs: {},
|
||||
setAttribute(k, v) { this.attrs[k] = v; },
|
||||
removeAttribute(k) { delete this.attrs[k]; },
|
||||
classList: { add() {}, remove() {} },
|
||||
};
|
||||
const syncIcon = { classList: { add() {}, remove() {} } };
|
||||
"""
|
||||
|
||||
_CHUNK_NODE_TAIL = """
|
||||
const show = (tag) => {
|
||||
console.log(JSON.stringify([
|
||||
tag, syncBtn.title, syncResult.textContent,
|
||||
syncLabelMain.textContent,
|
||||
syncLabelCount.textContent, syncLabelCount.hidden,
|
||||
syncLabelChunk.textContent, syncLabelChunk.hidden,
|
||||
]));
|
||||
};
|
||||
// 1. import, batch in flight (the task's fixture values, 14/38):
|
||||
enterSyncRunningState("sync", "src/a/b.md", 2, 3, {
|
||||
phase: "import", current_file: "src/a/b.md", files_done: 2, files_total: 3,
|
||||
current_chunk: 14, chunks_total: 38,
|
||||
});
|
||||
show("import-inflight");
|
||||
// 2. import before the first batch (0/0) — the chunk part hides:
|
||||
enterSyncRunningState("sync", "src/a/b.md", 2, 3, {
|
||||
phase: "import", current_file: "src/a/b.md", files_done: 2, files_total: 3,
|
||||
current_chunk: 0, chunks_total: 0,
|
||||
});
|
||||
show("import-prebatch");
|
||||
// 3. the prelude (null phase, no file yet):
|
||||
enterSyncRunningState("sync", null, 0, 0, {
|
||||
phase: null, current_file: null, files_done: 0, files_total: 0,
|
||||
current_chunk: 0, chunks_total: 0,
|
||||
});
|
||||
show("prelude");
|
||||
// 4. the 202/409 click (no status yet):
|
||||
enterSyncRunningState("sync", null, 0, 0);
|
||||
show("click");
|
||||
// 5. the upload run (no status — its endpoint has no chunk fields, D2):
|
||||
enterSyncRunningState("upload", null, 0, 0);
|
||||
show("upload");
|
||||
// 6. overview (the chunk fields are cleared at this transition):
|
||||
enterSyncRunningState("sync", null, 4, 4, {
|
||||
phase: "overview", current_file: null, files_done: 4, files_total: 4,
|
||||
current_chunk: 0, chunks_total: 0,
|
||||
});
|
||||
show("overview");
|
||||
// 7. summaries (the phase's own (n/m) is its position):
|
||||
enterSyncRunningState("sync", null, 4, 4, {
|
||||
phase: "summaries", current_summary: "src/alpha", files_done: 4, files_total: 4,
|
||||
summaries_done: 2, summaries_total: 3, current_chunk: 0, chunks_total: 0,
|
||||
});
|
||||
show("summaries");
|
||||
// 8. the idle settle goes through the writer with no chunk text:
|
||||
setSyncLabel("Sync sources");
|
||||
show("idle");
|
||||
"""
|
||||
|
||||
|
||||
def _run_node(script: str) -> str:
|
||||
proc = subprocess.run(["node", "-e", script], capture_output=True, text=True, timeout=60)
|
||||
assert proc.returncode == 0, f"node failed: {proc.stderr}"
|
||||
return proc.stdout
|
||||
|
||||
|
||||
@pytest.mark.skipif(not HAVE_NODE, reason="node not available")
|
||||
def test_chunk_part_renders_only_in_the_import_phase_under_node() -> None:
|
||||
"""Behavioral check (node) of the REAL label path — the extracted
|
||||
fmtSyncLabel / fmtSyncPhaseLabel / setSyncLabel /
|
||||
enterSyncRunningState run against stub element refs (the
|
||||
test-document-viewer pattern). The pinned "· chunk n/m" part (the
|
||||
task's fixture values):
|
||||
|
||||
* import, batch in flight (chunks_total 38 > 0): the chunk part
|
||||
shows "· chunk 14/38" and the UNTRUNCATED title + #sync-result
|
||||
read exactly "Syncing… src/a/b.md (2/3) · chunk 14/38" (A4);
|
||||
* import before the first batch (0/0): the chunk part hides and
|
||||
the label stays "Syncing… src/a/b.md (2/3)";
|
||||
* every OTHER state — prelude (null phase), the 202/409 click (no
|
||||
status), the upload run (no status, D2), overview, summaries (its
|
||||
own (n/m) is its position), the idle settle — leaves the chunk
|
||||
part hidden + empty.
|
||||
"""
|
||||
js = _js()
|
||||
fns = "\n".join(
|
||||
_fn(js, name)
|
||||
for name in ("fmtSyncLabel", "fmtSyncPhaseLabel", "setSyncLabel", "enterSyncRunningState")
|
||||
)
|
||||
script = _CHUNK_NODE_HEAD + fns + _CHUNK_NODE_TAIL
|
||||
rows = [json.loads(line) for line in _run_node(script).splitlines() if line]
|
||||
by_tag = {row[0]: row for row in rows}
|
||||
# row = [tag, title, announcer, main, count, countHidden, chunk, chunkHidden]
|
||||
|
||||
# 1. the in-flight import — the chunk part renders, the full
|
||||
# untruncated label rides the title + the announcer (A4).
|
||||
row = by_tag["import-inflight"]
|
||||
assert row[3] == "Syncing… src/a/b.md", row
|
||||
assert row[4] == "(2/3)" and row[5] is False, row
|
||||
assert row[6] == "· chunk 14/38" and row[7] is False, (
|
||||
"the chunk part shows the batch position (1-based first chunk / file total)"
|
||||
)
|
||||
assert row[1] == "Syncing… src/a/b.md (2/3) · chunk 14/38", row[1]
|
||||
assert row[2] == row[1], "the announcer hears the whole thing (A4)"
|
||||
|
||||
# 2. the import before the first batch (0/0) — the part hides, the
|
||||
# label is the byte-identical phase-64 string.
|
||||
row = by_tag["import-prebatch"]
|
||||
assert row[1] == "Syncing… src/a/b.md (2/3)", row[1]
|
||||
assert row[2] == row[1]
|
||||
assert row[6] == "" and row[7] is True, row
|
||||
|
||||
# 3. every other state — the chunk part is hidden + empty, and the
|
||||
# title is the byte-identical pre-phase-129 label for that state.
|
||||
for tag, expected_title in (
|
||||
("prelude", "Syncing…"),
|
||||
("click", "Syncing…"),
|
||||
("upload", "Importing"),
|
||||
("overview", "Writing KB overview…"),
|
||||
("summaries", "Summarizing folders… src/alpha (2/3)"),
|
||||
("idle", "Sync sources"),
|
||||
):
|
||||
row = by_tag[tag]
|
||||
assert row[1] == expected_title, (tag, row)
|
||||
assert row[2] == row[1], (tag, row)
|
||||
assert row[6] == "" and row[7] is True, f"{tag} must hide the chunk part"
|
||||
|
||||
|
||||
def test_every_other_state_passes_no_chunk_text() -> None:
|
||||
"""Source-level pin of the render-only-while-chunks_total>0
|
||||
contract: the import fall-through is the ONLY place in sources.js
|
||||
that reads the status's chunk fields (the summaries/overview
|
||||
branches and the bare builders never touch them), and every
|
||||
NON-running label emission passes no chunk text (the part hides):
|
||||
the settle/idle labels go through setSyncLabel(label) — the only
|
||||
three-arg call site is the running entry."""
|
||||
js = _js()
|
||||
body = _fn(js, "fmtSyncPhaseLabel")
|
||||
i_fall = body.find("fmtSyncLabel(")
|
||||
pre = body[:i_fall]
|
||||
assert "chunks_total" not in pre and "current_chunk" not in pre, (
|
||||
"the overview/summaries branches never read the chunk fields"
|
||||
)
|
||||
fall = body[i_fall:]
|
||||
assert "status.chunks_total" in fall and "status.current_chunk" in fall
|
||||
assert js.count("setSyncLabel(main, count, chunk)") == 1
|
||||
assert "setSyncLabel(label)" in js, "the settle/idle labels stay two-arg (no chunk)"
|
||||
|
||||
|
||||
# ---------- the two-job tick (startSyncPolling) ----------
|
||||
|
||||
|
||||
|
||||
@@ -47,6 +47,7 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
@@ -730,7 +731,12 @@ class _EmbedFailingVision(FakeEmbedder):
|
||||
first = user.split()
|
||||
return "Summary of " + (first[0] if first else "<empty>")
|
||||
|
||||
async def embed(self, texts):
|
||||
async def embed(
|
||||
self,
|
||||
texts: list[str],
|
||||
*,
|
||||
on_progress: Callable[[int, int], None] | None = None,
|
||||
) -> list[list[float]]:
|
||||
raise EmbeddingError("simulated embed endpoint failure (test sentinel)")
|
||||
|
||||
|
||||
|
||||
@@ -66,6 +66,7 @@ import asyncio
|
||||
import base64
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
@@ -199,13 +200,18 @@ class _SummaryEmbedFailsLLM(FakeEmbedder):
|
||||
super().__init__()
|
||||
self._embed_batches = 0
|
||||
|
||||
async def embed(self, texts: list[str]) -> list[list[float]]:
|
||||
async def embed(
|
||||
self,
|
||||
texts: list[str],
|
||||
*,
|
||||
on_progress: Callable[[int, int], None] | None = None,
|
||||
) -> list[list[float]]:
|
||||
self._embed_batches += 1
|
||||
if self._embed_batches == 2:
|
||||
raise EmbeddingError(
|
||||
"simulated summary-chunk embed failure (test sentinel)"
|
||||
)
|
||||
return await super().embed(texts)
|
||||
return await super().embed(texts, on_progress=on_progress)
|
||||
|
||||
|
||||
class _VisionMockLLM(FakeEmbedder):
|
||||
|
||||
+218
-4
@@ -16,6 +16,7 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
@@ -27,6 +28,7 @@ from sqlalchemy.orm import Session
|
||||
import app.rag.importer as importer
|
||||
from app.config import Settings
|
||||
from app.models import Chunk, Document
|
||||
from app.rag.chunker import chunk_document
|
||||
from app.rag.importer import (
|
||||
EXCLUDED_DIRS,
|
||||
ImportSummary,
|
||||
@@ -48,10 +50,15 @@ DEFAULT_EXTS = frozenset({".md", ".markdown", ".txt", ".yaml", ".yml", ".json",
|
||||
class _PoisonEmbedder(FakeEmbedder):
|
||||
"""Fails (like a real endpoint) on any text containing 'poison'."""
|
||||
|
||||
async def embed(self, texts: list[str]) -> list[list[float]]:
|
||||
async def embed(
|
||||
self,
|
||||
texts: list[str],
|
||||
*,
|
||||
on_progress: Callable[[int, int], None] | None = None,
|
||||
) -> list[list[float]]:
|
||||
if any("poison" in t for t in texts):
|
||||
raise EmbeddingError("embeddings endpoint refused the input (simulated)")
|
||||
return await super().embed(texts)
|
||||
return await super().embed(texts, on_progress=on_progress)
|
||||
|
||||
|
||||
class _FailingChatEmbedder(FakeEmbedder):
|
||||
@@ -67,13 +74,18 @@ class _CapEmbedder(FakeEmbedder):
|
||||
"""Simulates the endpoint's ~1024-token input cap at ~1.1 chars/token:
|
||||
any single text over 1000 chars is rejected (URL-dense worst case)."""
|
||||
|
||||
async def embed(self, texts: list[str]) -> list[list[float]]:
|
||||
async def embed(
|
||||
self,
|
||||
texts: list[str],
|
||||
*,
|
||||
on_progress: Callable[[int, int], None] | None = None,
|
||||
) -> list[list[float]]:
|
||||
if any(len(t) > 1000 for t in texts):
|
||||
raise EmbeddingError(
|
||||
"a single 1100-char chunk exceeded the endpoint's per-request "
|
||||
"input token cap — lower BOR_CHUNK_TARGET_CHARS and re-import"
|
||||
)
|
||||
return await super().embed(texts)
|
||||
return await super().embed(texts, on_progress=on_progress)
|
||||
|
||||
|
||||
def _embedder_with_extensions(extensions: str) -> FakeEmbedder:
|
||||
@@ -1430,3 +1442,205 @@ def test_prune_records_changed_paths_only_when_summary_given(db, tmp_path: Path)
|
||||
assert counters.changed_paths == [(source, "gone2.md")]
|
||||
finally:
|
||||
_cleanup_source(db, root.name)
|
||||
|
||||
|
||||
# ---------- phase 129 (task 02): on_chunk_progress plumbing ----------
|
||||
|
||||
|
||||
class _BatchingEmbedder(FakeEmbedder):
|
||||
"""Mimics ``LLMClient``'s top-level token-budgeted batching (the
|
||||
task-01 contract on the importer side): *texts* is split into
|
||||
consecutive groups of *batch_size* and the hook fires
|
||||
``(first_chunk_1based, total)`` before EACH group — the group is then
|
||||
embedded as one request (``super().embed`` with ``on_progress=None``:
|
||||
this class owns the hook sequence, so the base's single-batch
|
||||
announce is suppressed per group)."""
|
||||
|
||||
def __init__(self, batch_size: int) -> None:
|
||||
super().__init__()
|
||||
self.batch_size = batch_size
|
||||
|
||||
async def embed(
|
||||
self,
|
||||
texts: list[str],
|
||||
*,
|
||||
on_progress: Callable[[int, int], None] | None = None,
|
||||
) -> list[list[float]]:
|
||||
out: list[list[float]] = []
|
||||
for start in range(0, len(texts), self.batch_size):
|
||||
group = texts[start : start + self.batch_size]
|
||||
if on_progress is not None:
|
||||
on_progress(start + 1, len(texts))
|
||||
out.extend(await super().embed(group, on_progress=None))
|
||||
return out
|
||||
|
||||
|
||||
def _markdown_with_sections(stem: str, n: int) -> str:
|
||||
"""One markdown file with a preamble (h1 title + intro) and *n* short
|
||||
``##`` sections — each section is under the 2000-char target, so the
|
||||
file chunks to exactly *n* + 1 content chunks (the preamble + one per
|
||||
section; verified against ``chunk_document`` below)."""
|
||||
lines = [f"# {stem}", "", f"Intro line for {stem.lower()}.", ""]
|
||||
for i in range(1, n + 1):
|
||||
lines += [f"## {stem} {i}", "", f"Body text for section {i}.", ""]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _chunk_count(content: str, rel: str) -> int:
|
||||
"""The content-chunk count the importer's pipeline will produce for
|
||||
*content* (the same ``chunk_document`` call and defaults as
|
||||
``_replace_content_chunks``: ``max(400, chunk_target_chars)`` target,
|
||||
``chunk_overlap_chars`` overlap)."""
|
||||
s = Settings(_env_file=None) # pyright: ignore[reportCallIssue]
|
||||
return len(chunk_document(content, rel, max(400, s.chunk_target_chars), s.chunk_overlap_chars))
|
||||
|
||||
|
||||
def test_on_chunk_progress_reports_per_file_batch_sequences(
|
||||
db, tmp_path: Path
|
||||
) -> None:
|
||||
"""A multi-batch file's embed position is fully observable: the hook
|
||||
fires before each embed batch with ``(first_chunk_1based, total)`` —
|
||||
``total`` is the file's content-chunk count, the first indices climb
|
||||
by the previous batch's size, and a SECOND file in the same run gets
|
||||
its OWN sequence (the two files' totals never mix)."""
|
||||
root = tmp_path / "chunkprog"
|
||||
root.mkdir()
|
||||
content_a = _markdown_with_sections("Alpha", 8) # preamble + 8 = 9 chunks
|
||||
content_b = _markdown_with_sections("Beta", 4) # preamble + 4 = 5 chunks
|
||||
(root / "alpha.md").write_text(content_a)
|
||||
(root / "beta.md").write_text(content_b)
|
||||
n_a, n_b = _chunk_count(content_a, "alpha.md"), _chunk_count(content_b, "beta.md")
|
||||
assert (n_a, n_b) == (9, 5)
|
||||
|
||||
batch_size = 3
|
||||
|
||||
def expected(n: int) -> list[tuple[int, int]]:
|
||||
return [(start + 1, n) for start in range(0, n, batch_size)]
|
||||
|
||||
# The task's multi-batch case: file A spans >= 2 batches.
|
||||
assert len(expected(n_a)) >= 2
|
||||
|
||||
llm = _BatchingEmbedder(batch_size=batch_size)
|
||||
chunk_events: list[tuple[int, int]] = []
|
||||
file_events: list[tuple[str, str, int, int]] = []
|
||||
try:
|
||||
summary = asyncio.run(
|
||||
import_sources(
|
||||
[root], llm, session=db,
|
||||
progress=lambda s, r, d, t: file_events.append((s, r, d, t)),
|
||||
on_chunk_progress=lambda f, t: chunk_events.append((f, t)),
|
||||
)
|
||||
)
|
||||
assert (summary.added, summary.chunks) == (2, n_a + n_b)
|
||||
finally:
|
||||
_cleanup_source(db, root.name)
|
||||
|
||||
# The per-file ``progress`` hook names each file in walk order ...
|
||||
assert file_events == [
|
||||
(root.name, "alpha.md", 1, 2),
|
||||
(root.name, "beta.md", 2, 2),
|
||||
]
|
||||
# ... and the chunk sequence is exactly file A's batches followed by
|
||||
# file B's OWN restarted sequence (its totals never leak into A's).
|
||||
assert chunk_events == expected(n_a) + expected(n_b)
|
||||
|
||||
|
||||
def test_on_chunk_progress_single_batch_file_and_summary_excluded(
|
||||
db, tmp_path: Path
|
||||
) -> None:
|
||||
"""A file whose chunks fit in ONE embed batch reports exactly one
|
||||
``(1, N)`` event, and the document's summary-chunk embed (the SECOND
|
||||
``llm.embed`` call) NEVER fires the hook — the counter counts content
|
||||
chunks only (A6: no ``1/1`` noise after the real progress ends)."""
|
||||
root = tmp_path / "chunkprog-one"
|
||||
root.mkdir()
|
||||
content = _markdown_with_sections("Gamma", 3) # preamble + 3 = 4 chunks
|
||||
(root / "gamma.md").write_text(content)
|
||||
n = _chunk_count(content, "gamma.md")
|
||||
assert n == 4
|
||||
events: list[tuple[int, int]] = []
|
||||
llm = FakeEmbedder()
|
||||
try:
|
||||
summary = asyncio.run(
|
||||
import_sources(
|
||||
[root], llm, session=db,
|
||||
on_chunk_progress=lambda f, t: events.append((f, t)),
|
||||
)
|
||||
)
|
||||
assert summary.added == 1
|
||||
finally:
|
||||
_cleanup_source(db, root.name)
|
||||
assert events == [(1, n)]
|
||||
# Two embed calls happened (the content batch + the excluded summary
|
||||
# chunk) — the hook fired for the first only.
|
||||
assert [len(c) for c in llm.calls] == [n, 1]
|
||||
|
||||
|
||||
def test_on_chunk_progress_image_branch_reports_one_of_one(
|
||||
db, tmp_path: Path
|
||||
) -> None:
|
||||
"""The image branch's ONE description-chunk embed reports exactly
|
||||
``(1, 1)`` — it IS the file's only chunk — and the mirrored
|
||||
summary-chunk embed never fires the hook. The REAL description seam
|
||||
is used (the ``_VisionLLM`` multimodal ``chat``)."""
|
||||
root = tmp_path / "imgchunk"
|
||||
root.mkdir()
|
||||
(root / "pic.png").write_bytes(_png_bytes(4, 4, (0.8, 0.2, 0.2)))
|
||||
events: list[tuple[int, int]] = []
|
||||
llm = _VisionLLM()
|
||||
llm.settings = Settings(
|
||||
_env_file=None, # pyright: ignore[reportCallIssue]
|
||||
images=True,
|
||||
image_dir=str(tmp_path / "images"),
|
||||
)
|
||||
try:
|
||||
summary = asyncio.run(
|
||||
import_sources(
|
||||
[root], llm, session=db,
|
||||
on_chunk_progress=lambda f, t: events.append((f, t)),
|
||||
)
|
||||
)
|
||||
assert (summary.files, summary.added) == (1, 1)
|
||||
finally:
|
||||
_cleanup_source(db, root.name)
|
||||
assert events == [(1, 1)]
|
||||
# The content embed (the 1 description chunk) + the excluded
|
||||
# summary-chunk embed — the hook fired for the first only.
|
||||
assert [len(c) for c in llm.calls] == [1, 1]
|
||||
|
||||
|
||||
def test_on_chunk_progress_pdf_branch_reports_content_chunks(
|
||||
db, tmp_path: Path
|
||||
) -> None:
|
||||
"""The PDF branch's CONTENT embed is the primary target: a two-page
|
||||
PDF (two ``Page N:`` chunks) reports ``(1, 2)`` once through the
|
||||
hook, and the summary-chunk embed never fires it."""
|
||||
root = tmp_path / "pdfchunk"
|
||||
root.mkdir()
|
||||
doc = fitz.open()
|
||||
doc.new_page().insert_text((72, 72), "First page text about networking.")
|
||||
doc.new_page().insert_text((72, 72), "Second page text about storage.")
|
||||
doc.save(root / "report.pdf")
|
||||
doc.close()
|
||||
events: list[tuple[int, int]] = []
|
||||
llm = _VisionLLM()
|
||||
llm.settings = Settings(
|
||||
_env_file=None, # pyright: ignore[reportCallIssue]
|
||||
images=True,
|
||||
image_dir=str(tmp_path / "images"),
|
||||
import_extensions="md,txt,pdf",
|
||||
)
|
||||
try:
|
||||
summary = asyncio.run(
|
||||
import_sources(
|
||||
[root], llm, session=db,
|
||||
on_chunk_progress=lambda f, t: events.append((f, t)),
|
||||
)
|
||||
)
|
||||
assert (summary.files, summary.added) == (1, 1)
|
||||
finally:
|
||||
_cleanup_source(db, root.name)
|
||||
assert events == [(1, 2)]
|
||||
# The content embed (the 2 page chunks, one batch) + the excluded
|
||||
# summary-chunk embed — the hook fired for the first only.
|
||||
assert [len(c) for c in llm.calls] == [2, 1]
|
||||
|
||||
@@ -54,12 +54,14 @@ class _FakeEmbeddingsService:
|
||||
self,
|
||||
dim: int = 768,
|
||||
fail: Exception | None = None,
|
||||
fail_from: int = 1,
|
||||
drop_index: int = -1,
|
||||
http_error: int | None = None,
|
||||
too_large_min: int | None = None,
|
||||
) -> None:
|
||||
self.dim = dim
|
||||
self.fail = fail
|
||||
self.fail_from = fail_from # 1-based: fail on this call number onward
|
||||
self.drop_index = drop_index
|
||||
self.http_error = http_error
|
||||
self.too_large_min = too_large_min
|
||||
@@ -67,7 +69,7 @@ class _FakeEmbeddingsService:
|
||||
|
||||
async def create(self, *, model: str, input: list[str]) -> _Response:
|
||||
self.calls.append(list(input))
|
||||
if self.fail is not None:
|
||||
if self.fail is not None and len(self.calls) >= self.fail_from:
|
||||
raise self.fail
|
||||
return _Response(
|
||||
[
|
||||
@@ -275,6 +277,118 @@ def test_single_oversized_text_fails_actionably() -> None:
|
||||
assert llm.embed_batches == 0
|
||||
|
||||
|
||||
# ---------- embed on_progress hook (phase 129, task 01) ----------
|
||||
|
||||
|
||||
def test_embed_on_progress_reports_batch_starts_against_batch_texts() -> None:
|
||||
"""The hook fires once per TOP-LEVEL batch, before the request, with
|
||||
``(first_chunk_1based, total)`` — asserted against ``_batch_texts``'s
|
||||
actual partition: each first index = the previous first + the
|
||||
previous batch's size, total always ``len(texts)``."""
|
||||
service = _FakeEmbeddingsService()
|
||||
llm, _ = _make_client(service, embed_batch_size=4) # count cap binds
|
||||
texts = [f"t{i}" for i in range(11)] # 4 + 4 + 3 ⇒ three top-level batches
|
||||
events: list[tuple[int, int]] = []
|
||||
vecs = asyncio.run(
|
||||
llm.embed(texts, on_progress=lambda first, total: events.append((first, total)))
|
||||
)
|
||||
|
||||
batches = llm._batch_texts(texts)
|
||||
assert [len(b) for b in batches] == [4, 4, 3]
|
||||
# Walk the partition: each first index = the previous first + the
|
||||
# previous batch's size (offset counts chunks of earlier batches).
|
||||
expected: list[tuple[int, int]] = []
|
||||
offset = 0
|
||||
for batch in batches:
|
||||
expected.append((offset + 1, len(texts)))
|
||||
offset += len(batch)
|
||||
assert events == expected == [(1, 11), (5, 11), (9, 11)]
|
||||
# first-indices strictly increasing, total always the full input count
|
||||
firsts = [first for first, _ in events]
|
||||
assert firsts == sorted(firsts) and len(set(firsts)) == len(firsts)
|
||||
assert all(total == 11 for _, total in events)
|
||||
assert len(vecs) == 11
|
||||
assert llm.embed_batches == 3
|
||||
|
||||
|
||||
def test_embed_on_progress_token_budget_partition() -> None:
|
||||
"""The same contract holds when the TOKEN budget (not the count cap)
|
||||
binds the partition — one oversized text per request."""
|
||||
service = _FakeEmbeddingsService()
|
||||
llm, _ = _make_client(service, embed_batch_size=16) # token budget binds
|
||||
texts = ["x" * 2000 for _ in range(5)] # 2×2000 > 2800-char budget
|
||||
events: list[tuple[int, int]] = []
|
||||
asyncio.run(
|
||||
llm.embed(texts, on_progress=lambda first, total: events.append((first, total)))
|
||||
)
|
||||
batches = llm._batch_texts(texts)
|
||||
assert [len(b) for b in batches] == [1, 1, 1, 1, 1]
|
||||
assert events == [(1, 5), (2, 5), (3, 5), (4, 5), (5, 5)]
|
||||
|
||||
|
||||
def test_embed_on_progress_fires_for_a_failing_batch() -> None:
|
||||
"""The hook precedes the request's try/except: a failing batch still
|
||||
announced its start — the position is visible at the failure point."""
|
||||
service = _FakeEmbeddingsService(
|
||||
fail=RuntimeError("connection refused"), fail_from=2
|
||||
) # first batch succeeds, the second fails
|
||||
llm, _ = _make_client(service) # embed_batch_size=2 ⇒ [a,b] + [c]
|
||||
events: list[tuple[int, int]] = []
|
||||
|
||||
def hook(first: int, total: int) -> None:
|
||||
events.append((first, total))
|
||||
|
||||
with pytest.raises(EmbeddingError, match="connection refused"):
|
||||
asyncio.run(llm.embed(["a", "b", "c"], on_progress=hook))
|
||||
assert events == [(1, 3), (3, 3)]
|
||||
assert llm.embed_batches == 1 # the failing batch did not count
|
||||
|
||||
|
||||
def test_embed_on_progress_halved_batches_do_not_subreport() -> None:
|
||||
"""The halving backstop is rare by construction and does NOT sub-report:
|
||||
a halved batch leaves the announced position approximate (documented in
|
||||
the ``embed`` docstring) — one top-level batch ⇒ exactly one hook call."""
|
||||
service = _FakeEmbeddingsService(too_large_min=3)
|
||||
llm, _ = _make_client(service, embed_batch_size=16) # all 8 fit one request
|
||||
texts = [f"t{i}" for i in range(8)]
|
||||
events: list[tuple[int, int]] = []
|
||||
vecs = asyncio.run(
|
||||
llm.embed(texts, on_progress=lambda first, total: events.append((first, total)))
|
||||
)
|
||||
assert events == [(1, 8)]
|
||||
assert len(vecs) == 8
|
||||
|
||||
|
||||
def test_embed_on_progress_none_is_a_noop_identical_to_default() -> None:
|
||||
"""Explicit ``None`` (and the default) keep the pre-hook client: the
|
||||
same request sequence, the same vectors, the same counter."""
|
||||
service_default = _FakeEmbeddingsService()
|
||||
llm_default, _ = _make_client(service_default)
|
||||
service_none = _FakeEmbeddingsService()
|
||||
llm_none, _ = _make_client(service_none)
|
||||
texts = [f"t{i}" for i in range(5)]
|
||||
vecs_default = asyncio.run(llm_default.embed(texts))
|
||||
vecs_none = asyncio.run(llm_none.embed(texts, on_progress=None))
|
||||
assert vecs_default == vecs_none
|
||||
assert service_default.calls == service_none.calls
|
||||
assert [len(c) for c in service_default.calls] == [2, 2, 1]
|
||||
assert llm_default.embed_batches == llm_none.embed_batches == 3
|
||||
|
||||
|
||||
def test_embed_empty_texts_never_calls_on_progress() -> None:
|
||||
"""The early return for an empty input list precedes the batch loop —
|
||||
no hook calls, no requests."""
|
||||
service = _FakeEmbeddingsService()
|
||||
llm, http = _make_client(service)
|
||||
events: list[tuple[int, int]] = []
|
||||
vecs = asyncio.run(
|
||||
llm.embed([], on_progress=lambda first, total: events.append((first, total)))
|
||||
)
|
||||
assert vecs == []
|
||||
assert events == []
|
||||
assert http.bodies == []
|
||||
|
||||
|
||||
# ---------- chat streaming (phase 03) ----------
|
||||
|
||||
|
||||
|
||||
@@ -12,7 +12,7 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
import hashlib
|
||||
import logging
|
||||
from collections.abc import Sequence
|
||||
from collections.abc import Callable, Sequence
|
||||
from datetime import timedelta
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
@@ -116,10 +116,15 @@ class _FirstImageEmbedFailsLLM(FakeEmbedder):
|
||||
first = user.split()
|
||||
return "Summary of " + (first[0] if first else "<empty>")
|
||||
|
||||
async def embed(self, texts: list[str]) -> list[list[float]]:
|
||||
async def embed(
|
||||
self,
|
||||
texts: list[str],
|
||||
*,
|
||||
on_progress: Callable[[int, int], None] | None = None,
|
||||
) -> list[list[float]]:
|
||||
if any(self.EMBED_FAIL in t for t in texts):
|
||||
raise EmbeddingError("simulated embed failure (test sentinel)")
|
||||
return await super().embed(texts)
|
||||
return await super().embed(texts, on_progress=on_progress)
|
||||
|
||||
|
||||
def _pdf_llm[
|
||||
|
||||
+326
-27
@@ -38,16 +38,34 @@ file" (remove-when-null) rule is gone.
|
||||
Phase 128 (task 01): the label becomes PARTS — #sync-label ships a
|
||||
main span (the idle text pre-JS) + a hidden count span (the pinned
|
||||
"(n/m)"), the builders return { main, count }, and the single writer
|
||||
setSyncLabel(main, count) lands the parts + the untruncated full
|
||||
label (main + " " + count) on the title + #sync-result; the
|
||||
running-state / settle pins move onto that shape (their labels stay
|
||||
byte-identical as full strings).
|
||||
setSyncLabel(main, count, chunk — the chunk part is phase 129) lands
|
||||
the parts + the untruncated full label (main + " " + count +
|
||||
" " + chunk) on the title + #sync-result; the running-state /
|
||||
settle pins move onto that shape (their labels stay byte-identical
|
||||
as full strings). Phase 129 (task 04): #sync-label ships a THIRD
|
||||
hidden span (the pinned "· chunk n/m" — the in-flight file's
|
||||
embed-batch position), rendered only while a file's embed batch is in
|
||||
flight (the import phase after the first batch — hidden in every
|
||||
other state); the markup pin below pins the exact three-part ship
|
||||
shape.
|
||||
|
||||
Phase 129 (task 03): the status contract gains the in-flight file's
|
||||
EMBED-chunk position — ``current_chunk`` (the 1-based first chunk of
|
||||
the in-flight token-budgeted batch) + ``chunks_total`` (the file's
|
||||
total content chunks): the idle dict pins them 0/0, the running
|
||||
dict's pins gain them, and the new mid-run pair test reads the
|
||||
positions between ticks (1/38 → 13/38) through the runner's own
|
||||
``_chunk_hook`` closure; the convention split is pinned — the chunk
|
||||
position clears at every point the import phase ends (the
|
||||
overview/summaries transitions + both terminals) while the FILE
|
||||
counts keep their final values.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Callable, Iterator, Sequence
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
@@ -130,23 +148,31 @@ def test_sync_button_has_icon_and_label_parts() -> None:
|
||||
|
||||
Phase 128 (task 01): #sync-label is PARTS — the MAIN span
|
||||
(.sync-label-main) carries the idle text (the pre-JS label renders
|
||||
from the static markup; the part CSS may ellipsize it), and the
|
||||
COUNT span (.sync-label-count — the pinned "(n/m)" position, never
|
||||
clipped) ships hidden + empty (there is no count at idle). The
|
||||
class/id stay (the module query + the E2E label assertions)."""
|
||||
from the static markup; the part CSS may ellipsize it), the
|
||||
COUNT span (.sync-label-count — the pinned "(n/m)" position,
|
||||
never clipped) ships hidden + empty (there is no count at idle),
|
||||
and — phase 129 (task 04) — the CHUNK span (.sync-label-chunk —
|
||||
the pinned "· chunk n/m" embed-batch position) ships hidden +
|
||||
empty too (no chunk position at idle — the part renders only
|
||||
while a file's embed batch is in flight). The class/id stay (the
|
||||
module query + the E2E label assertions)."""
|
||||
text = _text(SHELL_HTML)
|
||||
btn = text[text.find('id="sync-btn"') : text.find("</button>", text.find('id="sync-btn"'))]
|
||||
assert re.search(r'<svg[^>]*class="sync-icon"[^>]*aria-hidden="true"', btn)
|
||||
# Phase 128 (task 01): the parts markup — the exact shape (the main
|
||||
# span holds the idle text, the count span is hidden + empty).
|
||||
# Phase 128 (task 01) + Phase 129 (task 04): the parts markup —
|
||||
# the exact three-part shape (the main span holds the idle text,
|
||||
# the count + chunk spans are hidden + empty).
|
||||
label = re.search(
|
||||
r'<span\b[^>]*id="sync-label"[^>]*>'
|
||||
r'<span class="sync-label-main">Sync sources</span>'
|
||||
r'<span class="sync-label-count" hidden></span>'
|
||||
r'<span class="sync-label-chunk" hidden></span>'
|
||||
r"</span>",
|
||||
btn,
|
||||
)
|
||||
assert label, ("the #sync-label parts markup (main + hidden count) is missing")
|
||||
assert label, (
|
||||
"the #sync-label parts markup (main + hidden count + hidden chunk) is missing"
|
||||
)
|
||||
assert 'class="sync-label"' in label.group(0)
|
||||
|
||||
|
||||
@@ -299,9 +325,11 @@ def test_sources_js_running_state_is_never_stale() -> None:
|
||||
Phase 128 (task 01): the entry hands the label PARTS (the
|
||||
phase-aware split — sync + status → fmtSyncPhaseLabel, the
|
||||
fall-through → the byte-identical fmtSyncLabel) to the SINGLE
|
||||
writer, setSyncLabel(main, count), which lands the main part, the
|
||||
pinned count part (hidden when absent), and the FULL untruncated
|
||||
label on the button title + #sync-result (A4) — the direct span /
|
||||
writer, setSyncLabel(main, count, chunk — phase 129: the chunk
|
||||
part destructures with a "" default), which lands the main part,
|
||||
the pinned count part (hidden when absent), the pinned chunk part
|
||||
(phase 129 — hidden when absent), and the FULL untruncated label
|
||||
on the button title + #sync-result (A4) — the direct span /
|
||||
title / announcer writes are GONE from the entry (they live in
|
||||
setSyncLabel)."""
|
||||
js = _text(SOURCES_JS)
|
||||
@@ -313,7 +341,7 @@ def test_sources_js_running_state_is_never_stale() -> None:
|
||||
assert "fmtSyncLabel(kind, currentFile, done, total)" in body
|
||||
# …and the single writer lands the parts (A4 in it — the title is
|
||||
# set for EVERY running label: the file-only rule is adjusted away).
|
||||
assert "setSyncLabel(main, count)" in body
|
||||
assert "setSyncLabel(main, count, chunk)" in body
|
||||
assert body.count("setSyncLabel(") == 1, "the entry writes ONLY through the parts writer"
|
||||
assert "syncBtn.title =" not in body, ("the title write lives in setSyncLabel (A4)")
|
||||
assert "syncLabel.textContent" not in body, ("no direct span write — the parts")
|
||||
@@ -759,7 +787,14 @@ class _GatedImport:
|
||||
progress-shaped call goes through the real hook closure — the
|
||||
closure under test), parks on a threading gate so the test can read
|
||||
the status mid-run, then returns the canned summary (or raises
|
||||
``fail``)."""
|
||||
``fail``).
|
||||
|
||||
Phase 129 (task 03): optionally ALSO fires the in-flight file's
|
||||
chunk-position pairs (``chunk_pairs``) through the runner's OWN
|
||||
``_chunk_hook`` closure: pair one lands before the main park
|
||||
(readable mid-run), and every later pair lands after the main
|
||||
park + its own park (``chunk_release``), so each position is
|
||||
readable on the status between ticks."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -767,6 +802,8 @@ class _GatedImport:
|
||||
started: threading.Event,
|
||||
release: threading.Event,
|
||||
fail: BaseException | None = None,
|
||||
chunk_pairs: Sequence[tuple[int, int]] = (),
|
||||
chunk_release: threading.Event | None = None,
|
||||
) -> None:
|
||||
self.summary = summary
|
||||
self.started = started
|
||||
@@ -774,6 +811,9 @@ class _GatedImport:
|
||||
self.fail = fail
|
||||
self.hook_calls: list[tuple[str, str, int, int]] = []
|
||||
self.prune_flags: list[bool] = []
|
||||
self.chunk_pairs = list(chunk_pairs)
|
||||
self.chunk_release = chunk_release
|
||||
self.chunk_hook_calls: list[tuple[int, int]] = []
|
||||
|
||||
async def __call__(
|
||||
self,
|
||||
@@ -784,6 +824,7 @@ class _GatedImport:
|
||||
limit: int | None = None,
|
||||
session: object = None,
|
||||
progress: Callable[[str, str, int, int], None] | None = None,
|
||||
on_chunk_progress: Callable[[int, int], None] | None = None, # phase 129
|
||||
ignore_by_root: dict[str, list[str]] | None = None, # phase 89
|
||||
include_hidden_by_root: dict[str, bool] | None = None, # phase 105
|
||||
doc_dates_by_root: dict[str, dict[str, datetime]] | None = None, # phase 106
|
||||
@@ -792,8 +833,26 @@ class _GatedImport:
|
||||
if progress is not None:
|
||||
progress("repo", "notes/deep.md", 1, 3)
|
||||
self.hook_calls.append(("repo", "notes/deep.md", 1, 3))
|
||||
self.started.set()
|
||||
await asyncio.to_thread(self.release.wait) # park without freezing the worker loop
|
||||
if on_chunk_progress is not None and self.chunk_pairs:
|
||||
# Phase 129 (task 03): the file's chunk-position pairs,
|
||||
# fired through the runner's OWN hook closure (the closure
|
||||
# under test) — pair one before the main park, each later
|
||||
# pair after the main park + its own park (each park lets
|
||||
# the test read the status mid-run). The runner passes the
|
||||
# hook only when the wiring is intact — a regression to
|
||||
# None skips this branch and the pair tests time out.
|
||||
on_chunk_progress(*self.chunk_pairs[0])
|
||||
self.chunk_hook_calls.append(self.chunk_pairs[0])
|
||||
self.started.set()
|
||||
await asyncio.to_thread(self.release.wait)
|
||||
for pair in self.chunk_pairs[1:]:
|
||||
on_chunk_progress(*pair)
|
||||
self.chunk_hook_calls.append(pair)
|
||||
if self.chunk_release is not None:
|
||||
await asyncio.to_thread(self.chunk_release.wait)
|
||||
else:
|
||||
self.started.set()
|
||||
await asyncio.to_thread(self.release.wait) # park without freezing the worker loop
|
||||
if self.fail is not None:
|
||||
raise self.fail
|
||||
return self.summary
|
||||
@@ -896,9 +955,10 @@ def test_idle_status_pins_full_shape_including_progress_keys(
|
||||
fresh_sync_status: None,
|
||||
) -> None:
|
||||
"""Idle: the three phase-64 progress keys ride along as null/0/0,
|
||||
the four phase-98 phase keys as null/0/0/0, and EVERY pre-existing
|
||||
key is unchanged — the full response dict is pinned, so the
|
||||
current UI and every existing consumer keep working."""
|
||||
the two phase-129 chunk keys as 0/0, the four phase-98 phase keys
|
||||
as null/0/0/0, and EVERY pre-existing key is unchanged — the full
|
||||
response dict is pinned, so the current UI and every existing
|
||||
consumer keep working."""
|
||||
assert sync_api.sync_status() == {
|
||||
"state": "idle",
|
||||
"started_at": None,
|
||||
@@ -908,6 +968,10 @@ def test_idle_status_pins_full_shape_including_progress_keys(
|
||||
"current_file": None,
|
||||
"files_done": 0,
|
||||
"files_total": 0,
|
||||
# Phase 129 (task 03): the chunk-position keys — 0/0 idle
|
||||
# (the dataclass defaults, no special-casing).
|
||||
"current_chunk": 0,
|
||||
"chunks_total": 0,
|
||||
# Phase 98 (task 01): the phase-machine keys — null/0/0/0 idle.
|
||||
"phase": None,
|
||||
"current_summary": None,
|
||||
@@ -938,7 +1002,8 @@ def test_mid_run_status_reports_current_file(
|
||||
assert clone.started.wait(5.0), "the run never reached the clone phase"
|
||||
# Clone/pull prelude: running — but no file yet (A4: bare
|
||||
# "Syncing…") and no phase yet either (D1: the prelude stays
|
||||
# null — the bare label's pin).
|
||||
# null — the bare label's pin), and no chunk position yet
|
||||
# (phase 129: 0/0 before the first embed batch).
|
||||
s = sync_api.sync_status()
|
||||
assert s["state"] == "running"
|
||||
assert s["phase"] is None
|
||||
@@ -946,11 +1011,14 @@ def test_mid_run_status_reports_current_file(
|
||||
assert s["current_summary"] is None
|
||||
assert s["files_done"] == 0 and s["files_total"] == 0
|
||||
assert s["summaries_done"] == 0 and s["summaries_total"] == 0
|
||||
assert s["current_chunk"] == 0 and s["chunks_total"] == 0
|
||||
clone.release.set()
|
||||
assert fake_import.started.wait(5.0), "the run never reached the import"
|
||||
# Import phase: the hook's file is live on the status, and the
|
||||
# phase machine names the phase (the summary span has not
|
||||
# started — its counters are still 0/0).
|
||||
# started — its counters are still 0/0); no embed batch has
|
||||
# fired the chunk hook (this canned import passes no pairs),
|
||||
# so the chunk position stays 0/0.
|
||||
s = sync_api.sync_status()
|
||||
assert s["state"] == "running"
|
||||
assert s["phase"] == "import"
|
||||
@@ -959,6 +1027,7 @@ def test_mid_run_status_reports_current_file(
|
||||
assert s["files_total"] == 3
|
||||
assert s["current_summary"] is None
|
||||
assert s["summaries_done"] == 0 and s["summaries_total"] == 0
|
||||
assert s["current_chunk"] == 0 and s["chunks_total"] == 0
|
||||
fake_import.release.set()
|
||||
finally:
|
||||
clone.release.set()
|
||||
@@ -974,7 +1043,9 @@ def test_mid_run_status_reports_current_file(
|
||||
# Success terminal: current_file null, final file counts retained;
|
||||
# the phase machine clears phase + current_summary — the fake
|
||||
# folder step never fired the summary hook, so the summary counts
|
||||
# stay 0/0 (the keep-final-counts convention keeps what fired).
|
||||
# stay 0/0 (the keep-final-counts convention keeps what fired);
|
||||
# the chunk position clears with the file (phase 129 — it is a
|
||||
# sub-position of a done file, never a keep-final counter).
|
||||
s = sync_api.sync_status()
|
||||
assert s["state"] == "success"
|
||||
assert s["phase"] is None
|
||||
@@ -982,6 +1053,7 @@ def test_mid_run_status_reports_current_file(
|
||||
assert s["current_file"] is None
|
||||
assert s["files_done"] == 1 and s["files_total"] == 3
|
||||
assert s["summaries_done"] == 0 and s["summaries_total"] == 0
|
||||
assert s["current_chunk"] == 0 and s["chunks_total"] == 0
|
||||
assert s["error"] is None
|
||||
assert s["started_at"] is not None and s["finished_at"] is not None
|
||||
|
||||
@@ -993,7 +1065,13 @@ def test_failed_terminal_clears_current_file_keeps_counts(
|
||||
) -> None:
|
||||
"""Terminal (failed): a run that dies inside the import clears
|
||||
``current_file`` but keeps the hook's final counts — the last
|
||||
position is useful context next to the (sanitized) error."""
|
||||
position is useful context next to the (sanitized) error.
|
||||
|
||||
Phase 129 (task 03): the convention split extends to the chunk
|
||||
position — the (about-to-fail) import reports a live pair
|
||||
(1/38), the failed terminal clears it to 0/0 (a sub-position of a
|
||||
file that's done/abandoned), while the FILE counts keep their
|
||||
final values next to the error."""
|
||||
clone = _GatedClone(threading.Event(), threading.Event())
|
||||
fake_import = _GatedImport(
|
||||
ImportSummary(),
|
||||
@@ -1003,6 +1081,7 @@ def test_failed_terminal_clears_current_file_keeps_counts(
|
||||
"embeddings request to https://u:p@aipi.example.com/v1 "
|
||||
"failed: connection refused"
|
||||
),
|
||||
chunk_pairs=[(1, 38)], # phase 129: the live pair before the raise
|
||||
)
|
||||
_patch_sync_seams(monkeypatch, tmp_path, fake_import, clone)
|
||||
|
||||
@@ -1010,11 +1089,15 @@ def test_failed_terminal_clears_current_file_keeps_counts(
|
||||
try:
|
||||
assert clone.started.wait(5.0), "the run never reached the clone phase"
|
||||
clone.release.set()
|
||||
assert fake_import.started.wait(5.0), "the progress hook never fired"
|
||||
# While the (about-to-fail) import is parked: the file is live.
|
||||
assert fake_import.started.wait(5.0), "the chunk pair never landed"
|
||||
# While the (about-to-fail) import is parked: the file is live,
|
||||
# and so is its chunk position (the pair the fake fired through
|
||||
# the runner's own closure).
|
||||
s = sync_api.sync_status()
|
||||
assert s["state"] == "running"
|
||||
assert s["current_file"] == "repo/notes/deep.md"
|
||||
assert s["files_done"] == 1 and s["files_total"] == 3
|
||||
assert s["current_chunk"] == 1 and s["chunks_total"] == 38
|
||||
fake_import.release.set()
|
||||
finally:
|
||||
clone.release.set()
|
||||
@@ -1022,6 +1105,8 @@ def test_failed_terminal_clears_current_file_keeps_counts(
|
||||
thread.join(10.0)
|
||||
assert not thread.is_alive()
|
||||
assert errors == []
|
||||
# Wiring: the pair fired through the runner's own closure.
|
||||
assert fake_import.chunk_hook_calls == [(1, 38)]
|
||||
s = sync_api.sync_status()
|
||||
assert s["state"] == "failed"
|
||||
assert s["current_file"] is None # cleared in the terminal state
|
||||
@@ -1029,8 +1114,222 @@ def test_failed_terminal_clears_current_file_keeps_counts(
|
||||
assert s["current_summary"] is None
|
||||
assert s["files_done"] == 1 and s["files_total"] == 3 # final counts kept
|
||||
assert s["summaries_done"] == 0 and s["summaries_total"] == 0
|
||||
# Phase 129 (task 03): the chunk position clears in the FAILED
|
||||
# terminal too — the convention split, pinned (the FILE counts
|
||||
# above keep their final values, the sub-position does not).
|
||||
assert s["current_chunk"] == 0 and s["chunks_total"] == 0
|
||||
assert s["detail"] == {}
|
||||
error = s["error"] or ""
|
||||
assert "*****@aipi.example.com" in error # credentials masked
|
||||
assert "u:p" not in error
|
||||
assert "connection refused" in error # the reason survives
|
||||
|
||||
|
||||
# ---------- phase 129 (task 03): the in-flight file's embed-chunk ----------
|
||||
# ---------- position (current_chunk / chunks_total) ----------------------
|
||||
#
|
||||
# Same seams as the phase-64 block above: the fake import fires the
|
||||
# runner's OWN ``_chunk_hook`` closure (the task under test) with the
|
||||
# file's chunk-position pairs, parking between pairs so the test reads
|
||||
# the status BETWEEN ticks (the A6 batch-granularity contract: 14 =
|
||||
# the first chunk of the in-flight token-budgeted batch).
|
||||
|
||||
|
||||
def _poll_until(
|
||||
predicate: Callable[[], bool],
|
||||
timeout: float = 5.0,
|
||||
what: str = "condition",
|
||||
) -> None:
|
||||
"""Poll (main thread) until *predicate* — the worker-thread run's
|
||||
state is only observable between its parks."""
|
||||
deadline = time.monotonic() + timeout
|
||||
while not predicate():
|
||||
if time.monotonic() >= deadline:
|
||||
raise AssertionError(f"timed out waiting for {what}")
|
||||
time.sleep(0.01)
|
||||
|
||||
|
||||
def test_mid_run_status_reports_chunk_position_between_ticks(
|
||||
fresh_sync_status: None,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""Phase 129 (task 03): the status carries the in-flight file's
|
||||
embed-chunk position — the hook's (first, total) pair climbs
|
||||
through the file's batches (1/38 → 13/38, read between ticks),
|
||||
paired with the file the ``progress`` hook named; the prelude
|
||||
reports 0/0 (no embed batch yet), and the success terminal clears
|
||||
the position to 0/0 while the FILE counts keep their final values
|
||||
(the convention split — the chunk position is a sub-position of a
|
||||
done file, never a keep-final counter)."""
|
||||
clone = _GatedClone(threading.Event(), threading.Event())
|
||||
import_release = threading.Event()
|
||||
chunk_release = threading.Event()
|
||||
fake_import = _GatedImport(
|
||||
ImportSummary(files=3, added=1, updated=1, unchanged=1),
|
||||
threading.Event(),
|
||||
import_release,
|
||||
chunk_pairs=[(1, 38), (13, 38)], # two batches of a 38-chunk file
|
||||
chunk_release=chunk_release,
|
||||
)
|
||||
_patch_sync_seams(monkeypatch, tmp_path, fake_import, clone)
|
||||
|
||||
thread, errors = _start_run()
|
||||
try:
|
||||
assert clone.started.wait(5.0), "the run never reached the clone phase"
|
||||
# Prelude: the chunk position is 0/0 (no file, no batch yet —
|
||||
# alongside the bare-label pins of the phase-64 test above).
|
||||
s = sync_api.sync_status()
|
||||
assert s["state"] == "running"
|
||||
assert s["current_file"] is None
|
||||
assert s["current_chunk"] == 0 and s["chunks_total"] == 0
|
||||
clone.release.set()
|
||||
assert fake_import.started.wait(5.0), "the first chunk pair never landed"
|
||||
# Import phase, between ticks: pair one is live on the status,
|
||||
# paired with the file the progress hook named (the two hooks
|
||||
# are paired — the position belongs to that file).
|
||||
s = sync_api.sync_status()
|
||||
assert s["state"] == "running"
|
||||
assert s["phase"] == "import"
|
||||
assert s["current_file"] == "repo/notes/deep.md"
|
||||
assert s["files_done"] == 1 and s["files_total"] == 3
|
||||
assert s["current_chunk"] == 1 and s["chunks_total"] == 38
|
||||
# Release pair one: the import fires pair two (13/38 — the
|
||||
# second batch of the 38-chunk file) and parks again.
|
||||
import_release.set()
|
||||
_poll_until(
|
||||
lambda: sync_api.sync_status()["current_chunk"] == 13,
|
||||
what="the second chunk pair on the status",
|
||||
)
|
||||
s = sync_api.sync_status()
|
||||
assert s["state"] == "running"
|
||||
assert s["phase"] == "import"
|
||||
# The position CLIMBED (batch granularity) — total unchanged
|
||||
# (the file's chunk count, not the batch size).
|
||||
assert s["current_chunk"] == 13 and s["chunks_total"] == 38
|
||||
assert s["current_file"] == "repo/notes/deep.md"
|
||||
assert s["files_done"] == 1 and s["files_total"] == 3
|
||||
chunk_release.set()
|
||||
finally:
|
||||
clone.release.set()
|
||||
import_release.set()
|
||||
chunk_release.set()
|
||||
thread.join(10.0)
|
||||
assert not thread.is_alive()
|
||||
assert errors == [], f"the runner raised: {errors!r}"
|
||||
# Wiring: the pairs fired through the runner's OWN closure (the
|
||||
# status reads above came through it; a wiring regression to
|
||||
# None would have skipped the firing branch and timed out).
|
||||
assert fake_import.chunk_hook_calls == [(1, 38), (13, 38)]
|
||||
# Success terminal: the chunk position clears (0/0) ...
|
||||
s = sync_api.sync_status()
|
||||
assert s["state"] == "success"
|
||||
assert s["phase"] is None
|
||||
assert s["current_chunk"] == 0 and s["chunks_total"] == 0
|
||||
# ... while the FILE counts keep their final values (the
|
||||
# convention split, pinned alongside the phase-64 terminal pin).
|
||||
assert s["current_file"] is None
|
||||
assert s["files_done"] == 1 and s["files_total"] == 3
|
||||
|
||||
|
||||
def test_chunk_position_clears_at_phase_transitions_and_terminal(
|
||||
fresh_sync_status: None,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""Phase 129 (task 03): the chunk position is CLEARED (0/0) at
|
||||
every point the import phase ends. This run's summary changes the
|
||||
KB, so BOTH post-import transitions run and are pinned: the
|
||||
import→``"overview"`` transition and the import→``"summaries"``
|
||||
transition (the changed-KB branch) each clear it to 0/0, and the
|
||||
success terminal keeps it 0/0 — through the whole span the FILE
|
||||
counts keep their final values (the convention split: the file
|
||||
counts survive for the result line's context, the chunk position
|
||||
is a sub-position of a file that's done)."""
|
||||
clone = _GatedClone(threading.Event(), threading.Event())
|
||||
import_release = threading.Event()
|
||||
overview_started = threading.Event()
|
||||
overview_release = threading.Event()
|
||||
folders_started = threading.Event()
|
||||
folders_release = threading.Event()
|
||||
fake_import = _GatedImport(
|
||||
ImportSummary(files=3, added=1, updated=1, unchanged=1),
|
||||
threading.Event(),
|
||||
import_release,
|
||||
chunk_pairs=[(7, 38)], # a live pair to clear at each transition
|
||||
)
|
||||
_patch_sync_seams(monkeypatch, tmp_path, fake_import, clone)
|
||||
|
||||
# Park the post-import steps (re-monkeypatched over the seams
|
||||
# helper's no-ops): the test reads the status DURING the
|
||||
# "overview" phase and DURING the "summaries" phase — the two
|
||||
# import-phase-ending transitions the task pins.
|
||||
async def gated_overview(llm: object, session: object = None) -> bool:
|
||||
overview_started.set()
|
||||
await asyncio.to_thread(overview_release.wait)
|
||||
return True
|
||||
|
||||
async def gated_folders(
|
||||
db: object,
|
||||
llm: object,
|
||||
*,
|
||||
skip: bool = False,
|
||||
only_missing: bool = False,
|
||||
changed_docs: Sequence[tuple[str, str]] | None = None, # phase 127
|
||||
on_progress: Callable[[int, int, str, str], None] | None = None,
|
||||
) -> dict[str, int]:
|
||||
folders_started.set()
|
||||
await asyncio.to_thread(folders_release.wait)
|
||||
return {"generated": 0, "failed": 0, "pruned": 0}
|
||||
|
||||
monkeypatch.setattr(sync_api, "regenerate_overview", gated_overview)
|
||||
monkeypatch.setattr(sync_api, "generate_folder_summaries", gated_folders)
|
||||
|
||||
thread, errors = _start_run()
|
||||
try:
|
||||
assert clone.started.wait(5.0), "the run never reached the clone phase"
|
||||
clone.release.set()
|
||||
assert fake_import.started.wait(5.0), "the chunk pair never landed"
|
||||
# Mid-import: the live pair (7/38) is on the status.
|
||||
s = sync_api.sync_status()
|
||||
assert s["phase"] == "import"
|
||||
assert s["current_chunk"] == 7 and s["chunks_total"] == 38
|
||||
# Release the import: it returns (added+updated > 0) → the
|
||||
# phase machine moves to "overview" and the position clears.
|
||||
import_release.set()
|
||||
assert overview_started.wait(5.0), "the run never reached the overview"
|
||||
s = sync_api.sync_status()
|
||||
assert s["state"] == "running"
|
||||
assert s["phase"] == "overview"
|
||||
# CLEARED at the import→overview transition (0/0) ... the file
|
||||
# counts KEEP their final values (the convention split).
|
||||
assert s["current_chunk"] == 0 and s["chunks_total"] == 0
|
||||
assert s["files_done"] == 1 and s["files_total"] == 3
|
||||
overview_release.set()
|
||||
# ... and the changed-KB branch moves to "summaries" (the
|
||||
# second import-phase-ending transition — cleared there too,
|
||||
# idempotently) before parking.
|
||||
assert folders_started.wait(5.0), "the run never reached the summaries"
|
||||
s = sync_api.sync_status()
|
||||
assert s["state"] == "running"
|
||||
assert s["phase"] == "summaries"
|
||||
assert s["current_chunk"] == 0 and s["chunks_total"] == 0
|
||||
assert s["files_done"] == 1 and s["files_total"] == 3
|
||||
folders_release.set()
|
||||
finally:
|
||||
clone.release.set()
|
||||
import_release.set()
|
||||
overview_release.set()
|
||||
folders_release.set()
|
||||
thread.join(10.0)
|
||||
assert not thread.is_alive()
|
||||
assert errors == [], f"the runner raised: {errors!r}"
|
||||
assert fake_import.chunk_hook_calls == [(7, 38)]
|
||||
# Success terminal: still 0/0 (the terminal clears too — same
|
||||
# spots as current_file), file counts kept.
|
||||
s = sync_api.sync_status()
|
||||
assert s["state"] == "success"
|
||||
assert s["phase"] is None
|
||||
assert s["current_file"] is None
|
||||
assert s["current_chunk"] == 0 and s["chunks_total"] == 0
|
||||
assert s["files_done"] == 1 and s["files_total"] == 3
|
||||
|
||||
Reference in New Issue
Block a user