feat(sources): real-time file progress for sync and upload — background upload with success toast

This commit is contained in:
2026-09-01 23:51:43 -04:00
parent cddc84c7db
commit 4677d86f49
103 changed files with 5914 additions and 456 deletions
+663
View File
@@ -0,0 +1,663 @@
"""Unit: the RAG-page sync button's phase-64 (task 04) live-file contract.
The browser behavior is E2E-covered (tests/e2e/test_sync_upload_progress.py,
task 06); here we pin the source-level wiring in sources.js, styles.css,
and sources.html — the fmtSyncLabel contract (both kinds, file
present/absent, counts only when total > 0), enterSyncRunningState
writing the full untruncated path to the button title + #sync-result,
the two-job tick decision tree (sync running > upload running > sync
success > sync failed > upload success > upload failed > idle; the A3
settle never renders upload counts into #sync-result), and the
load-time re-attach of an in-flight upload scan — so a silent
regression is caught without a browser.
Phase 64 task 05 adds the Sources-page (git-sources.js) upload
contract: the "Successfully uploaded — <file>" toast on the 202
(page-local, phase-55 pattern, success-only), the live
"Processing… <file> (n/m)" label + full-path title driven by the 2 s
GET /api/git-sources/upload/status poll, the 409 re-attach (no error
banner), the poll's terminal decision tree (success → result line +
announce + reload, NO second toast; failed → sanitized error banner
+ reload; idle → defensive restore), the finally's never-restore-
while-polling guard (PLAN §7.4), and the boot re-attach branches.
"""
from __future__ import annotations
import re
from pathlib import Path
FRONTEND = Path(__file__).resolve().parents[2] / "frontend"
SOURCES_JS = FRONTEND / "assets" / "sources.js"
STYLES_CSS = FRONTEND / "assets" / "styles.css"
SOURCES_HTML = FRONTEND / "sources.html"
GIT_SOURCES_JS = FRONTEND / "assets" / "git-sources.js"
GIT_SOURCES_HTML = FRONTEND / "git-sources.html"
def _js() -> str:
return SOURCES_JS.read_text(encoding="utf-8")
def _css() -> str:
return STYLES_CSS.read_text(encoding="utf-8")
def _html() -> str:
return SOURCES_HTML.read_text(encoding="utf-8")
def _gjs() -> str:
return GIT_SOURCES_JS.read_text(encoding="utf-8")
def _ghtml() -> str:
return GIT_SOURCES_HTML.read_text(encoding="utf-8")
def _gfn(js: str, name: str) -> str:
"""The source of the first `function <name>` in git-sources.js
(up to the first line-leading closing brace — the house pin
pattern)."""
fn = js.find(f"function {name}")
assert fn != -1, f"{name} must be defined in git-sources.js"
return js[fn : js.find("\n}", fn)]
def _utick(js: str) -> str:
"""The upload poll tick inside startUploadPolling — from `const
tick = async () => {` to the next top-level function
(initUploadStatus), so the whole decision tree is in the slice."""
fn = js.find("function startUploadPolling")
assert fn != -1, "startUploadPolling must be defined in git-sources.js"
tick = js.find("const tick = async () => {", fn)
assert tick != -1, "the tick must live inside startUploadPolling"
end = js.find("\nasync function initUploadStatus", tick)
assert end != -1, "initUploadStatus must follow startUploadPolling"
return js[tick:end]
def _usubmit(js: str) -> str:
"""The upload form's submit handler — from the addEventListener to
the next top-level function (focusNewRow), so every branch (202 /
409 / other non-2xx / network / finally) is in the slice."""
start = js.find('uploadFormEl.addEventListener("submit"')
assert start != -1, "the upload form must wire a submit handler"
end = js.find("function focusNewRow", start)
assert end != -1, "focusNewRow must follow the upload handler"
return js[start:end]
def _fn(js: str, name: str) -> str:
"""The source of the first `function <name>` in js (up to the first
line-leading closing brace — the house pin pattern from
tests/unit/test_sync_button.py)."""
fn = js.find(f"function {name}")
assert fn != -1, f"{name} must be defined in sources.js"
return js[fn : js.find("\n}", fn)]
def _tick(js: str) -> str:
"""The poll tick inside startSyncPolling — from `const tick = async
() => {` to the next top-level function (startSync), so the whole
two-job decision tree is in the slice."""
fn = js.find("function startSyncPolling")
assert fn != -1, "startSyncPolling must be defined in sources.js"
tick = js.find("const tick = async () => {", fn)
assert tick != -1, "the tick must live inside startSyncPolling"
end = js.find("\nasync function startSync", tick)
assert end != -1, "startSync must follow startSyncPolling"
return js[tick:end]
# ---------- fmtSyncLabel: the live-file label contract ----------
def test_fmt_sync_label_signature_and_prefixes() -> None:
"""fmtSyncLabel(kind, currentFile, done, total): `kind` picks the
prefix — "upload" → "Importing" (the background scan's word, A3),
anything else → "Syncing…". The file is appended only when present
(the bare prefix shows during clone/pull or unpack, before any file
is indexed — A4); the counts are appended only when total > 0 (the
import has started); file before counts."""
body = _fn(_js(), "fmtSyncLabel")
assert "function fmtSyncLabel(kind, currentFile, done, total)" in body
assert 'kind === "upload"' in body
assert '"Importing"' in body and '"Syncing…"' in body
# the upload prefix must come from the kind check (ternary, in order)
i_kind = body.find('kind === "upload"')
i_importing = body.find('"Importing"')
i_syncing = body.find('"Syncing…"')
assert -1 < i_kind < i_importing < i_syncing
# file appended only when present
assert "currentFile ?" in body
assert "`${prefix} ${currentFile}`" in body
# counts only when the import has started (total > 0)
assert "total > 0" in body
assert "` (${done}/${total})`" in body
i_file = body.find("currentFile ?")
i_counts = body.find("total > 0")
assert -1 < i_file < i_counts, "the file lands before the counts"
# ---------- enterSyncRunningState: full path to title + announcer ----------
def test_enter_running_state_writes_full_path_to_title_and_announcer() -> None:
"""The running-state entry keeps the §7.4 never-stale mechanics
(disabled, aria-busy, spinning icon, the stale .is-error removed)
and — phase 64 — writes the FULL untruncated current file to the
button title (removed when null: no file yet) and the full
untruncated fmtSyncLabel text to BOTH the label span and
#sync-result (the aria-live announcer reads the full live path;
CSS only ellipsizes the button's span)."""
body = _fn(_js(), "enterSyncRunningState")
assert "syncBtn.disabled = true" in body
assert 'syncBtn.setAttribute("aria-busy", "true")' in body
assert "syncIcon.classList.add(\"is-spinning\")" in body
assert "syncBtn.classList.remove(\"is-error\")" in body
assert "syncBtn.title = currentFile" in body, "the full path on hover"
assert 'syncBtn.removeAttribute("title")' in body, "removed when no file yet"
i_set = body.find("if (currentFile) syncBtn.title = currentFile")
i_remove = body.find('syncBtn.removeAttribute("title")')
assert -1 < i_set < i_remove, "title is set (not removed) only when a file exists"
assert "fmtSyncLabel(kind, currentFile, done, total)" in body
i_label = body.find("fmtSyncLabel(kind, currentFile, done, total)")
i_span = body.find("syncLabel.textContent = label")
i_result = body.find("syncResult.textContent = label")
assert -1 < i_label < i_span < i_result, (
"one label: built once, written to the span AND the announcer"
)
# ---------- the two-job tick (startSyncPolling) ----------
def test_tick_fetches_both_jobs_with_403_and_blip_rules() -> None:
"""Each tick fetches BOTH status endpoints (the sync and the
background upload scan). The 403 backstop (button hidden) applies
to the SYNC fetch only — a 403 on the upload fetch is simply "no
upload" (never a hide); a network blip on either fetch retries next
tick (the tick reschedules, it never dies on a failed fetch)."""
tick = _tick(_js())
assert 'fetch("/api/sync/status")' in tick
assert 'fetch("/api/git-sources/upload/status")' in tick
assert "r.status === 403" in tick, "the sync fetch keeps the whoami backstop"
assert "ur.status === 403" not in tick, "upload 403 = no upload (A3)"
assert "ur.ok" in tick, "the upload fetch is read only when ok"
assert tick.count("catch {") >= 2, "both fetches are blip-tolerant"
assert tick.count("setTimeout(tick, SYNC_POLL_MS)") >= 3, (
"the blip and both running branches all reschedule"
)
# the sync-status blip must not settle the button: it reschedules.
i_nostatus = tick.find("if (!syncStatus)")
assert i_nostatus != -1
branch = tick[i_nostatus : i_nostatus + 200]
assert "setTimeout(tick, SYNC_POLL_MS)" in branch
assert "applySyncIdle" not in branch, "a blip is not an idle"
def test_tick_decision_tree_order_and_branches() -> None:
"""The phase-64 decision tree, in order (A3/A4): 1. sync running →
2. upload running → 3. sync success → 4. sync failed → 5. upload
success → 6. upload failed → 7. both idle. The running branches
enter the running state with THEIR job's kind + live file/counts;
the sync terminals are the unchanged phase-32 appliers; the upload
terminals settle "Sync sources" + clear #sync-result + hide a stale
error + emit a synthetic idle frame — never the upload's status
object, never fmtSyncResult (no upload counts in #sync-result, A3);
only the upload SUCCESS refreshes the catalog (the KB changed); the
upload failure raises no error surface on this page (the banner is
the Sources page's)."""
tick = _tick(_js())
b_sync_run = tick.find('syncStatus.state === "running"')
b_up_run = tick.find('uploadStatus && uploadStatus.state === "running"')
b_sync_ok = tick.find('syncStatus.state === "success"')
b_sync_fail = tick.find('syncStatus.state === "failed"')
b_up_ok = tick.find('uploadStatus && uploadStatus.state === "success"')
b_up_fail = tick.find('uploadStatus && uploadStatus.state === "failed"')
b_idle = tick.find("applySyncIdle(syncStatus)")
assert (
-1 < b_sync_run < b_up_run < b_sync_ok < b_sync_fail < b_up_ok < b_up_fail < b_idle
), "the tree must fire in the documented order"
# 1. sync running: the sync-kind live label, reschedule.
branch1 = tick[b_sync_run:b_up_run]
assert (
'"sync", syncStatus.current_file, syncStatus.files_done, syncStatus.files_total'
in branch1
)
assert "setTimeout(tick, SYNC_POLL_MS)" in branch1
# 2. upload running: the upload-kind live label (A3), reschedule.
branch2 = tick[b_up_run:b_sync_ok]
assert (
'"upload", uploadStatus.current_file, uploadStatus.files_done, uploadStatus.files_total'
in branch2
)
assert "setTimeout(tick, SYNC_POLL_MS)" in branch2
# 3 + 4. the sync terminals are the unchanged phase-32 appliers.
assert "applySyncSuccess(syncStatus)" in tick[b_sync_ok:b_sync_fail]
assert "applySyncFailure(syncStatus)" in tick[b_sync_fail:b_up_ok]
# 5. upload success: settle + clear + hide stale error + emit idle
# + the catalog refresh (the new documents must appear).
up_ok = tick[b_up_ok:b_up_fail]
for line in (
'settleSyncButton("Sync sources")',
'syncResult.textContent = ""',
"hideSyncError()",
'emitSyncStatus({ state: "idle" })',
"loadDocs()",
):
assert line in up_ok, f"the upload-success settle must carry {line!r}"
assert "fmtSyncResult" not in up_ok, "no upload counts in #sync-result (A3)"
assert "applySyncSuccess" not in up_ok, "the sync applier is never an upload branch"
# 6. upload failed: settle only — no catalog refresh (the KB did
# not change) and no error surface on this page (A3).
up_fail = tick[b_up_fail:b_idle]
for line in (
'settleSyncButton("Sync sources")',
'syncResult.textContent = ""',
"hideSyncError()",
'emitSyncStatus({ state: "idle" })',
):
assert line in up_fail, f"the upload-failed settle must carry {line!r}"
assert "loadDocs()" not in up_fail, "no KB change on a failed upload"
assert "showSyncError" not in up_fail, "no banner on this page (A3)"
assert "applySyncFailure" not in up_fail and "showSyncModal" not in up_fail
# 7. both idle: the unchanged idle settle.
assert "stopSyncPolling()" in tick[b_idle - 60 : b_idle + 40]
# ---------- the click branch + the load-time re-attach ----------
def test_click_branch_enters_sync_running_without_a_file() -> None:
"""The 202/409 branch of startSync enters the running state with
the sync kind and no file yet (the run is just starting — model
check / clone-pull: bare "Syncing…", A4); the emitSyncStatus({
state: "running" }) dedup via lastSyncState stays, and the poll
starts."""
js = _js()
idx = js.find("r.status === 202 || r.status === 409")
assert idx != -1
branch = js[idx : idx + 500]
assert "enterSyncRunningState(\"sync\", null, 0, 0)" in branch
assert 'emitSyncStatus({ state: "running" })' in branch
assert "lastSyncState !== \"running\"" in branch, "the dedup stays"
assert "startSyncPolling()" in branch
def test_reattach_adopts_a_running_upload_only() -> None:
"""initSyncButton: the sync branches are unchanged (running
re-enters with the live file; the terminals render the last
result). With the sync IDLE it fetches the upload status: a RUNNING
upload scan re-attaches (running state, upload kind + live file,
the synthetic running frame, the poll starts); a terminal upload is
a no-op — the fall-through is the plain idle settle (the boot-time
loadDocs() already shows the current catalog)."""
body = _fn(_js(), "initSyncButton")
assert "await fetchIsAdmin()" in body, "admin-only (no extra fetch)"
assert 'fetch("/api/git-sources/upload/status")' in body
# the sync running re-attach now carries the live file too
assert '"sync", status.current_file, status.files_done, status.files_total' in body
# the ONLY upload branch is the running one (A3)
assert "upload && upload.state === \"running\"" in body
assert 'upload.state === "success"' not in body, "a terminal upload never re-attaches"
assert 'upload.state === "failed"' not in body, "a terminal upload never re-attaches"
i_check = body.find("upload && upload.state === \"running\"")
i_fall = body.find("applySyncIdle(status)", i_check)
assert -1 < i_check < i_fall
branch = body[i_check:i_fall]
assert '"upload", upload.current_file, upload.files_done, upload.files_total' in branch
assert 'emitSyncStatus({ state: "running" })' in branch
assert "startSyncPolling()" in branch
# the idle settle is the fall-through (the last statement)
assert body.rstrip().endswith("applySyncIdle(status);")
# ---------- the section header + the page comment ----------
def test_section_header_documents_the_two_job_contract() -> None:
"""The sync-button section marker comment documents the phase-64
contract: the live file label, the two-job decision tree (both
status endpoints), and the A3 settle behavior (catalog refresh; the
upload counts never render here)."""
js = _js()
marker = js.find("Sync sources button (Sources page only)")
assert marker != -1, "the sync section marker comment must stay"
header = js[marker : js.find("const syncBtn")]
assert "Phase 64 (task 04)" in header
assert "/api/git-sources/upload/status" in header, "the second job's endpoint"
assert "loadDocs" in header, "the A3 catalog refresh"
assert "A3" in header and "A4" in header
def test_sources_html_comment_documents_the_live_announcer() -> None:
"""The #sync-result comment in sources.html documents the phase-64
dual role: the live file label (both kinds) while either job runs,
untruncated for the aria-live announcer, and empty after an upload
settles (A3 — the upload's counts live on the Sources page)."""
html = _html()
idx = html.find('id="sync-result"')
assert idx != -1
comment = html[max(0, idx - 900):idx]
assert "Importing <file>" in comment, "the upload kind is documented"
assert "Syncing…" in comment, "the sync kind is documented"
assert "A3" in comment, "the settle contract is documented"
# ---------- styles.css: the ellipsized live label ----------
def test_sync_label_css_ellipsis_truncation() -> None:
""".sync-label: the live-file label ellipsizes a long
source/relative/path inside the pill (A4) — inline-block with the
min(16rem, 40vw) cap, overflow hidden, text-overflow ellipsis, no
wrap, baseline-aligned; the mobile squeeze's display:none override
(icon-only button) stays."""
css = _css()
block = re.search(r"\.sync-label\s*\{([^}]*)\}", css)
assert block, "styles.css must style .sync-label"
body = block.group(1)
for prop in (
"display: inline-block",
"max-width: min(16rem, 40vw)",
"overflow: hidden",
"text-overflow: ellipsis",
"white-space: nowrap",
"vertical-align: bottom",
):
assert prop in body, f".sync-label must carry {prop!r}"
mobile = re.search(r"@media \(max-width: 640px\) \{([\s\S]*?)\n\}", css)
assert mobile, "the ≤640px media query must stay"
assert ".sync-label { display: none; }" in mobile.group(1), (
"the mobile icon-only override must survive the ellipsis rule"
)
# =====================================================================
# Phase 64 task 05 — the Sources-page upload (git-sources.js)
# =====================================================================
# ---------- the "Successfully uploaded" toast (A2) ----------
def test_upload_toast_single_node_status_and_auto_dismiss() -> None:
"""showUploadToast (A2 owner-locked): the phase-55 share-toast
pattern made page-local — a SINGLE node, lazy-created on the first
202 and reused (toasts never stack): a plain ``<div class="toast">``
appended to ``document.body``; ``role="status" aria-live="polite"``
(on THIS page the toast is the a11y announcer for the 202); the
text lands via ``textContent`` (XSS-safe — never innerHTML). A new
toast replaces a pending one: clear the prior dismiss timer, remove
the visible class, force a reflow (``offsetWidth`` — restarts the
CSS transition), re-add the class. Auto-dismiss: the 5000ms
(UPLOAD_TOAST_MS) timer is armed AFTER the visible class is added
and removes the class on fire. The existing .toast CSS is reused
as-is (no new styles)."""
js = _gjs()
body = _gfn(js, "showUploadToast")
assert "if (!uploadToastEl)" in body, "the node is created once, on first use"
assert 'document.createElement("div")' in body
assert 'uploadToastEl.className = "toast"' in body, "the phase-55 .toast CSS, as-is"
assert 'uploadToastEl.setAttribute("role", "status")' in body
assert 'uploadToastEl.setAttribute("aria-live", "polite")' in body
assert "document.body.appendChild(uploadToastEl)" in body
assert "uploadToastEl.textContent = message" in body
assert "innerHTML" not in body, "XSS contract: textContent only"
# Single instance: module-scope node + timer.
assert re.search(r"^let uploadToastEl = null", js, re.M), "the node is module scope"
assert re.search(r"^let uploadToastTimer = 0", js, re.M), "the timer is module scope"
assert "const UPLOAD_TOAST_MS = 5000" in js, "the ~5 s auto-dismiss (A2)"
# Re-trigger order: clear dismiss → remove class → force reflow →
# re-add the visible class.
clear_i = body.find("clearTimeout(uploadToastTimer)")
remove_i = body.find('uploadToastEl.classList.remove("is-visible")')
reflow_i = body.find("void uploadToastEl.offsetWidth")
add_i = body.find('uploadToastEl.classList.add("is-visible")')
assert -1 < clear_i < remove_i < reflow_i < add_i, (
"dismiss cleared → class removed → reflow forced → visible re-added"
)
# The auto-dismiss timer is armed AFTER the visible class is set.
timer_i = body.find("setTimeout")
assert -1 < add_i < timer_i and "UPLOAD_TOAST_MS" in body[timer_i:]
assert 'uploadToastEl.classList.remove("is-visible")' in body[timer_i:], (
"the pending dismiss removes the visible state"
)
def test_toast_fires_on_202_with_the_safe_name() -> None:
"""The 202 branch of the upload submit (A1/A2): the 202 body
(UploadAccepted) is parsed for the safe source name — a body parse
failure degrades to the picked file's name — and the toast fires
with `Successfully uploaded — <name>` BEFORE the scan finishes:
the file input clears, the processing state enters, and the poll
starts. The toast is the SINGLE success surface: exactly one call
site in the whole page (definition + one call — never a failure
branch, never the poll)."""
js = _gjs()
sub = _usubmit(js)
i202 = sub.find("r.status === 202")
i409 = sub.find("r.status === 409")
assert -1 < i202 < i409, "the 202 branch precedes the 409 re-attach"
branch = sub[i202:i409]
assert "let name = file.name;" in branch, "the degrade-to-picked-name fallback"
assert "await r.json()" in branch, "the UploadAccepted body is parsed"
assert "data.name" in branch, "the safe source name comes from the 202 body"
assert "showUploadToast(`Successfully uploaded — ${name}`)" in branch
assert 'uploadFileInput.value = ""' in branch, "the file input clears at 202"
assert "enterUploadProcessingState()" in branch
assert "startUploadPolling()" in branch
# Success-only: exactly the definition + the single 202 call.
assert js.count("showUploadToast(") == 2, (
"the definition + exactly ONE call site (the 202 branch)"
)
# ---------- the processing state + the live label ----------
def test_processing_state_and_live_label_builder() -> None:
"""The button's processing entry (202 / 409): disabled,
"Processing…", title cleared (the poll owns it from here). The
tick's running branch builds the live label — the base prefix,
the file appended ONLY when present, the counts appended ONLY
when total > 0 (A4 — bare "Processing…" during the unpack phase,
before any file is indexed) — and rides the FULL untruncated path
on the button title (empty until a file exists), then
reschedules at the 2 s house cadence."""
js = _gjs()
state = _gfn(js, "enterUploadProcessingState")
assert "uploadBtn.disabled = true" in state
assert 'uploadBtn.textContent = "Processing…"' in state
assert 'uploadBtn.title = "";' in state, "a live file lands on the title at the first tick"
assert "const UPLOAD_POLL_MS = 2000" in js, "the SYNC_POLL_MS house value"
tick = _utick(js)
i_run = tick.find('status.state === "running"')
i_stop = tick.find("stopUploadPolling();")
assert -1 < i_run < i_stop, "the running branch precedes the terminal stop"
run = tick[i_run:i_stop]
assert '"Processing…"' in run, "the base prefix"
assert "(status.current_file ? ` ${status.current_file}` : \"\")" in run, (
"the file is appended only when present"
)
counts_expr = '(status.files_total > 0 ? ` (${status.files_done}/${status.files_total})` : "")'
assert counts_expr in run, "the counts appear only when total > 0"
assert 'uploadBtn.title = status.current_file || "";' in run, (
"the full path on hover (empty until a file exists)"
)
assert "uploadPollTimer = setTimeout(tick, UPLOAD_POLL_MS)" in run, "reschedule"
def test_start_upload_polling_double_start_guard() -> None:
"""startUploadPolling: single timer, one loop at a time — the
first statement bails when a poll is already active (the guard a
double 202/409 cannot bypass)."""
js = _gjs()
i = js.find("function startUploadPolling")
assert i != -1
head = js[i : i + 120]
assert "if (uploadPollTimer !== null) return;" in head
assert "let uploadPollTimer = null" in js, "null = no poll active"
# ---------- the 409 re-attach + the kept error branches ----------
def test_409_reattaches_without_an_error_banner() -> None:
"""409 (an upload is already in progress): NO error banner — the
phase-49 "server detail inline" branch does not apply to 409
anymore. It re-attaches to the in-flight run: the processing state
+ the poll (never stale). The OTHER non-2xx (422 format/name, 413
cap, 5xx) keep the phase-49 apiDetail banner + the kept file
selection; the network-failure fixed line stays."""
js = _gjs()
sub = _usubmit(js)
i409 = sub.find("r.status === 409")
assert i409 != -1
branch = sub[i409 : sub.find("// Other non-2xx", i409)]
assert "enterUploadProcessingState()" in branch
assert "startUploadPolling()" in branch
assert "uploadError" not in branch, "409 never raises the error banner"
assert "apiDetail" not in branch, "no server-detail branch for 409 anymore"
# The other non-2xx keeps the phase-49 convention (after the 409).
i_err = sub.find('await apiDetail(r, "Could not upload the archive — try again.")')
assert i_err > i409, "the other non-2xx branch follows the 409 re-attach (and was found)"
assert "uploadError.hidden = false" in sub[i_err:], "the banner shows for the other non-2xx"
assert "Could not upload the archive — is the app reachable?" in sub, "the network line stays"
# The no-file guard + the short transfer label stay.
assert "Choose an archive file to upload." in sub
assert 'uploadBtn.textContent = "Uploading…"' in sub
# ---------- the poll's terminal decision tree ----------
def test_upload_polling_decision_tree() -> None:
"""The upload poll tick (task 05): fetches
GET /api/git-sources/upload/status; a blip (non-ok / network /
unparseable) reschedules — the tick never dies on a failed fetch.
Then: running → live label + reschedule; ONE stop for the
terminals; success → the result line (fmtUploadResult — the
existing helper reads exactly these keys) + the announce + the row
reload + the cleared file input + the restored button — NO toast
(it already fired at the 202, A2); failed → the sanitized server
error banner (A2 failure UI) + the restored button + the row
reload (a post-swap failure keeps the row — the list state may
have changed; the selection is kept for a one-click re-upload);
idle → the defensive restore (a started run never returns to
idle)."""
tick = _utick(_gjs())
assert 'fetch("/api/git-sources/upload/status")' in tick
# The blip branch reschedules.
i_blip = tick.find("if (!status)")
assert i_blip != -1
assert "uploadPollTimer = setTimeout(tick, UPLOAD_POLL_MS)" in tick[i_blip:i_blip + 150]
# One stop, placed after the running branch and before the terminals.
assert tick.count("stopUploadPolling()") == 1
i_run = tick.find('status.state === "running"')
i_stop = tick.find("stopUploadPolling();")
i_ok = tick.find('status.state === "success"')
i_fail = tick.find('status.state === "failed"')
assert -1 < i_run < i_stop < i_ok < i_fail, "running < stop < success < failed"
# success: result line + announce + reload, NO toast.
ok = tick[i_ok:i_fail]
for line in (
"fmtUploadResult(detail)",
"uploadResult.hidden = false",
"announce(`Archive uploaded: ${detail.source}.`)",
'uploadFileInput.value = ""',
"restoreUploadButton()",
"loadSources()",
):
assert line in ok, f"the success settle must carry {line!r}"
assert "showUploadToast" not in tick, "no toast in the poll — it fired at the 202 (A2)"
# failed: the sanitized error banner + reload.
fail = tick[i_fail:]
assert "status.error" in fail
assert "uploadError.hidden = false" in fail
assert "restoreUploadButton()" in fail
assert "loadSources()" in fail, "the list state may have changed"
# idle: the defensive fall-through — a final restore, no more state
# checks after the failed branch.
i_idle = tick.rfind("restoreUploadButton()")
assert i_idle > i_fail
assert "state ===" not in tick[i_idle:], "the idle settle is the fall-through"
# ---------- the finally's never-restore-while-polling guard ----------
def test_finally_restores_only_when_no_poll_active() -> None:
"""The submit finally (PLAN §7.4): the button is restored ONLY
when no poll is active (``uploadPollTimer === null``) — while
startUploadPolling owns the button (the 202 / 409 paths) it stays
disabled / "Processing…", so an unconditional finally restore
would race the poll and leave a stale-looking idle button under a
running scan."""
sub = _usubmit(_gjs())
i = sub.find("} finally {")
assert i != -1
fin = sub[i:]
assert "if (uploadPollTimer === null)" in fin, "the guard: no poll → the button is ours"
assert "restoreUploadButton()" in fin
assert "uploadBtn.disabled = false" not in fin, "no unconditional restore in the finally"
# ---------- the boot re-attach ----------
def test_boot_reattach_branches() -> None:
"""initUploadStatus (the admin branch of the boot): the upload
status is fetched ONCE. running → the processing state + the poll
(a reload mid-scan re-attaches — no second upload, no error, no
toast); success → the last result line ONLY (no announce, no
toast — A2); failed → the error banner; idle → nothing (no
branch). The boot IIFE awaits it right after loadSources()."""
js = _gjs()
body = _gfn(js, "initUploadStatus")
assert body.count('fetch("/api/git-sources/upload/status")') == 1, "fetched ONCE at boot"
i_run = body.find('status.state === "running"')
i_ok = body.find('status.state === "success"')
i_fail = body.find('status.state === "failed"')
assert -1 < i_run < i_ok < i_fail
run = body[i_run:i_ok]
assert "enterUploadProcessingState()" in run
assert "startUploadPolling()" in run
assert "uploadError" not in run and "showUploadToast" not in run
ok = body[i_ok:i_fail]
assert "fmtUploadResult(status.detail)" in ok, "the last result line"
assert "uploadResult.hidden = false" in ok
assert "announce(" not in ok, "no announce at boot (A2)"
assert "showUploadToast" not in ok, "no toast at boot (A2)"
fail = body[i_fail:]
assert "status.error" in fail
assert "uploadError.hidden = false" in fail
assert 'status.state === "idle"' not in body, "idle does nothing — no branch"
# The boot IIFE: after the list loads, the re-attach runs (admin
# branch only — the anonymous path returns before it).
i_boot = js.rfind("await loadSources();")
tail = js[i_boot:i_boot + 400]
assert "await initUploadStatus();" in tail
assert "})();" in tail
# ---------- the page comment ----------
def test_git_sources_html_comment_documents_the_202_contract() -> None:
"""The #archive-upload-form comment in git-sources.html documents
the phase-64 202 contract (the phase-49 synchronous paragraph
marked superseded): the 202 = "safely on disk" + the JS-created
toast (no markup), the live "Processing…" label via the status
poll, and the 409 re-attach without an error banner."""
html = _ghtml()
idx = html.find('id="archive-upload-form"')
assert idx != -1
comment = html[max(0, idx - 1600):idx]
assert "Phase 64" in comment
assert "superseded" in comment, "the phase-49 synchronous paragraph is marked superseded"
assert "Successfully uploaded" in comment, "the toast is documented"
assert "Processing…" in comment, "the live label is documented"
assert "GET /api/git-sources/upload/status" in comment, "the polling endpoint"
assert "409 re-attaches" in comment, "the re-attach without an error banner"
+138
View File
@@ -18,6 +18,7 @@ from pathlib import Path
import pytest
from sqlalchemy import func, select
import app.rag.importer as importer
from app.config import Settings
from app.models import Chunk, Document
from app.rag.importer import (
@@ -708,3 +709,140 @@ def test_prune_removes_files_now_excluded_by_format_filter(db, tmp_path: Path) -
) is not None
finally:
_cleanup_source(db, root.name)
# ---------- phase 64 (task 01): optional per-file progress hook ----------
def test_progress_hook_reports_every_file_in_order_across_roots(
db, tmp_path: Path
) -> None:
"""Multi-root, multi-file: the hook receives the exact
``(source, rel, done, total)`` sequence — roots in *sources* order,
``rel`` the same POSIX path the doc rows use, ``done`` the 1-based
index across **all** sources, ``total`` the combined count."""
root_a = tmp_path / "Alpha"
root_b = tmp_path / "Beta"
root_a.mkdir()
(root_b / "sub").mkdir(parents=True)
(root_a / "a1.md").write_text("# A1\n\na one\n")
(root_a / "a2.md").write_text("# A2\n\na two\n")
(root_a / "a1.md").write_text("# A1\n\na one\n")
(root_b / "sub" / "b1.md").write_text("# B1\n\nb one\n")
events: list[tuple[str, str, int, int]] = []
def progress(source: str, rel: str, done: int, total: int) -> None:
events.append((source, rel, done, total))
try:
summary = asyncio.run(
import_sources([root_a, root_b], FakeEmbedder(), session=db, progress=progress)
)
assert summary.files == 3 and summary.added == 3
assert events == [
("Alpha", "a1.md", 1, 3),
("Alpha", "a2.md", 2, 3),
("Beta", "sub/b1.md", 3, 3), # POSIX rel, sorted within the root
]
finally:
_cleanup_source(db, "Alpha")
_cleanup_source(db, "Beta")
def test_no_progress_means_no_prewalk(
db, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""``progress=None`` callers pay no extra pass: the real walker is hit
exactly once per source root (one pass — as before phase 64), proven
with a counting sentinel; with the hook it is hit twice (pre-walk for
``total`` + the processing pass)."""
root = tmp_path / "nowalk"
root.mkdir()
(root / "a.md").write_text("# A\n\na\n")
real_walker = importer.iter_importable_files
walk_calls = 0
def counting(
r: Path, extensions: frozenset[str], excluded: frozenset[str] = EXCLUDED_DIRS
) -> list[Path]:
nonlocal walk_calls
walk_calls += 1
return real_walker(r, extensions, excluded)
monkeypatch.setattr(importer, "iter_importable_files", counting)
try:
summary = asyncio.run(import_sources([root], FakeEmbedder(), session=db))
assert summary.files == 1
assert walk_calls == 1 # exactly one pass — the pre-change behaviour
walk_calls = 0
events: list[tuple[str, str, int, int]] = []
summary2 = asyncio.run(
import_sources(
[root], FakeEmbedder(), session=db,
progress=lambda s, r, d, t: events.append((s, r, d, t)),
)
)
assert summary2.files == 1
assert walk_calls == 2 # pre-walk (total) + processing pass
assert events == [(root.name, "a.md", 1, 1)]
finally:
_cleanup_source(db, root.name)
def test_progress_hook_counts_unchanged_and_error_files(db, tmp_path: Path) -> None:
"""The hook fires *before* ``_index_file``: a file whose embedding
fails (and one that re-imports as unchanged) is still reported as the
current file — the sequence covers every importable file."""
root = tmp_path / "progress-mixed"
root.mkdir()
(root / "bad.md").write_text("# Bad\n\npoison content the endpoint refuses\n")
(root / "good.md").write_text("# Good\n\nperfectly fine content\n")
expected = [(root.name, "bad.md", 1, 2), (root.name, "good.md", 2, 2)]
try:
events: list[tuple[str, str, int, int]] = []
first = asyncio.run(
import_sources(
[root], _PoisonEmbedder(), session=db,
progress=lambda s, r, d, t: events.append((s, r, d, t)),
)
)
# bad.md was already reported (done=1) when its embed raised —
# no file silently disappears from the sequence.
assert events == expected
assert first.errors == 1 and first.added == 1
# Re-run: good.md is now unchanged, bad.md is retried and fails
# again — both still count in the sequence.
events.clear()
second = asyncio.run(
import_sources(
[root], _PoisonEmbedder(), session=db,
progress=lambda s, r, d, t: events.append((s, r, d, t)),
)
)
assert events == expected
assert second.errors == 1 and second.unchanged == 1
finally:
_cleanup_source(db, root.name)
def test_progress_hook_with_limit_keeps_full_total(db, tmp_path: Path) -> None:
"""The debug ``limit`` path is unchanged for the hook: it fires only
for processed files (``done`` never exceeds the limit), while
``total`` stays the FULL pre-walk count — an incomplete walk must not
misreport the denominator."""
root = tmp_path / "progress-limited"
root.mkdir()
for name in ("a.md", "b.md", "c.md"):
(root / name).write_text(f"# {name}\n\nbody {name}\n")
events: list[tuple[str, str, int, int]] = []
try:
summary = asyncio.run(
import_sources(
[root], FakeEmbedder(), limit=2, session=db,
progress=lambda s, r, d, t: events.append((s, r, d, t)),
)
)
assert summary.files == 2
assert events == [(root.name, "a.md", 1, 3), (root.name, "b.md", 2, 3)]
finally:
_cleanup_source(db, root.name)
+314 -9
View File
@@ -13,12 +13,31 @@ client-side hard timeout), the page's #sync-result line +
§7.4 never-stale CSS (spin + reduced-motion opt-out,
disabled state, 44px floor, contrast pair) — so a silent regression is
caught without a browser.
Phase 64 (task 02): unit coverage of the ``app/api/sync.py`` status
contract — the idle response dict is pinned in full (every pre-existing
key unchanged, plus ``current_file``/``files_done``/``files_total`` as
null/0/0); mid-run the status reports the file the (mock) import is
processing, through the runner's own hook closure (no file yet during
the clone/pull phase — A4); the terminal states clear
``current_file`` while keeping the run's final counts.
"""
from __future__ import annotations
import asyncio
import re
import threading
from collections.abc import Callable, Iterator
from pathlib import Path
import pytest
from app.api import sync as sync_api
from app.config import Settings
from app.models import GitSource
from app.rag.importer import ImportSummary
from app.rag.llm import EmbeddingError
FRONTEND = Path(__file__).resolve().parents[2] / "frontend"
ASSETS = FRONTEND / "assets"
@@ -178,12 +197,15 @@ def test_sources_js_polls_every_2000ms() -> None:
def test_sources_js_adopts_409_and_starts_on_202() -> None:
"""202 (started) and 409 (a run started elsewhere — e.g. a second
tab) both enter the running state and start polling: the UI never
starts a second run, it adopts the in-flight one."""
starts a second run, it adopts the in-flight one.
Phase 64 (task 04): the entry is the sync-kind live label with no
file yet (the run is just starting — bare "Syncing…", A4)."""
js = _text(SOURCES_JS)
assert "r.status === 202 || r.status === 409" in js
idx = js.find("r.status === 202 || r.status === 409")
branch = js[idx : idx + 400]
assert "enterSyncRunningState()" in branch
branch = js[idx : idx + 600]
assert "enterSyncRunningState(\"sync\", null, 0, 0)" in branch
assert "startSyncPolling()" in branch
@@ -202,19 +224,29 @@ def test_sources_js_hides_the_button_on_403() -> None:
def test_sources_js_running_state_is_never_stale() -> None:
"""Entering the running state disables the button, sets aria-busy,
spins the icon, and swaps the label to 'Syncing…' (the §7.4
feedback while the poll waits) — and a fresh run starts clean: the
previous failure's title / aria-label / .is-error come off NOW,
not when the run settles."""
spins the icon, and swaps the label to the live file label
(fmtSyncLabel — the §7.4 feedback while the poll waits) — and a
fresh run starts clean: the previous failure's title / aria-label /
.is-error come off NOW, not when the run settles.
Phase 64 (task 04): the button title carries the FULL untruncated
current file (removed when null — no file yet) and #sync-result
(the aria-live announcer) carries the same untruncated label."""
js = _text(SOURCES_JS)
body = _body(js, "enterSyncRunningState")
assert "syncBtn.disabled = true" in body
assert 'syncBtn.setAttribute("aria-busy", "true")' in body
assert 'syncBtn.removeAttribute("title")' in body
assert "syncBtn.title = currentFile" in body, "the full path on hover"
assert 'syncBtn.removeAttribute("title")' in body, "removed when no file yet"
title_if = body.find("if (currentFile) syncBtn.title = currentFile")
title_else = body.find("syncBtn.removeAttribute(\"title\")")
assert -1 < title_if < title_else, "title is set, not removed, only when a file exists"
assert 'syncBtn.setAttribute("aria-label", "Sync sources")' in body
assert "syncBtn.classList.remove(\"is-error\")" in body
assert "syncIcon.classList.add(\"is-spinning\")" in body
assert '"Syncing…"' in body
assert "fmtSyncLabel(kind, currentFile, done, total)" in body
assert "syncLabel.textContent = label" in body
assert "syncResult.textContent = label" in body, ("the announcer reads the full live path")
def test_sources_js_terminal_states() -> None:
@@ -595,3 +627,276 @@ def test_sync_modal_respects_reduced_motion() -> None:
)
assert reduced, "the backdrop fade must opt out under prefers-reduced-motion"
assert "transition: none" in reduced.group(1)
# ---------- phase 64 (task 02): per-file progress on the sync status ----------
#
# The GET /api/sync/status contract in app/api/sync.py (task 04 renders
# the live file label on the button — its pins live in
# tests/unit/test_frontend_sync_upload.py). The runner's seams are
# monkeypatched on ``app.api.sync`` (the house mock-import pattern from
# tests/integration/test_sync_api.py); the background task runs on a
# worker thread's own loop so the test can read the status mid-run.
# No DB, no HTTP: every seam is faked.
@pytest.fixture()
def fresh_sync_status() -> Iterator[None]:
"""The module-level status object + task are process-global: reset
them before AND after every progress test (the integration suite's
``_fresh_sync_state`` pattern)."""
sync_api._status = sync_api.SyncStatus()
sync_api._task = None
yield
sync_api._status = sync_api.SyncStatus()
sync_api._task = None
class _GatedClone:
"""A ``clone_or_pull`` that parks between start and finish on a
threading gate, so the test can read the status during the
clone/pull phase (A4: no file yet)."""
def __init__(self, started: threading.Event, release: threading.Event) -> None:
self.started = started
self.release = release
self.calls: list[tuple[str, Path]] = []
def __call__(self, url: str, dest: Path | str) -> Path:
dest = Path(dest)
self.calls.append((url, dest))
dest.mkdir(parents=True, exist_ok=True)
(dest / "notes.md").write_text("# repo\ncontent for the KB\n", encoding="utf-8")
self.started.set()
self.release.wait() # blocking is fine: the clone is a sync call
return dest
class _GatedImport:
"""The mock import: fires the runner's OWN progress hook once (the
progress-shaped call goes through the real hook closure — the
closure under test), parks on a threading gate so the test can read
the status mid-run, then returns the canned summary (or raises
``fail``)."""
def __init__(
self,
summary: ImportSummary,
started: threading.Event,
release: threading.Event,
fail: BaseException | None = None,
) -> None:
self.summary = summary
self.started = started
self.release = release
self.fail = fail
self.hook_calls: list[tuple[str, str, int, int]] = []
self.prune_flags: list[bool] = []
async def __call__(
self,
sources: list[Path],
llm: object,
*,
prune: bool = False,
limit: int | None = None,
session: object = None,
progress: Callable[[str, str, int, int], None] | None = None,
) -> ImportSummary:
self.prune_flags.append(prune)
if progress is not None:
progress("repo", "notes/deep.md", 1, 3)
self.hook_calls.append(("repo", "notes/deep.md", 1, 3))
self.started.set()
await asyncio.to_thread(self.release.wait) # park without freezing the worker loop
if self.fail is not None:
raise self.fail
return self.summary
def _patch_sync_seams(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
fake_import: _GatedImport,
fake_clone: _GatedClone,
) -> None:
"""The runner's seams, monkeypatched on ``app.api.sync`` (the house
mock-import pattern): fresh settings (no ``.env`` leak), a no-op
model probe, a sentinel LLM client, one git row, the gated clone +
import, a no-op overview, and the DB-free sources-version step
(dummy session + pinned counters)."""
monkeypatch.setattr(
sync_api,
"get_settings",
lambda: Settings(_env_file=None, sources_dir=str(tmp_path / "bor")), # pyright: ignore[reportCallIssue]
)
async def fake_probe(llm: object) -> None:
pass
monkeypatch.setattr(sync_api, "check_models", fake_probe)
monkeypatch.setattr(sync_api, "LLMClient", lambda: object())
monkeypatch.setattr(
sync_api,
"effective_sources",
lambda session: (
# kind explicit: the Python-side default applies at INSERT
# flush, not on an in-memory instance
[GitSource(url="https://git.example.com/repo.git", kind="git")],
"db",
),
)
monkeypatch.setattr(sync_api, "clone_or_pull", fake_clone)
monkeypatch.setattr(sync_api, "import_sources", fake_import)
async def fake_overview(llm: object, session: object = None) -> bool:
return False
monkeypatch.setattr(sync_api, "regenerate_overview", fake_overview)
class _DummySession:
def close(self) -> None:
pass
def commit(self) -> None:
pass
monkeypatch.setattr(sync_api, "SessionLocal", _DummySession)
monkeypatch.setattr(sync_api, "bump_sources_version", lambda session: 1)
monkeypatch.setattr(sync_api, "current_sources_version", lambda session: 1)
def _start_run() -> tuple[threading.Thread, list[BaseException]]:
"""Run the module-level runner on a worker thread's own event loop
(the house background-task pattern), capturing any unexpected
exception — the runner is supposed to die in state, never raise."""
errors: list[BaseException] = []
def _run() -> None:
try:
asyncio.run(sync_api._run_sync())
except BaseException as e: # noqa: BLE001 — surfaced to the test
errors.append(e)
thread = threading.Thread(target=_run, daemon=True)
thread.start()
return thread, errors
def test_idle_status_pins_full_shape_including_progress_keys(
fresh_sync_status: None,
) -> None:
"""Idle: the three phase-64 progress keys ride along as null/0/0,
and EVERY pre-existing key is unchanged — the full response dict is
pinned, so the current UI and every existing consumer keep working."""
assert sync_api.sync_status() == {
"state": "idle",
"started_at": None,
"finished_at": None,
"detail": {},
"error": None,
"current_file": None,
"files_done": 0,
"files_total": 0,
}
def test_mid_run_status_reports_current_file(
fresh_sync_status: None,
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
"""Mid-run: the status carries the file the (mock) import is
processing — assigned through the runner's own hook closure; during
the clone/pull phase (A4) no file is reported yet. The success
terminal clears ``current_file`` but keeps the final counts."""
clone = _GatedClone(threading.Event(), threading.Event())
fake_import = _GatedImport(
ImportSummary(files=3, added=1, updated=1, unchanged=1),
threading.Event(),
threading.Event(),
)
_patch_sync_seams(monkeypatch, tmp_path, fake_import, clone)
thread, errors = _start_run()
try:
assert clone.started.wait(5.0), "the run never reached the clone phase"
# Clone/pull phase: running — but no file yet (A4: bare "Syncing…").
s = sync_api.sync_status()
assert s["state"] == "running"
assert s["current_file"] is None
assert s["files_done"] == 0 and s["files_total"] == 0
clone.release.set()
assert fake_import.started.wait(5.0), "the run never reached the import"
# Import phase: the hook's file is live on the status.
s = sync_api.sync_status()
assert s["state"] == "running"
assert s["current_file"] == "repo/notes/deep.md"
assert s["files_done"] == 1
assert s["files_total"] == 3
fake_import.release.set()
finally:
clone.release.set()
fake_import.release.set()
thread.join(10.0)
assert not thread.is_alive()
assert errors == [], f"the runner raised: {errors!r}"
# Wiring: prune=True is preserved, and the hook fired through the
# runner's own closure (the recorded call is what the closure
# assigned to the status above).
assert fake_import.prune_flags == [True]
assert fake_import.hook_calls == [("repo", "notes/deep.md", 1, 3)]
# Success terminal: current_file null, final counts retained.
s = sync_api.sync_status()
assert s["state"] == "success"
assert s["current_file"] is None
assert s["files_done"] == 1 and s["files_total"] == 3
assert s["error"] is None
assert s["started_at"] is not None and s["finished_at"] is not None
def test_failed_terminal_clears_current_file_keeps_counts(
fresh_sync_status: None,
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
"""Terminal (failed): a run that dies inside the import clears
``current_file`` but keeps the hook's final counts — the last
position is useful context next to the (sanitized) error."""
clone = _GatedClone(threading.Event(), threading.Event())
fake_import = _GatedImport(
ImportSummary(),
threading.Event(),
threading.Event(),
fail=EmbeddingError(
"embeddings request to https://u:p@aipi.example.com/v1 "
"failed: connection refused"
),
)
_patch_sync_seams(monkeypatch, tmp_path, fake_import, clone)
thread, errors = _start_run()
try:
assert clone.started.wait(5.0), "the run never reached the clone phase"
clone.release.set()
assert fake_import.started.wait(5.0), "the progress hook never fired"
# While the (about-to-fail) import is parked: the file is live.
s = sync_api.sync_status()
assert s["state"] == "running"
assert s["current_file"] == "repo/notes/deep.md"
fake_import.release.set()
finally:
clone.release.set()
fake_import.release.set()
thread.join(10.0)
assert not thread.is_alive()
assert errors == []
s = sync_api.sync_status()
assert s["state"] == "failed"
assert s["current_file"] is None # cleared in the terminal state
assert s["files_done"] == 1 and s["files_total"] == 3 # final counts kept
assert s["detail"] == {}
error = s["error"] or ""
assert "*****@aipi.example.com" in error # credentials masked
assert "u:p" not in error
assert "connection refused" in error # the reason survives