Experimental/brain-of-reese
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(sources): real-time file progress for sync and upload — background upload with success toast

Browse Source
This commit is contained in:
ducoterra
2026-09-01 23:51:43 -04:00
parent cddc84c7db
commit 4677d86f49
103 changed files with 5914 additions and 456 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/e2e/slow_llm.py
+93
View File
@@ -0,0 +1,93 @@
"""Phase 64 E2E helper — a delay-injecting reverse proxy in front of the
mock LLM.
The mock LLM (``mock_llm.py``) answers instantly: a real archive scan of
dozens of files finishes in well under a second and never outlives the
UI's 2 s status poll — the phase-64 live file labels ("Processing…
<file>", "Importing <file>", "Syncing… <file>"), the at-202 toast →
navigate-away contract, and the mid-scan reload re-attach would all be
races against the mock. This proxy sits between the app under test and
the mock LLM and sleeps ``SLOW_LLM_DELAY_S`` seconds (default 0.15)
before forwarding each request, so a scan's duration is deterministic
(≈ the run's number of LLM requests × the delay — for an N-file
archive/sync that is N + 3: the ``check_models`` embed + chat probe,
one embed per file, and the change-gated overview chat). Both the
tests' ~100 ms status polling (the deterministic layer) and the UI's
2 s poll (the UI layer) then observe the running state, the current
file, and the counts reliably.
Everything else is byte-transparent: method, path, query, headers, and
body are forwarded verbatim; the upstream response's status and body
come back as-is (``content-encoding`` / ``content-length`` are dropped
— httpx has already decoded the body and starlette recomputes the
length). The mock's responses are all finite (its SSE streams end with
``[DONE]``), so the proxy reads each body to completion before
answering.
Run it the conftest way (a suite's module ``app_server`` fixture spawns
it as a subprocess):
uv run python -m uvicorn tests.e2e.slow_llm:app --port 8902
with ``E2E_MOCK_PORT`` (upstream, default 8901) and ``SLOW_LLM_DELAY_S``
(delay, default 0.15) in its environment.
"""
from __future__ import annotations
import asyncio
import os
import httpx
from fastapi import FastAPI, Request, Response
#: The mock LLM this proxy forwards to (the conftest's MOCK_PORT).
MOCK_PORT = int(os.environ.get("E2E_MOCK_PORT", "8901"))
UPSTREAM = f"http://127.0.0.1:{MOCK_PORT}"
#: Per-request delay in seconds — each suite's proxy fixture picks its
#: own (passed through the subprocess env).
DELAY_S = float(os.environ.get("SLOW_LLM_DELAY_S", "0.15"))
app = FastAPI()
_client: httpx.AsyncClient | None = None
def _get_client() -> httpx.AsyncClient:
global _client
if _client is None:
_client = httpx.AsyncClient(base_url=UPSTREAM, timeout=60.0)
return _client
@app.on_event("shutdown")
async def _close_client() -> None:
global _client
if _client is not None:
await _client.aclose()
_client = None
@app.api_route("/{path:path}", methods=["GET", "POST", "PUT", "DELETE", "PATCH"])
async def proxy(path: str, request: Request) -> Response:
"""Sleep ``DELAY_S``, then forward the request to the mock LLM."""
await asyncio.sleep(DELAY_S)
body = await request.body()
headers = {
k: v for k, v in request.headers.items() if k.lower() not in ("host", "content-length")
}
upstream = await _get_client().request(
request.method,
f"/{path}",
content=body,
headers=headers,
params=dict(request.query_params),
)
resp_headers = {
k: v
for k, v in upstream.headers.items()
if k.lower()
not in ("content-length", "content-encoding", "transfer-encoding", "connection")
}
return Response(
content=upstream.content, status_code=upstream.status_code, headers=resp_headers
)
tests/e2e/test_archive_upload_sources.py
+169 -39
View File
@@ -11,10 +11,26 @@ form is gone, replaced by this form): an uploaded ``.tar``/``.tar.gz``/
``.tgz``/``.zip`` is safely unpacked under ``BOR_UPLOAD_DIR/<name>/``
(name = filename minus the archive suffix), the ``git_sources`` row is
upserted (``kind='local'``, no duplicates), and the source is **scanned
synchronously in the request** (single-source ``import_sources`` with
``prune=True`` + the change-gated overview refresh) — the real pipeline,
against the deterministic mock LLM (no real models, no network beyond
the app itself).
in a background task** (phase 64, task 03 — owner-locked A1: the POST
answers **202 the moment the archive is safely on disk** — the
"Successfully uploaded — <source>" toast fires then and the user may
navigate away — while unpack → swap → row upsert → model check →
single-source ``import_sources`` with ``prune=True`` + the change-gated
overview refresh run server-side behind
``GET /api/git-sources/upload/status``, the phase-32 ``SyncStatus``
pattern with the phase-64 ``current_file``/counts). The result line and
the row land from the status ``success`` (same ``UploadOut`` counts,
uncompressed in shape) — the real pipeline, against the deterministic
mock LLM (no real models, no network beyond the app itself).
**Timing fixture (phase 64):** the mock LLM answers instantly, so this
module's app boots behind ``tests/e2e/slow_llm.py`` — a delay-injecting
reverse proxy in front of it (``SLOW_DELAY_S`` per request). A 2-file
scan is 5 LLM requests ≈ 5 × 0.6 s ≈ 3 s: long enough to outlive the
UI's 2 s status poll, so the button's literal
"Uploading… → Processing… → restored" lifecycle is observable
(the "Processing…" tick even carries the live file, A4) instead of
racing the mock.
The archives are **built in-test** with Python's ``tarfile`` over
``tmp_path`` fixture files carrying markdown sentinels (``ALPHA-…`` /
@@ -38,24 +54,33 @@ Contract under test:
the upload form is in its place with the labeled file input (accept
= the four archive extensions), the "Upload & scan" button, and the
hint explains unpack/scan + in-place replace;
* **upload → scan → list** (§7.4 never-stale): the button shows
"Uploading…" while the POST is in flight (the request is held in the
browser via ``page.route`` so the in-flight state is deterministic),
then restores; the result line shows the added count; the list gains
exactly one row for ``e2e-upload`` with the **Local** badge;
``GET /api/docs`` lists both sentinel files under source
``e2e-upload``; the RAG catalog (``/sources.html``) shows them;
* **upload → 202 + toast → background scan → list** (§7.4 never-stale,
phase-64 A1/A2): the button shows "Uploading…" while the POST is in
flight (the request is held in the browser via ``page.route`` so the
in-flight state is deterministic); at the 202 the "Successfully
uploaded — <source>" toast fires (``.toast.is-visible``,
``role="status"``) WHILE the scan is still running, and the button
hands over to the scan — "Processing…" (the live-file tick carries
the current file, A4) — then restores when the status ``success``
lands: the result line shows the added count; the list gains exactly
one row for ``e2e-upload`` with the **Local** badge; ``GET
/api/docs`` lists both sentinel files under source ``e2e-upload``;
the RAG catalog (``/sources.html``) shows them;
* **re-upload, same filename** → in-place replace: the result line
shows the prune, the list still has exactly ONE ``e2e-upload`` row
(no duplicate), the KB shows the changed ``alpha`` + the new
shows the prune, the SECOND RUN'S STATUS ``detail`` carries the
prune/refresh counts, the list still has exactly ONE ``e2e-upload``
row (no duplicate), the KB shows the changed ``alpha`` + the new
``gamma`` and NOT the dropped ``beta``, and the on-disk folder holds
only the new archive's files;
* **bad file** → inline 422 (role=alert) naming the accepted formats,
button restored, the file selection kept, the list unchanged, and a
subsequent good upload still works (the form is not wedged);
* **bad file** → inline 422 (role=alert) naming the accepted formats
(UNCHANGED — the name/format/cap gates are inline, pre-202, exactly
as before), button restored, the file selection kept, the list
unchanged, and a subsequent good upload still works (the form is not
wedged);
* **anonymous** → the sign-in gate (``#git-sources-gate``) shows, the
manager (and thus the upload form) stays hidden, and
``POST /api/git-sources/upload`` is 403.
``POST /api/git-sources/upload`` is 403 — as is the phase-64
``GET /api/git-sources/upload/status`` (same wall).
Test → story mapping (Playwright Mapping Rule):
1. ``test_form_swapped``
@@ -86,6 +111,7 @@ from e2e.auth_helpers import login
from e2e.conftest import (
ADMIN_PASSWORD,
APP_PORT,
MOCK_PORT,
SESSION_SECRET,
USE_REAL_LLM,
_wait_http,
@@ -94,6 +120,17 @@ from e2e.conftest import (
REPO = Path(__file__).resolve().parents[2]
APP_URL = f"http://127.0.0.1:{APP_PORT}"
#: The slow-LLM proxy's port (the conftest's mock LLM stays on MOCK_PORT).
SLOW_PORT = int(os.environ.get("E2E_SLOW_LLM_PORT", "8902"))
SLOW_URL = f"http://127.0.0.1:{SLOW_PORT}"
#: Per-LLM-request delay on the proxy — a 2-file scan is 5 LLM requests
#: (the check_models embed + chat probe, one embed per file, the
#: change-gated overview chat) ≈ 5 × 0.6 s ≈ 3 s: the scan outlives the
#: UI's 2 s status poll, so the button's "Uploading… → Processing… →
#: restored" lifecycle (with the live-file tick, A4) is observable.
SLOW_DELAY_S = "0.6"
GIT_SOURCES_URL = "/git-sources.html"
SOURCES_URL = "/sources.html"
@@ -184,17 +221,47 @@ def tarball_v2(tmp_path_factory: pytest.TempPathFactory) -> Path:
return _build_targz(root / f"{SOURCE_NAME}.tar.gz", V2_FILES)
@pytest.fixture(scope="module")
def slow_llm(mock_llm: int) -> Iterator[int]:
"""The delay-injecting reverse proxy in front of the mock LLM
(tests/e2e/slow_llm.py) — this suite's timing fixture: the phase-64
button lifecycle ("Uploading… → Processing… → restored") needs the
2-file scan to outlive the UI's 2 s status poll (see
``SLOW_DELAY_S``)."""
env = dict(os.environ)
env.pop("DEBUGPY", None)
env["SLOW_LLM_DELAY_S"] = SLOW_DELAY_S
env["E2E_MOCK_PORT"] = str(MOCK_PORT)
proc = subprocess.Popen(
[sys.executable, "-m", "uvicorn", "tests.e2e.slow_llm:app",
"--host", "127.0.0.1", "--port", str(SLOW_PORT), "--log-level", "warning"],
cwd=REPO,
env=env,
)
try:
_wait_http(f"{SLOW_URL}/v1/models")
yield SLOW_PORT
finally:
proc.terminate()
try:
proc.wait(timeout=10)
except subprocess.TimeoutExpired:
proc.kill()
@pytest.fixture(scope="module")
def app_server(
mock_llm: int,
slow_llm: int,
upload_dir: Path,
tmp_path_factory: pytest.TempPathFactory,
) -> Iterator[str]:
"""The real app under test — per-module env: uploads unpack into a
scratch dir and the env git list is forced empty (the dev ``.env``'s
``BOR_GIT_SOURCES`` must not render as env rows on the initially
empty table). No sync is triggered here — the upload's own scan is
the pipeline under test."""
"""The real app under test — per-module env: the LLM base URL is the
SLOW PROXY in front of the mock (the timing fixture), uploads unpack
into a scratch dir and the env git list is forced empty (the dev
``.env``'s ``BOR_GIT_SOURCES`` must not render as env rows on the
initially empty table). No sync is triggered here — the upload's own
scan is the pipeline under test."""
env = dict(os.environ)
env.pop("DEBUGPY", None)
env["BOR_ENVIRONMENT"] = "e2e"
@@ -202,7 +269,7 @@ def app_server(
env["BOR_LLM_BASE_URL"] = (
"https://aipi.reeseapps.com/v1"
if USE_REAL_LLM
else f"http://127.0.0.1:{mock_llm}/v1"
else f"{SLOW_URL}/v1"
)
# Mock-calibrated threshold (conftest pattern) — no chat turn is
# ever sent in this suite, but the app boots with the same env shape.
@@ -286,9 +353,10 @@ def _docs(page: Page, app_url: str) -> list[tuple[str, str]]:
def _upload_via_page(page: Page, archive: Path) -> str:
"""Pick the archive, submit the form, and wait for the result line
(the 200 path) — returns its text. The failing path is asserted
explicitly by the bad-file test, so any non-result outcome here is
a test error."""
(the phase-64 202 path: toast at the 202, then the button's status
polling renders the line from the run's ``success``) — returns its
text. The failing path is asserted explicitly by the bad-file test,
so any non-result outcome here is a test error."""
page.set_input_files("#archive-upload-file", str(archive))
page.click("#archive-upload-btn")
result = page.locator("#archive-upload-result")
@@ -298,6 +366,24 @@ def _upload_via_page(page: Page, archive: Path) -> str:
return text
def _wait_upload_running(page: Page, app_url: str, timeout_s: float = 15.0) -> dict[str, Any]:
"""Poll (cookie-authenticated) the upload status endpoint until the
run is ``running`` — the phase-64 single source of truth for the
background scan (A1)."""
deadline = time.monotonic() + timeout_s
body: dict[str, Any] = {}
while time.monotonic() < deadline:
r = page.request.get(f"{app_url}/api/git-sources/upload/status")
assert r.status == 200, r.text
body = r.json()
if body["state"] == "running":
return body
if body["state"] in ("success", "failed"):
raise AssertionError(f"the scan settled too fast to observe: {body}")
time.sleep(0.1)
raise AssertionError(f"the scan never entered running: {body}")
def _hold_upload_request(page: Page, hold_s: float) -> None:
"""Intercept the upload POST and hold the REQUEST in the browser for
``hold_s`` seconds before letting it reach the server. While it is
@@ -365,11 +451,15 @@ def test_upload_scans_and_lists(
page: Page, app_url: str, db_ready: None, tarball_v1: Path, upload_dir: Path
) -> None:
"""One real upload through the page: while the POST is in flight the
button is disabled and reads "Uploading…"; on the 200 it restores,
the result line shows the added count (2), the file input clears,
the list gains exactly ONE row for ``e2e-upload`` with the Local
badge, ``/api/docs`` lists both sentinel files under the source, and
the RAG catalog shows them where the admin expects them."""
button is disabled and reads "Uploading…"; at the 202 the
"Successfully uploaded — <source>" toast fires (A2) WHILE the
background scan is still running and the button hands over to it —
"Processing…" (the 2 s poll tick carries the live file, A4); when
the status ``success`` lands the button restores, the result line
shows the added count (2), the file input clears, the list gains
exactly ONE row for ``e2e-upload`` with the Local badge,
``/api/docs`` lists both sentinel files under the source, and the
RAG catalog shows them where the admin expects them."""
page.set_default_timeout(30_000)
_admin_git_sources_page(page, app_url)
expect(page.locator("#git-sources-tbody tr")).to_have_count(0)
@@ -378,7 +468,7 @@ def test_upload_scans_and_lists(
result = page.locator("#archive-upload-result")
# Hold the upload request in the browser: the in-flight state below
# cannot race the (fast) mock-LLM scan while it is held.
# cannot race the receive while it is held.
_hold_upload_request(page, hold_s=0.8)
page.set_input_files("#archive-upload-file", str(tarball_v1))
btn.click()
@@ -388,12 +478,35 @@ def test_upload_scans_and_lists(
expect(btn).to_have_text("Uploading…")
expect(result).to_be_hidden()
# The request goes out, the server unpacks + scans (mock LLM) and
# answers 200 → the result line shows the added count.
# The request goes out; the server stores the archive and answers
# 202 the moment it is safely on disk (A1) → the toast fires NOW
# (A2) — while the scan is still running — and the button hands
# over to the scan (bare "Processing…" — A4: no file yet during the
# unpack phase).
toast = page.locator(".toast")
expect(toast).to_have_count(1, timeout=UPLOAD_TIMEOUT_MS)
expect(toast).to_have_class(re.compile(r"\bis-visible\b"))
assert toast.get_attribute("role") == "status"
expect(toast).to_have_text(f"Successfully uploaded — {SOURCE_NAME}")
expect(result).to_be_hidden()
expect(btn).to_be_disabled()
expect(btn).to_have_text("Processing…", timeout=5_000)
# The scan is running server-side (the status endpoint is the
# single source of truth, A1) — the run the UI's poll tracks.
_wait_upload_running(page, app_url)
# …and the button's 2 s poll tick renders the live file label
# ("Processing… <file> (n/m)", A4).
expect(btn).to_have_text(
re.compile(rf"Processing… {re.escape(SOURCE_NAME)}/.+\.md"),
timeout=UPLOAD_TIMEOUT_MS,
)
# The status success lands → the result line (the same UploadOut
# counts) + the never-stale restore (input cleared).
expect(result).to_be_visible(timeout=UPLOAD_TIMEOUT_MS)
expect(result).to_have_text("2 added")
# Never stale: the button restored on success and the input cleared.
expect(btn).to_be_enabled()
expect(btn).to_have_text("Upload & scan")
expect(page.locator("#archive-upload-file")).to_have_value("")
@@ -434,7 +547,9 @@ def test_reupload_replaces_in_place(
upload_dir: Path,
) -> None:
"""v1 then v2 under the SAME filename (``e2e-upload.tar.gz``): the
result line shows the prune, the list still has exactly ONE
result line shows the prune, the SECOND RUN'S STATUS ``detail``
carries the prune/refresh counts (phase 64 — the line is rendered
from the status success), the list still has exactly ONE
``e2e-upload`` row (the row count for that source is invariant — no
duplicate), the KB shows the changed ``alpha`` + the new ``gamma``
and NOT the dropped ``beta``, and the on-disk folder holds only the
@@ -442,7 +557,7 @@ def test_reupload_replaces_in_place(
page.set_default_timeout(30_000)
_admin_git_sources_page(page, app_url)
# Baseline: v1 through the page (200 → "2 added", one row).
# Baseline: v1 through the page (202 → "2 added", one row).
assert _upload_via_page(page, tarball_v1) == "2 added"
expect(page.locator("#git-sources-tbody tr", has_text=SOURCE_NAME)).to_have_count(1)
@@ -452,6 +567,19 @@ def test_reupload_replaces_in_place(
result = page.locator("#archive-upload-result")
expect(result).to_have_text(re.compile(r"\d+ pruned"))
# The SECOND RUN's status ``detail`` shows the prune/refresh counts
# (phase 64: the result line is rendered from this success).
r = page.request.get(f"{app_url}/api/git-sources/upload/status")
assert r.status == 200, r.text
status = r.json()
assert status["state"] == "success", status
detail = status["detail"]
assert detail["source"] == SOURCE_NAME
assert detail["files"] == 2
assert detail["added"] == 1 # gamma — new in v2
assert detail["updated"] == 1 # alpha — changed in v2
assert detail["pruned"] == 1 # beta — dropped in v2
# No duplicate: exactly ONE row for that source (and one row total).
expect(page.locator("#git-sources-tbody tr", has_text=SOURCE_NAME)).to_have_count(1)
expect(page.locator("#git-sources-tbody tr")).to_have_count(1)
@@ -552,6 +680,8 @@ def test_anonymous_gate(page: Page, app_url: str, db_ready: None) -> None:
# The upload route 403s anonymous callers (require_admin runs before
# the multipart body is parsed — the body is a stand-in, the
# test_local_directory_sources.py pattern for this route).
# test_local_directory_sources.py pattern for this route)…
r = page.request.post(f"{app_url}/api/git-sources/upload", data={"file": ""})
assert r.status == 403
# …and so does the phase-64 upload STATUS endpoint (same wall).
assert page.request.get(f"{app_url}/api/git-sources/upload/status").status == 403
tests/e2e/test_sync_upload_progress.py
+758
View File
@@ -0,0 +1,758 @@
"""Phase 64 story E2E (Playwright): real-time progress for sync + upload.
Run in isolation (DB must be up: ``podman compose up -d db``):
uv run pytest tests/e2e/test_sync_upload_progress.py -v --no-cov
The story gate for the phase's executable proof (owner-locked A1–A5):
both long-running KB jobs report **which file is being processed right
now** — not just "Syncing…"/"Uploading…" — and the archive upload is
fully **backgrounded**: ``POST /api/git-sources/upload`` answers 202 the
moment the archive is on disk (the "Successfully uploaded — <file>"
toast fires — the user may navigate away), the unpack/scan continues
server-side behind ``GET /api/git-sources/upload/status`` (the phase-32
``SyncStatus`` pattern), and the RAG-page sync button
(``/sources.html``) animates with the upload's current file while that
scan runs.
**Timing fixture (the phase's fixture note):** the mock LLM indexes
fast — the in-progress state is real but brief (a 25-file scan against
it takes ≈0.4 s, well under the UI's 2 s status poll). This module's
app therefore boots behind ``tests/e2e/slow_llm.py`` — a delay-injecting
reverse proxy in front of the mock LLM (``SLOW_DELAY_S`` per request →
a 25-file scan is 28 LLM requests ≈ 4.2 s), so the scan outlives the
2 s poll and the live-file label is asserted at BOTH layers the task
pins:
* **deterministic** — the status endpoints (``page.request`` / the
concurrent recorder, ~100 ms cadence): ``state == "running"`` with a
non-null ``current_file`` (``source/relative/path``) observed at some
tick, the counts advancing, and the file-less ticks (unpack/row/
model probe — A4) preceding the first file tick;
* **UI** — polling the button labels for the ``Importing`` /
``Syncing…`` / ``Processing…`` prefix plus a file path (generous
timeout), which the pages' own 2 s poll ticks render.
The upload archive is built in-test with Python's ``tarfile`` from
**25 small ``.md`` files** (``e2e-prog.tar.gz`` → source ``e2e-prog``);
the sync subject is a host temp dir (``sync-corpus/``, 25 small
``.md`` files under ``notes/``) registered as a ``kind=local`` row —
the ``test_sync_button.py`` / ``test_local_directory_sources.py``
fixture styles. Per-module app env (the conftest pattern):
``BOR_UPLOAD_DIR`` scratch, ``BOR_GIT_SOURCES`` forced empty (the sync
sources are this suite's own local row), ``BOR_LLM_BASE_URL`` the slow
proxy.
Contract under test:
* **toast → navigate away (A2 + A3)**: on ``/git-sources.html`` the
"Successfully uploaded — <source>" toast (``.toast.is-visible``,
``role="status"``) appears while the scan is still running;
navigating to ``/sources.html`` shows the sync button animating
(spinner + ``aria-busy``) with the ``Importing <file>`` label; on
completion the button settles to "Sync sources" (no error UI,
``#sync-result`` stays empty — the upload's counts never render
there, A3) and the catalog shows the uploaded documents (the
phase-63 listing, untouched);
* **upload progress (A4)**: during the scan the status endpoint
reports a non-null ``current_file`` (``source/relative/path`` shape,
full denominator, advancing counts) at running ticks, the upload
button shows "Processing… <file>" (bare "Processing…" during unpack)
before the result line lands, and the toast fired earlier in the run
— the result line itself comes from the status ``success``;
* **sync live file (A4)**: a multi-file local source; clicking
**Sync sources** on ``/sources.html`` shows "Syncing…" (bare, the
pre-64 click state) then "Syncing… <file> (n/m)" (both layers), then
the pre-64 success settle — "Synced HH:MM" + the counts result line —
preserved, plus the file in the label;
* **reload re-attach (A1 + A2)**: starting an upload and reloading
``/git-sources.html`` mid-scan leaves the button in the Processing
state (disabled) with no error banner and NO second upload (the
status endpoint's single run is still the one from before the
reload — pinned on its ``started_at``); it then settles with the
result line and the list shows exactly one row for the archive.
Test → story mapping (Playwright Mapping Rule):
1. ``test_upload_toast_then_navigate_away``
2. ``test_upload_progress_shows_current_file``
3. ``test_sync_live_file_label``
4. ``test_upload_reattach_after_reload``
"""
from __future__ import annotations
import io
import os
import re
import subprocess
import sys
import tarfile
import threading
import time
from collections.abc import Iterator
from pathlib import Path
from typing import Any
import httpx
import pytest
from playwright.sync_api import Page, expect
from sqlalchemy import text
from app.db import SessionLocal
from app.models import GitSource
from e2e.auth_helpers import login
from e2e.conftest import (
ADMIN_PASSWORD,
APP_PORT,
MOCK_PORT,
SESSION_SECRET,
USE_REAL_LLM,
_wait_http,
)
REPO = Path(__file__).resolve().parents[2]
APP_URL = f"http://127.0.0.1:{APP_PORT}"
GIT_SOURCES_URL = "/git-sources.html"
SOURCES_URL = "/sources.html"
#: The slow-LLM proxy's port (the conftest's mock LLM stays on MOCK_PORT).
SLOW_PORT = int(os.environ.get("E2E_SLOW_LLM_PORT", "8902"))
SLOW_URL = f"http://127.0.0.1:{SLOW_PORT}"
#: Per-LLM-request delay on the proxy — the scan's duration becomes
#: deterministic: an N-file archive scan issues N + 3 LLM requests
#: (the check_models embed + chat probe, one embed per file, the
#: change-gated overview chat), so a 25-file upload takes ≈ 28 × 0.15 s
#: ≈ 4.2 s — long enough to outlive the UI's 2 s status poll (see the
#: module docstring's timing-fixture note).
SLOW_DELAY_S = "0.15"
#: The uploaded archive: 25 small docs under ``docs/`` (the phase's
#: fixture note — 20+ files so the scan outlives the 2 s poll).
UPLOAD_NAME = "e2e-prog"
UPLOAD_ARCHIVE = f"{UPLOAD_NAME}.tar.gz"
N_FILES = 25
UPLOAD_FILES: dict[str, str] = {
f"docs/{i:02d}.md": f"# Doc {i:02d}\n\nDeterministic upload content {i:02d}.\n"
for i in range(N_FILES)
}
#: The local sync source (host temp dir — the app runs on the same
#: machine): 25 small docs under ``notes/`` (the current_file's
#: source/relative/path shape has a directory level).
SYNC_SOURCE_DIR = "sync-corpus"
SYNC_FILES: dict[str, str] = {
f"notes/{i:02d}.md": f"# Sync doc {i:02d}\n\nDeterministic sync content {i:02d}.\n"
for i in range(N_FILES)
}
#: "Synced HH:MM" — the local-time last-result label (sources.js's
#: fmtSyncTime), any hour/minute (test_sync_button.py's pattern).
SYNCED_LABEL = re.compile(r"Synced \d{1,2}:\d{2}")
#: Generous settle budget: a 25-file scan against the slowed LLM is
#: ≈4.2 s; the UI's 2 s poll settles at most one tick after the
#: terminal state lands.
SETTLE_TIMEOUT_MS = 45_000
# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------
def _build_targz(path: Path, files: dict[str, str]) -> Path:
"""A deterministic ``.tar.gz`` (mtime 0) over the given files."""
with tarfile.open(path, "w:gz") as tf:
for rel, content in files.items():
data = content.encode("utf-8")
info = tarfile.TarInfo(rel)
info.size = len(data)
info.mtime = 0
tf.addfile(info, io.BytesIO(data))
return path
@pytest.fixture(scope="module")
def slow_llm(mock_llm: int) -> Iterator[int]:
"""The delay-injecting reverse proxy in front of the mock LLM
(tests/e2e/slow_llm.py) — this suite's timing fixture: the live-file
contract needs the scan to outlive the UI's 2 s poll (see
``SLOW_DELAY_S``)."""
env = dict(os.environ)
env.pop("DEBUGPY", None)
env["SLOW_LLM_DELAY_S"] = SLOW_DELAY_S
env["E2E_MOCK_PORT"] = str(MOCK_PORT)
proc = subprocess.Popen(
[sys.executable, "-m", "uvicorn", "tests.e2e.slow_llm:app",
"--host", "127.0.0.1", "--port", str(SLOW_PORT), "--log-level", "warning"],
cwd=REPO,
env=env,
)
try:
_wait_http(f"{SLOW_URL}/v1/models")
yield SLOW_PORT
finally:
proc.terminate()
try:
proc.wait(timeout=10)
except subprocess.TimeoutExpired:
proc.kill()
@pytest.fixture(scope="module")
def upload_dir(tmp_path_factory: pytest.TempPathFactory) -> Path:
"""The app's ``BOR_UPLOAD_DIR`` for this suite — a scratch dir the
host-side assertions inspect (the app server runs on the same
machine)."""
return tmp_path_factory.mktemp("bor_uploads") / "uploads"
@pytest.fixture(scope="module")
def upload_archive(tmp_path_factory: pytest.TempPathFactory) -> Path:
"""The 25-file upload archive (``e2e-prog.tar.gz``)."""
root = tmp_path_factory.mktemp("bor_archive")
return _build_targz(root / UPLOAD_ARCHIVE, UPLOAD_FILES)
@pytest.fixture(scope="module")
def sync_local_dir(tmp_path_factory: pytest.TempPathFactory) -> Path:
"""The host temp dir the sync test registers as a ``kind=local``
source — ``sync-corpus/notes/NN.md`` (25 files)."""
root = tmp_path_factory.mktemp("bor_sync_local") / SYNC_SOURCE_DIR
for rel, content in SYNC_FILES.items():
path = root / rel
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(content, encoding="utf-8")
return root
@pytest.fixture(scope="module")
def app_server(
mock_llm: int,
slow_llm: int,
upload_dir: Path,
tmp_path_factory: pytest.TempPathFactory,
) -> Iterator[str]:
"""The real app under test — per-module env: the LLM base URL is the
SLOW PROXY in front of the mock (the timing fixture), uploads unpack
into a scratch dir, and the env git list is forced empty (the sync
sources are this suite's own ``kind=local`` row, seeded per test)."""
env = dict(os.environ)
env.pop("DEBUGPY", None)
env["BOR_ENVIRONMENT"] = "e2e"
env["BOR_STATIC_DIR"] = str(REPO / "frontend")
env["BOR_LLM_BASE_URL"] = (
"https://aipi.reeseapps.com/v1"
if USE_REAL_LLM
else f"{SLOW_URL}/v1"
)
# Mock-calibrated threshold (conftest pattern) — no chat turn is
# ever sent in this suite, but the app boots with the same env shape.
env["BOR_RELEVANCE_THRESHOLD"] = "0.30"
env.setdefault(
"BOR_DATABASE_URL",
"postgresql+psycopg://reese:reese@localhost:5432/brain_of_reese",
)
# Phase 16: admin auth must be set or create_app() refuses to boot.
env["BOR_ADMIN_PASSWORD"] = ADMIN_PASSWORD
env["BOR_SESSION_SECRET"] = SESSION_SECRET
env["BOR_GIT_SOURCES"] = ""
env["BOR_UPLOAD_DIR"] = str(upload_dir)
env["BOR_SOURCES_DIR"] = str(tmp_path_factory.mktemp("bor_checkouts"))
proc = subprocess.Popen(
[sys.executable, "-m", "uvicorn", "app.main:app",
"--host", "127.0.0.1", "--port", str(APP_PORT), "--log-level", "warning"],
cwd=REPO,
env=env,
)
try:
_wait_http(f"{APP_URL}/api/health")
yield APP_URL
finally:
proc.terminate()
try:
proc.wait(timeout=10)
except subprocess.TimeoutExpired:
proc.kill()
@pytest.fixture(scope="module")
def app_url(app_server: str) -> str:
return app_server
def _truncate_all() -> None:
"""Fresh registry + KB per test (the E2E isolation pattern): the
upload's/sync's counts and every ``/api/docs`` assertion must be
this test's own doing. The E2E suites share one Postgres, and a
leftover git_sources row or document would corrupt the row-count
and doc-list assertions (and a leftover row would be a SECOND sync
source, skewing the per-file counts)."""
with SessionLocal() as db:
db.execute(text("TRUNCATE chunks, documents, query_log, kb_overview, git_sources"))
db.commit()
def _wait_no_running_jobs(app_url: str) -> None:
"""No background job may leak across tests (the run states live in
the app's memory, and a still-running scan would keep importing
into the NEXT test's truncated KB): wait for both status endpoints
to be non-running BEFORE the truncate. Own admin session (the
endpoints are admin-only) — usually a no-op: the tests settle only
after their run's terminal state."""
with httpx.Client(timeout=5.0) as client:
client.post(f"{app_url}/api/login", json={"password": ADMIN_PASSWORD})
for path in ("/api/sync/status", "/api/git-sources/upload/status"):
body: dict[str, Any] = {}
deadline = time.monotonic() + 90
while time.monotonic() < deadline:
r = client.get(f"{app_url}{path}")
if r.status_code == 200:
body = r.json()
if body["state"] != "running":
break
time.sleep(0.2)
assert body["state"] != "running", (
f"a background job was still running at test boundary: {path} {body}"
)
@pytest.fixture(autouse=True)
def _clean(app_url: str, db_ready: None) -> Iterator[None]:
_wait_no_running_jobs(app_url)
_truncate_all()
yield
_truncate_all()
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _seed_local_source(path: str) -> None:
"""Register a ``kind=local`` source row directly (deterministic —
the phase-49 page form is upload-only; a plain directory is an
API/DB-only operation, the test_local_directory_sources.py note)."""
with SessionLocal() as db:
db.add(GitSource(url=path, kind="local", path=path))
db.commit()
def _admin_git_sources_page(page: Page, app_url: str) -> None:
"""Real form login landing on the git sources page (admin settled:
Sign out visible, the manager revealed by the page module)."""
login(page, app_url, next=GIT_SOURCES_URL)
expect(page).to_have_url(app_url + GIT_SOURCES_URL, timeout=30_000)
expect(page.locator("#sign-out-btn")).to_be_visible(timeout=15_000)
expect(page.locator("#git-sources-gate")).to_be_hidden()
expect(page.locator("#git-sources-content")).to_be_visible()
def _status(page: Page, app_url: str, path: str) -> dict[str, Any]:
"""One (cookie-authenticated) status-endpoint fetch."""
r = page.request.get(f"{app_url}{path}")
assert r.status == 200, r.text
return r.json()
def _wait_running_started_at(
page: Page, app_url: str, path: str, timeout_s: float = 15.0
) -> str:
"""Poll the status endpoint until the run is ``running``; return its
``started_at`` (the run's identity — a second run would reset it)."""
deadline = time.monotonic() + timeout_s
body: dict[str, Any] = {}
while time.monotonic() < deadline:
body = _status(page, app_url, path)
if body["state"] == "running":
assert body["started_at"] is not None
return str(body["started_at"])
if body["state"] in ("success", "failed"):
raise AssertionError(f"the run settled too fast to observe: {body}")
time.sleep(0.1)
raise AssertionError(f"the run never entered running: {body}")
class _TickRecorder:
"""The deterministic layer, concurrent with the UI assertions.
A daemon thread that tight-polls (~100 ms cadence) the status
endpoint with its OWN admin session (``httpx`` — the browser page
drives itself in the meantime; the Playwright sync API is not
thread-safe, so the thread never touches it), recording every tick
from the first poll: the idle prelude, the running ticks (file-less
unpack/row/probe phase, then the per-file ticks), and the terminal
body. The thread only READS the same endpoint the UI's 2 s poll
reads — it starts no jobs and cannot skew the run."""
def __init__(self, app_url: str, path: str) -> None:
self._url = f"{app_url}{path}"
self._login_url = f"{app_url}/api/login"
self._ticks: list[dict[str, Any]] = []
self._terminal: dict[str, Any] | None = None
self._stop = threading.Event()
self._thread: threading.Thread | None = None
def start(self) -> None:
def run() -> None:
with httpx.Client(timeout=5.0) as client:
# Its own admin session — the recorder's reads must not
# depend on (or disturb) the browser context's cookie.
client.post(self._login_url, json={"password": ADMIN_PASSWORD})
saw_running = False
while not self._stop.is_set():
try:
r = client.get(self._url)
if r.status_code == 200:
body = r.json()
self._ticks.append(body)
# The run status is in the app's memory and
# SURVIVES across this module's tests: a
# residual terminal state (a previous test's
# run) must not be mistaken for this test's
# own terminal — accept it only AFTER this
# run's "running" has been observed.
if body["state"] == "running":
saw_running = True
elif body["state"] in ("success", "failed") and saw_running:
self._terminal = body
return
except Exception: # noqa: BLE001 — blip: retry next tick
pass
self._stop.wait(0.1)
self._thread = threading.Thread(target=run, daemon=True)
self._thread.start()
def stop(self, timeout_s: float = 60.0) -> dict[str, Any]:
"""Join until a terminal tick is recorded (or fail the test)."""
deadline = time.monotonic() + timeout_s
while time.monotonic() < deadline:
if self._terminal is not None:
break
time.sleep(0.05)
self._stop.set()
if self._thread is not None:
self._thread.join(timeout=5)
assert self._terminal is not None, (
"the recorder saw no terminal state "
f"(last ticks: {self._ticks[-3:] if self._ticks else 'none'})"
)
return self._terminal
@property
def running_ticks(self) -> list[dict[str, Any]]:
return [t for t in self._ticks if t["state"] == "running"]
def _assert_live_file_ticks(
ticks: list[dict[str, Any]], source: str, n_files: int
) -> None:
"""A4 against the recorded running ticks (the deterministic layer):
* the file-less ticks (unpack/row/model probe — before any file is
indexed) come FIRST (the label is the bare prefix then);
* SOME tick reports a non-null ``current_file`` in the
``source/relative/path`` shape;
* ``files_total`` is the full pre-walk count from the first file
tick, and ``files_done`` advances monotonically to it.
"""
with_file = [t for t in ticks if t["current_file"]]
assert with_file, f"no running tick carried a current_file: {ticks[:6]}"
first_with = with_file[0]
assert first_with["current_file"].startswith(f"{source}/"), first_with
assert re.fullmatch(
rf"{re.escape(source)}/.+\.(md|markdown)", first_with["current_file"]
), first_with
assert first_with["files_total"] == n_files, first_with
assert 1 <= first_with["files_done"] <= n_files, first_with
dones = [t["files_done"] for t in with_file]
assert dones == sorted(dones), f"files_done not monotonic: {dones}"
# The last file tick is (n-1, n): the hook fires before the final
# file's index, so the final count itself can land in the terminal
# body instead of a recorded tick (100 ms cadence vs ≈160 ms file).
assert max(dones) >= n_files - 1, f"files_done never advanced: {dones}"
pre = [t for t in ticks if t["current_file"] is None]
assert pre, f"no file-less running tick (the unpack phase): {ticks[:6]}"
assert ticks.index(pre[0]) < ticks.index(first_with), (
"a file tick preceded the file-less unpack ticks"
)
# ---------------------------------------------------------------------------
# 1. Toast on 202 → navigate away → sync button animates with the
# upload's current file → settle + catalog refresh (A2 + A3)
# ---------------------------------------------------------------------------
def test_upload_toast_then_navigate_away(
page: Page, app_url: str, db_ready: None, upload_archive: Path
) -> None:
"""On ``/git-sources.html``: pick the multi-file archive, submit →
the "Successfully uploaded — <source>" toast appears WHILE the scan
is still running; immediately navigate to ``/sources.html`` → the
sync button is present, animating (icon ``is-spinning``,
``aria-busy``) with the ``Importing`` label; wait for the settle →
button idle ("Sync sources"), no error UI, and the catalog table
shows the uploaded documents (the phase-63 listing, untouched)."""
page.set_default_timeout(30_000)
_admin_git_sources_page(page, app_url)
# (A previous test's terminal run may re-render its result line at
# boot — the task-05 re-attach contract; the submit below clears
# it, and that is the "clean start" asserted after the click.)
page.set_input_files("#archive-upload-file", str(upload_archive))
page.click("#archive-upload-btn")
# The 202 moment (A2): a single .toast node, visible, role=status,
# naming the safe source name…
toast = page.locator(".toast")
expect(toast).to_have_count(1, timeout=SETTLE_TIMEOUT_MS)
expect(toast).to_have_class(re.compile(r"\bis-visible\b"))
assert toast.get_attribute("role") == "status"
expect(toast).to_have_text(f"Successfully uploaded — {UPLOAD_NAME}")
# …and the scan is still running — the result line is not up yet
# (the toast precedes the scan's completion, A2).
expect(page.locator("#archive-upload-result")).to_be_hidden()
started_at = _wait_running_started_at(page, app_url, "/api/git-sources/upload/status")
assert started_at is not None
# Navigate away immediately (A1: the scan no longer dies with the
# page).
page.goto(app_url + SOURCES_URL)
# The RAG-page sync button re-attaches to the in-flight upload scan
# (A3): present, animating (spinner + aria-busy), disabled, with
# the live "Importing <file>" label — no error UI on this page
# (the upload's failure UI lives on the Sources page, A3).
btn = page.locator("#sync-btn")
expect(btn).to_be_visible(timeout=30_000)
expect(btn).to_be_disabled()
expect(btn).to_have_attribute("aria-busy", "true")
expect(btn.locator(".sync-icon")).to_have_class(re.compile(r"\bis-spinning\b"))
expect(page.locator("#sync-label")).to_have_text(
re.compile(rf"Importing {re.escape(UPLOAD_NAME)}/"), timeout=SETTLE_TIMEOUT_MS
)
expect(page.locator("#sync-error-banner")).to_be_hidden()
# Settle: the button returns to idle, the sync-result line never
# rendered the upload's counts (A3), and the catalog refreshes with
# the uploaded documents — the phase-63 listing, untouched.
expect(page.locator("#sync-label")).to_have_text("Sync sources", timeout=SETTLE_TIMEOUT_MS)
expect(btn).to_be_enabled()
expect(btn).not_to_have_attribute("aria-busy")
expect(btn.locator(".sync-icon")).not_to_have_class(re.compile(r"\bis-spinning\b"))
expect(page.locator("#sync-result")).to_have_text("")
expect(page.locator("#docs-tbody tr")).to_have_count(N_FILES, timeout=30_000)
expect(page.locator("#docs-tbody tr", has_text="docs/00.md")).to_have_count(1)
expect(page.locator("#docs-tbody tr", has_text="docs/24.md")).to_have_count(1)
expect(page.locator("#docs-tbody tr", has_text=UPLOAD_NAME)).to_have_count(N_FILES)
# ---------------------------------------------------------------------------
# 2. Upload progress: live current file at BOTH layers; the toast fired
# earlier than the result (A4)
# ---------------------------------------------------------------------------
def test_upload_progress_shows_current_file(
page: Page, app_url: str, db_ready: None, upload_archive: Path
) -> None:
"""During the scan: the status endpoint reports a non-null
``current_file`` (``source/relative/path`` shape) at some running
tick; the upload button label shows "Processing…" with a file path
(UI layer) BEFORE the result line lands; the toast fired earlier in
the run (not after the result). The result line + the settle come
from the status ``success``."""
page.set_default_timeout(30_000)
_admin_git_sources_page(page, app_url)
recorder = _TickRecorder(app_url, "/api/git-sources/upload/status")
recorder.start()
page.set_input_files("#archive-upload-file", str(upload_archive))
page.click("#archive-upload-btn")
# A new attempt starts clean (the submit handler hides the result
# line — any previous run's re-rendered line is gone by now).
expect(page.locator("#archive-upload-result")).to_be_hidden()
# The toast fires at the 202 — earlier in the run, NOT after the
# result (the result line is still down when the toast is up).
toast = page.locator(".toast")
expect(toast).to_have_count(1, timeout=SETTLE_TIMEOUT_MS)
expect(toast).to_have_class(re.compile(r"\bis-visible\b"))
expect(toast).to_have_text(f"Successfully uploaded — {UPLOAD_NAME}")
expect(page.locator("#archive-upload-result")).to_be_hidden()
# UI layer: the button hands over to the scan — bare "Processing…"
# at the 202 (A4: no file yet during the unpack phase)…
btn = page.locator("#archive-upload-btn")
expect(btn).to_be_disabled()
expect(btn).to_have_text("Processing…", timeout=5_000)
# …then the live file label ("Processing… <file> (n/m)") at the
# page's next 2 s poll tick, still before the result line lands.
expect(btn).to_have_text(
re.compile(rf"Processing… {re.escape(UPLOAD_NAME)}/.+\.md \(\d+/{N_FILES}\)"),
timeout=SETTLE_TIMEOUT_MS,
)
expect(page.locator("#archive-upload-result")).to_be_hidden()
# Deterministic layer: the recorder's full tick series (the same
# endpoint the UI's 2 s poll reads) — file-less unpack ticks first,
# then the per-file ticks with the full denominator.
terminal = recorder.stop()
assert terminal["state"] == "success", terminal
_assert_live_file_ticks(recorder.running_ticks, UPLOAD_NAME, N_FILES)
# Phase 64: current_file is null in terminal states (the final
# counts survive).
assert terminal["current_file"] is None
assert terminal["files_done"] == N_FILES
assert terminal["files_total"] == N_FILES
# The result line is rendered from the status success (the
# UploadOut-shaped detail, counts unchanged in shape).
assert terminal["detail"]["files"] == N_FILES
assert terminal["detail"]["added"] == N_FILES
assert terminal["detail"]["source"] == UPLOAD_NAME
# Settle: the result line lands, the button restores, the input
# cleared, and the list has exactly one row for the archive.
result = page.locator("#archive-upload-result")
expect(result).to_have_text(f"{N_FILES} added", timeout=30_000)
expect(btn).to_be_enabled()
expect(btn).to_have_text("Upload & scan")
expect(page.locator("#archive-upload-file")).to_have_value("")
expect(page.locator("#git-sources-tbody tr", has_text=UPLOAD_NAME)).to_have_count(1)
# ---------------------------------------------------------------------------
# 3. Sync live file label: both layers, then the preserved pre-64
# success settle (A4)
# ---------------------------------------------------------------------------
def test_sync_live_file_label(
page: Page, app_url: str, db_ready: None, sync_local_dir: Path
) -> None:
"""A multi-file local source (the ``test_sync_button.py`` fixture
style): on ``/sources.html`` click **Sync sources** → the label
shows "Syncing…" (bare, the pre-64 click state) then "Syncing…
<file> (n/m)" while running (endpoint layer: ``current_file``
non-null at running ticks; UI layer: the label poll), then the
success settle with the counts result line — the pre-phase-64 sync
UX is preserved, plus the file."""
page.set_default_timeout(30_000)
_seed_local_source(str(sync_local_dir))
login(page, app_url) # lands on /sources.html (the button's home)
btn = page.locator("#sync-btn")
expect(btn).to_be_visible(timeout=30_000)
expect(page.locator("#sync-label")).to_have_text("Sync sources")
recorder = _TickRecorder(app_url, "/api/sync/status")
recorder.start()
btn.click()
# The click's immediate state is the pre-64 one (A4 — the bare
# prefix until the import's first file): disabled, aria-busy,
# spinning icon, no error…
expect(btn).to_be_disabled()
expect(btn).to_have_attribute("aria-busy", "true")
expect(btn.locator(".sync-icon")).to_have_class(re.compile(r"\bis-spinning\b"))
expect(page.locator("#sync-label")).to_have_text("Syncing…")
expect(page.locator("#sync-error-banner")).to_be_hidden()
# UI layer: the label gains the live file at the page's 2 s poll
# tick ("Syncing… <source/relative/path> (n/m)").
expect(page.locator("#sync-label")).to_have_text(
re.compile(rf"Syncing… {re.escape(SYNC_SOURCE_DIR)}/.+\.md \(\d+/{N_FILES}\)"),
timeout=SETTLE_TIMEOUT_MS,
)
# Deterministic layer: the recorder's full tick series — file-less
# model-check ticks first, then the per-file ticks (full
# denominator, advancing counts).
terminal = recorder.stop()
assert terminal["state"] == "success", terminal
_assert_live_file_ticks(recorder.running_ticks, SYNC_SOURCE_DIR, N_FILES)
assert terminal["current_file"] is None
assert terminal["files_done"] == N_FILES
assert terminal["files_total"] == N_FILES
assert terminal["detail"]["added"] == N_FILES
# The pre-64 success settle, preserved: "Synced HH:MM" + the counts
# result line, button re-enabled, no error — and the catalog lists
# the imported docs.
expect(page.locator("#sync-label")).to_have_text(SYNCED_LABEL, timeout=30_000)
expect(btn).to_be_enabled()
expect(btn).not_to_have_attribute("aria-busy")
expect(page.locator("#sync-result")).to_have_text(f"{N_FILES} added")
expect(page.locator("#docs-tbody tr")).to_have_count(N_FILES, timeout=30_000)
expect(page.locator("#docs-tbody tr", has_text="notes/00.md")).to_have_count(1)
expect(page.locator("#docs-tbody tr", has_text=SYNC_SOURCE_DIR)).to_have_count(N_FILES)
# ---------------------------------------------------------------------------
# 4. Reload mid-scan → re-attach: no error, no second upload (A1 + A2)
# ---------------------------------------------------------------------------
def test_upload_reattach_after_reload(
page: Page, app_url: str, db_ready: None, upload_archive: Path
) -> None:
"""Start the upload and, DURING the scan, reload
``/git-sources.html`` → the button is in the Processing state
(disabled) with no error banner and NO second upload (the status
endpoint's single run is still the one from before the reload —
pinned on its ``started_at``); it then settles with the result line
and the list shows exactly one row for the archive (in-place
identity preserved)."""
page.set_default_timeout(30_000)
_admin_git_sources_page(page, app_url)
page.set_input_files("#archive-upload-file", str(upload_archive))
page.click("#archive-upload-btn")
# The 202 toast (the run is in flight)…
toast = page.locator(".toast")
expect(toast).to_have_count(1, timeout=SETTLE_TIMEOUT_MS)
expect(toast).to_have_class(re.compile(r"\bis-visible\b"))
# …and the run's identity: the status's started_at (a second run
# would reset it — the single-run claim is pinned on it).
started_at = _wait_running_started_at(page, app_url, "/api/git-sources/upload/status")
# Reload mid-scan — the page must re-attach, not dead-end.
page.reload()
expect(page).to_have_url(app_url + GIT_SOURCES_URL, timeout=30_000)
expect(page.locator("#sign-out-btn")).to_be_visible(timeout=15_000)
expect(page.locator("#git-sources-content")).to_be_visible()
# The boot re-attach (task 05): the button is in the Processing
# state (disabled, "Processing…") with no error banner and no
# result line yet…
btn = page.locator("#archive-upload-btn")
expect(btn).to_be_disabled(timeout=15_000)
expect(btn).to_have_text(re.compile(r"Processing…"), timeout=15_000)
expect(page.locator("#archive-upload-error")).to_be_hidden()
expect(page.locator("#archive-upload-result")).to_be_hidden()
# …and it settles with the result line from the status success…
expect(page.locator("#archive-upload-result")).to_have_text(
f"{N_FILES} added", timeout=SETTLE_TIMEOUT_MS
)
expect(btn).to_be_enabled()
expect(btn).to_have_text("Upload & scan")
# …and the list shows exactly one row for the archive.
expect(page.locator("#git-sources-tbody tr")).to_have_count(1)
row = page.locator("#git-sources-tbody tr", has_text=UPLOAD_NAME)
expect(row).to_have_count(1)
expect(row.locator("span.git-source-kind")).to_have_text("Local")
# No second upload: the terminal status is the SAME run — its
# started_at is the one from before the reload.
terminal = _status(page, app_url, "/api/git-sources/upload/status")
assert terminal["state"] == "success", terminal
assert str(terminal["started_at"]) == started_at, (
f"the run's identity changed (a second upload ran): {terminal['started_at']}"
)
assert terminal["current_file"] is None
tests/integration/test_git_sources_upload.py
+749 -183
View File
File diff suppressed because it is too large Load Diff
tests/integration/test_sync_api.py
+15 -1
View File
@@ -63,7 +63,7 @@ from __future__ import annotations
import asyncio
import logging
import time
from collections.abc import Iterator
from collections.abc import Callable, Iterator
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
@@ -235,6 +235,9 @@ class FakeImportSources:
self.sources: list[list[Path]] = []
self.llms: list[LLMClient] = []
self.prune_flags: list[bool] = []
# Phase 64 (task 02): the progress hook the runner passes (a live
# closure while wired, None if the wiring regresses).
self.progress_hooks: list[object] = []
async def __call__(
self,
@@ -244,10 +247,12 @@ class FakeImportSources:
prune: bool = False,
limit: int | None = None,
session: Session | None = None,
progress: Callable[[str, str, int, int], None] | None = None,
) -> ImportSummary:
self.sources.append(list(sources))
self.llms.append(llm)
self.prune_flags.append(prune)
self.progress_hooks.append(progress)
if self.delay:
await asyncio.sleep(self.delay)
return self.summary
@@ -324,6 +329,10 @@ def test_admin_sync_success_reports_full_detail(
"finished_at": None,
"detail": {},
"error": None,
# Phase 64 (task 02): the per-file progress keys — null/0/0 idle.
"current_file": None,
"files_done": 0,
"files_total": 0,
}
r = sync_client.post("/api/sync")
@@ -353,6 +362,10 @@ def test_admin_sync_success_reports_full_detail(
assert fake_import.prune_flags == [True]
assert len(fake_import.llms) == 1
assert isinstance(fake_import.llms[0], LLMClient)
# Phase 64 (task 02): the runner wires the per-file progress hook
# (the live closure the status endpoint reads while the import runs).
assert len(fake_import.progress_hooks) == 1
assert callable(fake_import.progress_hooks[0])
# Overview: refreshed (added + updated > 0) with the same client.
assert fake_overview.llms == [fake_import.llms[0]]
# Phase 41: the probe ran first and got the very client the import
@@ -733,6 +746,7 @@ def test_import_error_is_reported_with_credentials_masked(
prune: bool = False,
limit: int | None = None,
session: Session | None = None,
progress: Callable[[str, str, int, int], None] | None = None,
) -> ImportSummary:
raise EmbeddingError(
"embeddings request to https://user:secret@aipi.reeseapps.com/v1 "
tests/unit/test_frontend_sync_upload.py
+663
View File
@@ -0,0 +1,663 @@
"""Unit: the RAG-page sync button's phase-64 (task 04) live-file contract.
The browser behavior is E2E-covered (tests/e2e/test_sync_upload_progress.py,
task 06); here we pin the source-level wiring in sources.js, styles.css,
and sources.html — the fmtSyncLabel contract (both kinds, file
present/absent, counts only when total > 0), enterSyncRunningState
writing the full untruncated path to the button title + #sync-result,
the two-job tick decision tree (sync running > upload running > sync
success > sync failed > upload success > upload failed > idle; the A3
settle never renders upload counts into #sync-result), and the
load-time re-attach of an in-flight upload scan — so a silent
regression is caught without a browser.
Phase 64 task 05 adds the Sources-page (git-sources.js) upload
contract: the "Successfully uploaded — <file>" toast on the 202
(page-local, phase-55 pattern, success-only), the live
"Processing… <file> (n/m)" label + full-path title driven by the 2 s
GET /api/git-sources/upload/status poll, the 409 re-attach (no error
banner), the poll's terminal decision tree (success → result line +
announce + reload, NO second toast; failed → sanitized error banner
+ reload; idle → defensive restore), the finally's never-restore-
while-polling guard (PLAN §7.4), and the boot re-attach branches.
"""
from __future__ import annotations
import re
from pathlib import Path
FRONTEND = Path(__file__).resolve().parents[2] / "frontend"
SOURCES_JS = FRONTEND / "assets" / "sources.js"
STYLES_CSS = FRONTEND / "assets" / "styles.css"
SOURCES_HTML = FRONTEND / "sources.html"
GIT_SOURCES_JS = FRONTEND / "assets" / "git-sources.js"
GIT_SOURCES_HTML = FRONTEND / "git-sources.html"
def _js() -> str:
return SOURCES_JS.read_text(encoding="utf-8")
def _css() -> str:
return STYLES_CSS.read_text(encoding="utf-8")
def _html() -> str:
return SOURCES_HTML.read_text(encoding="utf-8")
def _gjs() -> str:
return GIT_SOURCES_JS.read_text(encoding="utf-8")
def _ghtml() -> str:
return GIT_SOURCES_HTML.read_text(encoding="utf-8")
def _gfn(js: str, name: str) -> str:
"""The source of the first `function <name>` in git-sources.js
(up to the first line-leading closing brace — the house pin
pattern)."""
fn = js.find(f"function {name}")
assert fn != -1, f"{name} must be defined in git-sources.js"
return js[fn : js.find("\n}", fn)]
def _utick(js: str) -> str:
"""The upload poll tick inside startUploadPolling — from `const
tick = async () => {` to the next top-level function
(initUploadStatus), so the whole decision tree is in the slice."""
fn = js.find("function startUploadPolling")
assert fn != -1, "startUploadPolling must be defined in git-sources.js"
tick = js.find("const tick = async () => {", fn)
assert tick != -1, "the tick must live inside startUploadPolling"
end = js.find("\nasync function initUploadStatus", tick)
assert end != -1, "initUploadStatus must follow startUploadPolling"
return js[tick:end]
def _usubmit(js: str) -> str:
"""The upload form's submit handler — from the addEventListener to
the next top-level function (focusNewRow), so every branch (202 /
409 / other non-2xx / network / finally) is in the slice."""
start = js.find('uploadFormEl.addEventListener("submit"')
assert start != -1, "the upload form must wire a submit handler"
end = js.find("function focusNewRow", start)
assert end != -1, "focusNewRow must follow the upload handler"
return js[start:end]
def _fn(js: str, name: str) -> str:
"""The source of the first `function <name>` in js (up to the first
line-leading closing brace — the house pin pattern from
tests/unit/test_sync_button.py)."""
fn = js.find(f"function {name}")
assert fn != -1, f"{name} must be defined in sources.js"
return js[fn : js.find("\n}", fn)]
def _tick(js: str) -> str:
"""The poll tick inside startSyncPolling — from `const tick = async
() => {` to the next top-level function (startSync), so the whole
two-job decision tree is in the slice."""
fn = js.find("function startSyncPolling")
assert fn != -1, "startSyncPolling must be defined in sources.js"
tick = js.find("const tick = async () => {", fn)
assert tick != -1, "the tick must live inside startSyncPolling"
end = js.find("\nasync function startSync", tick)
assert end != -1, "startSync must follow startSyncPolling"
return js[tick:end]
# ---------- fmtSyncLabel: the live-file label contract ----------
def test_fmt_sync_label_signature_and_prefixes() -> None:
"""fmtSyncLabel(kind, currentFile, done, total): `kind` picks the
prefix — "upload" → "Importing" (the background scan's word, A3),
anything else → "Syncing…". The file is appended only when present
(the bare prefix shows during clone/pull or unpack, before any file
is indexed — A4); the counts are appended only when total > 0 (the
import has started); file before counts."""
body = _fn(_js(), "fmtSyncLabel")
assert "function fmtSyncLabel(kind, currentFile, done, total)" in body
assert 'kind === "upload"' in body
assert '"Importing"' in body and '"Syncing…"' in body
# the upload prefix must come from the kind check (ternary, in order)
i_kind = body.find('kind === "upload"')
i_importing = body.find('"Importing"')
i_syncing = body.find('"Syncing…"')
assert -1 < i_kind < i_importing < i_syncing
# file appended only when present
assert "currentFile ?" in body
assert "`${prefix} ${currentFile}`" in body
# counts only when the import has started (total > 0)
assert "total > 0" in body
assert "` (${done}/${total})`" in body
i_file = body.find("currentFile ?")
i_counts = body.find("total > 0")
assert -1 < i_file < i_counts, "the file lands before the counts"
# ---------- enterSyncRunningState: full path to title + announcer ----------
def test_enter_running_state_writes_full_path_to_title_and_announcer() -> None:
"""The running-state entry keeps the §7.4 never-stale mechanics
(disabled, aria-busy, spinning icon, the stale .is-error removed)
and — phase 64 — writes the FULL untruncated current file to the
button title (removed when null: no file yet) and the full
untruncated fmtSyncLabel text to BOTH the label span and
#sync-result (the aria-live announcer reads the full live path;
CSS only ellipsizes the button's span)."""
body = _fn(_js(), "enterSyncRunningState")
assert "syncBtn.disabled = true" in body
assert 'syncBtn.setAttribute("aria-busy", "true")' in body
assert "syncIcon.classList.add(\"is-spinning\")" in body
assert "syncBtn.classList.remove(\"is-error\")" in body
assert "syncBtn.title = currentFile" in body, "the full path on hover"
assert 'syncBtn.removeAttribute("title")' in body, "removed when no file yet"
i_set = body.find("if (currentFile) syncBtn.title = currentFile")
i_remove = body.find('syncBtn.removeAttribute("title")')
assert -1 < i_set < i_remove, "title is set (not removed) only when a file exists"
assert "fmtSyncLabel(kind, currentFile, done, total)" in body
i_label = body.find("fmtSyncLabel(kind, currentFile, done, total)")
i_span = body.find("syncLabel.textContent = label")
i_result = body.find("syncResult.textContent = label")
assert -1 < i_label < i_span < i_result, (
"one label: built once, written to the span AND the announcer"
)
# ---------- the two-job tick (startSyncPolling) ----------
def test_tick_fetches_both_jobs_with_403_and_blip_rules() -> None:
"""Each tick fetches BOTH status endpoints (the sync and the
background upload scan). The 403 backstop (button hidden) applies
to the SYNC fetch only — a 403 on the upload fetch is simply "no
upload" (never a hide); a network blip on either fetch retries next
tick (the tick reschedules, it never dies on a failed fetch)."""
tick = _tick(_js())
assert 'fetch("/api/sync/status")' in tick
assert 'fetch("/api/git-sources/upload/status")' in tick
assert "r.status === 403" in tick, "the sync fetch keeps the whoami backstop"
assert "ur.status === 403" not in tick, "upload 403 = no upload (A3)"
assert "ur.ok" in tick, "the upload fetch is read only when ok"
assert tick.count("catch {") >= 2, "both fetches are blip-tolerant"
assert tick.count("setTimeout(tick, SYNC_POLL_MS)") >= 3, (
"the blip and both running branches all reschedule"
)
# the sync-status blip must not settle the button: it reschedules.
i_nostatus = tick.find("if (!syncStatus)")
assert i_nostatus != -1
branch = tick[i_nostatus : i_nostatus + 200]
assert "setTimeout(tick, SYNC_POLL_MS)" in branch
assert "applySyncIdle" not in branch, "a blip is not an idle"
def test_tick_decision_tree_order_and_branches() -> None:
"""The phase-64 decision tree, in order (A3/A4): 1. sync running →
2. upload running → 3. sync success → 4. sync failed → 5. upload
success → 6. upload failed → 7. both idle. The running branches
enter the running state with THEIR job's kind + live file/counts;
the sync terminals are the unchanged phase-32 appliers; the upload
terminals settle "Sync sources" + clear #sync-result + hide a stale
error + emit a synthetic idle frame — never the upload's status
object, never fmtSyncResult (no upload counts in #sync-result, A3);
only the upload SUCCESS refreshes the catalog (the KB changed); the
upload failure raises no error surface on this page (the banner is
the Sources page's)."""
tick = _tick(_js())
b_sync_run = tick.find('syncStatus.state === "running"')
b_up_run = tick.find('uploadStatus && uploadStatus.state === "running"')
b_sync_ok = tick.find('syncStatus.state === "success"')
b_sync_fail = tick.find('syncStatus.state === "failed"')
b_up_ok = tick.find('uploadStatus && uploadStatus.state === "success"')
b_up_fail = tick.find('uploadStatus && uploadStatus.state === "failed"')
b_idle = tick.find("applySyncIdle(syncStatus)")
assert (
-1 < b_sync_run < b_up_run < b_sync_ok < b_sync_fail < b_up_ok < b_up_fail < b_idle
), "the tree must fire in the documented order"
# 1. sync running: the sync-kind live label, reschedule.
branch1 = tick[b_sync_run:b_up_run]
assert (
'"sync", syncStatus.current_file, syncStatus.files_done, syncStatus.files_total'
in branch1
)
assert "setTimeout(tick, SYNC_POLL_MS)" in branch1
# 2. upload running: the upload-kind live label (A3), reschedule.
branch2 = tick[b_up_run:b_sync_ok]
assert (
'"upload", uploadStatus.current_file, uploadStatus.files_done, uploadStatus.files_total'
in branch2
)
assert "setTimeout(tick, SYNC_POLL_MS)" in branch2
# 3 + 4. the sync terminals are the unchanged phase-32 appliers.
assert "applySyncSuccess(syncStatus)" in tick[b_sync_ok:b_sync_fail]
assert "applySyncFailure(syncStatus)" in tick[b_sync_fail:b_up_ok]
# 5. upload success: settle + clear + hide stale error + emit idle
# + the catalog refresh (the new documents must appear).
up_ok = tick[b_up_ok:b_up_fail]
for line in (
'settleSyncButton("Sync sources")',
'syncResult.textContent = ""',
"hideSyncError()",
'emitSyncStatus({ state: "idle" })',
"loadDocs()",
):
assert line in up_ok, f"the upload-success settle must carry {line!r}"
assert "fmtSyncResult" not in up_ok, "no upload counts in #sync-result (A3)"
assert "applySyncSuccess" not in up_ok, "the sync applier is never an upload branch"
# 6. upload failed: settle only — no catalog refresh (the KB did
# not change) and no error surface on this page (A3).
up_fail = tick[b_up_fail:b_idle]
for line in (
'settleSyncButton("Sync sources")',
'syncResult.textContent = ""',
"hideSyncError()",
'emitSyncStatus({ state: "idle" })',
):
assert line in up_fail, f"the upload-failed settle must carry {line!r}"
assert "loadDocs()" not in up_fail, "no KB change on a failed upload"
assert "showSyncError" not in up_fail, "no banner on this page (A3)"
assert "applySyncFailure" not in up_fail and "showSyncModal" not in up_fail
# 7. both idle: the unchanged idle settle.
assert "stopSyncPolling()" in tick[b_idle - 60 : b_idle + 40]
# ---------- the click branch + the load-time re-attach ----------
def test_click_branch_enters_sync_running_without_a_file() -> None:
"""The 202/409 branch of startSync enters the running state with
the sync kind and no file yet (the run is just starting — model
check / clone-pull: bare "Syncing…", A4); the emitSyncStatus({
state: "running" }) dedup via lastSyncState stays, and the poll
starts."""
js = _js()
idx = js.find("r.status === 202 || r.status === 409")
assert idx != -1
branch = js[idx : idx + 500]
assert "enterSyncRunningState(\"sync\", null, 0, 0)" in branch
assert 'emitSyncStatus({ state: "running" })' in branch
assert "lastSyncState !== \"running\"" in branch, "the dedup stays"
assert "startSyncPolling()" in branch
def test_reattach_adopts_a_running_upload_only() -> None:
"""initSyncButton: the sync branches are unchanged (running
re-enters with the live file; the terminals render the last
result). With the sync IDLE it fetches the upload status: a RUNNING
upload scan re-attaches (running state, upload kind + live file,
the synthetic running frame, the poll starts); a terminal upload is
a no-op — the fall-through is the plain idle settle (the boot-time
loadDocs() already shows the current catalog)."""
body = _fn(_js(), "initSyncButton")
assert "await fetchIsAdmin()" in body, "admin-only (no extra fetch)"
assert 'fetch("/api/git-sources/upload/status")' in body
# the sync running re-attach now carries the live file too
assert '"sync", status.current_file, status.files_done, status.files_total' in body
# the ONLY upload branch is the running one (A3)
assert "upload && upload.state === \"running\"" in body
assert 'upload.state === "success"' not in body, "a terminal upload never re-attaches"
assert 'upload.state === "failed"' not in body, "a terminal upload never re-attaches"
i_check = body.find("upload && upload.state === \"running\"")
i_fall = body.find("applySyncIdle(status)", i_check)
assert -1 < i_check < i_fall
branch = body[i_check:i_fall]
assert '"upload", upload.current_file, upload.files_done, upload.files_total' in branch
assert 'emitSyncStatus({ state: "running" })' in branch
assert "startSyncPolling()" in branch
# the idle settle is the fall-through (the last statement)
assert body.rstrip().endswith("applySyncIdle(status);")
# ---------- the section header + the page comment ----------
def test_section_header_documents_the_two_job_contract() -> None:
"""The sync-button section marker comment documents the phase-64
contract: the live file label, the two-job decision tree (both
status endpoints), and the A3 settle behavior (catalog refresh; the
upload counts never render here)."""
js = _js()
marker = js.find("Sync sources button (Sources page only)")
assert marker != -1, "the sync section marker comment must stay"
header = js[marker : js.find("const syncBtn")]
assert "Phase 64 (task 04)" in header
assert "/api/git-sources/upload/status" in header, "the second job's endpoint"
assert "loadDocs" in header, "the A3 catalog refresh"
assert "A3" in header and "A4" in header
def test_sources_html_comment_documents_the_live_announcer() -> None:
"""The #sync-result comment in sources.html documents the phase-64
dual role: the live file label (both kinds) while either job runs,
untruncated for the aria-live announcer, and empty after an upload
settles (A3 — the upload's counts live on the Sources page)."""
html = _html()
idx = html.find('id="sync-result"')
assert idx != -1
comment = html[max(0, idx - 900):idx]
assert "Importing <file>" in comment, "the upload kind is documented"
assert "Syncing…" in comment, "the sync kind is documented"
assert "A3" in comment, "the settle contract is documented"
# ---------- styles.css: the ellipsized live label ----------
def test_sync_label_css_ellipsis_truncation() -> None:
""".sync-label: the live-file label ellipsizes a long
source/relative/path inside the pill (A4) — inline-block with the
min(16rem, 40vw) cap, overflow hidden, text-overflow ellipsis, no
wrap, baseline-aligned; the mobile squeeze's display:none override
(icon-only button) stays."""
css = _css()
block = re.search(r"\.sync-label\s*\{([^}]*)\}", css)
assert block, "styles.css must style .sync-label"
body = block.group(1)
for prop in (
"display: inline-block",
"max-width: min(16rem, 40vw)",
"overflow: hidden",
"text-overflow: ellipsis",
"white-space: nowrap",
"vertical-align: bottom",
):
assert prop in body, f".sync-label must carry {prop!r}"
mobile = re.search(r"@media \(max-width: 640px\) \{([\s\S]*?)\n\}", css)
assert mobile, "the ≤640px media query must stay"
assert ".sync-label { display: none; }" in mobile.group(1), (
"the mobile icon-only override must survive the ellipsis rule"
)
# =====================================================================
# Phase 64 task 05 — the Sources-page upload (git-sources.js)
# =====================================================================
# ---------- the "Successfully uploaded" toast (A2) ----------
def test_upload_toast_single_node_status_and_auto_dismiss() -> None:
"""showUploadToast (A2 owner-locked): the phase-55 share-toast
pattern made page-local — a SINGLE node, lazy-created on the first
202 and reused (toasts never stack): a plain ``<div class="toast">``
appended to ``document.body``; ``role="status" aria-live="polite"``
(on THIS page the toast is the a11y announcer for the 202); the
text lands via ``textContent`` (XSS-safe — never innerHTML). A new
toast replaces a pending one: clear the prior dismiss timer, remove
the visible class, force a reflow (``offsetWidth`` — restarts the
CSS transition), re-add the class. Auto-dismiss: the 5000ms
(UPLOAD_TOAST_MS) timer is armed AFTER the visible class is added
and removes the class on fire. The existing .toast CSS is reused
as-is (no new styles)."""
js = _gjs()
body = _gfn(js, "showUploadToast")
assert "if (!uploadToastEl)" in body, "the node is created once, on first use"
assert 'document.createElement("div")' in body
assert 'uploadToastEl.className = "toast"' in body, "the phase-55 .toast CSS, as-is"
assert 'uploadToastEl.setAttribute("role", "status")' in body
assert 'uploadToastEl.setAttribute("aria-live", "polite")' in body
assert "document.body.appendChild(uploadToastEl)" in body
assert "uploadToastEl.textContent = message" in body
assert "innerHTML" not in body, "XSS contract: textContent only"
# Single instance: module-scope node + timer.
assert re.search(r"^let uploadToastEl = null", js, re.M), "the node is module scope"
assert re.search(r"^let uploadToastTimer = 0", js, re.M), "the timer is module scope"
assert "const UPLOAD_TOAST_MS = 5000" in js, "the ~5 s auto-dismiss (A2)"
# Re-trigger order: clear dismiss → remove class → force reflow →
# re-add the visible class.
clear_i = body.find("clearTimeout(uploadToastTimer)")
remove_i = body.find('uploadToastEl.classList.remove("is-visible")')
reflow_i = body.find("void uploadToastEl.offsetWidth")
add_i = body.find('uploadToastEl.classList.add("is-visible")')
assert -1 < clear_i < remove_i < reflow_i < add_i, (
"dismiss cleared → class removed → reflow forced → visible re-added"
)
# The auto-dismiss timer is armed AFTER the visible class is set.
timer_i = body.find("setTimeout")
assert -1 < add_i < timer_i and "UPLOAD_TOAST_MS" in body[timer_i:]
assert 'uploadToastEl.classList.remove("is-visible")' in body[timer_i:], (
"the pending dismiss removes the visible state"
)
def test_toast_fires_on_202_with_the_safe_name() -> None:
"""The 202 branch of the upload submit (A1/A2): the 202 body
(UploadAccepted) is parsed for the safe source name — a body parse
failure degrades to the picked file's name — and the toast fires
with `Successfully uploaded — <name>` BEFORE the scan finishes:
the file input clears, the processing state enters, and the poll
starts. The toast is the SINGLE success surface: exactly one call
site in the whole page (definition + one call — never a failure
branch, never the poll)."""
js = _gjs()
sub = _usubmit(js)
i202 = sub.find("r.status === 202")
i409 = sub.find("r.status === 409")
assert -1 < i202 < i409, "the 202 branch precedes the 409 re-attach"
branch = sub[i202:i409]
assert "let name = file.name;" in branch, "the degrade-to-picked-name fallback"
assert "await r.json()" in branch, "the UploadAccepted body is parsed"
assert "data.name" in branch, "the safe source name comes from the 202 body"
assert "showUploadToast(`Successfully uploaded — ${name}`)" in branch
assert 'uploadFileInput.value = ""' in branch, "the file input clears at 202"
assert "enterUploadProcessingState()" in branch
assert "startUploadPolling()" in branch
# Success-only: exactly the definition + the single 202 call.
assert js.count("showUploadToast(") == 2, (
"the definition + exactly ONE call site (the 202 branch)"
)
# ---------- the processing state + the live label ----------
def test_processing_state_and_live_label_builder() -> None:
"""The button's processing entry (202 / 409): disabled,
"Processing…", title cleared (the poll owns it from here). The
tick's running branch builds the live label — the base prefix,
the file appended ONLY when present, the counts appended ONLY
when total > 0 (A4 — bare "Processing…" during the unpack phase,
before any file is indexed) — and rides the FULL untruncated path
on the button title (empty until a file exists), then
reschedules at the 2 s house cadence."""
js = _gjs()
state = _gfn(js, "enterUploadProcessingState")
assert "uploadBtn.disabled = true" in state
assert 'uploadBtn.textContent = "Processing…"' in state
assert 'uploadBtn.title = "";' in state, "a live file lands on the title at the first tick"
assert "const UPLOAD_POLL_MS = 2000" in js, "the SYNC_POLL_MS house value"
tick = _utick(js)
i_run = tick.find('status.state === "running"')
i_stop = tick.find("stopUploadPolling();")
assert -1 < i_run < i_stop, "the running branch precedes the terminal stop"
run = tick[i_run:i_stop]
assert '"Processing…"' in run, "the base prefix"
assert "(status.current_file ? ` ${status.current_file}` : \"\")" in run, (
"the file is appended only when present"
)
counts_expr = '(status.files_total > 0 ? ` (${status.files_done}/${status.files_total})` : "")'
assert counts_expr in run, "the counts appear only when total > 0"
assert 'uploadBtn.title = status.current_file || "";' in run, (
"the full path on hover (empty until a file exists)"
)
assert "uploadPollTimer = setTimeout(tick, UPLOAD_POLL_MS)" in run, "reschedule"
def test_start_upload_polling_double_start_guard() -> None:
"""startUploadPolling: single timer, one loop at a time — the
first statement bails when a poll is already active (the guard a
double 202/409 cannot bypass)."""
js = _gjs()
i = js.find("function startUploadPolling")
assert i != -1
head = js[i : i + 120]
assert "if (uploadPollTimer !== null) return;" in head
assert "let uploadPollTimer = null" in js, "null = no poll active"
# ---------- the 409 re-attach + the kept error branches ----------
def test_409_reattaches_without_an_error_banner() -> None:
"""409 (an upload is already in progress): NO error banner — the
phase-49 "server detail inline" branch does not apply to 409
anymore. It re-attaches to the in-flight run: the processing state
+ the poll (never stale). The OTHER non-2xx (422 format/name, 413
cap, 5xx) keep the phase-49 apiDetail banner + the kept file
selection; the network-failure fixed line stays."""
js = _gjs()
sub = _usubmit(js)
i409 = sub.find("r.status === 409")
assert i409 != -1
branch = sub[i409 : sub.find("// Other non-2xx", i409)]
assert "enterUploadProcessingState()" in branch
assert "startUploadPolling()" in branch
assert "uploadError" not in branch, "409 never raises the error banner"
assert "apiDetail" not in branch, "no server-detail branch for 409 anymore"
# The other non-2xx keeps the phase-49 convention (after the 409).
i_err = sub.find('await apiDetail(r, "Could not upload the archive — try again.")')
assert i_err > i409, "the other non-2xx branch follows the 409 re-attach (and was found)"
assert "uploadError.hidden = false" in sub[i_err:], "the banner shows for the other non-2xx"
assert "Could not upload the archive — is the app reachable?" in sub, "the network line stays"
# The no-file guard + the short transfer label stay.
assert "Choose an archive file to upload." in sub
assert 'uploadBtn.textContent = "Uploading…"' in sub
# ---------- the poll's terminal decision tree ----------
def test_upload_polling_decision_tree() -> None:
"""The upload poll tick (task 05): fetches
GET /api/git-sources/upload/status; a blip (non-ok / network /
unparseable) reschedules — the tick never dies on a failed fetch.
Then: running → live label + reschedule; ONE stop for the
terminals; success → the result line (fmtUploadResult — the
existing helper reads exactly these keys) + the announce + the row
reload + the cleared file input + the restored button — NO toast
(it already fired at the 202, A2); failed → the sanitized server
error banner (A2 failure UI) + the restored button + the row
reload (a post-swap failure keeps the row — the list state may
have changed; the selection is kept for a one-click re-upload);
idle → the defensive restore (a started run never returns to
idle)."""
tick = _utick(_gjs())
assert 'fetch("/api/git-sources/upload/status")' in tick
# The blip branch reschedules.
i_blip = tick.find("if (!status)")
assert i_blip != -1
assert "uploadPollTimer = setTimeout(tick, UPLOAD_POLL_MS)" in tick[i_blip:i_blip + 150]
# One stop, placed after the running branch and before the terminals.
assert tick.count("stopUploadPolling()") == 1
i_run = tick.find('status.state === "running"')
i_stop = tick.find("stopUploadPolling();")
i_ok = tick.find('status.state === "success"')
i_fail = tick.find('status.state === "failed"')
assert -1 < i_run < i_stop < i_ok < i_fail, "running < stop < success < failed"
# success: result line + announce + reload, NO toast.
ok = tick[i_ok:i_fail]
for line in (
"fmtUploadResult(detail)",
"uploadResult.hidden = false",
"announce(`Archive uploaded: ${detail.source}.`)",
'uploadFileInput.value = ""',
"restoreUploadButton()",
"loadSources()",
):
assert line in ok, f"the success settle must carry {line!r}"
assert "showUploadToast" not in tick, "no toast in the poll — it fired at the 202 (A2)"
# failed: the sanitized error banner + reload.
fail = tick[i_fail:]
assert "status.error" in fail
assert "uploadError.hidden = false" in fail
assert "restoreUploadButton()" in fail
assert "loadSources()" in fail, "the list state may have changed"
# idle: the defensive fall-through — a final restore, no more state
# checks after the failed branch.
i_idle = tick.rfind("restoreUploadButton()")
assert i_idle > i_fail
assert "state ===" not in tick[i_idle:], "the idle settle is the fall-through"
# ---------- the finally's never-restore-while-polling guard ----------
def test_finally_restores_only_when_no_poll_active() -> None:
"""The submit finally (PLAN §7.4): the button is restored ONLY
when no poll is active (``uploadPollTimer === null``) — while
startUploadPolling owns the button (the 202 / 409 paths) it stays
disabled / "Processing…", so an unconditional finally restore
would race the poll and leave a stale-looking idle button under a
running scan."""
sub = _usubmit(_gjs())
i = sub.find("} finally {")
assert i != -1
fin = sub[i:]
assert "if (uploadPollTimer === null)" in fin, "the guard: no poll → the button is ours"
assert "restoreUploadButton()" in fin
assert "uploadBtn.disabled = false" not in fin, "no unconditional restore in the finally"
# ---------- the boot re-attach ----------
def test_boot_reattach_branches() -> None:
"""initUploadStatus (the admin branch of the boot): the upload
status is fetched ONCE. running → the processing state + the poll
(a reload mid-scan re-attaches — no second upload, no error, no
toast); success → the last result line ONLY (no announce, no
toast — A2); failed → the error banner; idle → nothing (no
branch). The boot IIFE awaits it right after loadSources()."""
js = _gjs()
body = _gfn(js, "initUploadStatus")
assert body.count('fetch("/api/git-sources/upload/status")') == 1, "fetched ONCE at boot"
i_run = body.find('status.state === "running"')
i_ok = body.find('status.state === "success"')
i_fail = body.find('status.state === "failed"')
assert -1 < i_run < i_ok < i_fail
run = body[i_run:i_ok]
assert "enterUploadProcessingState()" in run
assert "startUploadPolling()" in run
assert "uploadError" not in run and "showUploadToast" not in run
ok = body[i_ok:i_fail]
assert "fmtUploadResult(status.detail)" in ok, "the last result line"
assert "uploadResult.hidden = false" in ok
assert "announce(" not in ok, "no announce at boot (A2)"
assert "showUploadToast" not in ok, "no toast at boot (A2)"
fail = body[i_fail:]
assert "status.error" in fail
assert "uploadError.hidden = false" in fail
assert 'status.state === "idle"' not in body, "idle does nothing — no branch"
# The boot IIFE: after the list loads, the re-attach runs (admin
# branch only — the anonymous path returns before it).
i_boot = js.rfind("await loadSources();")
tail = js[i_boot:i_boot + 400]
assert "await initUploadStatus();" in tail
assert "})();" in tail
# ---------- the page comment ----------
def test_git_sources_html_comment_documents_the_202_contract() -> None:
"""The #archive-upload-form comment in git-sources.html documents
the phase-64 202 contract (the phase-49 synchronous paragraph
marked superseded): the 202 = "safely on disk" + the JS-created
toast (no markup), the live "Processing…" label via the status
poll, and the 409 re-attach without an error banner."""
html = _ghtml()
idx = html.find('id="archive-upload-form"')
assert idx != -1
comment = html[max(0, idx - 1600):idx]
assert "Phase 64" in comment
assert "superseded" in comment, "the phase-49 synchronous paragraph is marked superseded"
assert "Successfully uploaded" in comment, "the toast is documented"
assert "Processing…" in comment, "the live label is documented"
assert "GET /api/git-sources/upload/status" in comment, "the polling endpoint"
assert "409 re-attaches" in comment, "the re-attach without an error banner"
tests/unit/test_importer.py
+138
View File
@@ -18,6 +18,7 @@ from pathlib import Path
import pytest
from sqlalchemy import func, select
import app.rag.importer as importer
from app.config import Settings
from app.models import Chunk, Document
from app.rag.importer import (
@@ -708,3 +709,140 @@ def test_prune_removes_files_now_excluded_by_format_filter(db, tmp_path: Path) -
) is not None
finally:
_cleanup_source(db, root.name)
# ---------- phase 64 (task 01): optional per-file progress hook ----------
def test_progress_hook_reports_every_file_in_order_across_roots(
db, tmp_path: Path
) -> None:
"""Multi-root, multi-file: the hook receives the exact
``(source, rel, done, total)`` sequence — roots in *sources* order,
``rel`` the same POSIX path the doc rows use, ``done`` the 1-based
index across **all** sources, ``total`` the combined count."""
root_a = tmp_path / "Alpha"
root_b = tmp_path / "Beta"
root_a.mkdir()
(root_b / "sub").mkdir(parents=True)
(root_a / "a1.md").write_text("# A1\n\na one\n")
(root_a / "a2.md").write_text("# A2\n\na two\n")
(root_a / "a1.md").write_text("# A1\n\na one\n")
(root_b / "sub" / "b1.md").write_text("# B1\n\nb one\n")
events: list[tuple[str, str, int, int]] = []
def progress(source: str, rel: str, done: int, total: int) -> None:
events.append((source, rel, done, total))
try:
summary = asyncio.run(
import_sources([root_a, root_b], FakeEmbedder(), session=db, progress=progress)
)
assert summary.files == 3 and summary.added == 3
assert events == [
("Alpha", "a1.md", 1, 3),
("Alpha", "a2.md", 2, 3),
("Beta", "sub/b1.md", 3, 3), # POSIX rel, sorted within the root
]
finally:
_cleanup_source(db, "Alpha")
_cleanup_source(db, "Beta")
def test_no_progress_means_no_prewalk(
db, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""``progress=None`` callers pay no extra pass: the real walker is hit
exactly once per source root (one pass — as before phase 64), proven
with a counting sentinel; with the hook it is hit twice (pre-walk for
``total`` + the processing pass)."""
root = tmp_path / "nowalk"
root.mkdir()
(root / "a.md").write_text("# A\n\na\n")
real_walker = importer.iter_importable_files
walk_calls = 0
def counting(
r: Path, extensions: frozenset[str], excluded: frozenset[str] = EXCLUDED_DIRS
) -> list[Path]:
nonlocal walk_calls
walk_calls += 1
return real_walker(r, extensions, excluded)
monkeypatch.setattr(importer, "iter_importable_files", counting)
try:
summary = asyncio.run(import_sources([root], FakeEmbedder(), session=db))
assert summary.files == 1
assert walk_calls == 1 # exactly one pass — the pre-change behaviour
walk_calls = 0
events: list[tuple[str, str, int, int]] = []
summary2 = asyncio.run(
import_sources(
[root], FakeEmbedder(), session=db,
progress=lambda s, r, d, t: events.append((s, r, d, t)),
)
)
assert summary2.files == 1
assert walk_calls == 2 # pre-walk (total) + processing pass
assert events == [(root.name, "a.md", 1, 1)]
finally:
_cleanup_source(db, root.name)
def test_progress_hook_counts_unchanged_and_error_files(db, tmp_path: Path) -> None:
"""The hook fires *before* ``_index_file``: a file whose embedding
fails (and one that re-imports as unchanged) is still reported as the
current file — the sequence covers every importable file."""
root = tmp_path / "progress-mixed"
root.mkdir()
(root / "bad.md").write_text("# Bad\n\npoison content the endpoint refuses\n")
(root / "good.md").write_text("# Good\n\nperfectly fine content\n")
expected = [(root.name, "bad.md", 1, 2), (root.name, "good.md", 2, 2)]
try:
events: list[tuple[str, str, int, int]] = []
first = asyncio.run(
import_sources(
[root], _PoisonEmbedder(), session=db,
progress=lambda s, r, d, t: events.append((s, r, d, t)),
)
)
# bad.md was already reported (done=1) when its embed raised —
# no file silently disappears from the sequence.
assert events == expected
assert first.errors == 1 and first.added == 1
# Re-run: good.md is now unchanged, bad.md is retried and fails
# again — both still count in the sequence.
events.clear()
second = asyncio.run(
import_sources(
[root], _PoisonEmbedder(), session=db,
progress=lambda s, r, d, t: events.append((s, r, d, t)),
)
)
assert events == expected
assert second.errors == 1 and second.unchanged == 1
finally:
_cleanup_source(db, root.name)
def test_progress_hook_with_limit_keeps_full_total(db, tmp_path: Path) -> None:
"""The debug ``limit`` path is unchanged for the hook: it fires only
for processed files (``done`` never exceeds the limit), while
``total`` stays the FULL pre-walk count — an incomplete walk must not
misreport the denominator."""
root = tmp_path / "progress-limited"
root.mkdir()
for name in ("a.md", "b.md", "c.md"):
(root / name).write_text(f"# {name}\n\nbody {name}\n")
events: list[tuple[str, str, int, int]] = []
try:
summary = asyncio.run(
import_sources(
[root], FakeEmbedder(), limit=2, session=db,
progress=lambda s, r, d, t: events.append((s, r, d, t)),
)
)
assert summary.files == 2
assert events == [(root.name, "a.md", 1, 3), (root.name, "b.md", 2, 3)]
finally:
_cleanup_source(db, root.name)
tests/unit/test_sync_button.py
+314 -9
View File
@@ -13,12 +13,31 @@ client-side hard timeout), the page's #sync-result line +
§7.4 never-stale CSS (spin + reduced-motion opt-out,
disabled state, 44px floor, contrast pair) — so a silent regression is
caught without a browser.
Phase 64 (task 02): unit coverage of the ``app/api/sync.py`` status
contract — the idle response dict is pinned in full (every pre-existing
key unchanged, plus ``current_file``/``files_done``/``files_total`` as
null/0/0); mid-run the status reports the file the (mock) import is
processing, through the runner's own hook closure (no file yet during
the clone/pull phase — A4); the terminal states clear
``current_file`` while keeping the run's final counts.
"""
from __future__ import annotations
import asyncio
import re
import threading
from collections.abc import Callable, Iterator
from pathlib import Path
import pytest
from app.api import sync as sync_api
from app.config import Settings
from app.models import GitSource
from app.rag.importer import ImportSummary
from app.rag.llm import EmbeddingError
FRONTEND = Path(__file__).resolve().parents[2] / "frontend"
ASSETS = FRONTEND / "assets"
@@ -178,12 +197,15 @@ def test_sources_js_polls_every_2000ms() -> None:
def test_sources_js_adopts_409_and_starts_on_202() -> None:
"""202 (started) and 409 (a run started elsewhere — e.g. a second
tab) both enter the running state and start polling: the UI never
starts a second run, it adopts the in-flight one."""
starts a second run, it adopts the in-flight one.
Phase 64 (task 04): the entry is the sync-kind live label with no
file yet (the run is just starting — bare "Syncing…", A4)."""
js = _text(SOURCES_JS)
assert "r.status === 202 || r.status === 409" in js
idx = js.find("r.status === 202 || r.status === 409")
branch = js[idx : idx + 400]
assert "enterSyncRunningState()" in branch
branch = js[idx : idx + 600]
assert "enterSyncRunningState(\"sync\", null, 0, 0)" in branch
assert "startSyncPolling()" in branch
@@ -202,19 +224,29 @@ def test_sources_js_hides_the_button_on_403() -> None:
def test_sources_js_running_state_is_never_stale() -> None:
"""Entering the running state disables the button, sets aria-busy,
spins the icon, and swaps the label to 'Syncing…' (the §7.4
feedback while the poll waits) — and a fresh run starts clean: the
previous failure's title / aria-label / .is-error come off NOW,
not when the run settles."""
spins the icon, and swaps the label to the live file label
(fmtSyncLabel — the §7.4 feedback while the poll waits) — and a
fresh run starts clean: the previous failure's title / aria-label /
.is-error come off NOW, not when the run settles.
Phase 64 (task 04): the button title carries the FULL untruncated
current file (removed when null — no file yet) and #sync-result
(the aria-live announcer) carries the same untruncated label."""
js = _text(SOURCES_JS)
body = _body(js, "enterSyncRunningState")
assert "syncBtn.disabled = true" in body
assert 'syncBtn.setAttribute("aria-busy", "true")' in body
assert 'syncBtn.removeAttribute("title")' in body
assert "syncBtn.title = currentFile" in body, "the full path on hover"
assert 'syncBtn.removeAttribute("title")' in body, "removed when no file yet"
title_if = body.find("if (currentFile) syncBtn.title = currentFile")
title_else = body.find("syncBtn.removeAttribute(\"title\")")
assert -1 < title_if < title_else, "title is set, not removed, only when a file exists"
assert 'syncBtn.setAttribute("aria-label", "Sync sources")' in body
assert "syncBtn.classList.remove(\"is-error\")" in body
assert "syncIcon.classList.add(\"is-spinning\")" in body
assert '"Syncing…"' in body
assert "fmtSyncLabel(kind, currentFile, done, total)" in body
assert "syncLabel.textContent = label" in body
assert "syncResult.textContent = label" in body, ("the announcer reads the full live path")
def test_sources_js_terminal_states() -> None:
@@ -595,3 +627,276 @@ def test_sync_modal_respects_reduced_motion() -> None:
)
assert reduced, "the backdrop fade must opt out under prefers-reduced-motion"
assert "transition: none" in reduced.group(1)
# ---------- phase 64 (task 02): per-file progress on the sync status ----------
#
# The GET /api/sync/status contract in app/api/sync.py (task 04 renders
# the live file label on the button — its pins live in
# tests/unit/test_frontend_sync_upload.py). The runner's seams are
# monkeypatched on ``app.api.sync`` (the house mock-import pattern from
# tests/integration/test_sync_api.py); the background task runs on a
# worker thread's own loop so the test can read the status mid-run.
# No DB, no HTTP: every seam is faked.
@pytest.fixture()
def fresh_sync_status() -> Iterator[None]:
"""The module-level status object + task are process-global: reset
them before AND after every progress test (the integration suite's
``_fresh_sync_state`` pattern)."""
sync_api._status = sync_api.SyncStatus()
sync_api._task = None
yield
sync_api._status = sync_api.SyncStatus()
sync_api._task = None
class _GatedClone:
"""A ``clone_or_pull`` that parks between start and finish on a
threading gate, so the test can read the status during the
clone/pull phase (A4: no file yet)."""
def __init__(self, started: threading.Event, release: threading.Event) -> None:
self.started = started
self.release = release
self.calls: list[tuple[str, Path]] = []
def __call__(self, url: str, dest: Path | str) -> Path:
dest = Path(dest)
self.calls.append((url, dest))
dest.mkdir(parents=True, exist_ok=True)
(dest / "notes.md").write_text("# repo\ncontent for the KB\n", encoding="utf-8")
self.started.set()
self.release.wait() # blocking is fine: the clone is a sync call
return dest
class _GatedImport:
"""The mock import: fires the runner's OWN progress hook once (the
progress-shaped call goes through the real hook closure — the
closure under test), parks on a threading gate so the test can read
the status mid-run, then returns the canned summary (or raises
``fail``)."""
def __init__(
self,
summary: ImportSummary,
started: threading.Event,
release: threading.Event,
fail: BaseException | None = None,
) -> None:
self.summary = summary
self.started = started
self.release = release
self.fail = fail
self.hook_calls: list[tuple[str, str, int, int]] = []
self.prune_flags: list[bool] = []
async def __call__(
self,
sources: list[Path],
llm: object,
*,
prune: bool = False,
limit: int | None = None,
session: object = None,
progress: Callable[[str, str, int, int], None] | None = None,
) -> ImportSummary:
self.prune_flags.append(prune)
if progress is not None:
progress("repo", "notes/deep.md", 1, 3)
self.hook_calls.append(("repo", "notes/deep.md", 1, 3))
self.started.set()
await asyncio.to_thread(self.release.wait) # park without freezing the worker loop
if self.fail is not None:
raise self.fail
return self.summary
def _patch_sync_seams(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
fake_import: _GatedImport,
fake_clone: _GatedClone,
) -> None:
"""The runner's seams, monkeypatched on ``app.api.sync`` (the house
mock-import pattern): fresh settings (no ``.env`` leak), a no-op
model probe, a sentinel LLM client, one git row, the gated clone +
import, a no-op overview, and the DB-free sources-version step
(dummy session + pinned counters)."""
monkeypatch.setattr(
sync_api,
"get_settings",
lambda: Settings(_env_file=None, sources_dir=str(tmp_path / "bor")), # pyright: ignore[reportCallIssue]
)
async def fake_probe(llm: object) -> None:
pass
monkeypatch.setattr(sync_api, "check_models", fake_probe)
monkeypatch.setattr(sync_api, "LLMClient", lambda: object())
monkeypatch.setattr(
sync_api,
"effective_sources",
lambda session: (
# kind explicit: the Python-side default applies at INSERT
# flush, not on an in-memory instance
[GitSource(url="https://git.example.com/repo.git", kind="git")],
"db",
),
)
monkeypatch.setattr(sync_api, "clone_or_pull", fake_clone)
monkeypatch.setattr(sync_api, "import_sources", fake_import)
async def fake_overview(llm: object, session: object = None) -> bool:
return False
monkeypatch.setattr(sync_api, "regenerate_overview", fake_overview)
class _DummySession:
def close(self) -> None:
pass
def commit(self) -> None:
pass
monkeypatch.setattr(sync_api, "SessionLocal", _DummySession)
monkeypatch.setattr(sync_api, "bump_sources_version", lambda session: 1)
monkeypatch.setattr(sync_api, "current_sources_version", lambda session: 1)
def _start_run() -> tuple[threading.Thread, list[BaseException]]:
"""Run the module-level runner on a worker thread's own event loop
(the house background-task pattern), capturing any unexpected
exception — the runner is supposed to die in state, never raise."""
errors: list[BaseException] = []
def _run() -> None:
try:
asyncio.run(sync_api._run_sync())
except BaseException as e: # noqa: BLE001 — surfaced to the test
errors.append(e)
thread = threading.Thread(target=_run, daemon=True)
thread.start()
return thread, errors
def test_idle_status_pins_full_shape_including_progress_keys(
fresh_sync_status: None,
) -> None:
"""Idle: the three phase-64 progress keys ride along as null/0/0,
and EVERY pre-existing key is unchanged — the full response dict is
pinned, so the current UI and every existing consumer keep working."""
assert sync_api.sync_status() == {
"state": "idle",
"started_at": None,
"finished_at": None,
"detail": {},
"error": None,
"current_file": None,
"files_done": 0,
"files_total": 0,
}
def test_mid_run_status_reports_current_file(
fresh_sync_status: None,
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
"""Mid-run: the status carries the file the (mock) import is
processing — assigned through the runner's own hook closure; during
the clone/pull phase (A4) no file is reported yet. The success
terminal clears ``current_file`` but keeps the final counts."""
clone = _GatedClone(threading.Event(), threading.Event())
fake_import = _GatedImport(
ImportSummary(files=3, added=1, updated=1, unchanged=1),
threading.Event(),
threading.Event(),
)
_patch_sync_seams(monkeypatch, tmp_path, fake_import, clone)
thread, errors = _start_run()
try:
assert clone.started.wait(5.0), "the run never reached the clone phase"
# Clone/pull phase: running — but no file yet (A4: bare "Syncing…").
s = sync_api.sync_status()
assert s["state"] == "running"
assert s["current_file"] is None
assert s["files_done"] == 0 and s["files_total"] == 0
clone.release.set()
assert fake_import.started.wait(5.0), "the run never reached the import"
# Import phase: the hook's file is live on the status.
s = sync_api.sync_status()
assert s["state"] == "running"
assert s["current_file"] == "repo/notes/deep.md"
assert s["files_done"] == 1
assert s["files_total"] == 3
fake_import.release.set()
finally:
clone.release.set()
fake_import.release.set()
thread.join(10.0)
assert not thread.is_alive()
assert errors == [], f"the runner raised: {errors!r}"
# Wiring: prune=True is preserved, and the hook fired through the
# runner's own closure (the recorded call is what the closure
# assigned to the status above).
assert fake_import.prune_flags == [True]
assert fake_import.hook_calls == [("repo", "notes/deep.md", 1, 3)]
# Success terminal: current_file null, final counts retained.
s = sync_api.sync_status()
assert s["state"] == "success"
assert s["current_file"] is None
assert s["files_done"] == 1 and s["files_total"] == 3
assert s["error"] is None
assert s["started_at"] is not None and s["finished_at"] is not None
def test_failed_terminal_clears_current_file_keeps_counts(
fresh_sync_status: None,
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
"""Terminal (failed): a run that dies inside the import clears
``current_file`` but keeps the hook's final counts — the last
position is useful context next to the (sanitized) error."""
clone = _GatedClone(threading.Event(), threading.Event())
fake_import = _GatedImport(
ImportSummary(),
threading.Event(),
threading.Event(),
fail=EmbeddingError(
"embeddings request to https://u:p@aipi.example.com/v1 "
"failed: connection refused"
),
)
_patch_sync_seams(monkeypatch, tmp_path, fake_import, clone)
thread, errors = _start_run()
try:
assert clone.started.wait(5.0), "the run never reached the clone phase"
clone.release.set()
assert fake_import.started.wait(5.0), "the progress hook never fired"
# While the (about-to-fail) import is parked: the file is live.
s = sync_api.sync_status()
assert s["state"] == "running"
assert s["current_file"] == "repo/notes/deep.md"
fake_import.release.set()
finally:
clone.release.set()
fake_import.release.set()
thread.join(10.0)
assert not thread.is_alive()
assert errors == []
s = sync_api.sync_status()
assert s["state"] == "failed"
assert s["current_file"] is None # cleared in the terminal state
assert s["files_done"] == 1 and s["files_total"] == 3 # final counts kept
assert s["detail"] == {}
error = s["error"] or ""
assert "*****@aipi.example.com" in error # credentials masked
assert "u:p" not in error
assert "connection refused" in error # the reason survives