feat(chat): stop an in-flight answer — Send becomes Stop, the partial is kept and persisted, the model stream is torn down
This commit is contained in:
@@ -211,8 +211,9 @@ def _last_query_log() -> QueryLog:
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
#: Records every value #send-label takes during the turn (a
|
||||
#: MutationObserver on the element), so the transient "Calling tool…"
|
||||
#: state is captured deterministically — no polling race.
|
||||
#: MutationObserver on the element), so the in-flight label state is
|
||||
#: captured deterministically — no polling race. Phase 48: the label is
|
||||
#: the Send↔Stop morph ("Stop" holds for the whole in-flight turn).
|
||||
LABEL_RECORDER = """
|
||||
() => {
|
||||
if (window.__labelsInstalled) return;
|
||||
@@ -232,6 +233,29 @@ LABEL_RECORDER = """
|
||||
}
|
||||
"""
|
||||
|
||||
#: Records every value #send-status takes during the turn (phase 48:
|
||||
#: the transient "… is listing documents" / "… is reading <source/path>"
|
||||
#: calling-tool states moved here from the button label), so their order
|
||||
#: is captured deterministically — no polling race.
|
||||
STATUS_RECORDER = """
|
||||
() => {
|
||||
if (window.__statusesInstalled) return;
|
||||
window.__statusesInstalled = true;
|
||||
window.__statuses = [];
|
||||
const el = document.querySelector('#send-status');
|
||||
if (!el) return;
|
||||
const rec = (v) => {
|
||||
const l = window.__statuses;
|
||||
if (!l.length || l[l.length - 1] !== v) l.push(v);
|
||||
};
|
||||
rec(el.textContent);
|
||||
new MutationObserver(() => rec(el.textContent)).observe(el, {
|
||||
childList: true,
|
||||
subtree: true,
|
||||
});
|
||||
}
|
||||
"""
|
||||
|
||||
#: Captures the raw SSE ``data:`` payloads of the /api/chat stream
|
||||
#: (a response clone read in the background) — wire-level assertions for
|
||||
#: the ``tool`` frames, independent of the UI rendering.
|
||||
@@ -266,12 +290,14 @@ def _install_page_hooks(page: Page) -> None:
|
||||
"""Install both hooks on the loaded page (post-goto, pre-submit).
|
||||
|
||||
The fetch wrapper only needs to be in place before the turn's
|
||||
``fetch("/api/chat")`` call; the label observer needs the rendered
|
||||
``#send-label``. (``add_init_script`` would not do — it binds to the
|
||||
NEXT navigation, and the story page is navigated exactly once.)
|
||||
``fetch("/api/chat")`` call; the observers need the rendered
|
||||
``#send-label`` / ``#send-status``. (``add_init_script`` would not do
|
||||
— it binds to the NEXT navigation, and the story page is navigated
|
||||
exactly once.)
|
||||
"""
|
||||
page.evaluate(SSE_HOOK)
|
||||
page.evaluate(LABEL_RECORDER)
|
||||
page.evaluate(STATUS_RECORDER)
|
||||
|
||||
|
||||
def _frames(page: Page) -> list[dict]:
|
||||
@@ -307,10 +333,16 @@ def _submit(page: Page, question: str) -> None:
|
||||
|
||||
|
||||
def _wait_settled(page: Page) -> None:
|
||||
"""The turn is complete: answer text in the bubble, button recovered."""
|
||||
"""The turn is complete: answer text in the bubble, button recovered.
|
||||
|
||||
Phase 48: the label assertion carries the settle wait with an
|
||||
explicit timeout — the in-flight button is the enabled Stop control
|
||||
(never disabled), so ``to_be_enabled`` no longer blocks until the
|
||||
turn settles, and Playwright expect's default (5s) does not inherit
|
||||
the page default."""
|
||||
expect(page.locator(".msg.brain .bubble").last).not_to_have_text("", timeout=30_000)
|
||||
expect(page.locator("#send-btn")).to_be_enabled(timeout=30_000)
|
||||
expect(page.locator("#send-label")).to_have_text("Send")
|
||||
expect(page.locator("#send-label")).to_have_text("Send", timeout=30_000)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
@@ -327,20 +359,31 @@ def test_marker_question_lists_reads_and_quotes(
|
||||
_install_page_hooks(page)
|
||||
|
||||
_submit(page, MARKER_QUESTION)
|
||||
# The "calling tool" label window is transient: the first `tool`
|
||||
# frame sets it and it holds until the FIRST answer delta (the agent
|
||||
# loop completes before the answer stream) — ~0.4 s at the mock's
|
||||
# 0.1 s tool-frame pacing. A polling expect can stride straight over
|
||||
# that window (observed flake, fixed in phase 44 task 03), so the
|
||||
# pre-submit MutationObserver record below is the deterministic
|
||||
# source of truth for the label transition.
|
||||
# The "calling tool" STATUS window is transient: the first `tool`
|
||||
# frame sets #send-status and it holds until the FIRST answer delta
|
||||
# (the agent loop completes before the answer stream) — ~0.4 s at
|
||||
# the mock's 0.1 s tool-frame pacing. A polling expect can stride
|
||||
# straight over that window (observed flake, fixed in phase 44 task
|
||||
# 03), so the pre-submit MutationObserver records below are the
|
||||
# deterministic source of truth for the label/status transitions.
|
||||
_wait_settled(page)
|
||||
|
||||
# The label transition, recorded deterministically (no race):
|
||||
# Thinking… → Calling tool… → … → Send.
|
||||
# Phase 48 (owner-locked): the in-flight button is the Stop control —
|
||||
# the label holds "Stop" for the whole turn (it no longer relabels to
|
||||
# "Calling tool…"), and the transient calling-tool state moved to
|
||||
# #send-status: "… is listing documents" then "… is reading <sp>",
|
||||
# in order (both recorded deterministically — no race).
|
||||
labels = page.evaluate("() => window.__labels")
|
||||
assert "Calling tool…" in labels, labels
|
||||
assert labels.index("Calling tool…") > labels.index("Thinking…")
|
||||
assert "Stop" in labels, labels
|
||||
statuses = page.evaluate("() => window.__statuses")
|
||||
i_list = next(
|
||||
(i for i, s in enumerate(statuses) if "is listing documents" in s), None
|
||||
)
|
||||
i_read = next(
|
||||
(i for i, s in enumerate(statuses) if f"is reading {READ_SP}" in s), None
|
||||
)
|
||||
assert i_list is not None and i_read is not None, statuses
|
||||
assert i_list < i_read, statuses
|
||||
|
||||
# Wire level: exactly two `tool` frames — list then read — and both
|
||||
# ahead of the first `delta` frame.
|
||||
|
||||
@@ -365,10 +365,16 @@ def _submit(page: Page, question: str) -> None:
|
||||
|
||||
|
||||
def _wait_settled(page: Page) -> None:
|
||||
"""The turn is complete: answer text in the bubble, button recovered."""
|
||||
"""The turn is complete: answer text in the bubble, button recovered.
|
||||
|
||||
Phase 48: the label assertion carries the settle wait with an
|
||||
explicit timeout — the in-flight button is the enabled Stop control
|
||||
(never disabled), so ``to_be_enabled`` no longer blocks until the
|
||||
turn settles, and Playwright expect's default (5s) does not inherit
|
||||
the page default."""
|
||||
expect(page.locator(".msg.brain .bubble").last).not_to_have_text("", timeout=30_000)
|
||||
expect(page.locator("#send-btn")).to_be_enabled(timeout=30_000)
|
||||
expect(page.locator("#send-label")).to_have_text("Send")
|
||||
expect(page.locator("#send-label")).to_have_text("Send", timeout=30_000)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
@@ -403,13 +409,14 @@ def test_multi_read_turn(
|
||||
done = next(f for f in frames if f.get("type") == "done")
|
||||
assert done["deflected"] is False
|
||||
|
||||
# The transient "calling tool" states, recorded deterministically:
|
||||
# the label shows "Calling tool…" and #send-status walked through
|
||||
# The transient "calling tool" states, recorded deterministically.
|
||||
# Phase 48 (owner-locked): the in-flight button is the Stop control —
|
||||
# the label holds "Stop" for the whole turn (it no longer relabels
|
||||
# to "Calling tool…"), and #send-status walked through
|
||||
# "… is listing documents" then "… is reading <sp>" for BOTH reads,
|
||||
# in order.
|
||||
labels = page.evaluate("() => window.__labels")
|
||||
assert "Calling tool…" in labels, labels
|
||||
assert labels.index("Calling tool…") > labels.index("Thinking…")
|
||||
assert "Stop" in labels, labels
|
||||
statuses = page.evaluate("() => window.__statuses")
|
||||
i_list = next(
|
||||
(i for i, s in enumerate(statuses) if "is listing documents" in s), None
|
||||
|
||||
@@ -9,7 +9,8 @@ Determinism comes from two mock-LLM behaviors (tests/e2e/mock_llm.py):
|
||||
|
||||
* ``pretend to think slowly`` in the user message → a 3s warm-up before
|
||||
the first token, wide enough to assert the pre-token UI (typing dots +
|
||||
busy "Thinking…" button) at a known timestamp;
|
||||
the enabled Stop button — phase 48 revised the busy-button contract) at
|
||||
a known timestamp;
|
||||
* the ``POST /__shutdown__`` hook → the ``llm_down`` fixture stops the
|
||||
shared mock to simulate an LLM outage, then restores a fresh instance
|
||||
on the same port so later tests keep working.
|
||||
@@ -205,8 +206,10 @@ def test_typing_indicator_during_slow_think(
|
||||
def test_button_state_machine(
|
||||
page: Page, app_url: str, mock_llm: int, db_ready: None
|
||||
) -> None:
|
||||
"""AC1/AC3: disabled + spinner + 'Thinking…' while in flight; enabled
|
||||
+ 'Send' + focused input after done."""
|
||||
"""AC1/AC3 (phase 48 revised contract, owner-locked 2026-08-29):
|
||||
in flight the button is the enabled Stop control — "Stop" label,
|
||||
.is-stop class, spinner hidden; after done it is enabled + "Send"
|
||||
(class removed) + the input is focused back."""
|
||||
_reset_db(mock_llm, seed=True)
|
||||
page.set_default_timeout(30_000)
|
||||
page.goto(app_url)
|
||||
@@ -216,15 +219,17 @@ def test_button_state_machine(
|
||||
|
||||
btn = page.locator("#send-btn")
|
||||
label = page.locator("#send-label")
|
||||
expect(btn).to_be_disabled(timeout=500)
|
||||
expect(label).to_have_text("Thinking…")
|
||||
expect(btn.locator(".spinner")).to_be_visible()
|
||||
expect(btn).to_be_enabled(timeout=500)
|
||||
expect(label).to_have_text("Stop")
|
||||
expect(btn).to_have_class(re.compile(r"is-stop"))
|
||||
expect(btn.locator(".spinner")).to_be_hidden()
|
||||
expect(page.locator("#send-status")).to_contain_text("thinking")
|
||||
|
||||
# Done: button recovers and the input is focused back.
|
||||
# Done: button recovers to the Send state and the input is focused back.
|
||||
page.locator(ANSWER).wait_for(state="visible", timeout=30_000)
|
||||
expect(label).to_have_text("Send", timeout=30_000)
|
||||
expect(btn).to_be_enabled()
|
||||
expect(btn).not_to_have_class(re.compile(r"is-stop"))
|
||||
expect(btn.locator(".spinner")).to_be_hidden()
|
||||
expect(page.locator("#message-input")).to_be_focused()
|
||||
|
||||
@@ -266,8 +271,10 @@ def test_streaming_appends_live(
|
||||
def test_reduced_motion_keeps_feedback(
|
||||
page: Page, app_url: str, mock_llm: int, db_ready: None
|
||||
) -> None:
|
||||
"""AC7: under prefers-reduced-motion the dots/spinner stay visible
|
||||
(static/slower) — feedback is never removed, only calmed."""
|
||||
"""AC7: under prefers-reduced-motion the typing dots stay visible
|
||||
(static) — feedback is never removed, only calmed. Phase 48: the
|
||||
in-flight button is the enabled Stop control (the spinner is no
|
||||
longer part of the in-flight feedback)."""
|
||||
_reset_db(mock_llm, seed=True)
|
||||
page.emulate_media(reduced_motion="reduce")
|
||||
page.set_default_timeout(30_000)
|
||||
@@ -278,7 +285,9 @@ def test_reduced_motion_keeps_feedback(
|
||||
|
||||
expect(page.locator(TYPING)).to_be_visible(timeout=500)
|
||||
expect(page.locator(TYPING).locator(".bubble span")).to_have_count(3)
|
||||
expect(page.locator("#send-btn .spinner")).to_be_visible()
|
||||
expect(page.locator("#send-btn")).to_have_class(re.compile(r"is-stop"))
|
||||
expect(page.locator("#send-label")).to_have_text("Stop")
|
||||
expect(page.locator("#send-btn .spinner")).to_be_hidden()
|
||||
|
||||
# And the turn still completes and recovers.
|
||||
expect(page.locator("#send-label")).to_have_text("Send", timeout=30_000)
|
||||
|
||||
@@ -165,9 +165,16 @@ def brain_bubble_longer_than(n: int, min_bubbles: int = 1) -> str:
|
||||
|
||||
|
||||
def wait_settled(page: Page) -> None:
|
||||
"""The turn is over: the never-stale contract re-enabled the button."""
|
||||
"""The turn is over: the label is back to "Send" (phase 48: the
|
||||
button is never disabled — the in-flight state is the enabled Stop
|
||||
control, so the label is the settle marker). Phase 48 note: the old
|
||||
contract let ``to_be_enabled`` block until the turn settled (the
|
||||
in-flight button was disabled); now the label assertion carries the
|
||||
wait — with an explicit timeout, since Playwright expect's default
|
||||
(5s) is shorter than the mock's long turn (~9.5s) and does not
|
||||
inherit the page default."""
|
||||
expect(page.locator("#send-btn")).to_be_enabled(timeout=30_000)
|
||||
expect(page.locator("#send-label")).to_have_text("Send")
|
||||
expect(page.locator("#send-label")).to_have_text("Send", timeout=30_000)
|
||||
|
||||
|
||||
def wait_scroll_still(page: Page, timeout: float = 10.0) -> float:
|
||||
@@ -259,15 +266,18 @@ def test_no_autoscroll_during_long_answer(
|
||||
assert abs(scroll_state(page)["y"] - target) <= STABLE_PX
|
||||
|
||||
# Sample the viewport across the rest of the stream ...
|
||||
# Phase 48: in-flight detection rides the Stop label (the button is
|
||||
# the enabled Stop control while a turn runs — it is no longer
|
||||
# disabled, so the disabled-state proxy is gone).
|
||||
samples: list[float] = []
|
||||
mid_stream = 0
|
||||
deadline = time.monotonic() + 40
|
||||
while time.monotonic() < deadline:
|
||||
time.sleep(0.25)
|
||||
samples.append(scroll_state(page)["y"])
|
||||
if not page.locator("#send-btn").is_enabled():
|
||||
if page.locator("#send-label").inner_text() == "Stop":
|
||||
mid_stream += 1
|
||||
if len(samples) >= 10 and page.locator("#send-btn").is_enabled():
|
||||
if len(samples) >= 10 and page.locator("#send-label").inner_text() == "Send":
|
||||
# ... and a few more AFTER `done` (the turn is over; nothing
|
||||
# queued behind the stream may move the page either).
|
||||
for _ in range(3):
|
||||
|
||||
@@ -188,8 +188,10 @@ def test_partial_answer_survives_sources_nav_midstream(
|
||||
bubble = page.locator(".msg.brain .bubble").last
|
||||
expect(bubble).to_contain_text(FIRST_LINE_DOM, timeout=30_000)
|
||||
# The turn is still in flight (the stream runs ~9s; navigation takes
|
||||
# well under that).
|
||||
expect(page.locator("#send-btn")).to_be_disabled()
|
||||
# well under that) — phase 48: the in-flight button is the enabled
|
||||
# Stop control, not the old disabled busy state.
|
||||
expect(page.locator("#send-btn")).to_be_enabled()
|
||||
expect(page.locator("#send-label")).to_have_text("Stop")
|
||||
|
||||
# THE BUG REPORT, VERBATIM: click "Sources" while chat is generating.
|
||||
page.click("#nav-sources")
|
||||
@@ -257,9 +259,10 @@ def test_no_orphan_brain_message_when_navigated_before_first_token(
|
||||
thinking = page.locator(".msg.brain").last.locator("details.thinking")
|
||||
thinking.wait_for(state="attached", timeout=10_000)
|
||||
expect(thinking.locator(".thinking-text")).to_contain_text(THINKING_TAIL)
|
||||
# Still pre-token: the button is busy with the Thinking state.
|
||||
expect(page.locator("#send-btn")).to_be_disabled()
|
||||
expect(page.locator("#send-label")).to_have_text("Thinking…")
|
||||
# Still pre-token: the button is the enabled Stop control (phase 48 —
|
||||
# the old disabled "Thinking…" busy state is gone).
|
||||
expect(page.locator("#send-btn")).to_be_enabled()
|
||||
expect(page.locator("#send-label")).to_have_text("Stop")
|
||||
|
||||
# Leave during the pause (no answer token has streamed — acc is empty,
|
||||
# so the pagehide save point must persist nothing brain-side).
|
||||
|
||||
@@ -0,0 +1,334 @@
|
||||
"""Phase 48 E2E (Playwright): stop / cancel an in-flight answer.
|
||||
|
||||
Source: ``TODO.md`` L3 — "Need a way to stop or cancel generation of text
|
||||
in the chat" (no user story file — TODO-derived phase).
|
||||
Run in isolation (DB must be up: ``podman compose up -d db``):
|
||||
|
||||
uv run pytest tests/e2e/test_stop_generation.py -v --no-cov
|
||||
|
||||
Mock-only, no admin login (chat is public). The owner-locked contract
|
||||
(2026-08-29) under test:
|
||||
|
||||
* while a turn is in flight the Send button is the enabled **Stop**
|
||||
control ("Stop" label, ``.is-stop`` rose treatment) — a click *or* an
|
||||
Enter in the focused input stops the turn;
|
||||
* a mid-stream stop keeps the partial answer on screen, shows the
|
||||
``.stopped-note`` "Stopped" marker, no error banner, and persists the
|
||||
partial with the optional ``stopped: true`` marker in ``bor.chat.v1``
|
||||
(a reload restores it, conversation order intact);
|
||||
* a pre-token stop leaves only the question in the conversation (no
|
||||
brain bubble, no brain-side record — phase-20 convention);
|
||||
* the server tears the model stream down on the client disconnect and
|
||||
writes **no** ``query_log`` row for a cancelled turn (unit-proven in
|
||||
``tests/unit/test_chat_cancel.py``; asserted here against the live
|
||||
database after the browser-side stop settles).
|
||||
|
||||
Determinism: the mock streams 12 chars / 0.02 s, so the ~900-word long
|
||||
answer ("write a long answer" trigger, the phase-11 on-topic phrasing)
|
||||
takes ~8 s — a comfortable stop window; the pre-token window is the
|
||||
mock's 3 s "pretend to think slowly" warm-up (phase-06 phrasing). The
|
||||
tests wait for observable states only (the one sanctioned 1.5 s
|
||||
re-read is the partial-stability check from the task).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
from pathlib import Path
|
||||
from threading import Thread
|
||||
from typing import Any
|
||||
|
||||
from playwright.sync_api import Page, expect
|
||||
from sqlalchemy import text
|
||||
|
||||
from app.config import Settings
|
||||
from app.db import SessionLocal
|
||||
from app.rag.importer import ImportSummary, import_sources
|
||||
from app.rag.llm import LLMClient
|
||||
|
||||
REPO = Path(__file__).resolve().parents[2]
|
||||
FIXTURES = REPO / "tests" / "fixtures" / "docs"
|
||||
|
||||
#: On-topic question + the mock's long-answer trigger — the phase-11
|
||||
#: phrasing, so the honesty gate is HIGH and the ~900-word answer
|
||||
#: streams for ~8 s (12 chars / 0.02 s): the stop window.
|
||||
LONG_QUESTION = "How is my Kubernetes cluster set up? write a long answer"
|
||||
#: Phase-06 phrasing: the mock's 3 s warm-up before the first token —
|
||||
#: the observable pre-token window.
|
||||
SLOW_QUESTION = "pretend to think slowly then tell me about kubernetes"
|
||||
SLOW_QUESTION_2 = "pretend to think slowly then tell me about backups"
|
||||
#: The long answer's unique final line — absent from any partial.
|
||||
LONG_ANSWER_END = "LONG-ANSWER-END"
|
||||
|
||||
STORAGE_KEY = "bor.chat.v1"
|
||||
# The typing indicator is itself a .msg.brain — exclude its bubble.
|
||||
ANSWER = ".msg.brain .bubble:not(.typing)"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# KB seeding (same pattern as the phase 02/03/06/14 story suites)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def _import_fixtures(mock_port: int) -> ImportSummary:
|
||||
kwargs: dict[str, Any] = {"_env_file": None, "llm_base_url": f"http://127.0.0.1:{mock_port}/v1"}
|
||||
settings = Settings(**kwargs) # pyright: ignore[reportCallIssue]
|
||||
return await import_sources([FIXTURES], LLMClient(settings))
|
||||
|
||||
|
||||
def _run_in_thread(coro: Any) -> Any:
|
||||
"""Run a coroutine on a worker thread.
|
||||
|
||||
Playwright's sync API keeps an asyncio loop running on the test
|
||||
thread, so ``asyncio.run`` cannot be called directly from a test body.
|
||||
"""
|
||||
box: dict[str, Any] = {}
|
||||
|
||||
def runner() -> None:
|
||||
try:
|
||||
box["value"] = asyncio.run(coro)
|
||||
except BaseException as e: # noqa: BLE001 — re-raised on the test thread
|
||||
box["error"] = e
|
||||
|
||||
t = Thread(target=runner)
|
||||
t.start()
|
||||
t.join()
|
||||
if "error" in box:
|
||||
raise box["error"]
|
||||
return box["value"]
|
||||
|
||||
|
||||
def _reset_db(mock_port: int) -> ImportSummary:
|
||||
with SessionLocal() as db:
|
||||
db.execute(text("TRUNCATE chunks, documents, query_log"))
|
||||
db.commit()
|
||||
return _run_in_thread(_import_fixtures(mock_port))
|
||||
|
||||
|
||||
def _query_log_count() -> int:
|
||||
with SessionLocal() as db:
|
||||
return db.execute(text("SELECT count(*) FROM query_log")).scalar_one()
|
||||
|
||||
|
||||
def _stored_parsed(page: Page) -> dict[str, Any]:
|
||||
raw = page.evaluate(f"() => localStorage.getItem('{STORAGE_KEY}')")
|
||||
assert raw is not None, "the conversation key must exist in localStorage"
|
||||
return json.loads(raw)
|
||||
|
||||
|
||||
def _assert_no_error_banner(page: Page) -> None:
|
||||
"""A stop settles to idle through the same finally path — never the
|
||||
red role=alert error banner (the KB-offline banner is a separate,
|
||||
health-driven state the db_ready fixture keeps away)."""
|
||||
banner = page.locator("#kb-banner")
|
||||
expect(banner).to_be_hidden()
|
||||
expect(banner).not_to_have_attribute("role", "alert")
|
||||
expect(banner).not_to_have_class(re.compile(r"is-error"))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Shared flows
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _stop_mid_stream(page: Page) -> str:
|
||||
"""Ask the long on-topic question and Stop it once a few words of
|
||||
answer have streamed. Returns the rendered partial text (stable
|
||||
after the stop)."""
|
||||
page.fill("#message-input", LONG_QUESTION)
|
||||
page.click("#send-btn")
|
||||
|
||||
# First deltas: the brain bubble exists and grows past a few words.
|
||||
answer = page.locator(ANSWER)
|
||||
answer.wait_for(state="visible", timeout=30_000)
|
||||
partial = ""
|
||||
deadline = time.monotonic() + 15
|
||||
while time.monotonic() < deadline:
|
||||
partial = answer.inner_text()
|
||||
if len(partial.split()) >= 8:
|
||||
break
|
||||
time.sleep(0.05)
|
||||
assert len(partial.split()) >= 8, "no answer deltas before the stop"
|
||||
|
||||
# In flight: the button IS the enabled Stop control.
|
||||
btn = page.locator("#send-btn")
|
||||
expect(btn).to_be_enabled()
|
||||
expect(page.locator("#send-label")).to_have_text("Stop")
|
||||
expect(btn).to_have_class(re.compile(r"is-stop"))
|
||||
|
||||
# Stop it.
|
||||
btn.click()
|
||||
|
||||
# Settled to idle: Send again, stop treatment gone, no error banner.
|
||||
expect(page.locator("#send-label")).to_have_text("Send", timeout=5_000)
|
||||
expect(btn).to_be_enabled()
|
||||
expect(btn).not_to_have_class(re.compile(r"is-stop"))
|
||||
_assert_no_error_banner(page)
|
||||
|
||||
# The partial is kept on screen — real answer text, and far shorter
|
||||
# than the mock's full long answer (its unique final line never
|
||||
# arrived).
|
||||
stopped_text = answer.inner_text()
|
||||
assert stopped_text.strip()
|
||||
assert "Step 1:" in stopped_text
|
||||
assert LONG_ANSWER_END not in stopped_text
|
||||
|
||||
# Stable: no further growth ~1.5 s after the stop.
|
||||
page.wait_for_timeout(1_500)
|
||||
assert answer.inner_text() == stopped_text, (
|
||||
"the partial answer kept growing after the stop"
|
||||
)
|
||||
return stopped_text
|
||||
|
||||
|
||||
def _stop_pre_token(page: Page, question: str, via_enter: bool) -> None:
|
||||
"""Ask a slow question and stop it during the 3 s pre-token warm-up —
|
||||
by clicking the Stop control, or by pressing Enter in the focused
|
||||
input (owner-locked: click *or* Enter)."""
|
||||
page.fill("#message-input", question)
|
||||
if via_enter:
|
||||
page.keyboard.press("Enter") # the fill focused the input; it submits
|
||||
else:
|
||||
page.click("#send-btn")
|
||||
|
||||
# In the warm-up window the button reads Stop (thinking state).
|
||||
expect(page.locator("#send-label")).to_have_text("Stop", timeout=5_000)
|
||||
if via_enter:
|
||||
# The input kept focus through the submit — Enter stops the turn.
|
||||
expect(page.locator("#message-input")).to_be_focused()
|
||||
page.keyboard.press("Enter")
|
||||
else:
|
||||
page.locator("#send-btn").click()
|
||||
|
||||
# Settled to idle: no error banner, no brain bubble at all (the typing
|
||||
# indicator is gone with it), the question survived, focus is back.
|
||||
expect(page.locator("#send-label")).to_have_text("Send", timeout=5_000)
|
||||
btn = page.locator("#send-btn")
|
||||
expect(btn).to_be_enabled()
|
||||
expect(btn).not_to_have_class(re.compile(r"is-stop"))
|
||||
_assert_no_error_banner(page)
|
||||
expect(page.locator(".msg.brain")).to_have_count(0)
|
||||
expect(page.locator(ANSWER)).to_have_count(0)
|
||||
expect(page.locator(".msg.user .bubble").last).to_contain_text(question)
|
||||
expect(page.locator("#message-input")).to_be_focused()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 1. Mid-stream stop: partial kept + "Stopped" note + stopped persistence
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_stop_mid_stream(
|
||||
page: Page, app_url: str, mock_llm: int, db_ready: None
|
||||
) -> None:
|
||||
summary = _reset_db(mock_llm)
|
||||
assert summary is not None and summary.added == 13 # A9 formats (phase 47 added quadlet+j2)
|
||||
page.set_default_timeout(30_000)
|
||||
page.goto(app_url)
|
||||
|
||||
stopped_text = _stop_mid_stream(page)
|
||||
|
||||
# The bubble keeps the partial and carries the "Stopped" note.
|
||||
expect(page.locator(ANSWER)).to_contain_text(stopped_text)
|
||||
note = page.locator(".msg.brain .stopped-note")
|
||||
expect(note).to_have_count(1)
|
||||
expect(note.first).to_contain_text("Stopped")
|
||||
# No sources on a stopped turn — it never settled.
|
||||
expect(page.locator(".msg.brain .source-chip")).to_have_count(0)
|
||||
# The live region confirms the stop (no error wording).
|
||||
expect(page.locator("#send-status")).to_contain_text("Answer stopped.")
|
||||
|
||||
# Persistence: question, then the partial with the optional
|
||||
# ``stopped`` marker (raw text — no HTML, no LONG-ANSWER-END).
|
||||
stored = _stored_parsed(page)
|
||||
assert [m["who"] for m in stored["messages"]] == ["user", "brain"]
|
||||
assert stored["messages"][0]["text"] == LONG_QUESTION
|
||||
last = stored["messages"][-1]
|
||||
assert last["stopped"] is True
|
||||
assert last["text"].strip()
|
||||
assert "Step 1:" in last["text"]
|
||||
assert LONG_ANSWER_END not in last["text"]
|
||||
|
||||
# Server side: the cancelled turn wrote no durable record (the
|
||||
# stability wait above already gave the teardown its margin).
|
||||
assert _query_log_count() == 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 2. Pre-token stop: question kept, nothing brain-side (click and Enter)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_stop_pre_token(
|
||||
page: Page, app_url: str, mock_llm: int, db_ready: None
|
||||
) -> None:
|
||||
_reset_db(mock_llm)
|
||||
page.set_default_timeout(30_000)
|
||||
page.goto(app_url)
|
||||
|
||||
_stop_pre_token(page, SLOW_QUESTION, via_enter=False)
|
||||
|
||||
# The conversation holds exactly the question — no brain-side record.
|
||||
stored = _stored_parsed(page)
|
||||
assert [m["who"] for m in stored["messages"]] == ["user"]
|
||||
assert stored["messages"][0]["text"] == SLOW_QUESTION
|
||||
assert "stopped" not in stored["messages"][0]
|
||||
|
||||
# The owner-locked "click *or* Enter": the same pre-token stop, this
|
||||
# time submitted and stopped through the focused input's Enter.
|
||||
_stop_pre_token(page, SLOW_QUESTION_2, via_enter=True)
|
||||
expect(page.locator(".msg.user .bubble")).to_have_count(2)
|
||||
expect(page.locator(".msg.brain")).to_have_count(0)
|
||||
stored = _stored_parsed(page)
|
||||
assert [m["who"] for m in stored["messages"]] == ["user", "user"]
|
||||
assert stored["messages"][-1]["text"] == SLOW_QUESTION_2
|
||||
|
||||
# Server side: neither pre-token stop left a durable record. The UI
|
||||
# settle was observed above; the short margin covers the server's
|
||||
# disconnect teardown (mock cadence 0.02 s).
|
||||
page.wait_for_timeout(500)
|
||||
assert _query_log_count() == 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 3. A stopped turn survives a reload
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_stopped_turn_survives_reload(
|
||||
page: Page, app_url: str, mock_llm: int, db_ready: None
|
||||
) -> None:
|
||||
_reset_db(mock_llm)
|
||||
page.set_default_timeout(30_000)
|
||||
page.goto(app_url)
|
||||
|
||||
stopped_text = _stop_mid_stream(page)
|
||||
assert _query_log_count() == 0
|
||||
|
||||
page.reload()
|
||||
|
||||
# Restored: the user message first, then the stopped brain bubble —
|
||||
# the same partial text, the "Stopped" note, and the idle Send button.
|
||||
expect(page.locator("#empty-state")).to_be_hidden()
|
||||
msgs = page.locator("#messages > .msg")
|
||||
expect(msgs).to_have_count(2)
|
||||
expect(msgs.nth(0)).to_have_class(re.compile(r"msg user"))
|
||||
expect(msgs.nth(1)).to_have_class(re.compile(r"msg brain"))
|
||||
expect(page.locator(".msg.user .bubble")).to_contain_text(LONG_QUESTION)
|
||||
bubble = page.locator(ANSWER)
|
||||
expect(bubble).to_have_count(1)
|
||||
assert bubble.inner_text() == stopped_text, "the restored partial differs"
|
||||
note = page.locator(".msg.brain .stopped-note")
|
||||
expect(note).to_have_count(1)
|
||||
expect(note.first).to_contain_text("Stopped")
|
||||
|
||||
expect(page.locator("#send-label")).to_have_text("Send")
|
||||
expect(page.locator("#send-btn")).to_be_enabled()
|
||||
expect(page.locator("#send-btn")).not_to_have_class(re.compile(r"is-stop"))
|
||||
|
||||
# The marker still rides the stored record after the restore.
|
||||
stored = _stored_parsed(page)
|
||||
assert [m["who"] for m in stored["messages"]] == ["user", "brain"]
|
||||
assert stored["messages"][-1]["stopped"] is True
|
||||
@@ -115,10 +115,13 @@ def send_and_wait(page: Page, question: str) -> None:
|
||||
page.evaluate("() => document.querySelector('#composer').requestSubmit()")
|
||||
expect(page.locator(".msg.user .bubble").last).to_contain_text(question)
|
||||
# The mock streams at 0.02s/chunk, so thinking + answer land in a few
|
||||
# seconds — 30s is generous on headless Chromium.
|
||||
# seconds — 30s is generous on headless Chromium. Phase 48: the label
|
||||
# assertion carries the settle wait with an explicit timeout — the
|
||||
# in-flight button is the enabled Stop control (never disabled), so
|
||||
# to_be_enabled no longer blocks until the turn settles.
|
||||
expect(page.locator(".msg.brain .bubble").last).not_to_have_text("", timeout=30_000)
|
||||
expect(page.locator("#send-btn")).to_be_enabled(timeout=30_000)
|
||||
expect(page.locator("#send-label")).to_have_text("Send")
|
||||
expect(page.locator("#send-label")).to_have_text("Send", timeout=30_000)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -176,10 +176,13 @@ def send_and_wait(page: Page, question: str) -> None:
|
||||
page.evaluate("() => document.querySelector('#composer').requestSubmit()")
|
||||
expect(page.locator(".msg.user .bubble").last).to_contain_text(question)
|
||||
# The mock streams at 0.02s/chunk, so thinking + answer land in a few
|
||||
# seconds — 30s is generous on headless Chromium.
|
||||
# seconds — 30s is generous on headless Chromium. Phase 48: the label
|
||||
# assertion carries the settle wait with an explicit timeout — the
|
||||
# in-flight button is the enabled Stop control (never disabled), so
|
||||
# to_be_enabled no longer blocks until the turn settles.
|
||||
expect(page.locator(".msg.brain .bubble").last).not_to_have_text("", timeout=30_000)
|
||||
expect(page.locator("#send-btn")).to_be_enabled(timeout=30_000)
|
||||
expect(page.locator("#send-label")).to_have_text("Send")
|
||||
expect(page.locator("#send-label")).to_have_text("Send", timeout=30_000)
|
||||
|
||||
|
||||
def submit(page: Page, question: str) -> None:
|
||||
@@ -787,8 +790,10 @@ def test_thinking_window_follows_across_paragraph_breaks(
|
||||
)
|
||||
|
||||
# The turn settles; the scratchpad text past the break is intact and
|
||||
# the answer landed.
|
||||
# the answer landed. Phase 48: the label assertion carries the settle
|
||||
# wait (explicit timeout — the in-flight Stop button is never
|
||||
# disabled, so to_be_enabled no longer blocks until the turn ends).
|
||||
expect(page.locator("#send-btn")).to_be_enabled(timeout=30_000)
|
||||
expect(page.locator("#send-label")).to_have_text("Send")
|
||||
expect(page.locator("#send-label")).to_have_text("Send", timeout=30_000)
|
||||
expect(details.locator(".thinking-text")).to_contain_text("Step 3")
|
||||
expect(page.locator(".msg.brain .bubble").last).to_contain_text(MOCK_ANSWER_MARKER)
|
||||
|
||||
@@ -0,0 +1,419 @@
|
||||
"""Unit: client-disconnect teardown of a chat turn (phase 48, task 01).
|
||||
|
||||
Drives ``POST /api/chat`` through the real ASGI app — a real
|
||||
``LLMClient`` with a slow, recording fake SDK stream behind it, plus the
|
||||
fake DB session / retriever pattern from ``tests/unit/test_chat_gate.py``
|
||||
— with an ASGI-level client disconnect: after a few SSE frames the
|
||||
``receive`` channel starts returning ``http.disconnect``, the ASGI 2.0
|
||||
contract this Starlette's ``StreamingResponse`` listens for (its task
|
||||
group cancels the body task, and the abandoned body generator chain is
|
||||
finalized by the loop's PEP 525 asyncgen hooks). The ``TestClient``
|
||||
transport buffers the whole body and cannot drop a connection
|
||||
mid-stream, so the disconnect is emulated at the ASGI boundary —
|
||||
exactly where a real server hands it over.
|
||||
|
||||
Owner-locked contract (2026-08-29): on a cancelled turn the model's
|
||||
stream is closed promptly, one ``chat: turn cancelled`` log line is
|
||||
written, **no** ``query_log`` row exists, and no ``done``/``error``
|
||||
frame follows the disconnect — while completed turns and the
|
||||
mid-stream ``LLMError`` path settle exactly as before (``done`` frame +
|
||||
query_log row; structured ``error`` frame, not logged as cancelled).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import gc
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import Callable, Iterator, MutableMapping
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from app.api import chat as chat_api
|
||||
from app.config import Settings
|
||||
from app.main import app as fastapi_app
|
||||
from app.models import Document, KbOverview, QueryLog
|
||||
from app.rag.llm import LLMClient
|
||||
from app.rag.retriever import RetrievedChunk
|
||||
|
||||
|
||||
def _doc(title: str, content: str) -> Document:
|
||||
return Document(
|
||||
id=uuid.uuid4(),
|
||||
source="Homelab",
|
||||
path=f"{title.lower().replace(' ', '-')}.md",
|
||||
full_path="/tmp/doc.md",
|
||||
title=title,
|
||||
content=content,
|
||||
content_hash="0" * 64,
|
||||
)
|
||||
|
||||
|
||||
def _chunk(doc: Document, cosine: float, fts_hit: bool = False) -> RetrievedChunk:
|
||||
return RetrievedChunk(
|
||||
chunk_id=uuid.uuid4(),
|
||||
position=0,
|
||||
content=doc.content[:32],
|
||||
score=cosine,
|
||||
document=doc,
|
||||
cosine=cosine,
|
||||
fts_hit=fts_hit,
|
||||
)
|
||||
|
||||
|
||||
def _fake_retriever(chunks: list[RetrievedChunk]) -> Any:
|
||||
def retrieve(_db: Any, _question: str, _vec: list[float]) -> list[RetrievedChunk]:
|
||||
return chunks
|
||||
|
||||
return retrieve
|
||||
|
||||
|
||||
# ---------- fakes: slow SDK stream behind a real LLMClient ----------
|
||||
|
||||
|
||||
def _sse_chunk(text: str) -> SimpleNamespace:
|
||||
"""One fake ChatCompletionChunk (``choices[].delta.content`` shape)."""
|
||||
return SimpleNamespace(choices=[SimpleNamespace(delta=SimpleNamespace(content=text))])
|
||||
|
||||
|
||||
class _SlowStream:
|
||||
"""A fake aipi SSE stream (the openai SDK ``AsyncStream`` shape):
|
||||
yields *chunks* with a small sleep between them (so a disconnect can
|
||||
land mid-iteration) and records ``close()`` calls — the SDK stream's
|
||||
deterministic teardown. ``fail_after`` makes ``__anext__`` raise a
|
||||
transport error after that many chunks (the mid-stream
|
||||
``LLMError`` path)."""
|
||||
|
||||
def __init__(
|
||||
self, chunks: list, fail_after: int | None = None, delay: float = 0.005
|
||||
) -> None:
|
||||
self._chunks = list(chunks)
|
||||
self._fail_after = fail_after
|
||||
self._delay = delay
|
||||
self._i = 0
|
||||
self.closed = False
|
||||
|
||||
def __aiter__(self) -> _SlowStream:
|
||||
return self
|
||||
|
||||
async def __anext__(self) -> SimpleNamespace:
|
||||
self._i += 1
|
||||
if self._fail_after is not None and self._i > self._fail_after:
|
||||
raise ConnectionError("simulated mid-stream drop")
|
||||
if self._i > len(self._chunks):
|
||||
raise StopAsyncIteration
|
||||
await asyncio.sleep(self._delay)
|
||||
return self._chunks[self._i - 1]
|
||||
|
||||
async def close(self) -> None:
|
||||
self.closed = True
|
||||
|
||||
|
||||
class _FakeCompletions:
|
||||
def __init__(self, stream: _SlowStream) -> None:
|
||||
self._stream = stream
|
||||
self.kwargs: dict | None = None
|
||||
|
||||
async def create(self, **kwargs: Any):
|
||||
self.kwargs = kwargs
|
||||
assert kwargs.get("stream") is True
|
||||
return self._stream
|
||||
|
||||
|
||||
def _make_llm(monkeypatch: pytest.MonkeyPatch, stream: _SlowStream) -> LLMClient:
|
||||
"""A real ``LLMClient`` (so the production ``chat_stream`` teardown
|
||||
runs) with the fake SDK stream behind it and a deterministic
|
||||
``embed_one`` (no embeddings HTTP)."""
|
||||
llm = LLMClient(Settings(_env_file=None)) # pyright: ignore[reportCallIssue]
|
||||
llm._client = SimpleNamespace( # pyright: ignore[reportAttributeAccessIssue]
|
||||
chat=SimpleNamespace(completions=_FakeCompletions(stream))
|
||||
)
|
||||
|
||||
async def _embed_one(self: Any, _text: str) -> list[float]:
|
||||
return [0.0] * 768
|
||||
|
||||
monkeypatch.setattr(LLMClient, "embed_one", _embed_one)
|
||||
return llm
|
||||
|
||||
|
||||
# ---------- fake DB session (test_chat_gate.py pattern) ----------
|
||||
|
||||
|
||||
class _FakeSteeringResult:
|
||||
def all(self) -> list[Any]:
|
||||
return []
|
||||
|
||||
|
||||
class _FakeSession:
|
||||
"""Records the QueryLog rows it is given; no steering notes, no
|
||||
stored KB overview."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.added: list[Any] = []
|
||||
self.commits = 0
|
||||
|
||||
def add(self, obj: Any) -> None:
|
||||
self.added.append(obj)
|
||||
|
||||
def commit(self) -> None:
|
||||
self.commits += 1
|
||||
|
||||
def scalars(self, _stmt: Any) -> _FakeSteeringResult:
|
||||
return _FakeSteeringResult()
|
||||
|
||||
def get(self, model: Any, pk: Any) -> Any:
|
||||
if model is KbOverview:
|
||||
return KbOverview(id=1, content="")
|
||||
return None
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def env(monkeypatch: pytest.MonkeyPatch) -> Iterator[_FakeSession]:
|
||||
"""``POST /api/chat`` with the DB session, retriever settings, and
|
||||
availability faked (the gate tests' wiring)."""
|
||||
monkeypatch.setattr(chat_api, "db_available", lambda: True)
|
||||
session = _FakeSession()
|
||||
monkeypatch.setitem(fastapi_app.dependency_overrides, chat_api.get_db, lambda: session)
|
||||
# A stable gate threshold, independent of the production default.
|
||||
monkeypatch.setattr(
|
||||
chat_api,
|
||||
"get_settings",
|
||||
lambda: Settings(_env_file=None, relevance_threshold=0.30), # pyright: ignore[reportCallIssue]
|
||||
)
|
||||
yield session
|
||||
fastapi_app.dependency_overrides.clear()
|
||||
|
||||
|
||||
def _install_llm(monkeypatch: pytest.MonkeyPatch, llm: LLMClient) -> None:
|
||||
monkeypatch.setitem(fastapi_app.dependency_overrides, chat_api.get_llm, lambda: llm)
|
||||
|
||||
|
||||
# ---------- the ASGI driver (client disconnect at the ASGI boundary) ----------
|
||||
|
||||
|
||||
def _scope() -> dict[str, Any]:
|
||||
return {
|
||||
"type": "http",
|
||||
"asgi": {"version": "3.0"},
|
||||
"http_version": "1.1",
|
||||
"method": "POST",
|
||||
"path": "/api/chat",
|
||||
"raw_path": b"/api/chat",
|
||||
"root_path": "",
|
||||
"scheme": "http",
|
||||
"query_string": b"",
|
||||
"headers": [
|
||||
(b"host", b"testserver"),
|
||||
(b"content-type", b"application/json"),
|
||||
],
|
||||
"client": ("testclient", 50000),
|
||||
"server": ("testserver", 80),
|
||||
"state": {},
|
||||
}
|
||||
|
||||
|
||||
async def _drive(
|
||||
question: str,
|
||||
stop_after: int | None,
|
||||
settle: Callable[[], bool] | None = None,
|
||||
) -> list[bytes]:
|
||||
"""Run one ``POST /api/chat`` through the real ASGI app.
|
||||
|
||||
``stop_after=None`` lets the turn complete; otherwise the client
|
||||
"disconnects" after ``stop_after`` body chunks (SSE frames) have
|
||||
been written — the ``receive`` channel starts returning
|
||||
``http.disconnect`` (the ASGI 2.0 contract; no ``spec_version`` in
|
||||
the scope, so ``StreamingResponse`` runs its task-group
|
||||
listen-for-disconnect path). When *settle* is given, spins the loop
|
||||
until it holds (the PEP 525 asyncgen finalizers run the abandoned
|
||||
generator chain's ``finally`` blocks a few loop turns after their
|
||||
frames are released) or a 5 s timeout runs out. Returns the body
|
||||
chunks written.
|
||||
"""
|
||||
body = json.dumps({"message": question}).encode()
|
||||
chunks: list[bytes] = []
|
||||
disconnect = asyncio.Event()
|
||||
request_done = False
|
||||
|
||||
async def receive() -> dict[str, Any]:
|
||||
nonlocal request_done
|
||||
if not request_done:
|
||||
request_done = True
|
||||
return {"type": "http.request", "body": body, "more_body": False}
|
||||
await disconnect.wait()
|
||||
return {"type": "http.disconnect"}
|
||||
|
||||
async def send(message: MutableMapping[str, Any]) -> None:
|
||||
if message["type"] != "http.response.body":
|
||||
return
|
||||
chunk = message.get("body", b"")
|
||||
if chunk:
|
||||
chunks.append(chunk)
|
||||
if stop_after is not None and len(chunks) >= stop_after:
|
||||
disconnect.set() # the client goes away
|
||||
|
||||
await fastapi_app(_scope(), receive, send)
|
||||
if settle is not None:
|
||||
gc.collect() # release any cycle-held frames up front
|
||||
deadline = time.monotonic() + 5.0
|
||||
while not settle():
|
||||
if time.monotonic() >= deadline:
|
||||
break
|
||||
await asyncio.sleep(0.01)
|
||||
return chunks
|
||||
|
||||
|
||||
def _frames(chunks: list[bytes]) -> list[dict[str, Any]]:
|
||||
"""Parse the SSE frames out of the written body chunks."""
|
||||
frames: list[dict[str, Any]] = []
|
||||
for raw in chunks:
|
||||
for frame in raw.decode("utf-8").split("\n\n"):
|
||||
frame = frame.strip()
|
||||
if frame.startswith("data:"):
|
||||
frames.append(json.loads(frame.removeprefix("data:").strip()))
|
||||
return frames
|
||||
|
||||
|
||||
# ---------- cancelled turns (the phase-48 contract) ----------
|
||||
|
||||
|
||||
def test_cancelled_grounded_turn_closes_stream_logs_cancel_and_skips_query_log(
|
||||
env: _FakeSession,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
"""Grounded turn (the agent loop): a mid-stream disconnect closes
|
||||
the model's stream, logs one cancel line, writes no query_log row,
|
||||
and emits no done/error frame after the disconnect."""
|
||||
stream = _SlowStream([_sse_chunk(f"word{i} ") for i in range(60)])
|
||||
llm = _make_llm(monkeypatch, stream)
|
||||
_install_llm(monkeypatch, llm)
|
||||
doc = _doc("Kubernetes Homelab Cluster", "TALOS_DOC_SENT")
|
||||
monkeypatch.setattr(chat_api, "retrieve", _fake_retriever([_chunk(doc, 0.90)]))
|
||||
|
||||
with caplog.at_level(logging.INFO, logger="app.chat"):
|
||||
chunks = asyncio.run(
|
||||
_drive(
|
||||
"How is my Kubernetes cluster set up?",
|
||||
stop_after=2,
|
||||
settle=lambda: stream.closed,
|
||||
)
|
||||
)
|
||||
|
||||
# The model's stream was closed promptly on abandon.
|
||||
assert stream.closed
|
||||
# The cancel log line — exactly once, with the question.
|
||||
cancel_lines = [
|
||||
r.getMessage() for r in caplog.records if "turn cancelled" in r.getMessage()
|
||||
]
|
||||
assert len(cancel_lines) == 1
|
||||
assert "How is my Kubernetes cluster set up?" in cancel_lines[0]
|
||||
assert "total_ms=" in cancel_lines[0]
|
||||
# No durable record for a cancelled turn.
|
||||
assert env.added == []
|
||||
# Frames: the streamed deltas only — no done, no error, after the
|
||||
# disconnect (a third delta may race the teardown; all deltas).
|
||||
frames = _frames(chunks)
|
||||
assert len(frames) >= 2
|
||||
assert all(f["type"] == "delta" for f in frames)
|
||||
|
||||
|
||||
def test_cancelled_deflected_turn_closes_stream(
|
||||
env: _FakeSession,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
"""Deflected turn (the direct ``chat_stream`` path — A8): the same
|
||||
teardown contract holds without the agent loop."""
|
||||
stream = _SlowStream([_sse_chunk(f"word{i} ") for i in range(60)])
|
||||
llm = _make_llm(monkeypatch, stream)
|
||||
_install_llm(monkeypatch, llm)
|
||||
doc = _doc("Deploying a New Service", "DOC_CONTENT_NEVER_SENT")
|
||||
monkeypatch.setattr(chat_api, "retrieve", _fake_retriever([_chunk(doc, 0.10)]))
|
||||
|
||||
with caplog.at_level(logging.WARNING, logger="app.chat"):
|
||||
chunks = asyncio.run(
|
||||
_drive(
|
||||
"How do I bake sourdough bread?",
|
||||
stop_after=2,
|
||||
settle=lambda: stream.closed,
|
||||
)
|
||||
)
|
||||
|
||||
assert stream.closed
|
||||
cancel_lines = [
|
||||
r.getMessage() for r in caplog.records if "turn cancelled" in r.getMessage()
|
||||
]
|
||||
assert len(cancel_lines) == 1
|
||||
assert env.added == []
|
||||
frames = _frames(chunks)
|
||||
assert len(frames) >= 2
|
||||
assert all(f["type"] == "delta" for f in frames)
|
||||
|
||||
|
||||
# ---------- regressions: settled turns behave exactly as before ----------
|
||||
|
||||
|
||||
def test_completed_turn_still_emits_done_and_writes_query_log(
|
||||
env: _FakeSession,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
"""A completed turn: the ``done`` frame, the query_log row, and the
|
||||
per-turn log line — and NO cancel line (it settled)."""
|
||||
stream = _SlowStream([_sse_chunk(f"word{i} ") for i in range(3)], delay=0.001)
|
||||
llm = _make_llm(monkeypatch, stream)
|
||||
_install_llm(monkeypatch, llm)
|
||||
doc = _doc("Kubernetes Homelab Cluster", "TALOS_DOC_SENT")
|
||||
monkeypatch.setattr(chat_api, "retrieve", _fake_retriever([_chunk(doc, 0.90)]))
|
||||
|
||||
with caplog.at_level(logging.INFO, logger="app.chat"):
|
||||
chunks = asyncio.run(_drive("How is my Kubernetes cluster set up?", None))
|
||||
|
||||
frames = _frames(chunks)
|
||||
assert [f["type"] for f in frames] == ["delta", "delta", "delta", "done"]
|
||||
assert frames[-1]["deflected"] is False
|
||||
assert frames[-1]["sources"][0]["title"] == "Kubernetes Homelab Cluster"
|
||||
(row,) = env.added
|
||||
assert isinstance(row, QueryLog)
|
||||
assert row.question == "How is my Kubernetes cluster set up?"
|
||||
assert env.commits == 1
|
||||
# The per-turn line still goes out; no cancel line for a settled turn.
|
||||
assert any("question=" in r.getMessage() for r in caplog.records)
|
||||
assert not any("turn cancelled" in r.getMessage() for r in caplog.records)
|
||||
|
||||
|
||||
def test_mid_stream_llm_error_settles_not_cancelled(
|
||||
env: _FakeSession,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
"""The mid-stream ``LLMError`` path (the fake stream drops after two
|
||||
pieces): the structured ``error`` frame is emitted, the turn is NOT
|
||||
logged as cancelled (it settled), the stream is closed on the
|
||||
exception path, and — as before — no query_log row is written."""
|
||||
stream = _SlowStream([_sse_chunk(f"word{i} ") for i in range(60)], fail_after=2)
|
||||
llm = _make_llm(monkeypatch, stream)
|
||||
_install_llm(monkeypatch, llm)
|
||||
doc = _doc("Kubernetes Homelab Cluster", "TALOS_DOC_SENT")
|
||||
monkeypatch.setattr(chat_api, "retrieve", _fake_retriever([_chunk(doc, 0.90)]))
|
||||
|
||||
with caplog.at_level(logging.INFO, logger="app.chat"):
|
||||
chunks = asyncio.run(_drive("How is my Kubernetes cluster set up?", None))
|
||||
|
||||
frames = _frames(chunks)
|
||||
assert [f["type"] for f in frames] == ["delta", "delta", "error"]
|
||||
assert frames[-1]["detail"] == "The chat model dropped the connection — try again?"
|
||||
# The exception path closes the model's stream too (synchronously —
|
||||
# no settle needed).
|
||||
assert stream.closed
|
||||
assert env.added == []
|
||||
# Settled: no cancel line (the LLM stream failure IS logged, though).
|
||||
assert not any("turn cancelled" in r.getMessage() for r in caplog.records)
|
||||
assert any(
|
||||
"LLM stream failed" in r.getMessage() for r in caplog.records
|
||||
)
|
||||
@@ -23,6 +23,10 @@ def _css() -> str:
|
||||
return STYLES_CSS.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def _html() -> str:
|
||||
return (FRONTEND / "index.html").read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_turn_timeout_constant_exported_at_120s() -> None:
|
||||
"""The 120s client-side guard (PLAN §7.4) must be an *exported*
|
||||
constant — testable, and the single value the E2E timeout story keys
|
||||
@@ -91,15 +95,22 @@ def test_reduced_motion_calm_not_removed() -> None:
|
||||
|
||||
|
||||
def test_busy_button_style_tokens() -> None:
|
||||
"""Story spec: busy send button is #a5b4fc with the 16px dark-arc
|
||||
spinner (--bg on #a5b4fc = 9.7:1, phase 08); label swaps Send ↔ Thinking…."""
|
||||
"""Phase 48 (revised contract, owner-locked 2026-08-29): in flight
|
||||
the button is the enabled Stop control — "Stop" label, .is-stop
|
||||
class (rose treatment, 6.3:1 with the #fff label), spinner hidden;
|
||||
idle/error keep the brand Send button (dark ink on brand 5.2:1).
|
||||
The spinner element stays in the markup + CSS (16px dark arc — the
|
||||
reduced-motion pin below) but the state machine never shows it: the
|
||||
Stop label + treatment carry the in-flight state."""
|
||||
css = _css()
|
||||
js = _js()
|
||||
assert ".send-btn:disabled" in css
|
||||
assert "#a5b4fc" in css
|
||||
assert ".send-btn.is-stop" in css
|
||||
assert "#be123c" in css, "the stop background: rose-700 (6.3:1 with #fff)"
|
||||
assert ".send-btn.is-stop:hover" in css, "the darker hover step"
|
||||
assert re.search(r"\.spinner \{[^}]*width: 16px", css)
|
||||
assert "Thinking…" in js
|
||||
assert 'sendLabel.textContent' in js
|
||||
assert 'sendLabel.textContent = inFlight ? "Stop" : "Send"' in js
|
||||
assert 'sendBtn.classList.toggle("is-stop", inFlight)' in js
|
||||
assert "sendBtn.disabled = false" in js, "the button is a control, never disabled"
|
||||
|
||||
|
||||
# ---------- thinking display (phase 17) ----------
|
||||
@@ -198,3 +209,160 @@ def test_thinking_chevron_stills_under_reduced_motion() -> None:
|
||||
"details.thinking summary::before" in b and "transition: none" in b
|
||||
for b in blocks
|
||||
), "chevron transition must still under reduced motion"
|
||||
|
||||
|
||||
# ---------- stop generation (phase 48, task 02) ----------
|
||||
|
||||
|
||||
def test_in_flight_button_is_the_stop_control() -> None:
|
||||
"""Phase 48 (owner-locked 2026-08-29): in flight the button is the
|
||||
enabled Stop control — "Stop" label, .is-stop class, spinner hidden
|
||||
(the label + the rose treatment carry the state); idle/error keep
|
||||
the Send label with the class removed. The state machine otherwise
|
||||
stays unchanged (same four states, same single entry point)."""
|
||||
js = _js()
|
||||
assert 'sendLabel.textContent = inFlight ? "Stop" : "Send"' in js
|
||||
assert 'sendBtn.classList.toggle("is-stop", inFlight)' in js
|
||||
assert 'sendBtn.querySelector(".spinner").hidden = true' in js, (
|
||||
"the spinner never shows — the Stop label carries the state"
|
||||
)
|
||||
assert "sendBtn.disabled = false" in js, "enabled in every state"
|
||||
|
||||
|
||||
def test_abort_plumbing_owns_the_fetch() -> None:
|
||||
"""The in-flight fetch is owned by an AbortController created at
|
||||
turn start (module scope, cleared in the finally), passed to the
|
||||
fetch as its signal; the 120s guard aborts the same controller as
|
||||
its backstop — with `aborted = true` FIRST, so the catch never reads
|
||||
the guard's abort as a user stop (one owner, same outcome)."""
|
||||
js = _js()
|
||||
assert "let turnAbort = null" in js, "module-scope abort owner"
|
||||
assert "turnAbort = new AbortController()" in js, "fresh controller per turn"
|
||||
assert "signal: turnAbort.signal" in js, "the fetch carries the signal"
|
||||
guard_start = js.find("armTurnTimeout(() => {")
|
||||
guard = js[guard_start : js.find("});", guard_start)]
|
||||
assert "aborted = true" in guard and "turnAbort?.abort()" in guard, (
|
||||
"the guard keeps cancelStream + the abort as backstops"
|
||||
)
|
||||
assert guard.index("aborted = true") < guard.index("turnAbort?.abort()"), (
|
||||
"aborted must be set before the guard's abort"
|
||||
)
|
||||
handle = js.find("async function handleSend")
|
||||
finally_idx = js.find("} finally {", handle)
|
||||
finally_block = js[finally_idx : finally_idx + 700]
|
||||
assert "turnAbort = null" in finally_block, "the abort owner is spent after the turn"
|
||||
|
||||
|
||||
def test_stop_turn_is_the_user_abort() -> None:
|
||||
"""stopTurn: a no-op unless a turn is in flight (thinking/streaming);
|
||||
it marks the turn as user-stopped and aborts. The in-flight guard at
|
||||
the top of handleSend routes a click / Enter-to-submit to it BEFORE
|
||||
the !text guard — the enabled in-flight button can never start a
|
||||
second turn."""
|
||||
js = _js()
|
||||
fn = js.find("function stopTurn")
|
||||
assert fn != -1, "stopTurn must exist"
|
||||
body = js[fn : js.find("\n}\n", fn)]
|
||||
assert "uiState !== UI_STATE.thinking" in body
|
||||
assert "uiState !== UI_STATE.streaming" in body
|
||||
assert "stoppedByUser = true" in body
|
||||
assert "turnAbort?.abort()" in body
|
||||
handle = js.find("async function handleSend")
|
||||
guard_idx = js.find("stopTurn();", handle)
|
||||
text_idx = js.find("const text = input.value.trim()", handle)
|
||||
assert handle < guard_idx < text_idx, (
|
||||
"the in-flight guard (→ stopTurn) must precede the !text guard"
|
||||
)
|
||||
|
||||
|
||||
def test_stop_branch_keeps_partial_and_persists_stopped() -> None:
|
||||
"""The stop path in handleSend's catch: no error state, no error
|
||||
banner; when answer text streamed the partial is kept on screen
|
||||
(thinking block closed, Tune + Stopped note appended — admin parity
|
||||
with the restore path) and persisted with the owner-locked optional
|
||||
`stopped: true` marker (+ optional thinking/tools); a pre-token stop
|
||||
persists nothing brain-side (phase-20 convention). The "Answer
|
||||
stopped." live-region confirmation is set in the finally, AFTER the
|
||||
single settle, so setUiState(idle) can't overwrite it."""
|
||||
js = _js()
|
||||
handle = js.find("async function handleSend")
|
||||
catch_idx = js.find("} catch (err) {", handle)
|
||||
stop_idx = js.find('stoppedByUser || err?.name === "AbortError"', catch_idx)
|
||||
finally_idx = js.find("} finally {", catch_idx)
|
||||
assert catch_idx < stop_idx < finally_idx, "the stop branch must live in the catch"
|
||||
# The stop branch only (the error `else` follows it and is not pinned here).
|
||||
branch = js[stop_idx : js.find("} else {", stop_idx)]
|
||||
assert "setUiState(UI_STATE.error" not in branch, "no error state on the stop path"
|
||||
assert "showErrorBanner" not in branch, "no error banner on the stop path"
|
||||
assert "if (wrap && acc && !persistedOnLeave)" in branch, (
|
||||
"only a partial WITH answer text is persisted (phase-20 dedupe)"
|
||||
)
|
||||
assert "closeThinkingBlock(wrap)" in branch
|
||||
assert "appendTuneButton(wrap)" in branch, "admin parity with the restore path"
|
||||
assert "appendStoppedNote(wrap)" in branch
|
||||
assert "stopped: true" in branch, "the owner-locked optional marker"
|
||||
assert "thinking: thinkingAcc || undefined" in branch
|
||||
assert "tools: toolAcc.length ? toolAcc : undefined" in branch
|
||||
# The confirmation rides the single settle in the finally.
|
||||
finally_block = js[finally_idx : finally_idx + 900]
|
||||
assert 'if (stoppedByUser) sendStatus.textContent = "Answer stopped."' in finally_block
|
||||
|
||||
|
||||
def test_stopped_note_helper_and_restore_path() -> None:
|
||||
"""appendStoppedNote: reuses/creates the .msg-meta row exactly like
|
||||
appendTuneButton (role=list → the span joins as a listitem), one
|
||||
.stopped-note per bubble — the aria-hidden stop-glyph SVG + the
|
||||
"Stopped" text (the accessible meaning). The restore path renders it
|
||||
for records with `m.stopped` (phase-14 optional-field convention —
|
||||
no version bump)."""
|
||||
js = _js()
|
||||
fn = js.find("function appendStoppedNote")
|
||||
assert fn != -1, "appendStoppedNote must exist"
|
||||
body = js[fn : js.find("\n}\n", fn)]
|
||||
assert 'querySelector(".msg-meta")' in body, "reuses the meta row when it exists"
|
||||
assert 'className = "msg-meta"' in body, "creates it otherwise"
|
||||
assert 'className = "stopped-note"' in body
|
||||
assert 'querySelector(".stopped-note")' in body, "one note per bubble"
|
||||
assert 'note.setAttribute("role", "listitem")' in body
|
||||
assert 'aria-hidden="true"' in body, "the glyph is decoration"
|
||||
assert '"Stopped"' in body, "the text carries the accessible meaning"
|
||||
# Restore path: the same helper, gated on the stored marker.
|
||||
rfn = js.find("function renderStoredMessage")
|
||||
rbody = js[rfn : js.find("\n}\n", rfn)]
|
||||
assert "if (m.stopped) appendStoppedNote(wrap)" in rbody
|
||||
|
||||
|
||||
def test_tool_branch_no_longer_writes_the_button_label() -> None:
|
||||
"""Phase 48 (owner-locked): the `tool` frame no longer relabels the
|
||||
button — it stays "Stop" for the whole in-flight turn; the
|
||||
calling-tool status lives in #send-status + the typing indicator's
|
||||
aria-label only (exactly where the phase-37 state used to write)."""
|
||||
js = _js()
|
||||
tool_idx = js.find('ev.type === "tool"')
|
||||
delta_idx = js.find('ev.type === "delta"')
|
||||
branch = js[tool_idx:delta_idx]
|
||||
assert "sendLabel" not in branch, "the button keeps its Stop label"
|
||||
assert "sendStatus.textContent = toolStatus" in branch
|
||||
assert 'setAttribute("aria-label", toolStatus)' in branch
|
||||
|
||||
|
||||
def test_composer_form_is_novalidate() -> None:
|
||||
"""Phase 48 (latent-defect fix, 2026-08-29): the composer form must
|
||||
skip browser constraint validation. The input is cleared after every
|
||||
send, so a `required` textarea would fail validation on the Stop
|
||||
click/Enter — the `submit` event never fires and handleSend's
|
||||
in-flight guard never runs, so the Stop control is dead. The `!text`
|
||||
guard in app.js is the real empty-input check (same precedent as the
|
||||
tuning form's noValidate)."""
|
||||
html = _html()
|
||||
composer = html.find('id="composer"')
|
||||
assert composer != -1, "index.html must contain #composer"
|
||||
form_tag = html[html.rfind("<form", 0, composer) : html.find(">", composer) + 1]
|
||||
assert "novalidate" in form_tag.lower(), (
|
||||
"the composer form must carry novalidate — a `required` input that "
|
||||
"is empty in flight would silently block the Stop submit"
|
||||
)
|
||||
textarea = html[composer: html.find("</textarea>", composer)]
|
||||
assert not re.search(r"\brequired\b", textarea), (
|
||||
"the composer textarea must not carry `required` (see novalidate)"
|
||||
)
|
||||
|
||||
@@ -193,7 +193,9 @@ def test_turn_end_focus_does_not_scroll() -> None:
|
||||
up would yank them to the bottom at the moment the turn ends.
|
||||
preventScroll keeps the keyboard flow without the scroll."""
|
||||
js = _js()
|
||||
finally_idx = js.find("// done | error → idle: always settle, always focus back")
|
||||
# Phase 48: the settle line gained the user-stop terminal (stop →
|
||||
# idle, no banner) — the focus-back contract is unchanged.
|
||||
finally_idx = js.find("// done | error | stop → idle: always settle, always focus back")
|
||||
assert finally_idx != -1, "the turn's finally block must exist"
|
||||
block = js[finally_idx : js.find("\n}", finally_idx)]
|
||||
assert 'input.focus({ preventScroll: true })' in block
|
||||
|
||||
@@ -53,15 +53,20 @@ def test_tool_branch_is_a_first_class_turn_branch() -> None:
|
||||
|
||||
|
||||
def test_calling_tool_label_strings() -> None:
|
||||
"""The 'calling tool' label strings the story keys off: the button
|
||||
text and the status/typing-indicator labels. Phase 39 centralizes
|
||||
the brand prefix: the name resolves from window.BOR_BRAND at call
|
||||
time via brand() (the default name renders the same bytes)."""
|
||||
"""The 'calling tool' label strings the story keys off: the
|
||||
status/typing-indicator labels. Phase 48 (owner-locked 2026-08-29)
|
||||
revised the phase-37 contract: the button no longer relabels to
|
||||
"Calling tool…" — it stays the enabled "Stop" control for the whole
|
||||
in-flight turn (no sendLabel write in the branch); the calling-tool
|
||||
status lives in #send-status + the typing-indicator aria-label only.
|
||||
Phase 39 centralizes the brand prefix: the name resolves from
|
||||
window.BOR_BRAND at call time via brand() (the default name renders
|
||||
the same bytes)."""
|
||||
js = _js()
|
||||
tool_idx = js.find('ev.type === "tool"')
|
||||
delta_idx = js.find('ev.type === "delta"')
|
||||
branch = js[tool_idx:delta_idx]
|
||||
assert '"Calling tool…"' in branch, "the button carries the calling-tool text"
|
||||
assert "sendLabel" not in branch, "phase 48: the button keeps its Stop label"
|
||||
assert "`${brand()} is listing documents`" in branch
|
||||
assert "`${brand()} is reading ${argument}`" in branch
|
||||
assert "sendStatus.textContent = toolStatus" in branch, (
|
||||
|
||||
@@ -289,6 +289,7 @@ def _chunk(
|
||||
class _FakeChatStream:
|
||||
def __init__(self, chunks: list) -> None:
|
||||
self._chunks = list(chunks)
|
||||
self.close_calls = 0
|
||||
|
||||
def __aiter__(self):
|
||||
self._i = 0
|
||||
@@ -301,6 +302,11 @@ class _FakeChatStream:
|
||||
self._i += 1
|
||||
return chunk
|
||||
|
||||
async def close(self) -> None:
|
||||
"""The openai SDK ``AsyncStream.close()`` (phase 48): ``chat_stream``
|
||||
awaits it on every exit after a successful ``create()``."""
|
||||
self.close_calls += 1
|
||||
|
||||
|
||||
class _FakeCompletion:
|
||||
"""One fake non-streaming ChatCompletion (``choices[].message`` shape).
|
||||
|
||||
@@ -0,0 +1,203 @@
|
||||
"""Unit: deterministic model-stream teardown (phase 48, task 01).
|
||||
|
||||
The openai SDK stream must be closed on **every** exit of
|
||||
``LLMClient.chat_stream`` after a successful ``create()`` — normal
|
||||
exhaustion, a wrapped mid-stream failure, and consumer abandon
|
||||
(``aclose()`` on the ``chat_stream`` generator — the stop/cancel path,
|
||||
2026-08-29, ``TODO.md`` L3). The fakes stand in at the SDK boundary: a
|
||||
recording async stream (small sleeps between chunks so an abandon can
|
||||
land mid-iteration) behind an ``AsyncOpenAI``-shaped client, following
|
||||
the fake patterns of ``tests/unit/test_llm_client.py``.
|
||||
|
||||
Note on the close method: the phase text says ``aclose()`` — the
|
||||
installed openai SDK's ``AsyncStream`` exposes the async ``close()``,
|
||||
which awaits the underlying httpx response's ``aclose()``; that is the
|
||||
method under test (a quiet no-op on an already-ended SDK stream, so the
|
||||
completed path stays byte-identical).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from types import SimpleNamespace
|
||||
from typing import Any, cast
|
||||
|
||||
import pytest
|
||||
|
||||
from app.config import Settings
|
||||
from app.rag.llm import LLMClient, LLMError, StreamPiece
|
||||
|
||||
|
||||
def _settings(**kwargs: Any) -> Settings:
|
||||
kwargs.setdefault("_env_file", None)
|
||||
return Settings(**kwargs) # pyright: ignore[reportCallIssue]
|
||||
|
||||
|
||||
def _chunk(content: str) -> SimpleNamespace:
|
||||
"""One fake ChatCompletionChunk (``choices[].delta.content`` shape)."""
|
||||
return SimpleNamespace(choices=[SimpleNamespace(delta=SimpleNamespace(content=content))])
|
||||
|
||||
|
||||
def _tool_chunk() -> SimpleNamespace:
|
||||
"""One chunk carrying a malformed-arguments tool call (index 0)."""
|
||||
fn = SimpleNamespace(name="read_document", arguments='{"source": "Homelab",')
|
||||
tc = SimpleNamespace(index=0, id="call_x", function=fn)
|
||||
delta = SimpleNamespace(content=None, tool_calls=[tc])
|
||||
return SimpleNamespace(choices=[SimpleNamespace(delta=delta)])
|
||||
|
||||
|
||||
class _RecordingStream:
|
||||
"""A fake SDK stream: yields *chunks* (small sleeps between them, so
|
||||
an abandon can land mid-iteration) and records ``close()`` calls."""
|
||||
|
||||
def __init__(self, chunks: list, delay: float = 0.005) -> None:
|
||||
self._chunks = list(chunks)
|
||||
self._delay = delay
|
||||
self._i = 0
|
||||
self.close_calls = 0
|
||||
|
||||
def __aiter__(self) -> _RecordingStream:
|
||||
return self
|
||||
|
||||
async def __anext__(self) -> SimpleNamespace:
|
||||
if self._i >= len(self._chunks):
|
||||
raise StopAsyncIteration
|
||||
await asyncio.sleep(self._delay)
|
||||
chunk = self._chunks[self._i]
|
||||
self._i += 1
|
||||
return chunk
|
||||
|
||||
async def close(self) -> None:
|
||||
self.close_calls += 1
|
||||
|
||||
|
||||
class _FailingStream(_RecordingStream):
|
||||
"""Same shape, but ``__anext__`` raises a transport error after
|
||||
*fail_after* chunks (the mid-iteration failure path)."""
|
||||
|
||||
def __init__(self, chunks: list, fail_after: int, delay: float = 0.005) -> None:
|
||||
super().__init__(chunks, delay)
|
||||
self._fail_after = fail_after
|
||||
|
||||
async def __anext__(self) -> SimpleNamespace:
|
||||
self._i += 1
|
||||
if self._i > self._fail_after:
|
||||
raise ConnectionError("simulated mid-stream drop")
|
||||
if self._i > len(self._chunks):
|
||||
raise StopAsyncIteration
|
||||
await asyncio.sleep(self._delay)
|
||||
chunk = self._chunks[self._i - 1]
|
||||
return chunk
|
||||
|
||||
|
||||
class _FakeCompletions:
|
||||
"""``chat.completions.create(stream=True)`` → the fake stream (or a
|
||||
create-level failure)."""
|
||||
|
||||
def __init__(self, stream: _RecordingStream | Exception) -> None:
|
||||
self._stream = stream
|
||||
self.kwargs: dict | None = None
|
||||
|
||||
async def create(self, **kwargs: Any):
|
||||
self.kwargs = kwargs
|
||||
assert kwargs.get("stream") is True
|
||||
if isinstance(self._stream, Exception):
|
||||
raise self._stream
|
||||
return self._stream
|
||||
|
||||
|
||||
def _make_client(
|
||||
stream: _RecordingStream | Exception,
|
||||
) -> tuple[LLMClient, _FakeCompletions]:
|
||||
completions = _FakeCompletions(stream)
|
||||
llm = LLMClient(_settings())
|
||||
llm._client = SimpleNamespace( # pyright: ignore[reportAttributeAccessIssue]
|
||||
chat=SimpleNamespace(completions=completions)
|
||||
)
|
||||
return llm, completions
|
||||
|
||||
|
||||
def _content_chunks(n: int) -> list:
|
||||
return [_chunk(f"piece {i} ") for i in range(1, n + 1)]
|
||||
|
||||
|
||||
async def _drain(llm: LLMClient) -> list[StreamPiece]:
|
||||
"""Tools-less drain: without a ``tools`` list no ToolCallPiece can
|
||||
appear (the phase-37 contract)."""
|
||||
pieces = [p async for p in llm.chat_stream([{"role": "user", "content": "q"}])]
|
||||
assert all(isinstance(p, StreamPiece) for p in pieces)
|
||||
return cast("list[StreamPiece]", pieces)
|
||||
|
||||
|
||||
def test_full_consumption_closes_stream_exactly_once() -> None:
|
||||
"""(a) A fully consumed stream still gets the explicit close — the
|
||||
completed path keeps its behavior (a quiet no-op on the real SDK
|
||||
stream) while the teardown is pinned."""
|
||||
stream = _RecordingStream(_content_chunks(3))
|
||||
llm, _ = _make_client(stream)
|
||||
pieces = asyncio.run(_drain(llm))
|
||||
assert [p.text for p in pieces] == ["piece 1 ", "piece 2 ", "piece 3 "]
|
||||
assert stream.close_calls == 1
|
||||
|
||||
|
||||
def test_mid_iteration_abandon_closes_stream_before_close_completes() -> None:
|
||||
"""(b) Abandoning the ``chat_stream`` generator after the first
|
||||
piece (``await gen.aclose()`` — the consumer-stop path) must await
|
||||
the SDK stream's close before the generator's close completes."""
|
||||
stream = _RecordingStream(_content_chunks(10))
|
||||
llm, _ = _make_client(stream)
|
||||
|
||||
async def abandon_after_first() -> None:
|
||||
gen = llm.chat_stream([{"role": "user", "content": "q"}])
|
||||
first = await gen.__anext__()
|
||||
assert isinstance(first, StreamPiece)
|
||||
assert first.text == "piece 1 "
|
||||
# The generator is suspended at its first yield; aclose must run
|
||||
# the finally (the SDK stream's close) before it returns.
|
||||
await gen.aclose()
|
||||
|
||||
asyncio.run(abandon_after_first())
|
||||
assert stream.close_calls == 1
|
||||
|
||||
|
||||
def test_create_failure_wraps_and_never_closes() -> None:
|
||||
"""A ``create()`` failure keeps today's wrap — generic exception →
|
||||
``LLMError`` with the base URL — and no stream exists to close."""
|
||||
llm, _ = _make_client(ConnectionError("connection reset by peer"))
|
||||
with pytest.raises(LLMError, match="chat stream from .* failed") as exc:
|
||||
asyncio.run(_drain(llm))
|
||||
assert "connection reset by peer" in str(exc.value)
|
||||
|
||||
|
||||
def test_mid_iteration_failure_wraps_and_closes() -> None:
|
||||
"""A failure inside the ``async for`` wraps exactly as before
|
||||
(``LLMError`` with the original message) — and the stream is closed
|
||||
on the exception path."""
|
||||
stream = _FailingStream(_content_chunks(10), fail_after=2)
|
||||
llm, _ = _make_client(stream)
|
||||
|
||||
async def drain() -> None:
|
||||
async for _ in llm.chat_stream([{"role": "user", "content": "q"}]):
|
||||
pass
|
||||
|
||||
with pytest.raises(LLMError, match="simulated mid-stream drop"):
|
||||
asyncio.run(drain())
|
||||
assert stream.close_calls == 1
|
||||
|
||||
|
||||
def test_llm_error_materialization_passes_through_and_closes() -> None:
|
||||
"""An ``LLMError`` from tool-call materialization (after the loop,
|
||||
before normal exhaustion) re-raises unwrapped — and the stream is
|
||||
still closed on the way out."""
|
||||
stream = _RecordingStream([_tool_chunk()])
|
||||
llm, _ = _make_client(stream)
|
||||
|
||||
async def drain() -> None:
|
||||
async for _ in llm.chat_stream(
|
||||
[{"role": "user", "content": "q"}],
|
||||
tools=[{"type": "function", "function": {"name": "read_document"}}],
|
||||
):
|
||||
pass
|
||||
|
||||
with pytest.raises(LLMError, match="malformed tool-call arguments"):
|
||||
asyncio.run(drain())
|
||||
assert stream.close_calls == 1
|
||||
Reference in New Issue
Block a user