fix(phased-execution): resume failed session on retry, surface errors, thinking/compaction indicators
- retries now resume the failed executor's session (pre/post session tracking replaces the broken mtime-vs-ref check) - report recovery from the session file when the JSON stream loses its tail (child signaled mid-flush) - progress.mjs: thinking indicators (◐ thinking… / ◑ thought for Ns), compaction (⧉ …) and provider auto-retry lines; trims trailing whitespace so no blank lines after LLM text; exits 1 on truncated stream, model error, or missing final text (pi --mode json always exits 0 even on errors) - explicit ✗ ERROR lines + exit codes: 0 ok, 1 phase failed, 130/143 interrupted (INT/TERM traps; post-pipeline check covers the case where bash suppresses the INT trap after a job dies from SIGINT) - PIPESTATUS captured in a single statement (any following command resets it) - skills dir: .gitignore README.md so pi's skill scanner (which honors .gitignore) stops warning 'description is required'
This commit is contained in:
@@ -14,13 +14,22 @@ set -uo pipefail
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$SCRIPT_DIR/lib.sh"
|
||||
|
||||
# Make interruptions visible: state stays in .agent/phases/todo, and the
|
||||
# failed executor's session is still resumable on the next run.
|
||||
trap 'echo; echo "✗ ERROR: interrupted (SIGINT) — ${phase:-the pipeline} is left in $PHASE_TODO/; re-run to continue where it stopped" >&2; exit 130' INT
|
||||
trap 'echo; echo "✗ ERROR: interrupted (SIGTERM) — ${phase:-the pipeline} is left in $PHASE_TODO/; re-run to continue where it stopped" >&2; exit 143' TERM
|
||||
|
||||
cd "$(find_root)" || die "no .agent/phases/todo found in this or parent directories (run /to-phase or /audit-create first)"
|
||||
|
||||
build_pi_args
|
||||
delivered=()
|
||||
while phase="$(next_phase)"; do
|
||||
[[ -n "$phase" ]] || break
|
||||
execute_phase "$phase" || exit 1
|
||||
if ! execute_phase "$phase"; then
|
||||
echo "✗ ERROR: pipeline stopped — $phase FAILED after $MAX_FIX_ATTEMPTS attempts" >&2
|
||||
echo " fix the issues above, then re-run this script to continue where it stopped" >&2
|
||||
exit 1
|
||||
fi
|
||||
delivered+=("$phase")
|
||||
done
|
||||
|
||||
|
||||
+111
-26
@@ -59,6 +59,35 @@ latest_child_session() {
|
||||
ls -t "$PHASE_SESSIONS"/*.jsonl 2>/dev/null | head -n1 || true
|
||||
}
|
||||
|
||||
# Write the last assistant text message of a session file to a report file.
|
||||
# Recovery path: when the child's JSON stream loses its tail (e.g. the child
|
||||
# is signaled mid-flush), the session file still holds the final report.
|
||||
# Returns 1 when there is no recoverable assistant text.
|
||||
recover_report() {
|
||||
local report="$1" session="$2"
|
||||
[[ -f "$session" ]] || return 1
|
||||
node -e '
|
||||
const [report, file] = process.argv.slice(1);
|
||||
const { readFileSync, writeFileSync } = require("node:fs");
|
||||
let last = "";
|
||||
for (const line of readFileSync(file, "utf8").split("\n")) {
|
||||
const t = line.trim();
|
||||
if (!t) continue;
|
||||
let e;
|
||||
try { e = JSON.parse(t); } catch { continue; }
|
||||
const m = e && e.type === "message" ? e.message : null;
|
||||
if (!m || m.role !== "assistant") continue;
|
||||
let text = "";
|
||||
if (typeof m.content === "string") text = m.content;
|
||||
else if (Array.isArray(m.content))
|
||||
text = m.content.filter((b) => b && b.type === "text").map((b) => b.text).join("");
|
||||
if (text.trim()) last = text.trim();
|
||||
}
|
||||
if (!last) process.exit(1);
|
||||
writeFileSync(report, last + "\n");
|
||||
' "$report" "$session"
|
||||
}
|
||||
|
||||
# --- prompts ------------------------------------------------------------------
|
||||
first_prompt() {
|
||||
local phase="$1"
|
||||
@@ -82,41 +111,63 @@ fix_prompt() {
|
||||
}
|
||||
|
||||
# --- child executor -----------------------------------------------------------
|
||||
# run_child <phase> <attempt> <prompt>
|
||||
# run_child <phase> <attempt> <prompt> [resume-session]
|
||||
# Attempt 1: fresh session in .agent/phase-sessions/.
|
||||
# Attempt N>1: resume attempt N-1's session (unless FRESH_FIX=1 or no session
|
||||
# was created — in that case a fresh ephemeral session with the failure context).
|
||||
# Progress (tool calls + assistant text) streams to the terminal live via
|
||||
# scripts/progress.mjs, which also writes the final assistant message to:
|
||||
# .agent/reports/<base>.a<attempt>.md
|
||||
# Attempt N>1: resumes the given session file — the failed executor's own
|
||||
# session, tracked by execute_phase (so retries keep its work). When no
|
||||
# session was captured (or FRESH_FIX=1) a fresh ephemeral session runs with
|
||||
# the failure context instead.
|
||||
# Progress (tool calls, assistant text, thinking, compaction) streams to the
|
||||
# terminal live via scripts/progress.mjs, which also writes the final
|
||||
# assistant message to: .agent/reports/<base>.a<attempt>.md
|
||||
# Child stderr → .agent/reports/<base>.a<attempt>.err
|
||||
# QUIET=1 suppresses the progress display (report file is still written).
|
||||
# Sets CHILD_RC.
|
||||
# Sets CHILD_RC (pi's exit code) and PROGRESS_RC (progress.mjs's exit code;
|
||||
# non-zero means the stream ended without a clean final report).
|
||||
|
||||
# Run `pi … | progress.mjs`, capturing both exit codes. Must be the direct
|
||||
# caller of the pipeline, and PIPESTATUS must be copied in ONE statement —
|
||||
# any following command (even a plain assignment) resets it.
|
||||
_run_pi_pipeline() {
|
||||
local errf="$1"; shift
|
||||
"$@" 2>"$errf" | "${PROGRESS[@]}"
|
||||
local rcs=( "${PIPESTATUS[@]}" )
|
||||
CHILD_RC=${rcs[0]}
|
||||
PROGRESS_RC=${rcs[1]}
|
||||
|
||||
# Ctrl+C sends INT to the whole foreground group: pi and progress.mjs die
|
||||
# with 130, and bash then SUPPRESSES the script's INT trap (it assumes the
|
||||
# job already handled the signal). Detect the kill here and exit as if the
|
||||
# trap had fired, so the failure is always visible.
|
||||
if (( CHILD_RC == 130 || PROGRESS_RC == 130 )); then
|
||||
echo
|
||||
echo "✗ ERROR: interrupted (SIGINT) — phase is left in $PHASE_TODO/; re-run to continue" >&2
|
||||
exit 130
|
||||
fi
|
||||
if (( CHILD_RC == 143 || PROGRESS_RC == 143 )); then
|
||||
echo
|
||||
echo "✗ ERROR: interrupted (SIGTERM) — phase is left in $PHASE_TODO/; re-run to continue" >&2
|
||||
exit 143
|
||||
fi
|
||||
}
|
||||
|
||||
run_child() {
|
||||
local phase="$1" attempt="$2" prompt="$3"
|
||||
local phase="$1" attempt="$2" prompt="$3" resume_session="${4:-}"
|
||||
local base="${phase%.md}"
|
||||
local out="$PHASE_REPORTS/$base.a$attempt.md"
|
||||
local errf="$PHASE_REPORTS/$base.a$attempt.err"
|
||||
local ref="$PHASE_REPORTS/.ref"
|
||||
local session
|
||||
local PROGRESS=(node "$SKILL_DIR/scripts/progress.mjs" "$out")
|
||||
[[ "${QUIET:-0}" == "1" ]] && PROGRESS+=(--quiet)
|
||||
|
||||
if (( attempt == 1 )); then
|
||||
touch "$ref"
|
||||
pi "${PI_ARGS[@]}" --session-dir "$PHASE_SESSIONS" --name "$base" --mode json "$prompt" 2>"$errf" | "${PROGRESS[@]}"
|
||||
_run_pi_pipeline "$errf" pi "${PI_ARGS[@]}" --session-dir "$PHASE_SESSIONS" --name "$base" --mode json "$prompt"
|
||||
elif [[ "${FRESH_FIX:-0}" == "1" || -z "$resume_session" || ! -f "$resume_session" ]]; then
|
||||
echo " (no resumable session — starting fresh)"
|
||||
_run_pi_pipeline "$errf" pi "${PI_ARGS[@]}" --no-session --mode json "$prompt"
|
||||
else
|
||||
touch "$ref"
|
||||
session="$(latest_child_session)"
|
||||
if [[ "${FRESH_FIX:-0}" != "1" && -n "$session" && "$session" -nt "$ref" ]]; then
|
||||
echo " (resuming failed executor session: ${session##*/})"
|
||||
pi "${PI_ARGS[@]}" --session "$session" --mode json "$prompt" 2>"$errf" | "${PROGRESS[@]}"
|
||||
else
|
||||
echo " (no resumable session — starting fresh)"
|
||||
pi "${PI_ARGS[@]}" --no-session --mode json "$prompt" 2>"$errf" | "${PROGRESS[@]}"
|
||||
fi
|
||||
echo " (resuming failed executor session: ${resume_session##*/})"
|
||||
_run_pi_pipeline "$errf" pi "${PI_ARGS[@]}" --session "$resume_session" --mode json "$prompt"
|
||||
fi
|
||||
CHILD_RC=${PIPESTATUS[0]}
|
||||
}
|
||||
|
||||
# --- validation gate ----------------------------------------------------------
|
||||
@@ -142,22 +193,47 @@ execute_phase() {
|
||||
local phase="$1"
|
||||
local base="${phase%.md}"
|
||||
local attempt=1 errors=""
|
||||
local last_session="" pre post
|
||||
command -v node >/dev/null 2>&1 || die "node not found on PATH (needed to render phase progress)"
|
||||
mkdir -p "$PHASE_DONE" "$PHASE_REPORTS" "$PHASE_SESSIONS"
|
||||
ensure_validate
|
||||
|
||||
while (( attempt <= MAX_FIX_ATTEMPTS )); do
|
||||
echo "━━ $phase — attempt $attempt/$MAX_FIX_ATTEMPTS ━━"
|
||||
pre="$(latest_child_session)"
|
||||
if (( attempt == 1 )); then
|
||||
run_child "$phase" 1 "$(first_prompt "$phase")"
|
||||
else
|
||||
run_child "$phase" "$attempt" "$(fix_prompt "$errors")"
|
||||
run_child "$phase" "$attempt" "$(fix_prompt "$errors")" "$last_session"
|
||||
fi
|
||||
# Track which session file this attempt used, so the next attempt resumes
|
||||
# exactly this phase's failed session (not just "latest in the directory").
|
||||
post="$(latest_child_session)"
|
||||
if [[ -n "$post" && "$post" != "$pre" ]]; then
|
||||
last_session="$post"
|
||||
fi
|
||||
|
||||
# The child can be signaled mid-flush: the session file then holds a final
|
||||
# report the stream lost. Recover it so a completed phase is not retried.
|
||||
if (( CHILD_RC == 0 )) && [[ -f "$PHASE_REPORTS/$base.a$attempt.md" ]] \
|
||||
&& grep -q "no final assistant message" "$PHASE_REPORTS/$base.a$attempt.md"; then
|
||||
if [[ -n "$last_session" ]] && recover_report "$PHASE_REPORTS/$base.a$attempt.md" "$last_session"; then
|
||||
warn "stream lost the final message — report recovered from ${last_session##*/}"
|
||||
PROGRESS_RC=0
|
||||
fi
|
||||
fi
|
||||
|
||||
errors=""
|
||||
if (( CHILD_RC != 0 )); then
|
||||
warn "child pi exited with code $CHILD_RC — see $PHASE_REPORTS/$base.a$attempt.err"
|
||||
errors+="[child pi exited with code $CHILD_RC]"$'\n'"$(tail -c 4000 "$PHASE_REPORTS/$base.a$attempt.err" 2>/dev/null)"
|
||||
fi
|
||||
if (( PROGRESS_RC != 0 )); then
|
||||
warn "child run ended without a clean final report — see $PHASE_REPORTS/$base.a$attempt.md"
|
||||
errors+="[child run ended without a clean final report]"$'\n'"$(tail -c 4000 "$PHASE_REPORTS/$base.a$attempt.err" 2>/dev/null)"
|
||||
fi
|
||||
if ! run_validation "$PHASE_REPORTS/$base.a$attempt.validate"; then
|
||||
warn ".agent/validate.sh FAILED — see $PHASE_REPORTS/$base.a$attempt.validate"
|
||||
errors+="[.agent/validate.sh FAILED]"$'\n'"$(tail -n 120 "$PHASE_REPORTS/$base.a$attempt.validate" 2>/dev/null)"
|
||||
fi
|
||||
|
||||
@@ -178,8 +254,17 @@ execute_phase() {
|
||||
attempt=$(( attempt + 1 ))
|
||||
done
|
||||
|
||||
echo "✗ $phase FAILED after $MAX_FIX_ATTEMPTS attempts — left in $PHASE_TODO/." >&2
|
||||
echo " logs: $PHASE_REPORTS/$base.a*.{md,err,validate}" >&2
|
||||
echo " resume: pi --session-dir $PHASE_SESSIONS -c" >&2
|
||||
{
|
||||
echo "✗ $phase FAILED after $MAX_FIX_ATTEMPTS attempts — left in $PHASE_TODO/."
|
||||
echo " last errors:"
|
||||
printf '%s\n' "$errors" | tail -n 40 | sed 's/^/ /'
|
||||
echo " logs: $PHASE_REPORTS/$base.a*.{md,err,validate}"
|
||||
if [[ -n "${last_session:-}" && -f "${last_session:-}" ]]; then
|
||||
echo " resume: re-run this script (it auto-resumes the failed session), or manually:"
|
||||
echo " pi --session $last_session -c \"review the failures above, fix them, re-validate\""
|
||||
else
|
||||
echo " resume: re-run this script to retry automatically"
|
||||
fi
|
||||
} >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
@@ -6,6 +6,15 @@
|
||||
//
|
||||
// Quiet mode still writes the report file (for harness logs); it only
|
||||
// suppresses the progress display.
|
||||
//
|
||||
// Shows: tool calls, assistant text, thinking indicators, context
|
||||
// compaction, and provider auto-retries.
|
||||
//
|
||||
// Exit code: 0 for a clean run (agent_end seen, no model errors, final
|
||||
// assistant text received). 1 when the stream ended without agent_end
|
||||
// (child crashed or was killed), the model reported an error, or no final
|
||||
// assistant text was produced. `pi --mode json` itself exits 0 even on
|
||||
// model errors, so the harness relies on this exit code to detect failures.
|
||||
|
||||
import { writeFileSync } from "node:fs";
|
||||
|
||||
@@ -95,12 +104,39 @@ let sessionId = "";
|
||||
let lastUsage = null;
|
||||
let lastAssistantText = "";
|
||||
let sawError = false;
|
||||
let sawAgentEnd = false;
|
||||
let thinkingSince = null;
|
||||
const failures = [];
|
||||
|
||||
// Streaming deltas (text / thinking / tool-call args) arrive inside
|
||||
// message_update.assistantMessageEvent; only thinking is surfaced live.
|
||||
function handleUpdate(ame) {
|
||||
if (!ame || typeof ame !== "object") return;
|
||||
switch (ame.type) {
|
||||
case "thinking_start":
|
||||
thinkingSince = Date.now();
|
||||
out(" " + c.dim("◐ thinking…"));
|
||||
break;
|
||||
case "thinking_end":
|
||||
if (thinkingSince !== null) {
|
||||
const secs = Math.max(1, Math.round((Date.now() - thinkingSince) / 1000));
|
||||
out(c.dim(` ◑ thought for ${secs}s`));
|
||||
thinkingSince = null;
|
||||
}
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
function handle(ev) {
|
||||
switch (ev.type) {
|
||||
case "session":
|
||||
sessionId = ev.id ?? "";
|
||||
break;
|
||||
case "agent_end":
|
||||
sawAgentEnd = true;
|
||||
break;
|
||||
case "tool_execution_start":
|
||||
out(" " + c.cyan("⏺ " + ev.toolName) + " " + c.dim(summarizeTool(ev.toolName, ev.args)));
|
||||
break;
|
||||
@@ -109,52 +145,104 @@ function handle(ev) {
|
||||
break;
|
||||
case "message_update":
|
||||
if (ev.usage) lastUsage = ev.usage;
|
||||
handleUpdate(ev.assistantMessageEvent);
|
||||
break;
|
||||
case "message_end": {
|
||||
const m = ev.message ?? {};
|
||||
if (m.role === "assistant") {
|
||||
const text = textOf(m);
|
||||
// Trim so trailing newlines in model output don't render as blank
|
||||
// lines after the message.
|
||||
const text = textOf(m).trim();
|
||||
if (text) {
|
||||
lastAssistantText = text;
|
||||
out(wrap(text, 2));
|
||||
}
|
||||
if (m.stopReason === "error") {
|
||||
if (m.stopReason === "error" || m.stopReason === "aborted") {
|
||||
sawError = true;
|
||||
out(c.red(" ✗ " + (m.errorMessage ?? "assistant error")));
|
||||
failures.push(`assistant ${m.stopReason}: ${m.errorMessage ?? ""}`);
|
||||
out(c.red(" ✗ " + (m.errorMessage ?? `assistant ${m.stopReason}`)));
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
case "compaction_start":
|
||||
out(" " + c.yellow("⧉ compacting context (" + (ev.reason ?? "auto") + ")…"));
|
||||
break;
|
||||
case "compaction_end":
|
||||
if (ev.aborted || ev.errorMessage) {
|
||||
sawError = true;
|
||||
failures.push(`compaction ${ev.aborted ? "aborted" : "failed"}: ${ev.errorMessage ?? ""}`);
|
||||
out(
|
||||
c.red(
|
||||
" ✗ compaction " + (ev.aborted ? "aborted" : "failed") + (ev.errorMessage ? ": " + ev.errorMessage : ""),
|
||||
),
|
||||
);
|
||||
} else {
|
||||
out(c.dim(" ✓ context compacted"));
|
||||
}
|
||||
break;
|
||||
case "auto_retry_start":
|
||||
out(
|
||||
c.yellow(
|
||||
` ↻ provider error, retrying ${ev.attempt}/${ev.maxAttempts} in ${Math.round((ev.delayMs ?? 0) / 1000)}s: ${truncate(
|
||||
ev.errorMessage ?? "",
|
||||
100,
|
||||
)}`,
|
||||
),
|
||||
);
|
||||
break;
|
||||
case "auto_retry_end":
|
||||
if (ev.success) {
|
||||
out(c.dim(" ✓ retry succeeded"));
|
||||
} else {
|
||||
sawError = true;
|
||||
failures.push(`retries exhausted: ${ev.finalError ?? ""}`);
|
||||
out(c.red(" ✗ retries exhausted: " + truncate(ev.finalError ?? "", 120)));
|
||||
}
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
function processLine(line) {
|
||||
line = line.trim();
|
||||
if (!line) return;
|
||||
let ev;
|
||||
try {
|
||||
ev = JSON.parse(line);
|
||||
} catch {
|
||||
out(c.yellow(" ? " + truncate(line, 120)));
|
||||
return;
|
||||
}
|
||||
handle(ev);
|
||||
}
|
||||
|
||||
let buf = "";
|
||||
process.stdin.setEncoding("utf8");
|
||||
process.stdin.on("data", (chunk) => {
|
||||
buf += chunk;
|
||||
let idx;
|
||||
while ((idx = buf.indexOf("\n")) >= 0) {
|
||||
const line = buf.slice(0, idx).trim();
|
||||
const line = buf.slice(0, idx);
|
||||
buf = buf.slice(idx + 1);
|
||||
if (!line) continue;
|
||||
let ev;
|
||||
try {
|
||||
ev = JSON.parse(line);
|
||||
} catch {
|
||||
out(c.yellow(" ? " + truncate(line, 120)));
|
||||
continue;
|
||||
}
|
||||
handle(ev);
|
||||
processLine(line);
|
||||
}
|
||||
});
|
||||
|
||||
process.stdin.on("end", () => {
|
||||
// A trailing line without a final newline (child killed mid-flush) still
|
||||
// counts — process whatever is left in the buffer.
|
||||
if (buf.trim()) {
|
||||
processLine(buf);
|
||||
buf = "";
|
||||
}
|
||||
|
||||
writeFileSync(
|
||||
reportPath,
|
||||
lastAssistantText ? lastAssistantText + "\n" : "(no final assistant message — see the .err log)\n",
|
||||
);
|
||||
|
||||
const u = lastUsage ?? {};
|
||||
const bits = [];
|
||||
if (u.input) bits.push(`↑${u.input}`);
|
||||
@@ -162,5 +250,17 @@ process.stdin.on("end", () => {
|
||||
if (u.cacheRead) bits.push(`R${u.cacheRead}`);
|
||||
if (sessionId) bits.push("session " + String(sessionId).slice(0, 8));
|
||||
if (bits.length) out(c.dim(" · " + bits.join(" ")));
|
||||
out(sawError ? c.red(" · child finished with errors") : c.green(" · child done"));
|
||||
|
||||
if (!sawAgentEnd) {
|
||||
failures.push("stream ended without agent_end (child crashed or was killed)");
|
||||
out(c.red(" ✗ stream ended without agent_end — child crashed or was killed (see .err log)"));
|
||||
} else if (sawError) {
|
||||
out(c.red(" ✗ child finished with errors: " + failures.join("; ")));
|
||||
} else if (!lastAssistantText) {
|
||||
failures.push("no final assistant text");
|
||||
out(c.red(" ✗ no final assistant text — no report available (see .err log)"));
|
||||
} else {
|
||||
out(c.green(" · child done"));
|
||||
}
|
||||
process.exitCode = sawError || !sawAgentEnd || !lastAssistantText ? 1 : 0;
|
||||
});
|
||||
|
||||
@@ -11,6 +11,11 @@ set -uo pipefail
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$SCRIPT_DIR/lib.sh"
|
||||
|
||||
# Make interruptions visible: state stays in .agent/phases/todo, and the
|
||||
# failed executor's session is still resumable on the next run.
|
||||
trap 'echo; echo "✗ ERROR: interrupted (SIGINT) — ${phase:-this phase} is left in $PHASE_TODO/; re-run to continue" >&2; exit 130' INT
|
||||
trap 'echo; echo "✗ ERROR: interrupted (SIGTERM) — ${phase:-this phase} is left in $PHASE_TODO/; re-run to continue" >&2; exit 143' TERM
|
||||
|
||||
cd "$(find_root)" || die "no .agent/phases/todo found in this or parent directories (run /to-phase or /audit-create first)"
|
||||
|
||||
phase="${1:-}"
|
||||
@@ -27,4 +32,10 @@ else
|
||||
fi
|
||||
|
||||
build_pi_args
|
||||
execute_phase "$phase"
|
||||
if execute_phase "$phase"; then
|
||||
exit 0
|
||||
fi
|
||||
echo "✗ ERROR: phase $phase FAILED after $MAX_FIX_ATTEMPTS attempts" >&2
|
||||
echo " reports: $PHASE_REPORTS/${phase%.md}.a*.{md,err,validate}" >&2
|
||||
echo " resume: re-run this script — the failed executor session is resumed automatically" >&2
|
||||
exit 1
|
||||
|
||||
Reference in New Issue
Block a user