#!/usr/bin/env bash # tests/unit/10_autophase.sh -- bin/ai-sandbox-autophase: the pipeline driver. # # Nothing here starts a container. The two things the driver is built on are both # stood in for, faithfully: # # * the LAUNCHER -- a sandbox-pi() function written into a fixture rc file, exactly # like the one people keep in ~/.bashrc.d. It records the argv it was handed (that # is what "the launcher, then --, then the harness command" has to mean) and then # runs what it was told to run, so the pipeline is exercised for real. It also # exports a canary token the way the real one calls `pass`, which is how these # tests check that no printed surface repeats a value. # * the PIPELINE LIBRARY -- tests/fixtures/phase-lib is a contract fixture: the same # function names, the same `pi ... | progress` call, the same gate call, the same # host-side commit+push at a phase's commit point. The last test runs the REAL # library as well, when it is installed. # # The git remote is a local bare repository: a push really happens, and really fails on # demand, with no network involved. source "$TESTS/lib.sh" APBIN="$(dirname "$AISB")/ai-sandbox-autophase" LIB_FIX="$TESTS/fixtures/phase-lib/lib.sh" CANARY="CANARY-TOKEN-9d41f6" REAL_GIT="$(command -v git)" mkdir -p "$WORK/bin" t_name "the driver exists" [[ -x "$APBIN" ]] && _ok "bin/ai-sandbox-autophase is executable" || _bad "bin/ai-sandbox-autophase is executable" # --- the launcher, as it lives in an rc file --------------------------------- mkdir -p "$WORK/rc" cat > "$WORK/rc/00-nothing.bash" <<'EOF' # defines nothing the driver asked for: the search must keep looking unrelated_setup() { PATH="$PATH:/nowhere"; } EOF cat > "$WORK/rc/sandbox.bash" <> "\$trace" (( \${#cmd[@]} )) || return 0 "\${cmd[@]}" } EOF cat > "$WORK/rc/aliases.bash" <<'EOF' alias sandbox-alias='sandbox-pi' EOF # The launcher exactly as it is often written for interactive use -- a fixed policy, and # nothing after it. It works when *you* type its name, and drops the whole in-container # command when something appends one, which is every run the driver makes. Kept as a # fixture so the driver is checked to name it instead of hanging on it. cat > "$WORK/rc/stubby.bash" <<'EOF' stubby-pi() { export AI_API_TOKEN='CANARY-STUBBY-4f2c81' ai-sandbox --allow api.example.com --env AI_API_TOKEN } EOF # An rc file that leaves a grandchild behind, the way one that starts an agent or a # helper daemon does. The definition probe must not wait for it: reading a definition is # not a conversation with the file's process tree. cat > "$WORK/rc/daemon.bash" <<'EOF' daemon-pi() { sleep 10 & ai-sandbox "$@" } EOF cat > "$WORK/rc/broken.bash" <<'EOF' broken-sandbox() { local -a cmd=() while (($#)); do case "$1" in --) shift; cmd=("$@"); break ;; *) shift ;; esac; done echo "Error: podman: executable not found" >&2 return 125 } EOF # --- the harness and the git shim ------------------------------------------- cat > "$WORK/bin/pi" <<'EOF' #!/usr/bin/env bash # fake pi: does the work, writes a session file, emits the stream a real pi emits. name=""; sessdir="."; prompt="" while (($#)); do case "$1" in --session-dir) sessdir="$2"; shift 2 ;; --name) name="$2"; shift 2 ;; --session|--mode|--model|--thinking) shift 2 ;; --no-session|--no-approve|--approve) shift ;; --*) shift ;; *) prompt="$1"; shift ;; esac done [[ -n "$prompt" ]] || { echo "fake pi: no prompt arrived" >&2; exit 3; } # the token has to reach the sandboxed agent -- recorded as a length, never a value printf 'token_bytes=%s\n' "${#AI_API_TOKEN}" >> "${FAKE_PI_MARKER:-/dev/null}" printf '%s\n---\n' "$prompt" >> "${FAKE_PI_PROMPTS:-/dev/null}" printf 'work by %s\n' "${name:-unit}" >> work.txt # an agent that tries to make the *host* run something when the harness commits if [[ -n "${FAKE_PI_PLANT_HOOK:-}" ]]; then mkdir -p .git/hooks printf '#!/bin/sh\ntouch "%s"\n' "$FAKE_PI_PLANT_HOOK" > .git/hooks/pre-commit chmod +x .git/hooks/pre-commit fi mkdir -p "$sessdir" { printf '{"type": "session", "id": "fake-%s"}\n' "${name:-s}" printf '{"type": "message_end", "message": {"role": "assistant", "content": [{"type": "text", "text": "report for %s"}]}}\n' "${name:-s}" printf '{"type": "agent_end"}\n' } | tee "${sessdir}/${name:-session}.jsonl" EOF cat > "$WORK/bin/git" <<'EOF' #!/usr/bin/env bash printf '%s\n' "$*" >> "${FAKE_GIT_TRACE:-/dev/null}" if [[ "$1" == push && -n "${FAKE_GIT_FAIL:-}" ]]; then echo "fatal: unable to access '${FAKE_GIT_FAIL}': Could not resolve host" >&2 exit 1 fi exec "$REAL_GIT" "$@" EOF chmod +x "$WORK/bin/pi" "$WORK/bin/git" export GIT_AUTHOR_NAME=test GIT_AUTHOR_EMAIL=t@example.invalid export GIT_COMMITTER_NAME=test GIT_COMMITTER_EMAIL=t@example.invalid # --- a phased project, with a local bare repo as its remote ------------------ # Every project gets its own bare remote: they all commit a branch called main, and a # shared remote would turn the second project's push into a non-fast-forward that has # nothing to do with what is being tested. mkproj() { # [phases] -> the remote it pushed its init commit to local p="$1" n="${2:-1}" REMOTE # Named after the project, not counted: a counter incremented in a command # substitution would stay in the subshell and hand every project the same remote, # which is a non-fast-forward waiting to happen. REMOTE="$WORK/remotes/$(basename "$p").git" mkdir -p "$WORK/remotes" "$REAL_GIT" init -q --bare "$REMOTE" >/dev/null 2>&1 mkdir -p "$p/.agents/phases/todo/01_one" printf '# task one\n\nDo the thing.\n' > "$p/.agents/phases/todo/01_one/01_task.md" printf '# final pass one\n\nWrap up phase 01_one.\n' > "$p/.agents/phases/todo/01_one/00_phase.md" if [[ "$n" == 2 ]]; then mkdir -p "$p/.agents/phases/todo/02_two" printf '# task two\n\nDo the next thing.\n' > "$p/.agents/phases/todo/02_two/01_task.md" printf '# final pass two\n\nWrap up phase 02_two.\n' > "$p/.agents/phases/todo/02_two/00_phase.md" fi printf '#!/usr/bin/env bash\nexit 0\n' > "$p/.agents/validate.sh" printf '.agents/phase-sessions/\n*.log\n' > "$p/.gitignore" ( cd "$p" && "$REAL_GIT" init -q -b main . && "$REAL_GIT" remote add origin "$REMOTE" \ && "$REAL_GIT" add -A && "$REAL_GIT" commit -q --no-gpg-sign -m 'init' \ && "$REAL_GIT" push -q -u origin main ) >/dev/null 2>&1 printf '%s\n' "$REMOTE" # so a test can count what actually reached the remote } TRACE="$WORK/launcher.trace" GTRACE="$WORK/git.trace" MARKER="$WORK/token.marker" PROMPTS="$WORK/prompts.log" : > "$TRACE"; : > "$GTRACE"; : > "$MARKER"; : > "$PROMPTS" printf 'the driver stdin, which no child is allowed to see\n' > "$WORK/stdin.marker" DEF_RC="$WORK/rc/sandbox.bash:$WORK/rc/aliases.bash:$WORK/rc/00-nothing.bash" # A prefix assignment in front of a shell function sticks around for the rest of the # script, so ap_run reads every T_* into a local and then unsets it: each run starts # from the defaults unless the test asked otherwise, and cannot inherit the last one. ap_run() { # [args...] local cmd="${T_CMD-sandbox-pi}" rc="${T_RC-$DEF_RC}" lib="${T_LIB-$LIB_FIX}" local validate="${T_VALIDATE-sandbox}" sargs="${T_SANDBOX_ARGS---no-tty --no-interactive}" local push="${T_PUSH-1}" pre="${T_PREFLIGHT-1}" notes="${T_NOTES-1}" gfail="${T_GITFAIL-}" local ppush="${T_PHASE_PUSH-}" dryout="${T_DRYOUT-}" plant="${T_PLANT-}" stdin_src="${T_STDIN-$WORK/stdin.marker}" unset T_CMD T_RC T_LIB T_VALIDATE T_SANDBOX_ARGS T_PUSH T_PREFLIGHT T_NOTES T_GITFAIL T_PHASE_PUSH T_DRYOUT T_PLANT T_STDIN # stdin is a file with a known name, never the test runner's own: the launcher records # what it was started with, and that is how "a child never inherits the terminal" is # checked. A driver that forwards stdin would show up as this path in the trace. OUT="$(env "PATH=$WORK/bin:$PATH" \ "AISB_AP_LIB=$lib" "AISB_AP_RC_FILES=$rc" "AISB_AP_PI_CMD=$cmd" \ "AISB_AP_VALIDATE=$validate" "AISB_AP_SANDBOX_ARGS=$sargs" \ "AISB_AP_PUSH=$push" "AISB_AP_PREFLIGHT=$pre" "AISB_AP_NOTES=$notes" \ "AISB_AP_TRACE=$TRACE" "FAKE_GIT_TRACE=$GTRACE" "FAKE_PI_MARKER=$MARKER" \ "FAKE_PI_PROMPTS=$PROMPTS" "REAL_GIT=$REAL_GIT" "PHASE_NOTIFY=0" \ ${gfail:+"FAKE_GIT_FAIL=$gfail"} ${ppush:+"PHASE_PUSH=$ppush"} ${dryout:+"AISB_AP_DRYOUT=$dryout"} \ ${plant:+"FAKE_PI_PLANT_HOOK=$plant"} \ "$APBIN" "$@" 2>&1 < "$stdin_src")" && RC=0 || RC=$? return 0 } gt() { grep -c "^$1" "$GTRACE" 2>/dev/null || true; } # count of git calls by verb trace_has() { grep -qF -- "$1" "$TRACE"; } PROJ="$WORK/proj"; mkproj "$PROJ" 1 >/dev/null cd "$PROJ" || exit 1 t_name "the launcher is resolved from the rc files, and is never run to find it" ap_run --status assert_rc 0 "--status" assert_contains "sandbox-pi" "names the configured command" assert_contains "function, from $WORK/rc/sandbox.bash" "found as a function, in the file that defines it" assert_absent "$CANARY" "the token never appears" if [[ -s "$TRACE" ]]; then _bad "resolving does not run the launcher" "the launcher was called"; else _ok "resolving does not run the launcher"; fi ap_run --dry-run assert_absent "$CANARY" "--dry-run does not either" t_name "an alias is a launcher too" T_CMD=sandbox-alias ap_run --status assert_rc 0 "--status with an alias" assert_contains "alias, from $WORK/rc/aliases.bash" "found as an alias" t_name "a program on PATH, and a whole command line" T_CMD=pi ap_run --status assert_rc 0 "a program on PATH" assert_contains "program, from $WORK/bin/pi" "and where it came from" T_CMD='ai-sandbox --allow api.example.com --env API_KEY' ap_run --status assert_rc 0 "a command line" assert_contains "arguments, from --pi-cmd" "used verbatim" T_CMD='ai-sandbox --env API_KEY=SECRETLITERALVALUE --allow api.example.com' ap_run --status assert_absent "SECRETLITERALVALUE" "a literal --env value in --pi-cmd is not echoed back" assert_contains "--env API_KEY" "while the name still is" t_name "an unresolvable launcher is refused with what was searched" T_CMD=no-such-sandbox-cmd ap_run --status assert_rc 1 "nothing by that name anywhere" assert_contains "cannot resolve the sandbox launcher" "says what failed" assert_contains "$WORK/rc/00-nothing.bash" "lists the rc files it searched" T_CMD='' ap_run --status assert_rc 1 "an explicitly empty launcher is an error, not a silent fallback" assert_contains "no sandbox launcher" "and says so" # Extract one function from the driver itself, so a test asks the real code a question # instead of restating its answer (the way aisb_fn() does for bin/ai-sandbox). ap_fn() { # awk -v fn="$1" '$0 ~ "^" fn "\\(\\) \\{" {f=1} f {print} f && /^\}$/ {exit}' "$APBIN" } t_name "a launcher that drops its arguments is named, and never hangs a run" # The shape every interactive sandbox-pi takes: a fixed policy and no "$@". Appended to, # it runs the image's default harness with no prompt, so an unattended run of the real # thing sat in front of an agent waiting for keystrokes, behind a pipe that showed # nothing. The driver has to say so, in the preview and before the first unit. : > "$TRACE" T_CMD=stubby-pi T_RC="$WORK/rc/stubby.bash" ap_run --status assert_rc 0 "--status with a launcher that takes no arguments" assert_contains "drops the arguments appended to it" "the preview says so" assert_contains "End the function with" "and says how to fix it" assert_absent "CANARY-STUBBY" "without repeating what the function exports" assert_absent "export AI_API_TOKEN" "and without printing its body" T_CMD=stubby-pi T_RC="$WORK/rc/stubby.bash" ap_run --max-units 1 assert_rc 1 "a run with it is refused, not left waiting" assert_contains "preflight failed" "at the preflight, before any unit" assert_contains "ignores its arguments" "with the reason named" assert_contains 'ai-sandbox --allow ... --env NAME "$@"' "and the fix written out" if grep -q 'cmd=pi' "$TRACE"; then _bad "no unit runs"; else _ok "no unit runs"; fi # the fixture launcher in the tests forwards "\$@", and must not be nagged about it ap_run --status assert_absent "drops the arguments" "a launcher that forwards them is not flagged" t_name "reading the rc files cannot be stopped, and cannot be held open" # An interactive bash started from a terminal takes that terminal: it calls tcsetpgrp(), # gets SIGTTOU because it is in the background, and stops -- and a stopped process does # not take SIGTERM, so the probe waited on it forever with the driver waiting behind it. # Everything a run does starts in this function, previews included, so the whole script # looked wedged with nothing printed. The three defences are asserted here by shape; # stopping a process is not something a test can arrange without a terminal of its own. probe_src="$(ap_fn ap_probe_rc)" for want in 'AP_SETSID' '< /dev/null' 'timeout -k 5 15' 'mktemp' '> "$tmp"'; do if printf '%s' "$probe_src" | grep -qF -- "$want"; then _ok "the probe is bounded, file-backed and off the terminal ($want)" else _bad "the probe is bounded, file-backed and off the terminal" "missing: $want" fi done if grep -qF 'AP_SETSID=(setsid)' "$APBIN"; then _ok "and its own session, so it has no controlling terminal to be stopped by" else _bad "and its own session, so it has no controlling terminal to be stopped by" "no AP_SETSID=(setsid) in $APBIN" fi # and the consequence a user would feel: an rc file that leaves a process behind must not # be able to hold the read open. sleep 10 in the fixture is the stand-in for the agent or # helper daemon a real rc file starts; if the probe reads a pipe instead of a file, this # test does not fail fast -- it fails 10 seconds later, which is the whole point. : > "$TRACE" start=$SECONDS T_CMD=daemon-pi T_RC="$WORK/rc/daemon.bash" ap_run --status elapsed=$(( SECONDS - start )) assert_rc 0 "--status with an rc file that backgrounds a process" assert_contains "function, from $WORK/rc/daemon.bash" "resolved all the same" (( elapsed < 8 )) && _ok "without waiting for what it left behind (${elapsed}s)" \ || _bad "without waiting for what it left behind" "took ${elapsed}s" t_name "a child is the launcher, then --, then the harness command" : > "$TRACE" PROJ_A="$WORK/pa"; mkproj "$PROJ_A" 1 >/dev/null cd "$PROJ_A" || exit 1 ap_run --dry-run assert_rc 0 "--dry-run" assert_contains "-- pi " "the pi command rides after --" assert_contains "--no-tty" "no TTY for the JSON stream to corrupt" assert_contains "--no-interactive" "and your keystrokes cannot reach the agent" : > "$TRACE" T_SANDBOX_ARGS='' ap_run --dry-run assert_absent "--no-tty" "--sandbox-args replaces the defaults rather than adding to them" assert_rc 0 "and an explicitly empty value means none, not the default" : > "$TRACE" ap_run --max-units 1 assert_rc 0 "one unit" trace_has "A:--no-interactive" "the launcher got the run flags" if grep -q '^LAUNCH argc=[0-9]* cmd=pi$' "$TRACE"; then _ok "pi is the in-container command" else _bad "pi is the in-container command" "$(grep '^LAUNCH' "$TRACE" | head -3)" fi # Every child starts with stdin on /dev/null, whatever this test's own stdin is: the # flag that normally keeps podman off the terminal (--no-interactive) is a flag, and a # launcher is free to drop it. if grep -q '^STDIN:/dev/null$' "$TRACE"; then _ok "the child's stdin is /dev/null, not the caller's" else _bad "the child's stdin is /dev/null, not the caller's" "$(grep '^STDIN' "$TRACE" | head -3)" fi if grep -qF "stdin.marker" "$TRACE"; then _bad "the run's own stdin never reaches a child"; else _ok "the run's own stdin never reaches a child"; fi argc="$(grep '^LAUNCH argc=.* cmd=pi$' "$TRACE" | head -1 | sed 's/.*argc=\([0-9]*\).*/\1/')" [[ -n "$argc" && "$argc" -ge 8 ]] && _ok "the whole harness command line reached the launcher ($argc arguments)" \ || _bad "the whole harness command line reached the launcher" "argc=$argc" if grep -A1 '^A:--$' "$TRACE" | grep -q '^A:pi$'; then _ok "-- is the separator, and nothing after it is a sandbox flag" else _bad "-- is the separator, and nothing after it is a sandbox flag" fi t_name "the prompt arrives as one argument, with the sandbox rules in it" if grep -qF 'Sandbox rules for this run' "$PROMPTS"; then _ok "the agent is told it is inside a sandbox" else _bad "the agent is told it is inside a sandbox" "$(tail -5 "$PROMPTS")" fi grep -qF 'git push' "$PROMPTS" && _ok "it is told the remote is unreachable, by name" \ || _bad "it is told the remote is unreachable, by name" grep -qF '01_one/01_task.md' "$PROMPTS" && _ok "the unit it is running is in the prompt" \ || _bad "the unit it is running is in the prompt" : > "$PROMPTS" T_NOTES=0 ap_run --unit 01_one/00_phase.md assert_rc 0 "one unit, notes off" if grep -qF 'Sandbox rules' "$PROMPTS"; then _bad "--no-notes drops the block"; else _ok "--no-notes drops the block"; fi grep -qF '01_one/00_phase.md' "$PROMPTS" && _ok "the library's own prompt still arrives" \ || _bad "the library's own prompt still arrives" "$(head -3 "$PROMPTS")" t_name "the token reaches the agent" if grep -q "token_bytes=${#CANARY}" "$MARKER"; then _ok "exported by the launcher, present in the child" else _bad "exported by the launcher, present in the child" "$(tail -1 "$MARKER" 2>/dev/null)" fi t_name "the gate runs where the agent runs" : > "$TRACE" PROJ_B="$WORK/pb"; mkproj "$PROJ_B" 1 >/dev/null cd "$PROJ_B" || exit 1 ap_run --unit 01_one/01_task.md assert_rc 0 "one task" if grep -q '^A:.agents/validate.sh$' "$TRACE"; then _ok ".agents/validate.sh is handed to the launcher" else _bad ".agents/validate.sh is handed to the launcher" "$(tail -5 "$TRACE")" fi : > "$TRACE" T_VALIDATE=host ap_run --unit 01_one/00_phase.md assert_rc 0 "one unit, host gate" if grep -q '^A:.agents/validate.sh$' "$TRACE"; then _bad "host mode does not consult the launcher"; else _ok "host mode does not consult the launcher"; fi assert_contains "outside the sandbox" "and the choice is announced, because it is the risky one" T_VALIDATE=host ap_run --status assert_contains "on the host (see --validate-in)" "and --status says where the gate would run" assert_contains "gate will run on the HOST" "and says so loudly: the gate file is agent-writable" T_VALIDATE=sideways ap_run --status assert_rc 1 "and a gate location that is neither is refused" t_name "a completed phase is committed and pushed by the host" PROJ1="$WORK/p1"; R1="$(mkproj "$PROJ1" 1)" : > "$GTRACE"; cd "$PROJ1" || exit 1 ap_run assert_rc 0 "the pipeline ran to completion" if grep -q 'report for 01_one__00_phase' "$PROJ1/.agents/reports/01_one/01_one__00_phase.a1.md" 2>/dev/null; then _ok "the final pass ran last, and its report is on disk" else _bad "the final pass ran last, and its report is on disk" "$(ls "$PROJ1/.agents/reports/01_one" 2>/dev/null)" fi if grep -q '^push' "$GTRACE"; then _ok "a push went out for the phase"; else _bad "a push went out for the phase"; fi assert_contains "up to date with origin/main" "and the end-of-run check confirmed the remote has it" if [[ -d "$PROJ1/.agents/phases/complete/01_one" && ! -d "$PROJ1/.agents/phases/todo/01_one" ]]; then _ok "the phase moved to complete/" else _bad "the phase moved to complete/" "$(ls "$PROJ1/.agents/phases"/* 2>/dev/null)" fi if ( cd "$PROJ1" && "$REAL_GIT" log --oneline | grep -q 'phase: 01_one' ); then _ok "the host made the phase commit" else _bad "the host made the phase commit" "$( cd "$PROJ1" && "$REAL_GIT" log --oneline | head -3 )" fi assert_eq 2 "$( git --git-dir="$R1" rev-list --count --all )" "the remote has init + the phase" t_name "the driver owns delivery: push at the phase boundary, re-check, push again at the end" PROJ2="$WORK/p2"; R2="$(mkproj "$PROJ2" 2)" : > "$GTRACE"; cd "$PROJ2" || exit 1 ap_run assert_rc 0 "two phases ran" commits="$(gt commit)"; pushes="$(gt push)" [[ "$commits" == 2 ]] && _ok "both phases committed on the host" || _bad "both phases committed on the host" "commits=$commits" [[ "$pushes" == "$commits" ]] && _ok "every commit is delivered, not left for a later run ($pushes)" \ || _bad "every commit is delivered, not left for a later run" "commits=$commits pushes=$pushes" assert_contains "phase 01_one" "the push is attributed to the phase that made it" assert_contains "up to date with origin/main" "and the next phase's start re-checks before running it" assert_eq 3 "$( git --git-dir="$R2" rev-list --count --all )" "the remote has init + both phases" # PHASE_PUSH is the library's switch; it means the same thing here, so it is honoured # rather than overridden -- and honoured means nothing goes out, in either direction. PROJ3="$WORK/p3"; R3="$(mkproj "$PROJ3" 2)" : > "$GTRACE"; cd "$PROJ3" || exit 1 T_PHASE_PUSH=0 ap_run assert_rc 0 "the pipeline still ran" if grep -q '^push' "$GTRACE"; then _bad "PHASE_PUSH=0 pushes nothing"; else _ok "PHASE_PUSH=0 pushes nothing"; fi assert_contains "push disabled" "and says so" assert_eq 1 "$( git --git-dir="$R3" rev-list --count --all )" "the remote is untouched" t_name "--no-push leaves the commits local, and says so" PROJ4="$WORK/p4"; R4="$(mkproj "$PROJ4" 1)" : > "$GTRACE"; cd "$PROJ4" || exit 1 ap_run --no-push assert_rc 0 "the pipeline ran" if grep -q '^push' "$GTRACE"; then _bad "nothing is pushed" "$(grep '^push' "$GTRACE")"; else _ok "nothing is pushed"; fi assert_contains "push disabled" "and that is stated, not silent" if [[ "$(gt commit)" == 1 ]] && ( cd "$PROJ4" && "$REAL_GIT" log --oneline | grep -q 'phase: 01_one' ); then _ok "the work is still committed locally" else _bad "the work is still committed locally" "$( cd "$PROJ4" && "$REAL_GIT" log --oneline | head -3 )" fi assert_eq 1 "$( git --git-dir="$R4" rev-list --count --all )" "the remote is untouched" t_name "a failed push stops the run, with the hand-fix command" PROJ5="$WORK/p5"; mkproj "$PROJ5" 1 >/dev/null : > "$GTRACE"; cd "$PROJ5" || exit 1 T_GITFAIL="https://user:pushcanary@remote.invalid/repo.git" ap_run assert_rc 1 "the run stops on a failed push" assert_contains "the push FAILED" "says what failed" assert_contains "finish by hand" "and how to finish it" assert_absent "pushcanary" "a credential in the remote URL is not repeated in the output" : > "$GTRACE" ap_run # the remote answers this time assert_contains "unpushed commit" "the next run notices what is still local" assert_contains "pushed:" "and takes it out" # and a run that could not deliver never reports success PROJ5B="$WORK/p5b"; mkproj "$PROJ5B" 1 >/dev/null; cd "$PROJ5B" || exit 1 T_GITFAIL="https://user:pushcanary@remote.invalid/repo.git" ap_run assert_rc 1 "a run that could not deliver does not report success" assert_contains "the push FAILED" "and says which part failed" if ( cd "$PROJ5B" && "$REAL_GIT" log --oneline | grep -q 'phase: 01_one' ); then _ok "its commit is still local and safe" else _bad "its commit is still local and safe" fi if ( cd "$PROJ5" && "$REAL_GIT" log --oneline | grep -q 'phase: 01_one' ); then _ok "the phase stayed complete through it" else _bad "the phase stayed complete through it" fi t_name "nothing a preview prints carries a value" PROJ6="$WORK/p6"; mkproj "$PROJ6" 1 >/dev/null cd "$PROJ6" || exit 1 : > "$TRACE" out_status="$(cd "$PROJ6" && env PATH="$WORK/bin:$PATH" AISB_AP_LIB="$LIB_FIX" \ AISB_AP_RC_FILES="$WORK/rc/sandbox.bash" AISB_AP_PI_CMD=sandbox-pi \ AISB_AP_TRACE="$TRACE" "$APBIN" --status 2>&1)" out_dry="$(cd "$PROJ6" && env PATH="$WORK/bin:$PATH" AISB_AP_LIB="$LIB_FIX" \ AISB_AP_RC_FILES="$WORK/rc/sandbox.bash" AISB_AP_PI_CMD=sandbox-pi \ AISB_AP_TRACE="$TRACE" "$APBIN" --dry-run 2>&1)" for v in "$out_status" "$out_dry"; do if printf '%s' "$v" | grep -qF -- "$CANARY"; then _bad "a preview does not echo the launcher's secret" else _ok "a preview does not echo the launcher's secret" fi if printf '%s' "$v" | grep -qF "export AI_API_TOKEN"; then _bad "a preview does not print the launcher's body" else _ok "a preview does not print the launcher's body" fi done if [[ -s "$TRACE" ]]; then _bad "a preview runs nothing"; else _ok "a preview runs nothing"; fi assert_eq "$( git -C "$PROJ6" rev-list --count HEAD )" "$( git -C "$PROJ6" rev-list --count HEAD )" "a preview changes no git state" if [[ -e "$PROJ6/.agents/reports" ]]; then _bad "a preview writes no reports"; else _ok "a preview writes no reports"; fi if [[ -e "$PROJ6/.agents/phases/complete/01_one" ]]; then _bad "a preview does not move units"; else _ok "a preview does not move units"; fi OUT="$out_status" assert_contains "01_one/01_task.md" "but it does list what is pending" assert_contains "01_one/00_phase.md" "task first, then the final pass" OUT="$out_dry" assert_contains "one container per unit" "and --dry-run says what it would do" t_name "preflight refuses to start a pipeline the sandbox cannot run" cd "$PROJ6" || exit 1 : > "$TRACE" T_CMD=broken-sandbox T_RC="$WORK/rc/broken.bash" ap_run --max-units 1 assert_rc 1 "preflight stops the run" assert_contains "preflight failed" "and says so" assert_contains "executable not found" "with the launcher's own error, for diagnosis" if grep -q 'cmd=pi' "$TRACE"; then _bad "no unit runs"; else _ok "no unit runs"; fi T_PREFLIGHT=0 ap_run --max-units 1 assert_rc 0 "--no-preflight is available for the runs that cannot afford it" t_name "bad usage is refused before anything happens" cd "$PROJ6" || exit 1 ap_run --not-a-flag assert_rc 2 "an unknown option" ap_run --max-units lots assert_rc 1 "--max-units wants a number" cd / || exit 1 ap_run --status assert_rc 1 "outside a phased project" assert_contains "no .agents/phases/todo" "and it says what is missing" cd "$PROJ6" || exit 1 t_name "a library that does not match the contract is refused" printf 'find_root() { printf "%%s\\n" "$PWD"; }\nnext_unit() { :; }\n' > "$WORK/rc/half-lib.sh" T_LIB="$WORK/rc/half-lib.sh" ap_run --status assert_rc 1 "a library missing pieces is not run against" assert_contains "does not provide" "and the missing function is named" t_name "a hook planted from inside the sandbox never runs on the host" # The commit is made out here, by us, after an agent had the tree writeable. A repository # does not merely record .git/hooks and .git/config -- it runs them. So the delivery # refuses when they changed, and even when it does not, our git never looks in .git/hooks. HOOKPROOF="$WORK/hook-ran" PROJ9="$WORK/p9"; R9="$(mkproj "$PROJ9" 1)" cd "$PROJ9" || exit 1 : > "$GTRACE" T_PLANT="$HOOKPROOF" ap_run assert_rc 1 "a run that rewrote the repository's own git config does not deliver" assert_contains "refusing to deliver" "and says why" assert_contains "--trust-git-changes" "and names the way through" if [[ -e "$HOOKPROOF" ]]; then _bad "the planted pre-commit hook did not run"; else _ok "the planted pre-commit hook did not run"; fi if grep -q '^push' "$GTRACE"; then _bad "nothing was pushed"; else _ok "nothing was pushed"; fi assert_eq 1 "$( git --git-dir="$R9" rev-list --count --all )" "the remote still has only the init commit" ap_run --trust-git-changes assert_rc 0 "the named escape hatch does deliver" assert_contains "pushed:" "and says so" if [[ -e "$HOOKPROOF" ]]; then _bad "and still the hook did not run"; else _ok "and still the hook did not run: our git uses an empty hooks path"; fi assert_eq 2 "$( git --git-dir="$R9" rev-list --count --all )" "the phase commit is on the remote now" # and it is refused even when this script is not the one that would push: the point is # that the operator hears about it, not that a push is blocked PROJ9B="$WORK/p9b"; mkproj "$PROJ9B" 1 >/dev/null; cd "$PROJ9B" || exit 1 T_PLANT="$HOOKPROOF" ap_run --no-push assert_rc 1 "--no-push does not make it safe to ignore" assert_contains "refusing to deliver" "and the same message explains it" if [[ -e "$HOOKPROOF" ]]; then _bad "the hook did not run here either"; else _ok "the hook did not run here either"; fi t_name "what the driver composes is a real ai-sandbox invocation" # The fixture launcher stands in for ai-sandbox everywhere else, because this suite runs # without containers. Here the launcher IS ai-sandbox, in --dry-run: if the flags land on # the wrong side of --, or --no-interactive were forwarded to the harness, or the project # were not the mount, ai-sandbox itself says so. cat > "$WORK/rc/drylaunch.bash" <> "\${AISB_AP_DRYOUT:-/dev/null}" 2>&1 } EOF PROJ8="$WORK/p8"; mkproj "$PROJ8" 1 >/dev/null cd "$PROJ8" || exit 1 DRYOUT="$WORK/ai-sandbox.dryrun"; : > "$DRYOUT" T_CMD=sandbox-dry T_RC="$WORK/rc/drylaunch.bash" T_PREFLIGHT=0 T_DRYOUT="$DRYOUT" \ ap_run --max-units 1 # the child produces no JSON (ai-sandbox printed an invocation instead), so the unit # does not complete -- what is under test is the command that produced that output if grep -q 'effective podman invocation' "$DRYOUT"; then _ok "ai-sandbox accepted the composed command" else _bad "ai-sandbox accepted the composed command" "$(head -4 "$DRYOUT")" fi if grep -q 'egress: net=none' "$DRYOUT"; then _ok "the launcher's own policy is still what is enforced" else _bad "the launcher's own policy is still what is enforced" "$(head -2 "$DRYOUT")" fi if grep -qF -- '--no-interactive' "$DRYOUT" || grep -qF -- '--no-tty' "$DRYOUT"; then _bad "the run's sandbox flags are consumed by ai-sandbox, not passed on to pi" else _ok "the run's sandbox flags are consumed by ai-sandbox, not passed on to pi" fi if grep -qE -- ' pi (--no-approve|--session-dir)' "$DRYOUT"; then _ok "pi is the in-container command, after --" else _bad "pi is the in-container command, after --" "$(tail -1 "$DRYOUT" | cut -c1-200)" fi if grep -qE -- '(^| )-i( |$)' "$DRYOUT" || grep -qE -- '(^| )-t( |$)' "$DRYOUT"; then _bad "nothing is attached to the terminal: no -t in the JSON stream, no -i to steal keystrokes" else _ok "nothing is attached to the terminal: no -t in the JSON stream, no -i to steal keystrokes" fi # The project is mounted at its own host path where the sandbox can mirror it, and at # /project where it cannot -- the driver composes neither, it just does not disturb it. MOUNT_AT="$([[ -n "$(mirror_block_of "$PROJ8")" ]] && echo /project || echo "$PROJ8")" if grep -qF -- "$PROJ8:$MOUNT_AT" "$DRYOUT"; then _ok "the project directory is the mount the agent writes into" else _bad "the project directory is the mount the agent writes into" fi if grep -qF -- 'bash .agents/validate.sh' "$DRYOUT"; then _ok "and so is the gate: validate.sh runs in the same sandbox, as the same user" else _bad "and so is the gate: validate.sh runs in the same sandbox, as the same user" "$(tail -2 "$DRYOUT" | cut -c1-200)" fi # --------------------------------------------------------------------------- # The real library, real renderer, real commit -- with the same fake sandbox. t_name "the real phased-execution library, end to end" REAL_LIB="$HOME/.agents/skills/phased-execution/scripts/lib.sh" if [[ ! -f "$REAL_LIB" ]]; then _ok "the skill is not installed here -- the real-library run is skipped" else PROJ7="$WORK/p7"; mkproj "$PROJ7" 1 >/dev/null : > "$GTRACE"; cd "$PROJ7" || exit 1 # shellcheck disable=SC2034 # assert_rc() in tests/lib.sh is what reads RC OUT="$(env PATH="$WORK/bin:$PATH" AISB_AP_LIB="$REAL_LIB" \ AISB_AP_RC_FILES="$WORK/rc/sandbox.bash" AISB_AP_PI_CMD=sandbox-pi \ AISB_AP_VALIDATE=sandbox AISB_AP_TRACE="$TRACE" FAKE_GIT_TRACE="$GTRACE" \ FAKE_PI_MARKER="$MARKER" REAL_GIT="$REAL_GIT" PHASE_NOTIFY=0 QUIET=1 \ "$APBIN" 2>&1)" && RC=0 || RC=$? assert_rc 0 "the real pipeline completed a phase inside the sandbox launcher" if grep -rq 'report for 01_one__00_phase' "$PROJ7/.agents/reports/" 2>/dev/null; then _ok "the real progress renderer read the child stream and wrote the report" else _bad "the real progress renderer read the child stream and wrote the report" "$(ls -R "$PROJ7/.agents/reports" 2>/dev/null | head)" fi if compgen -G "$PROJ7/.agents/reports/01_one/*.md" > /dev/null; then _ok "the executor reports were written where the skill puts them" else _bad "the executor reports were written where the skill puts them" "$(ls -R "$PROJ7/.agents" 2>/dev/null | head -20)" fi if ( cd "$PROJ7" && "$REAL_GIT" log --oneline | grep -q 'phase: 01_one' ); then _ok "the real commit_phase made the scoped commit" else _bad "the real commit_phase made the scoped commit" "$( cd "$PROJ7" && "$REAL_GIT" log --oneline | head -3 )" fi assert_contains "pushed:" "and the push went out" if printf '%s' "$OUT" | grep -qF -- "$CANARY"; then _bad "no token in the real pipeline output"; else _ok "no token in the real pipeline output"; fi if ( cd "$PROJ7" && "$REAL_GIT" ls-tree -r --name-only HEAD | grep -q '.agents/phases/complete/01_one/00_phase.md' ); then _ok "the moved phase file is in the commit" else _bad "the moved phase file is in the commit" "$( cd "$PROJ7" && "$REAL_GIT" ls-tree -r --name-only HEAD | head -10 )" fi if grep -qF 'Sandbox rules for this run' "$PROMPTS"; then _ok "and the real prompts carried the sandbox rules" else _bad "and the real prompts carried the sandbox rules" fi fi cd "$WORK" || exit 1