Files
ai-sandbox/tests/e2e/12_sandbox.sh
T
ducoterra a08a69f143 make xhigh the sandbox's thinking, and stop settings.json freezing
Two changes, one bug found through the other.

Thinking is policy now: defaultThinkingLevel is xhigh unless you say otherwise
(thinking / --thinking / AISB_THINKING, with pi's levels plus 'host' to decline
and 'off' meaning pi's own no-reasoning level). It is written into the bridged
settings and re-applied by the entrypoint on every run, so a run cannot set its
own reasoning down for the runs after it. Unknown levels are refused with the
list to pick from. Nothing is written for codex, claude or a command after --,
and --no-pi-config declines it along with the bridge, because pi's own defaults
are what that flag asks for.

The reason xhigh had never reached a sandbox is worse than a missing default:
settings.json was managed as one whole file, and pi rewrites its own settings
(a theme change, "seen the changelog"), so the first such write matched nothing
and froze every host preference out permanently -- provider, model, thinking
budgets and level included, a real session left holding just
{"lastChangelogVersion": "1.4.0"}. It is merged per key instead, against a copy
of what we last wrote: a key edited in the sandbox stays edited, a key never
touched follows the host again, an untouched key the host dropped is dropped, a
key pi invented survives, and a file with no base copy takes the host values --
which repairs the volumes already frozen. Unparseable JSON is re-seeded with a
warning since pi cannot use it either; a file the container cannot read is
reported with the fix, because nothing inside the sandbox can repair one.

One trap worth remembering: the seeding script is passed to bash -c in single
quotes, so a single quote inside it closes the argument early and the rest runs
in the outer shell -- valid bash, wrong program, clean bash -n and clean
shellcheck. tests/unit/06_entrypoint.sh extracts the seeding script and asserts
it is quote-free, and extracts the merge program to test its semantics without
a container; tests/e2e/12_sandbox.sh does the same in a real run, including
forcing the level back after the sandbox sets it to off.

LLM-Generated: true
2026-10-27 21:48:33 -04:00

312 lines
19 KiB
Bash

#!/usr/bin/env bash
# tests/e2e/12_sandbox.sh -- lifecycle, filesystem and environment behaviour of a real run.
source "$TESTS/lib.sh"
WORK_DIR="$WORK/project"; mkdir -p "$WORK_DIR"; cd "$WORK_DIR" || exit 1
echo "hello from the host" > "$WORK_DIR/host-file.txt"
export AISB_CANARY_IN=canary-value-42
unset AISB_CANARY_OUT
DARK="--net none"
t_name "the command runs and its exit code comes back"
sb $DARK -- bash -c 'echo ran; exit 0'
assert_rc 0 "exit 0 propagates"; assert_contains "ran" "stdout reaches the host"
sb $DARK -- bash -c 'exit 3'
assert_rc 3 "exit 3 propagates"
sb $DARK -- bash -c 'exit 127'
assert_rc 127 "exit 127 propagates"
t_name "default command follows the harness"
sb $DARK --harness none -- true >/dev/null; assert_rc 0 "harness none is accepted"
sb $DARK --harness pi -- bash -lc "command -v pi"
assert_rc 0 "the default pi binary exists in the image"
t_name "the project directory is mounted, writable, and owned by the host user"
sb $DARK -- bash -c 'pwd; touch /project/from-container.txt; echo wrote'
assert_contains "/project" "working directory is /project"
[[ -f "$WORK_DIR/from-container.txt" ]] && _ok "writes land on the host" || _bad "writes land on the host"
if [[ "$(stat -c %u "$WORK_DIR/from-container.txt")" == "$(id -u)" ]]; then
_ok "the file is owned by the host uid, not by root"
else
_bad "the file is owned by the host uid" "owner=$(stat -c %u "$WORK_DIR/from-container.txt") expected=$(id -u)"
fi
sb $DARK -- bash -c 'cat /project/host-file.txt'
assert_contains "hello from the host" "existing project files are readable"
t_name "nothing else on the host is visible"
echo secret > "$WORK/not-shared.txt"
sb $DARK -- bash -c "cat '$WORK/not-shared.txt' 2>&1 || echo NOT_VISIBLE"
assert_contains "NOT_VISIBLE" "the parent directory is not mounted"
sb $DARK -- bash -c 'ls /root 2>&1 | head -1; test -e /etc/shadow && echo has_shadow'
assert_absent "$(hostname)" "the host filesystem layout is gone"
t_name "environment: deny by default, allow on request"
sb $DARK -- bash -c 'echo "canary=[${AISB_CANARY_IN:-unset}]"; echo "home=$HOME"; echo "aisb=$AISB"'
assert_contains "canary=[unset]" "a host variable does not leak in"
assert_contains "home=/home/sandbox" "HOME is the container's own"
assert_contains "aisb=1" "the sandbox marks itself"
sb $DARK --env AISB_CANARY_IN -- bash -c 'echo "canary=[$AISB_CANARY_IN]"'
assert_contains "canary=[canary-value-42]" "--env passes exactly what was asked for"
sb $DARK --env AISB_CANARY_OUT -- bash -c 'true'
assert_rc 1 "an unset --env variable fails the run closed"
t_name "the value never appears in a command line"
( sb $DARK --env AISB_CANARY_IN -- bash -c 'sleep 2' >/dev/null 2>&1 ) &
RUNPID=$!
sleep 1.5
# Every process the value could legitimately have been handed to: the container
# runtime. argv[0] is matched exactly so the test's own shell (whose command line
# contains this very string) can never be the thing that matches.
# (the bracket keeps this grep from matching its own command line)
LEAKY="$(ps -ewwo comm=,args= | awk '{a=$1; sub(".*/","",a)} a ~ /^(podman|crun|conmon|catatonit)$/' \
| grep -c 'canar[y]-value-42' || true)"
assert_eq "0" "$LEAKY" "no podman/crun/conmon argv carries the secret"
wait $RUNPID 2>/dev/null
t_name "the env file is 0600 while running and gone afterwards"
( sb $DARK --env AISB_CANARY_IN -- bash -c 'sleep 2' >/dev/null 2>&1 ) &
RUNPID2=$!
sleep 1.5
ENVF="$(find "$AISB_STATE_DIR/sessions" -name env -print -quit 2>/dev/null)"
if [[ -n "$ENVF" ]]; then
assert_file_mode "$ENVF" 600 "the env file is owner-read/write only"
grep -q 'AISB_CANARY_IN=canary-value-42' "$ENVF" && _ok "and it holds the value we asked for" || _bad "env file content"
else
_bad "the env file is owner-read/write only" "no env file found while the container was running"
fi
wait $RUNPID2 2>/dev/null
sb $DARK --env AISB_CANARY_IN -- true
LEFT="$(find "$AISB_STATE_DIR/sessions" -name env 2>/dev/null | wc -l | tr -d ' ')"
assert_eq "0" "$LEFT" "the env file is deleted when the run ends, even with --keep"
sb $DARK --keep --env AISB_CANARY_IN -- true
LEFT2="$(find "$AISB_STATE_DIR/sessions" -name env 2>/dev/null | wc -l | tr -d ' ')"
assert_eq "0" "$LEFT2" "--keep keeps the container, not the secret"
t_name "extra mounts"
mkdir -p "$WORK/ro-src" "$WORK/rw-src"; echo data > "$WORK/ro-src/file"
sb $DARK --mount "$WORK/ro-src:/ro:ro" --mount "$WORK/rw-src:/rw" -- bash -c '
cat /ro/file
touch /rw/created && echo rw_ok
touch /ro/writable 2>/dev/null && echo "BAD: ro mount is writable" || echo ro_locked=ok
'
assert_contains "data" "read-only mount is readable"
assert_contains "rw_ok" "read-write mount is writable"
assert_contains "ro_locked=ok" "ro really means ro"
t_name "the home directory"
sb $DARK --ephemeral-home -- bash -c 'echo hi > "$HOME/scratch"; ls -A "$HOME" | tr "\n" " "'
assert_contains "scratch" "home is writable with an ephemeral mount"
sb $DARK --home-cache -- bash -c 'echo cached > "$HOME/cached-marker"'
sb $DARK --home-cache -- bash -c 'cat "$HOME/cached-marker"'
assert_contains "cached" "the home cache volume persists across runs"
sb $DARK --ephemeral-home -- bash -c 'test -f "$HOME/cached-marker" && echo LEAK || echo isolated'
assert_contains "isolated" "an ephemeral home does not see it"
t_name "the host ~/.agents arrives as \$HOME/.agents, read-only"
AG="$WORK/host-agents"
mkdir -p "$AG/skills/demo-skill" "$AG/prompts"
printf 'name: demo-skill\ninstructions: always use the demo skill\n' > "$AG/skills/demo-skill/SKILL.md"
printf 'a prompt template\n' > "$AG/prompts/demo-prompt.md"
printf '# standing instructions\n' > "$AG/AGENTS.md"
export AISB_AGENTS_DIR="$AG"
sb $DARK -- bash -c 'ls -A "$HOME/.agents" | tr "\n" " "; echo; cat "$HOME/.agents/skills/demo-skill/SKILL.md"; cat "$HOME/.agents/AGENTS.md"'
assert_contains "skills" "the skills directory is where a harness looks for user-level skills"
assert_contains "always use the demo skill" "a SKILL.md is readable at the real path"
assert_contains "# standing instructions" "and so is the global AGENTS.md"
sb $DARK -- bash -c '
touch "$HOME/.agents/skills/demo-skill/injected" 2>/dev/null && echo "BAD: skills writable" || echo skills_locked=ok
mkdir "$HOME/.agents/evil-skill" 2>/dev/null && echo "BAD: could add a skill" || echo add_locked=ok
touch "$HOME/scratch-still-works" && echo home_still_writable
'
assert_contains "skills_locked=ok" "a sandboxed agent cannot rewrite the skills it is reading"
assert_contains "add_locked=ok" "nor plant a new one"
assert_contains "home_still_writable" "while the rest of home keeps working"
sb $DARK --insecure-keep-root -- bash -c 'touch "$HOME/.agents/skills/demo-skill/injected" 2>/dev/null && echo "BAD: writable as root" || echo root_still_locked'
assert_contains "root_still_locked" "not even --insecure-keep-root gets to write your skills"
BEFORE="$(find "$AG" | sort)"
sb $DARK -- bash -c 'echo pwned >> "$HOME/.agents/AGENTS.md" 2>/dev/null; rm -f "$HOME/.agents/prompts/demo-prompt.md" 2>/dev/null; true'
AFTER="$(find "$AG" | sort)"
assert_eq "$BEFORE" "$AFTER" "the host directory is byte-for-byte untouched by the run"
grep -q pwned "$AG/AGENTS.md" && _bad "appends from the container do not land on the host" || _ok "appends from the container do not land on the host"
sb $DARK --ephemeral-home -- bash -c 'cat "$HOME/.agents/skills/demo-skill/SKILL.md"'
assert_contains "always use the demo skill" "a tmpfs home still gets the mount nested inside it"
sb $DARK --read-only -- bash -c 'cat "$HOME/.agents/AGENTS.md"'
assert_contains "# standing instructions" "and so does a read-only root filesystem"
# podman creates the mount point inside the home volume, so the directory can outlive a
# run that shared it; what --no-agents guarantees is that none of the host is in it.
sb $DARK --no-agents -- bash -c 'ls -A "$HOME/.agents" 2>/dev/null | tr "\n" " "; test -e "$HOME/.agents/skills/demo-skill" && echo PRESENT || echo ABSENT'
assert_contains "ABSENT" "--no-agents leaves no host skill visible in the container"
assert_absent "demo-skill" "not even as a directory name"
# a project's own .agents needs no bridge: it is inside the project mount. Both are
# visible at once, which is what a harness that reads both (pi) expects.
mkdir -p "$WORK_DIR/.agents/skills/project-skill"
printf 'name: project-skill\ninstructions: project only\n' > "$WORK_DIR/.agents/skills/project-skill/SKILL.md"
sb $DARK -- bash -c 'cat /project/.agents/skills/project-skill/SKILL.md; ls "$HOME/.agents/skills" | tr "\n" " "'
assert_contains "project only" "the project's own .agents comes in with the project mount"
assert_contains "demo-skill" "and the user-level one is still alongside it"
t_name "read-only root filesystem"
sb $DARK --read-only -- bash -c 'touch /usr/bin/nope 2>/dev/null && echo "BAD: wrote to rootfs" || echo rootfs_locked=ok; touch /project/rw-still-works && echo project_still_writable'
assert_contains "rootfs_locked=ok" "--read-only blocks writes outside mounts"
assert_contains "project_still_writable" "the project mount stays writable"
t_name "lifecycle"
BEFORE="$(podman ps -a --filter label=ai-sandbox=1 -q | wc -l | tr -d ' ')"
sb $DARK -- true
AFTER="$(podman ps -a --filter label=ai-sandbox=1 -q | wc -l | tr -d ' ')"
assert_eq "$BEFORE" "$AFTER" "the container is removed when the command exits"
sb $DARK --keep --name aisb-keeptest -- true
podman ps -a --format '{{.Names}}' | grep -qx aisb-keeptest && _ok "--keep leaves the container" || _bad "--keep leaves the container"
SESSIONS_KEPT="$(ls -d "$AISB_STATE_DIR"/sessions/* 2>/dev/null | wc -l | tr -d ' ')"
[[ "$SESSIONS_KEPT" -ge 1 ]] && _ok "--keep leaves the session dir for inspection" || _bad "--keep leaves the session dir"
sb clean --all > /dev/null
GONE="$(podman ps -a --filter name=aisb-keeptest -q | wc -l | tr -d ' ')"
assert_eq "0" "$GONE" "clean removes leftovers"
t_name "clean --volumes reclaims the home cache, carefully"
podman volume create not-aisb-decoy > /dev/null 2>&1
podman volume create aisb-other-dangling > /dev/null 2>&1
AISB_HOME_VOLUME=aisb-test-home sb $DARK -- true
if podman volume exists aisb-test-home > /dev/null 2>&1; then _ok "the run created its home cache volume"; else _bad "the run created its home cache volume"; fi
sb clean --volumes=aisb-test-home > /dev/null
if podman volume exists aisb-test-home > /dev/null 2>&1; then _bad "clean --volumes=NAME reclaimed the unused volume"; else _ok "clean --volumes=NAME reclaimed the unused volume"; fi
if podman volume exists aisb-other-dangling > /dev/null 2>&1; then _ok "an unmatched volume was left alone"; else _bad "an unmatched volume was left alone"; fi
# a live sandbox's home is never reclaimed while it runs
AISB_HOME_VOLUME=aisb-test-home2 sb $DARK --name aisb-vollive -- bash -c 'sleep 25' > /dev/null 2>&1 &
for _ in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20; do
podman ps --filter name=aisb-vollive -q | grep -q . && break
sleep 0.5
done
AISB_HOME_VOLUME=aisb-test-home2 sb clean --volumes=aisb-test-home2 > /dev/null
if podman volume exists aisb-test-home2 > /dev/null 2>&1; then _ok "a running sandbox keeps its home volume"; else _bad "a running sandbox keeps its home volume"; fi
podman rm -f aisb-vollive > /dev/null 2>&1
AISB_HOME_VOLUME=aisb-test-home2 sb clean --volumes=aisb-test-home2 > /dev/null
if podman volume exists aisb-test-home2 > /dev/null 2>&1; then _bad "it is reclaimed once nothing references it"; else _ok "it is reclaimed once nothing references it"; fi
podman volume rm not-aisb-decoy aisb-other-dangling > /dev/null 2>&1
t_name "the session log records what ran, without secrets"
LOG="$(cat "$AISB_STATE_DIR/log" 2>/dev/null || true)"
printf '%s' "$LOG" | grep -q 'net=none' && _ok "the log records the run" || _bad "the log records the run"
printf '%s' "$LOG" | grep -q 'canary-value-42' && _bad "secrets must never reach the log" || _ok "no secrets in the log"
t_name "the pi config bridge lands inside the container"
AGENT="$WORK/agent"; mkdir -p "$AGENT"
cat > "$AGENT/models.json" <<EOF
{ "providers": { "gateway": { "baseUrl": "https://aipi.reeseapps.com/v1",
"api": "openai-completions", "apiKey": "!pass show gateway",
"models": [ { "id": "turbo", "contextWindow": 262144 } ] } } }
EOF
printf '{ "defaultProvider": "gateway", "defaultModel": "turbo", "theme": "dark/dark" }\n' > "$AGENT/settings.json"
printf '{ "gateway": "HOSTSECRET-must-not-travel" }' > "$AGENT/auth.json"
export AISB_PI_AGENT_DIR="$AGENT" AISB_PI_TOKEN_ENV=AI_API_TOKEN AISB_PI_TOKEN_AUTO=1
export AI_API_TOKEN=container-token-777
export AISB_HOME_VOLUME=aisb-test-pihome
sb $DARK --harness pi -- bash -lc '
f="$HOME/.pi/agent/models.json"
[ -f "$f" ] && echo seeded=yes || echo seeded=no
[ "$(id -u)" = 1000 ] || echo uid=wrong
echo "mode=$(stat -c %a "$f" 2>/dev/null)"
grep -q "\$AI_API_TOKEN" "$f" && echo ref=env || echo ref=other
grep -q "pass show" "$f" && echo LEAKED_CMD || echo cmd=gone
grep -q "HOSTSECRET" "$f" && echo LEAKED_KEY || echo key=gone
[ -f "$HOME/.pi/agent/auth.json" ] && echo LEAKED_AUTH || echo auth=absent
grep -q "turbo" "$HOME/.pi/agent/settings.json" && echo settings=yes
[ -n "$AI_API_TOKEN" ] && echo token=present || echo token=missing'
assert_contains "seeded=yes" "pi's config is seeded into the sandbox home"
assert_contains "mode=600" "and only the sandbox user can read it"
assert_contains "ref=env" "the credential stays an environment reference"
assert_contains "cmd=gone" "the host '!command' did not cross"
assert_contains "key=gone" "nor did the literal key"
assert_contains "auth=absent" "auth.json was not bridged"
assert_contains "settings=yes" "settings.json is in place"
assert_contains "token=present" "and the token reaches pi by env"
assert_absent "container-token-777" "while its value is never printed"
assert_absent "uid=wrong" "seeded files belong to the sandbox user"
# an edit made inside the sandbox survives; a change on the host does not get lost
sb $DARK --harness pi -- bash -c 'sed -i "s/turbo/juggernaut/" "$HOME/.pi/agent/models.json"; echo edited' >/dev/null
sb $DARK --harness pi -- bash -lc 'grep -q juggernaut "$HOME/.pi/agent/models.json" && echo edit_kept || echo edit_lost'
assert_contains "edit_kept" "a change made inside the sandbox is not clobbered"
sb $DARK --harness pi -- bash -lc 'grep -q juggernaut "$HOME/.pi/agent/models.json" && echo still_here' 2>/dev/null
sed -i 's/262144/524288/' "$AGENT/models.json"
sb $DARK --harness pi -- bash -lc '
grep -q 524288 "$HOME/.pi/agent/models.json" && echo host_change_ignored_fully || echo host_change_partly_ignored
grep -q juggernaut "$HOME/.pi/agent/models.json" && echo edit_still_kept'
assert_contains "edit_still_kept" "the sandbox edit still wins over the bridged file"
sb $DARK --harness pi -- bash -c 'rm -f "$HOME/.pi/agent/models.json"; echo reset' >/dev/null
sb $DARK --harness pi -- bash -lc 'grep -q 524288 "$HOME/.pi/agent/models.json" && echo reseeded || echo not_reseeded'
assert_contains "reseeded" "deleting it inside the sandbox takes the host version again"
t_name "settings.json follows the host per key, and the thinking level is policy"
# The host file says medium and no ai-sandbox policy is in it: anything that arrives as
# xhigh can only have been written by the policy.
printf '{ "defaultProvider": "gateway", "defaultModel": "turbo", "theme": "dark/dark",\n "defaultThinkingLevel": "medium", "thinkingBudgets": { "xhigh": 65536 } }\n' > "$AGENT/settings.json"
sb $DARK --harness pi -- bash -lc 'jq -c . "$HOME/.pi/agent/settings.json"'
assert_contains '"defaultThinkingLevel":"xhigh"' "xhigh is enforced over the host file's medium"
assert_contains '"thinkingBudgets":{"xhigh":65536}' "and the rest of the preferences come along"
# What pi does to its own settings: rewrites the file, adding a key of its own. That used
# to freeze every host preference out of the sandbox for good.
sb $DARK --harness pi -- bash -c '
f="$HOME/.pi/agent/settings.json"
jq ".theme=\"light/light\" | .defaultThinkingLevel=\"off\" | .lastChangelogVersion=\"9.9.9\"" "$f" > "$f.n" && mv "$f.n" "$f"
echo edited' >/dev/null
# the host then moves the model and the budgets, drops the provider entirely, and says
# medium itself -- so a later --thinking host run has something to hand over
printf '{ "defaultModel": "deepthink", "theme": "sepia/sepia",\n "defaultThinkingLevel": "medium", "thinkingBudgets": { "xhigh": 131072 } }\n' > "$AGENT/settings.json"
sb $DARK --harness pi -- bash -lc 'jq -c . "$HOME/.pi/agent/settings.json"'
MERGED="$OUT"
printf '%s' "$MERGED" | grep -qF '"theme":"light/light"' && _ok "the theme edited in the sandbox survives" \
|| _bad "the theme edited in the sandbox survives" "$MERGED"
printf '%s' "$MERGED" | grep -qF '"lastChangelogVersion":"9.9.9"' && _ok "a key only pi wrote survives" \
|| _bad "a key only pi wrote survives" "$MERGED"
printf '%s' "$MERGED" | grep -qF '"defaultModel":"deepthink"' && _ok "an untouched model follows the host again" \
|| _bad "an untouched model follows the host again" "$MERGED"
printf '%s' "$MERGED" | grep -qF '"xhigh":131072' && _ok "and so do the thinking budgets" \
|| _bad "and so do the thinking budgets" "$MERGED"
printf '%s' "$MERGED" | grep -qF 'defaultProvider' && _bad "an untouched key the host dropped is dropped" \
|| _ok "an untouched key the host dropped is dropped"
printf '%s' "$MERGED" | grep -qF '"defaultThinkingLevel":"xhigh"' && _ok "thinking is forced back to xhigh after the sandbox set off" \
|| _bad "thinking is forced back to xhigh after the sandbox set off" "$MERGED"
# (the jq programs below avoid `{a, b}` object construction: bash brace-expands an
# unquoted word containing a comma before jq ever sees it)
sb $DARK --harness pi --thinking host -- bash -lc 'jq -r .defaultThinkingLevel "$HOME/.pi/agent/settings.json"'
assert_eq "medium" "$(printf '%s' "$OUT" | tail -1)" "with --thinking host the host's own medium is handed over"
sb $DARK --harness pi -- bash -c 'printf "{oops" > "$HOME/.pi/agent/settings.json"' >/dev/null
sb $DARK --harness pi -- bash -lc 'jq -c . "$HOME/.pi/agent/settings.json"'
assert_contains "deepthink" "a settings.json that is not JSON is re-seeded rather than left broken"
assert_contains '"defaultThinkingLevel":"xhigh"' "with the policy still applied"
sb $DARK --harness pi -- bash -lc 'jq -r .defaultThinkingLevel "$HOME/.pi/agent/settings.json"'
assert_eq "xhigh" "$(printf '%s' "$OUT" | tail -1)" "and it stays xhigh, not the medium that came back"
sb $DARK --harness pi -- bash -c 'chmod 000 "$HOME/.pi/agent/settings.json"' >/dev/null
sb $DARK --harness pi -- bash -lc 'echo ran'
assert_contains "cannot read settings.json" "a file this container cannot read is reported, not blamed on JSON"
assert_contains "--ephemeral-home" "and the report names a way out"
unset AI_API_TOKEN
sb_timeout 90 $DARK --harness pi
assert_rc 1 "a run without the credential is refused before anything starts"
assert_contains "not going to reach the container" "and it says why"
sb_env AI_API_TOKEN= --net none --harness pi
assert_rc 1 "an empty token is refused too -- pi would send an empty credential"
unset AISB_PI_AGENT_DIR AISB_PI_TOKEN_ENV AISB_PI_TOKEN_AUTO AISB_HOME_VOLUME
sb clean --all --volumes=aisb-test-pihome > /dev/null
t_name "doctor"
sb doctor
assert_rc 0 "doctor reports no problems"
assert_contains "bind mount + uid mapping" "doctor verifies uid mapping"
assert_contains "base pin:" "doctor reports which base digest the image was built on"
assert_contains "rootless" "doctor verifies rootless podman"