ducoterra
a08a69f143
Two changes, one bug found through the other.
Thinking is policy now: defaultThinkingLevel is xhigh unless you say otherwise
(thinking / --thinking / AISB_THINKING, with pi's levels plus 'host' to decline
and 'off' meaning pi's own no-reasoning level). It is written into the bridged
settings and re-applied by the entrypoint on every run, so a run cannot set its
own reasoning down for the runs after it. Unknown levels are refused with the
list to pick from. Nothing is written for codex, claude or a command after --,
and --no-pi-config declines it along with the bridge, because pi's own defaults
are what that flag asks for.
The reason xhigh had never reached a sandbox is worse than a missing default:
settings.json was managed as one whole file, and pi rewrites its own settings
(a theme change, "seen the changelog"), so the first such write matched nothing
and froze every host preference out permanently -- provider, model, thinking
budgets and level included, a real session left holding just
{"lastChangelogVersion": "1.4.0"}. It is merged per key instead, against a copy
of what we last wrote: a key edited in the sandbox stays edited, a key never
touched follows the host again, an untouched key the host dropped is dropped, a
key pi invented survives, and a file with no base copy takes the host values --
which repairs the volumes already frozen. Unparseable JSON is re-seeded with a
warning since pi cannot use it either; a file the container cannot read is
reported with the fix, because nothing inside the sandbox can repair one.
One trap worth remembering: the seeding script is passed to bash -c in single
quotes, so a single quote inside it closes the argument early and the rest runs
in the outer shell -- valid bash, wrong program, clean bash -n and clean
shellcheck. tests/unit/06_entrypoint.sh extracts the seeding script and asserts
it is quote-free, and extracts the merge program to test its semantics without
a container; tests/e2e/12_sandbox.sh does the same in a real run, including
forcing the level back after the sandbox sets it to off.
LLM-Generated: true
103 lines
4.2 KiB
Bash
Executable File
103 lines
4.2 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# tests/run.sh -- the ai-sandbox test runner.
|
|
#
|
|
# ./tests/run.sh unit tests (no containers started, no network)
|
|
# ./tests/run.sh --e2e everything, including real containers and egress tests
|
|
# ./tests/run.sh --only 04 just the files matching 04
|
|
# ./tests/run.sh --verbose show each test file's raw output as it runs
|
|
#
|
|
# Every test file runs with its own XDG state so nothing here touches the real
|
|
# machine's config, state or cache directories.
|
|
set -uo pipefail
|
|
|
|
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
ROOT="$(dirname "$HERE")"
|
|
AISB="${AISB:-$ROOT/bin/ai-sandbox}"
|
|
export AISB
|
|
|
|
WITH_E2E=0 ONLY="" VERBOSE=0
|
|
while (($#)); do
|
|
case "$1" in
|
|
--e2e|-e) WITH_E2E=1; shift ;;
|
|
--only) ONLY="${2:-}"; shift 2 ;;
|
|
--verbose|-v) export VERBOSE=1; shift ;; # lib.sh reads VERBOSE
|
|
--help|-h) sed -n '2,12p' "$0"; exit 0 ;;
|
|
*) echo "unknown option: $1" >&2; exit 2 ;;
|
|
esac
|
|
done
|
|
|
|
command -v podman >/dev/null || { echo "podman is required to run the tests" >&2; exit 2; }
|
|
|
|
# Syntax-check every shell file in the project before running anything.
|
|
SYNTAX_BAD=0
|
|
for f in "$AISB" "$HERE"/*.sh "$HERE"/unit/*.sh "$HERE"/e2e/*.sh "$ROOT"/install.sh; do
|
|
[[ -f "$f" ]] || continue
|
|
if ! syn="$(bash -n "$f" 2>&1)"; then
|
|
echo "SYNTAX: $f" >&2; printf '%s\n' "$syn" | sed 's/^/ /' >&2; SYNTAX_BAD=1
|
|
fi
|
|
done
|
|
(( SYNTAX_BAD )) && exit 2
|
|
|
|
FILES=( "$HERE"/unit/*.sh )
|
|
(( WITH_E2E )) && FILES+=( "$HERE"/e2e/*.sh )
|
|
if [[ -n "$ONLY" ]]; then
|
|
mapfile -t FILES < <(printf '%s\n' "${FILES[@]}" | grep "/[^/]*$ONLY")
|
|
fi
|
|
|
|
BASE="$(mktemp -d "${TMPDIR:-/tmp}/ai-sandbox-tests.XXXXXX")"
|
|
COUNTS="$BASE/counts"
|
|
: > "$COUNTS"
|
|
trap 'chmod -R u+rwX "$BASE" 2>/dev/null; rm -rf "$BASE" 2>/dev/null' EXIT
|
|
|
|
echo "ai-sandbox test suite"
|
|
echo " script : $AISB"
|
|
echo " mode : $([[ $WITH_E2E == 1 ]] && echo 'unit + e2e (real containers)' || echo 'unit only (add --e2e for containers)')"
|
|
echo " files : ${#FILES[@]}"
|
|
echo
|
|
|
|
FAILED_FILES=()
|
|
for f in "${FILES[@]}"; do
|
|
name="$(basename "$f")"
|
|
W="$BASE/$name"; mkdir -p "$W"/{state,cache,cfg}
|
|
echo "=================================================================="
|
|
echo "== $name"
|
|
echo "=================================================================="
|
|
# shellcheck disable=SC1090
|
|
(
|
|
export AISB_STATE_DIR="$W/state" AISB_CACHE_DIR="$W/cache" AISB_CONFIG_DIR="$W/cfg"
|
|
export WORK="$W" TESTS="$HERE" AISB_TEST_COUNTS="$COUNTS"
|
|
# Unit files must not touch a registry: pins come from the seeded cache and the
|
|
# digest lookup is a stub. e2e files keep the real behaviour.
|
|
case "$f" in
|
|
*/unit/*) export AISB_BASE_AUTO_UPDATE="${AISB_BASE_AUTO_UPDATE:-0}" \
|
|
AISB_BASE_PULL_FALLBACK="${AISB_BASE_PULL_FALLBACK:-0}" ;;
|
|
esac
|
|
# Never bridge the developer's own ~/.pi/agent into a test: tests that exercise the
|
|
# bridge point AISB_PI_AGENT_DIR at a fixture of their own. Same for a real token.
|
|
export AISB_PI_AGENT_DIR="${AISB_PI_AGENT_DIR:-$W/absent-pi-agent-dir}"
|
|
# Same for ~/.agents: its default is the developer's own skills directory, which no
|
|
# test should depend on or accidentally share. Files that exercise the .agents
|
|
# bridge point AISB_AGENTS_DIR at a fixture of their own.
|
|
export AISB_AGENTS_DIR="${AISB_AGENTS_DIR:-$W/absent-agents-dir}"
|
|
unset AI_API_TOKEN AISB_PI_TOKEN_ENV AISB_PI_CONFIG AISB_PI_TOKEN_AUTO AISB_THINKING
|
|
# NOTE: HOME is deliberately not overridden -- podman storage lives there.
|
|
unset AISB_ALLOW AISB_ENV AISB_MOUNT AISB_INSTALL AISB_NET AISB_EXTRA_ARGS AISB_SESSION_OVERRIDE
|
|
unset AISB_AGENTS AISB_AGENTS_DEST
|
|
source "$f"
|
|
summary
|
|
) || FAILED_FILES+=("$name")
|
|
echo
|
|
done
|
|
|
|
TOTAL_P=0 TOTAL_F=0
|
|
while read -r p fl; do TOTAL_P=$((TOTAL_P + p)); TOTAL_F=$((TOTAL_F + fl)); done < "$COUNTS"
|
|
|
|
echo "=================================================================="
|
|
if ((${#FAILED_FILES[@]})); then
|
|
printf '\033[31mFAILED\033[0m %d passed, %d failed across %d file(s)\n' "$TOTAL_P" "$TOTAL_F" "${#FAILED_FILES[@]}"
|
|
printf ' failing files: %s\n' "${FAILED_FILES[*]}"
|
|
exit 1
|
|
fi
|
|
printf '\033[32mALL PASS\033[0m %d assertions, %d file(s), %s mode\n' \
|
|
"$TOTAL_P" "${#FILES[@]}" "$([[ $WITH_E2E == 1 ]] && echo unit+e2e || echo unit)"
|