ducoterra
a08a69f143
Two changes, one bug found through the other.
Thinking is policy now: defaultThinkingLevel is xhigh unless you say otherwise
(thinking / --thinking / AISB_THINKING, with pi's levels plus 'host' to decline
and 'off' meaning pi's own no-reasoning level). It is written into the bridged
settings and re-applied by the entrypoint on every run, so a run cannot set its
own reasoning down for the runs after it. Unknown levels are refused with the
list to pick from. Nothing is written for codex, claude or a command after --,
and --no-pi-config declines it along with the bridge, because pi's own defaults
are what that flag asks for.
The reason xhigh had never reached a sandbox is worse than a missing default:
settings.json was managed as one whole file, and pi rewrites its own settings
(a theme change, "seen the changelog"), so the first such write matched nothing
and froze every host preference out permanently -- provider, model, thinking
budgets and level included, a real session left holding just
{"lastChangelogVersion": "1.4.0"}. It is merged per key instead, against a copy
of what we last wrote: a key edited in the sandbox stays edited, a key never
touched follows the host again, an untouched key the host dropped is dropped, a
key pi invented survives, and a file with no base copy takes the host values --
which repairs the volumes already frozen. Unparseable JSON is re-seeded with a
warning since pi cannot use it either; a file the container cannot read is
reported with the fix, because nothing inside the sandbox can repair one.
One trap worth remembering: the seeding script is passed to bash -c in single
quotes, so a single quote inside it closes the argument early and the rest runs
in the outer shell -- valid bash, wrong program, clean bash -n and clean
shellcheck. tests/unit/06_entrypoint.sh extracts the seeding script and asserts
it is quote-free, and extracts the merge program to test its semantics without
a container; tests/e2e/12_sandbox.sh does the same in a real run, including
forcing the level back after the sandbox sets it to off.
LLM-Generated: true
290 lines
14 KiB
Bash
290 lines
14 KiB
Bash
#!/usr/bin/env bash
|
|
# tests/unit/08_pi_config.sh -- the pi configuration bridge.
|
|
#
|
|
# The bridge rewrites the host agent directory instead of copying it, so what matters
|
|
# most here is what does NOT cross the boundary: the credential, host commands, host
|
|
# paths, auth.json. Everything is preview (`--print-policy` / `--dry-run`) plus the
|
|
# refusal paths, so no container is started and no token is real.
|
|
source "$TESTS/lib.sh"
|
|
|
|
AGENT="$WORK/agent"; mkdir -p "$AGENT"
|
|
export AISB_PI_AGENT_DIR="$AGENT" AISB_PI_TOKEN_AUTO=1
|
|
export AI_API_TOKEN=sandbox-only-token-4242
|
|
|
|
CANARY_SECRET='REALSECRET-nev3r-lef7s-the-host'
|
|
CANARY_CMD='pass show services/gateway | head -1'
|
|
|
|
cat > "$AGENT/models.json" <<EOF
|
|
{
|
|
"providers": {
|
|
"gateway": {
|
|
"baseUrl": "https://aipi.reeseapps.com/v1",
|
|
"api": "openai-completions",
|
|
"apiKey": "!$CANARY_CMD",
|
|
"headers": { "X-Trace": "!date +%s", "X-Team": "self" },
|
|
"models": [
|
|
{ "id": "juggernaut", "reasoning": true, "contextWindow": 131072 },
|
|
{ "id": "turbo", "reasoning": true, "contextWindow": 262144 }
|
|
]
|
|
},
|
|
"lab-ollama": {
|
|
"baseUrl": "http://10.20.0.7:11434/v1",
|
|
"api": "openai-completions",
|
|
"models": [ { "id": "small" } ]
|
|
}
|
|
}
|
|
}
|
|
EOF
|
|
# sessionDir points at the *real* host home on purpose: that is the case the bridge
|
|
# has to drop, and HOME must stay the real home for podman's sake.
|
|
# defaultThinkingLevel is deliberately NOT xhigh here: the host has to disagree with the
|
|
# ai-sandbox policy, otherwise "the policy wrote xhigh" cannot be told apart from "the
|
|
# host said xhigh and we copied it".
|
|
cat > "$AGENT/settings.json" <<EOF
|
|
{
|
|
"defaultProvider": "gateway",
|
|
"defaultModel": "turbo",
|
|
"theme": "dark/dark",
|
|
"defaultThinkingLevel": "medium",
|
|
"enableInstallTelemetry": false,
|
|
"thinkingBudgets": { "xhigh": 65536 },
|
|
"sessionDir": "$HOME/pi-sessions"
|
|
}
|
|
EOF
|
|
printf '{ "gateway": "%s" }' "$CANARY_SECRET" > "$AGENT/auth.json"
|
|
printf '{}' > "$AGENT/models-store.json"
|
|
|
|
clear_sessions() { rm -rf "${AISB_STATE_DIR:?}"/sessions/*; }
|
|
gen_file() { find "$AISB_STATE_DIR/sessions" -path "*/pi/$1" -print -quit 2>/dev/null; }
|
|
restore_models() { cat > "$AGENT/models.json" <<EOF
|
|
{ "providers": { "gateway": { "baseUrl": "https://aipi.reeseapps.com/v1",
|
|
"api": "openai-completions", "apiKey": "literal-key-never-copied",
|
|
"models": [ { "id": "turbo" } ] } } }
|
|
EOF
|
|
}
|
|
|
|
t_name "the credential becomes an environment reference, not a copy"
|
|
clear_sessions
|
|
sb --keep --print-policy --net none
|
|
assert_contains "pi_config=bridged from $AGENT" "the bridge reports where the config came from"
|
|
assert_contains "pi_token_env=AI_API_TOKEN (set)" "and whether the token is in reach"
|
|
M="$(gen_file models.json)"
|
|
[[ -n "$M" ]] && _ok "models.json was generated" || _bad "models.json was generated"
|
|
if [[ -n "$M" ]]; then
|
|
grep -qF '"apiKey": "$AI_API_TOKEN"' "$M" && _ok "apiKey is the variable; pi resolves it at request time" || _bad "apiKey is the variable; pi resolves it at request time"
|
|
grep -qF "$CANARY_CMD" "$M" && _bad "the host '!command' is gone" || _ok "the host '!command' is gone"
|
|
grep -qF "$CANARY_SECRET" "$M" && _bad "the literal key is gone" || _ok "the literal key is gone"
|
|
grep -qF 'literal-key-never-copied' "$M" && _bad "no literal key from the host file" || _ok "no literal key from the host file"
|
|
grep -qF '"X-Team": "self"' "$M" && _ok "an ordinary header survives" || _bad "an ordinary header survives"
|
|
grep -qF 'X-Trace' "$M" && _bad "a header built by a host command is dropped" || _ok "a header built by a host command is dropped"
|
|
fi
|
|
assert_contains "dropped header(s) X-Trace" "and it says which one it dropped"
|
|
if printf '%s' "$OUT" | grep -qF "$AI_API_TOKEN"; then
|
|
_bad "the token value never appears in the policy output"
|
|
else
|
|
_ok "the token value never appears in the policy output"
|
|
fi
|
|
|
|
t_name "providers and models are carried over untouched"
|
|
if [[ -n "$M" ]]; then
|
|
grep -qF '"baseUrl": "https://aipi.reeseapps.com/v1"' "$M" && _ok "the gateway endpoint" || _bad "the gateway endpoint"
|
|
grep -qF '"id": "juggernaut"' "$M" && grep -qF '"id": "turbo"' "$M" && _ok "both models" || _bad "both models"
|
|
grep -qF '"contextWindow": 262144' "$M" && _ok "with their limits" || _bad "with their limits"
|
|
grep -qF '"lab-ollama"' "$M" && _ok "and the second provider" || _bad "and the second provider"
|
|
fi
|
|
JSON_CHECK="$(python3 - "${M:-/nonexistent}" <<'PY'
|
|
import json, sys
|
|
try:
|
|
prov = json.load(open(sys.argv[1]))["providers"]
|
|
except Exception as e:
|
|
print(f"invalid:{e}"); raise SystemExit
|
|
if "apiKey" in prov.get("lab-ollama", {}):
|
|
print("gained-a-key")
|
|
elif prov["gateway"]["apiKey"] != "$AI_API_TOKEN":
|
|
print("wrong-key-ref")
|
|
else:
|
|
print("json-valid")
|
|
PY
|
|
)"
|
|
assert_eq "json-valid" "$JSON_CHECK" "valid JSON, and a provider without a key does not gain one"
|
|
assert_contains "pi_endpoints=10.20.0.7 aipi.reeseapps.com" "both endpoints are reported"
|
|
|
|
t_name "settings travel, host paths do not"
|
|
S="$(gen_file settings.json)"
|
|
if [[ -n "$S" ]]; then
|
|
for pair in '"defaultProvider": "gateway"' '"defaultModel": "turbo"' '"theme": "dark/dark"' \
|
|
'"xhigh": 65536' '"enableInstallTelemetry": false'; do # the thinking level is policy, not a preference that travels
|
|
grep -qF "$pair" "$S" && _ok "kept $pair" || _bad "kept $pair"
|
|
done
|
|
grep -qF 'sessionDir' "$S" && _bad "a value pointing at the host home is dropped" || _ok "a value pointing at the host home is dropped"
|
|
fi
|
|
assert_contains "dropped settings sessionDir" "and it says so"
|
|
|
|
# `--print-policy --keep` renders the bridge into the session directory and stops there:
|
|
# a unit file never starts a container, and --keep keeps the rendered files to look at.
|
|
rendered_level() { # rendered_level <level-or-default>
|
|
clear_sessions
|
|
sb --keep --print-policy --net none ${1:+--thinking "$1"}
|
|
local f; f="$(gen_file settings.json)"
|
|
[[ -n "$f" ]] || { printf 'none'; return; }
|
|
jq -r '.defaultThinkingLevel // "absent"' "$f" 2>/dev/null || printf 'invalid'
|
|
}
|
|
|
|
t_name "the thinking level is policy, written over the settings that travel"
|
|
sb --keep --print-policy --net none
|
|
assert_contains "thinking=xhigh (enforced on every run)" "xhigh is the default, not pi's medium"
|
|
assert_contains "pi config: thinking -> xhigh" "and it is announced"
|
|
# the fixture says medium, so an xhigh here can only have come from the policy
|
|
assert_eq "xhigh" "$(rendered_level '')" "the policy beats the host file's medium"
|
|
# the level is a preference, so it belongs to settings.json and nowhere else
|
|
clear_sessions
|
|
sb --keep --print-policy --net none
|
|
if grep -q 'defaultThinkingLevel' "${M:-/nonexistent}" 2>/dev/null; then
|
|
_bad "models.json is untouched by it" "the key appeared in the model catalog"
|
|
else
|
|
_ok "models.json is untouched by it"
|
|
fi
|
|
S="$(gen_file settings.json)"
|
|
grep -qF '"defaultProvider": "gateway"' "${S:-/nonexistent}" \
|
|
&& _ok "the rest of the settings still travel" || _bad "the rest of the settings still travel"
|
|
grep -qF '"xhigh": 65536' "${S:-/nonexistent}" \
|
|
&& _ok "including the host's thinking budgets" || _bad "the host's thinking budgets"
|
|
grep -qF 'sessionDir' "${S:-/nonexistent}" \
|
|
&& _bad "and a host path is still dropped while the level is added" || _ok "and a host path is still dropped while the level is added"
|
|
|
|
t_name "every level, including 'host' and 'off', and nothing in between"
|
|
assert_eq "low" "$(rendered_level low)" "a level of your choosing"
|
|
assert_eq "max" "$(rendered_level max)" "and the top of pi's range"
|
|
assert_eq "off" "$(rendered_level off)" "off is pi's own level, not 'do not manage it'"
|
|
assert_eq "medium" "$(rendered_level host)" "host leaves the key to settings.json (medium there)"
|
|
rendered_level host >/dev/null
|
|
assert_contains "thinking=host (not applied: the config decides)" "and says so"
|
|
rendered_level bogus >/dev/null
|
|
assert_rc 1 "an unknown level is refused"
|
|
assert_contains "must be off, minimal, low, medium, high, xhigh, max or host" "with the list to pick from"
|
|
sb_env AISB_THINKING=medium --print-policy --net none
|
|
assert_contains "thinking=medium" "AISB_THINKING works"
|
|
sb_env AISB_THINKING=medium --thinking max --print-policy --net none
|
|
assert_contains "thinking=max" "the flag beats the environment"
|
|
|
|
t_name "a host with no settings.json still gets the policy"
|
|
mv "$AGENT/settings.json" "$WORK/settings-away.json"
|
|
assert_eq "xhigh" "$(rendered_level '')" "the level does not depend on the host having the file"
|
|
clear_sessions
|
|
sb --keep --print-policy --net none
|
|
assert_contains "pi_endpoints" "the model bridge still happened alongside it"
|
|
mv "$WORK/settings-away.json" "$AGENT/settings.json"
|
|
|
|
t_name "no bridge means no policy either, and it says so"
|
|
clear_sessions
|
|
sb --keep --no-pi-config --print-policy --net none
|
|
assert_contains "thinking=xhigh (not applied: pi config bridging is off)" "--no-pi-config declines the thinking policy too"
|
|
if [[ -n "$(gen_file settings.json)" ]]; then _bad "nothing is rendered for pi at all"; else _ok "nothing is rendered for pi at all"; fi
|
|
|
|
t_name "auth.json and the credential store are never bridged"
|
|
if find "$AISB_STATE_DIR/sessions" \( -name auth.json -o -name models-store.json \) | grep -q .; then
|
|
_bad "only models.json and settings.json are bridged"
|
|
else
|
|
_ok "only models.json and settings.json are bridged"
|
|
fi
|
|
if grep -rlF "$CANARY_SECRET" "$AISB_STATE_DIR" "$AISB_CACHE_DIR" > /dev/null 2>&1; then
|
|
_bad "the host credential appears nowhere in ai-sandbox state"
|
|
else
|
|
_ok "the host credential appears nowhere in ai-sandbox state"
|
|
fi
|
|
|
|
t_name "the token is passed by name, once, and announced"
|
|
sb --keep --print-policy --net none
|
|
assert_contains "passing \$AI_API_TOKEN from the host" "the auto-pass is announced"
|
|
assert_eq "env_names=AI_API_TOKEN" "$(printf '%s' "$OUT" | grep '^env_names=' || true)" "exactly one variable is passed"
|
|
assert_absent "$AI_API_TOKEN" "and only its name is reported"
|
|
|
|
t_name "--no-pi-token-env leaves it out and says so"
|
|
clear_sessions
|
|
sb --keep --no-pi-token-env --print-policy --net none
|
|
assert_eq "env_names=" "$(printf '%s' "$OUT" | grep '^env_names=' || true)" "nothing is passed"
|
|
assert_contains "will not reach the container" "preview warns instead of launching a pi with no credential"
|
|
|
|
t_name "a bridged provider with no credential in reach is refused, not attempted"
|
|
unset AI_API_TOKEN
|
|
sb --net none --harness pi
|
|
assert_rc 1 "the run refuses to start"
|
|
assert_contains 'asks for $AI_API_TOKEN, but it is not going to reach the container' "it names the variable and why"
|
|
assert_contains "env = AI_API_TOKEN" "it shows the config-file way"
|
|
assert_contains "--no-pi-config" "and the way to opt out"
|
|
assert_eq "0" "$([[ -n "$(podman ps -q --filter label=ai-sandbox=1)" ]] && echo 1 || echo 0)" "and nothing was started"
|
|
|
|
t_name "an empty token is treated as missing"
|
|
sb_env AI_API_TOKEN= --net none --harness pi # set, but empty
|
|
assert_rc 1 "an empty credential is refused, not sent"
|
|
assert_contains "unset or empty" "and it says which"
|
|
sb_env AI_API_TOKEN= --net none --print-policy
|
|
assert_contains "pi_token_env=AI_API_TOKEN (set but empty)" "preview distinguishes the two states"
|
|
|
|
t_name "but only when pi is the thing being launched"
|
|
export AI_API_TOKEN=x
|
|
sb --net none --dry-run -- make test
|
|
assert_absent "not going to reach the container" "a replaced command needs no credential"
|
|
sb --net none --dry-run --shell
|
|
assert_rc 0 "--shell works without a token"
|
|
assert_absent "not going to reach the container" "and does not demand one"
|
|
sb --net none --dry-run --harness codex
|
|
assert_absent "not going to reach the container" "codex does not read pi's models.json"
|
|
|
|
t_name "--no-pi-config opts out cleanly"
|
|
sb --no-pi-config --net none --print-policy
|
|
assert_absent "pi_config=" "no bridge is reported"
|
|
assert_absent "not going to reach the container" "and nothing is required"
|
|
clear_sessions
|
|
sb --keep --no-pi-config --net none -- true
|
|
if [[ -n "$(gen_file models.json)" ]]; then _bad "nothing was generated"; else _ok "nothing was generated"; fi
|
|
|
|
t_name "the token variable name is yours to choose"
|
|
clear_sessions
|
|
sb --keep --pi-token-env GW_TOKEN --print-policy --net none
|
|
assert_contains "pi_token_env=GW_TOKEN (NOT SET)" "reported under the new name"
|
|
clear_sessions
|
|
sb_env GW_TOKEN=whatever-gw --keep --pi-token-env GW_TOKEN --print-policy --net none
|
|
assert_contains "pi_token_env=GW_TOKEN (set)" "it is recognised as in reach"
|
|
assert_contains "passing \$GW_TOKEN" "and announced under that name"
|
|
M2="$(gen_file models.json)"
|
|
grep -qF '"apiKey": "$GW_TOKEN"' "${M2:-/nonexistent}" && _ok "the generated config references it" || _bad "the generated config references it"
|
|
sb --net none --harness pi --pi-token-env GW_TOKEN # GW_TOKEN is not set
|
|
assert_rc 1 "and it is enforced under that name too"
|
|
|
|
t_name "a missing agent directory is a skip, not an error"
|
|
sb_env AISB_PI_AGENT_DIR="$WORK/nope" --net none --print-policy
|
|
assert_rc 0 "nothing to bridge, nothing demanded"
|
|
assert_absent "pi_config=" "and no bridge is claimed"
|
|
sb_env AISB_PI_AGENT_DIR="$WORK/nope" --net none --harness pi
|
|
assert_rc 0 "pi may still be launched, with its own defaults"
|
|
|
|
t_name "broken JSON is reported, not propagated"
|
|
printf '{ "providers": oops' > "$AGENT/models.json"
|
|
clear_sessions
|
|
sb --keep --net none --print-policy
|
|
assert_contains "models.json is not valid JSON" "it says the file is broken"
|
|
if grep -qF 'oops' "${AISB_STATE_DIR}/sessions"/*/mnt/pi/models.json 2>/dev/null; then
|
|
_bad "no invalid file is placed in front of pi"
|
|
else
|
|
_ok "no invalid file is placed in front of pi"
|
|
fi
|
|
restore_models
|
|
|
|
t_name "the model endpoint has to be allowed, and says when it is not"
|
|
sb --net allow --allow registry.npmjs.org --print-policy
|
|
assert_contains "the model endpoint is not in the allowlist" "a pi that cannot reach its gateway is a warning, not a mystery"
|
|
assert_contains "add --allow aipi.reeseapps.com" "and it names the host to add"
|
|
sb --net allow --allow aipi.reeseapps.com --allow 10.20.0.7 --print-policy
|
|
assert_absent "not in the allowlist" "silent once both endpoints are allowed"
|
|
|
|
t_name "the bridge never widens the allowlist by itself"
|
|
sb --net allow --allow aipi.reeseapps.com --print-policy
|
|
assert_contains "gate_domains=aipi.reeseapps.com" "what you asked for is what you got"
|
|
assert_absent "10.20.0.7 80,443" "the discovered endpoint was not added on your behalf"
|
|
|
|
t_name "an invalid token variable name is rejected"
|
|
sb --net none --pi-token-env '9bad name' --print-policy
|
|
assert_rc 1 "rejected"
|
|
assert_contains "not a valid identifier" "with a clear reason"
|