solo-c/test.sh

#!/bin/sh
# Offline regression tests for Solo-C. No API key and no network: every line
# handed to the agent starts with / or !, so nothing is ever sent to the model.
#
#   sh ./test.sh [binary]     # default ./solo
#
# Each test runs the agent in its own directory, because the session file
# (.solo_sess.json) lives in the current directory. Exits non-zero if any test
# failed, with the reason on stderr.
set -u

SOLO=${1:-./solo}
case $SOLO in /*) ;; *) SOLO=$PWD/$SOLO ;; esac
[ -x "$SOLO" ] || { echo "test: no such executable: $SOLO" >&2; exit 1; }
command -v perl >/dev/null 2>&1 || { echo "test: perl is required" >&2; exit 1; }

WORK=$(mktemp -d)
# Long enough that the environment block lands where the allocator dislikes it,
# short enough to stay a legal path component.
LONGDIR=/tmp/$(perl -e 'print "d" x 200')
trap 'rm -rf "$WORK" "$LONGDIR"' EXIT INT TERM
PASS=0
FAIL=0

# ok EXIT-CODE NAME [DETAIL] -- prints pass, or the detail on stderr.
ok() {
  if [ "$1" = 0 ]; then
    PASS=$((PASS + 1))
    printf 'PASS %s\n' "$2"
  else
    FAIL=$((FAIL + 1))
    printf 'FAIL %s: %s\n' "$2" "${3:-}" >&2
  fi
}

# solo_in DIR ENV... < lines
solo_in() {
  _dir=$1; shift
  mkdir -p "$_dir"
  ( cd "$_dir" && env "$@" "$SOLO" )
}

echo "# $SOLO"

# --- a setting taken from the environment must not be freed by the agent -------
# These slots used to point straight at getenv memory, so setstr() called free()
# on it and the process died inside the slash command that changed it.
printf '/endpoint http://x.example\n/quit\n' | \
  solo_in "$WORK/envcrash" SOLO_ENDPOINT="http://$(perl -e 'print "a" x 900').example.com/api" \
  >/dev/null 2>&1
ok $? 'SOLO_ENDPOINT then /endpoint does not crash'

printf '/model other/model\n/quit\n' | \
  solo_in "$WORK/envmodel" SOLO_MODEL="vendor/$(perl -e 'print "m" x 500')" >/dev/null 2>&1
ok $? 'SOLO_MODEL then /model does not crash'

printf '/reasoning_effort low\n/quit\n' | \
  solo_in "$WORK/envreason" SOLO_REASON="$(perl -e 'print "r" x 700')" >/dev/null 2>&1
ok $? 'SOLO_REASON then /reasoning_effort does not crash'

# TMP is freed at exit, so this one used to die without a slash command at all.
printf '/quit\n' | solo_in "$WORK/envtmp" SOLO_TMPDIR="$LONGDIR" >/dev/null 2>&1
ok $? 'SOLO_TMPDIR then /quit does not crash'

# --- an integer parameter must stay an integer in the request ------------------
# Every number went through json_object_new_double, so /max_tokens 4096 reached
# the endpoint as 4096.0.
D=$WORK/params
printf '/max_tokens 4096\n/seed 3\n/top_p 0.95\n/quit\n' | solo_in "$D" >/dev/null 2>&1
perl -0777 -ne 'exit(/"max_tokens":4096,/ ? 0 : 1)' "$D/.solo_sess.json"
ok $? '/max_tokens 4096 is sent as an integer'
perl -0777 -ne 'exit(/"seed":3,/ ? 0 : 1)' "$D/.solo_sess.json"
ok $? '/seed 3 is sent as an integer'
perl -0777 -ne 'exit(/"top_p":0\.9/ ? 0 : 1)' "$D/.solo_sess.json"
ok $? '/top_p 0.95 is sent as a number'
perl -0777 -ne 'exit(/"foo"/ ? 1 : 0)' "$D/.solo_sess.json"
ok $? 'a string parameter is still a string'

# --- the timeout is the timeout, not one second less --------------------------
# The old deadline was time(0) + TO, and time() counts whole seconds, so a
# command that happened to start late in a second was killed up to a second
# early. Stagger the start and require that no run dies before the limit.
D=$WORK/timing
MIN=99
for frac in 0 0.25 0.5 0.75; do
  out=$( { sleep "$frac"
           printf '!sh -c "date +%%s.%%N; sleep 6"\n!date +%%s.%%N\n/quit\n'; } | \
         solo_in "$D" SOLO_TIMEOUT=3 2>/dev/null )
  a=$(printf '%s\n' "$out" | grep -o '^[0-9][0-9]*\.[0-9]*' | head -1)
  b=$(printf '%s\n' "$out" | grep -o '^[0-9][0-9]*\.[0-9]*' | tail -1)
  d=$(perl -e 'printf "%.2f", $ARGV[1] - $ARGV[0]' "$a" "$b" 2>/dev/null || echo 0)
  MIN=$(perl -e 'printf "%.2f", $ARGV[1] < $ARGV[0] ? $ARGV[1] : $ARGV[0]' "$MIN" "$d")
done
[ "$(perl -e 'print $ARGV[0] >= 2.8 ? 1 : 0' "$MIN")" = 1 ]
ok $? "SOLO_TIMEOUT=3 never fires early" "a run died after ${MIN}s"

# --- SOLO_TIMEOUT below 1 means no limit, not "kill instantly" ----------------
printf '!echo hi\n/quit\n' | solo_in "$WORK/nolimit" SOLO_TIMEOUT=0 2>&1 | grep -q '^hi$'
ok $? 'SOLO_TIMEOUT=0 runs commands instead of killing them at once'
printf '!echo hi\n/quit\n' | solo_in "$WORK/neglimit" SOLO_TIMEOUT=-5 2>&1 | grep -q '^hi$'
ok $? 'a negative SOLO_TIMEOUT is not an instant timeout'

# --- a timed out command leaves nothing behind --------------------------------
# The child calls setpgid(0,0) and the parent kills the group, so the whole
# group has to be gone once the turn ends.
D=$WORK/group
printf '!sh -c "sleep 41 & sleep 41"\n/quit\n' | solo_in "$D" SOLO_TIMEOUT=1 >/dev/null 2>&1
sleep 1
LEFT=$(ps -o args= 2>/dev/null | grep -c '^sleep 41$')
[ "$LEFT" -eq 0 ]
ok $? 'a timed out command takes its process group with it' "$LEFT process(es) left"

# --- the tool description must quote the real timeout -------------------------
D=$WORK/desc
printf '/quit\n' | solo_in "$D" SOLO_TIMEOUT=42 >/dev/null 2>&1
perl -0777 -ne 'exit(/after 42s is killed/ ? 0 : 1)' "$D/.solo_sess.json"
ok $? 'the bash tool description quotes SOLO_TIMEOUT'

printf '%d passed, %d failed\n' "$PASS" "$FAIL"
[ "$FAIL" -eq 0 ]