solo-c/test.sh

#!/bin/sh
# Offline regression tests for Solo-C. No API key and no network: every line
# handed to the agent starts with / or !, so nothing is ever sent to the model.
#
#   sh ./test.sh [binary]     # default ./solo
#
# Each test runs the agent in its own directory, because the session file
# (.solo_sess.json) lives in the current directory. Exits non-zero if any test
# failed, with the reason on stderr.
set -u

SOLO=${1:-./solo}
# Canonical, like /proc/self/exe inside the agent, which names its own path to the
# model so a subagent can be re-launched.
SOLO=$(perl -MCwd -e 'print Cwd::abs_path($ARGV[0]) // $ARGV[0]' "$SOLO" 2>/dev/null) || SOLO=${1:-./solo}
[ -x "$SOLO" ] || { echo "test: no such executable: $SOLO" >&2; exit 1; }
command -v perl >/dev/null 2>&1 || { echo "test: perl is required" >&2; exit 1; }

WORK=$(mktemp -d)
# Long enough that the environment block lands where the allocator dislikes it,
# short enough to stay a legal path component.
LONGDIR=/tmp/$(perl -e 'print "d" x 200')
trap 'rm -rf "$WORK" "$LONGDIR"' EXIT INT TERM
PASS=0
FAIL=0

# ok EXIT-CODE NAME [DETAIL] -- prints pass, or the detail on stderr.
ok() {
  if [ "$1" = 0 ]; then
    PASS=$((PASS + 1))
    printf 'PASS %s\n' "$2"
  else
    FAIL=$((FAIL + 1))
    printf 'FAIL %s: %s\n' "$2" "${3:-}" >&2
  fi
}

# solo_in DIR ENV... < lines
solo_in() {
  _dir=$1; shift
  mkdir -p "$_dir"
  ( cd "$_dir" && env "$@" "$SOLO" )
}

# solo_args DIR ARG... -- the one-shot mode: no stdin, no REPL
solo_args() {
  _dir=$1; shift
  mkdir -p "$_dir"
  ( cd "$_dir" && "$SOLO" "$@" < /dev/null )
}

echo "# $SOLO"

# --- a setting taken from the environment must not be freed by the agent -------
# These slots used to point straight at getenv memory, so setstr() called free()
# on it and the process died inside the slash command that changed it.
printf '/endpoint http://x.example\n/quit\n' | \
  solo_in "$WORK/envcrash" SOLO_ENDPOINT="http://$(perl -e 'print "a" x 900').example.com/api" \
  >/dev/null 2>&1
ok $? 'SOLO_ENDPOINT then /endpoint does not crash'

printf '/model other/model\n/quit\n' | \
  solo_in "$WORK/envmodel" SOLO_MODEL="vendor/$(perl -e 'print "m" x 500')" >/dev/null 2>&1
ok $? 'SOLO_MODEL then /model does not crash'

printf '/reasoning_effort low\n/quit\n' | \
  solo_in "$WORK/envreason" SOLO_REASON="$(perl -e 'print "r" x 700')" >/dev/null 2>&1
ok $? 'SOLO_REASON then /reasoning_effort does not crash'

# TMP is freed at exit, so this one used to die without a slash command at all.
printf '/quit\n' | solo_in "$WORK/envtmp" SOLO_TMPDIR="$LONGDIR" >/dev/null 2>&1
ok $? 'SOLO_TMPDIR then /quit does not crash'

# --- an integer parameter must stay an integer in the request ------------------
# Every number went through json_object_new_double, so /max_tokens 4096 reached
# the endpoint as 4096.0.
D=$WORK/params
printf '/max_tokens 4096\n/seed 3\n/top_p 0.95\n/quit\n' | solo_in "$D" >/dev/null 2>&1
perl -0777 -ne 'exit(/"max_tokens":4096,/ ? 0 : 1)' "$D/.solo_sess.json"
ok $? '/max_tokens 4096 is sent as an integer'
perl -0777 -ne 'exit(/"seed":3,/ ? 0 : 1)' "$D/.solo_sess.json"
ok $? '/seed 3 is sent as an integer'
perl -0777 -ne 'exit(/"top_p":0\.9/ ? 0 : 1)' "$D/.solo_sess.json"
ok $? '/top_p 0.95 is sent as a number'
perl -0777 -ne 'exit(/"foo"/ ? 1 : 0)' "$D/.solo_sess.json"
ok $? 'a string parameter is still a string'

# --- the timeout is the timeout, not one second less --------------------------
# The old deadline was time(0) + TO, and time() counts whole seconds, so a
# command that happened to start late in a second was killed up to a second
# early. Stagger the start and require that no run dies before the limit.
D=$WORK/timing
MIN=99
for frac in 0 0.25 0.5 0.75; do
  out=$( { sleep "$frac"
           printf '!sh -c "date +%%s.%%N; sleep 6"\n!date +%%s.%%N\n/quit\n'; } | \
         solo_in "$D" SOLO_TIMEOUT=3 2>/dev/null )
  a=$(printf '%s\n' "$out" | grep -o '^[0-9][0-9]*\.[0-9]*' | head -1)
  b=$(printf '%s\n' "$out" | grep -o '^[0-9][0-9]*\.[0-9]*' | tail -1)
  d=$(perl -e 'printf "%.2f", $ARGV[1] - $ARGV[0]' "$a" "$b" 2>/dev/null || echo 0)
  MIN=$(perl -e 'printf "%.2f", $ARGV[1] < $ARGV[0] ? $ARGV[1] : $ARGV[0]' "$MIN" "$d")
done
[ "$(perl -e 'print $ARGV[0] >= 2.8 ? 1 : 0' "$MIN")" = 1 ]
ok $? "SOLO_TIMEOUT=3 never fires early" "a run died after ${MIN}s"

# --- SOLO_TIMEOUT below 1 means no limit, not "kill instantly" ----------------
printf '!echo hi\n/quit\n' | solo_in "$WORK/nolimit" SOLO_TIMEOUT=0 2>&1 | grep -q '^hi$'
ok $? 'SOLO_TIMEOUT=0 runs commands instead of killing them at once'
printf '!echo hi\n/quit\n' | solo_in "$WORK/neglimit" SOLO_TIMEOUT=-5 2>&1 | grep -q '^hi$'
ok $? 'a negative SOLO_TIMEOUT is not an instant timeout'

# --- a timed out command leaves nothing behind --------------------------------
# The child calls setpgid(0,0) and the parent kills the group, so the whole
# group has to be gone once the turn ends.
D=$WORK/group
printf '!sh -c "sleep 41 & sleep 41"\n/quit\n' | solo_in "$D" SOLO_TIMEOUT=1 >/dev/null 2>&1
sleep 1
LEFT=$(ps -o args= 2>/dev/null | grep -c '^sleep 41$')
[ "$LEFT" -eq 0 ]
ok $? 'a timed out command takes its process group with it' "$LEFT process(es) left"

# --- the tool description must quote the real timeout -------------------------
D=$WORK/desc
printf '/quit\n' | solo_in "$D" SOLO_TIMEOUT=42 >/dev/null 2>&1
perl -0777 -ne 'exit(/after 42s is killed/ ? 0 : 1)' "$D/.solo_sess.json"
ok $? 'the bash tool description quotes SOLO_TIMEOUT'

# --- a prompt as an argument runs one task and exits --------------------------
# The banner moved to stderr, so stdout holds the answer alone and a one-shot run
# can be piped into another program.
D=$WORK/oneshot
solo_args "$D" '!echo hello' >"$WORK/out" 2>"$WORK/err"
ok $? 'a prompt argument runs the task and exits'
grep -q '^hello$' "$WORK/out"
ok $? 'the answer arrives on stdout'
grep -q 'Agent Solo ready' "$WORK/err"
ok $? 'the ready banner goes to stderr in one-shot mode'
if grep -q 'Agent Solo ready' "$WORK/out"; then rc=1; else rc=0; fi
ok "$rc" 'stdout carries no banner in one-shot mode'
# Anything left on stdin is ignored: the one-shot run never opens the REPL.
( cd "$D" && printf '!echo from-stdin\n/quit\n' | "$SOLO" '!echo hello' ) >>"$WORK/out" 2>>"$WORK/err"
if grep -q 'from-stdin' "$WORK/out"; then rc=1; else rc=0; fi
ok "$rc" 'a one-shot run never reads stdin'

solo_args "$WORK/onestatus" '!sh -c "exit 7"' >/dev/null 2>&1
[ $? -eq 7 ]
ok $? 'a ! task hands its own exit status back'

solo_args "$WORK/joined" '!echo' one two >"$WORK/joined.out" 2>/dev/null
grep -q '^one two$' "$WORK/joined.out"
ok $? 'every word becomes part of one task, quoting optional'

# An empty or whitespace-only argument is no task at all, so the REPL opens.
mkdir -p "$WORK/blank"
( cd "$WORK/blank" && printf '/quit\n' | "$SOLO" '  ' ) >"$WORK/blank.out" 2>/dev/null
grep -q 'Agent Solo ready' "$WORK/blank.out"
ok $? 'a whitespace-only argument still opens the REPL'

# --- the usage line belongs to the last turn, not to every turn ----------------
# The token stats used to be printed after every turn, so they landed in the
# middle of a tool run and chopped up whatever was piped out of a one-shot run.
cat >"$WORK/mock.pl" <<'MOCK_EOF'
# A chat endpoint on the loopback: the first turn asks for a tool, the second one
# answers. Prints the port it got, then serves until it is killed.
use IO::Socket::INET;
my $srv = IO::Socket::INET->new(LocalAddr => '127.0.0.1', LocalPort => 0,
                                Listen => 5, ReuseAddr => 1) or exit 1;
$| = 1;
print $srv->sockport, "\n";
my $n = 0;
while(my $c = $srv->accept) {
  $c->autoflush(1);
  my $req = '';
  while(my $l = <$c>) { last if $l =~ /^\r?\n$/; $req .= $l; }
  print $c "HTTP/1.0 100 Continue\r\n\r\n" if $req =~ /^Expect:/mi;
  my ($len) = $req =~ /Content-Length:\s*(\d+)/i;
  my $body = '';
  read($c, $body, $len) if $len;
  $n++;
  my $call = '{"role":"assistant","content":"","tool_calls":[{"id":"c1","type":"function",'
    . '"function":{"name":"bash","arguments":"{\"command\":\"echo tool-ran\"}"}}]}';
  my $msg = $n == 1 ? $call : '{"role":"assistant","content":"The answer is 42."}';
  my $out = '{"model":"mock/free","usage":{"total_tokens":7,"prompt_tokens":5,"completion_tokens":2},'
    . '"choices":[{"message":' . $msg . '}]}';
  print $c "HTTP/1.0 200 OK\r\nContent-Type: application/json\r\nContent-Length: "
    . length($out) . "\r\n\r\n" . $out;
  close $c;
}
MOCK_EOF
perl "$WORK/mock.pl" >"$WORK/port" 2>/dev/null &
MOCK=$!
i=0
while [ ! -s "$WORK/port" ] && [ "$i" -lt 50 ]; do sleep 0.1; i=$((i + 1)); done
PORT=$(cat "$WORK/port" 2>/dev/null)
mkdir -p "$WORK/tokens"
( cd "$WORK/tokens" && SOLO_ENDPOINT="http://127.0.0.1:$PORT/api" "$SOLO" 'do it' ) \
  >"$WORK/tokens.out" 2>"$WORK/tokens.err"
kill "$MOCK" 2>/dev/null
[ "$(grep -c 'tokens:' "$WORK/tokens.out")" -eq 1 ]
ok $? 'the token stats are printed once, on the last turn' \
  "$(grep -c 'tokens:' "$WORK/tokens.out") line(s)"
awk '/tool-ran/{t=NR} /tokens:/{s=NR} END{exit !(t && s && s > t)}' "$WORK/tokens.out"
ok $? 'the tool run is not chopped up by stats' "$(tail -1 "$WORK/tokens.out")"
grep -q 'The answer is 42\.$' "$WORK/tokens.out"
ok $? 'the answer of the last turn reaches stdout'
tail -1 "$WORK/tokens.out" | grep -q '^--- \[mock/free\] tokens:'
ok $? 'the stats line is the last thing on stdout'

# --- the model has to know how to delegate to a subagent ----------------------
D=$WORK/subagent
printf '/quit\n' | solo_in "$D" SOLO_ENDPOINT=http://e.example/api/// >/dev/null 2>&1
perl -0777 -ne 'exit(/split into subagents/ ? 0 : 1)' "$D/.solo_sess.json"
ok $? 'the system prompt tells the model it can spawn subagents'
tr -d '\\' <"$D/.solo_sess.json" >"$D/plain"   # json-c escapes every slash
grep -qF "SOLO_ENDPOINT=http://e.example/api/ $SOLO 'the subtask, stated in full'" "$D/plain"
ok $? 'the subagent line names the endpoint and Solo own path'
if grep -q 'api///' "$D/.solo_sess.json"; then rc=1; else rc=0; fi
ok "$rc" 'a trailing slash on SOLO_ENDPOINT is stripped'

printf '%d passed, %d failed\n' "$PASS" "$FAIL"
[ "$FAIL" -eq 0 ]