Luxferre
141c6cc414
Port the two latest changes of the Perl original. Non-interactive mode: a prompt as an argument runs that one task and exits. Every word is joined into a single task, so quoting is optional, and an empty or whitespace-only argument still opens the REPL. The ready banner moves to stderr so stdout carries the answer, progress and token stats alone, stdout is line buffered so a piped run arrives as it is written, and the process exits with the task status: the command's own for !cmd, 1 on failure, 130 on Ctrl-C. dispatch() returns an int for that. Subagent delegation: the system prompt now tells the model it can split a multi-stage task by copying .solo_sess.json into a fresh directory under the scratch dir and re-launching this binary from there with the same endpoint, model, tools and prompt, which the copy carries over. The prompt names our own path, resolved through /proc/self/exe, and notes that a subagent answers on stdout while its banner goes to stderr, so independent subtasks run in parallel with &. The prompt buffer is now sized from the expanded paths, and trailing slashes are stripped from SOLO_ENDPOINT as in the original, so the command in the prompt is always well formed. Also print the token stats only on the turn that ends the tool loop, as the original does: they used to land in the middle of a tool run and chop up whatever was piped out of a one-shot run. Tests go to 28, all offline: 15 new ones, including a loopback mock endpoint that drives a tool call and checks where the stats land.
222 lines
9.9 KiB
Bash
Executable File
222 lines
9.9 KiB
Bash
Executable File
#!/bin/sh
|
|
# Offline regression tests for Solo-C. No API key and no network: every line
|
|
# handed to the agent starts with / or !, so nothing is ever sent to the model.
|
|
#
|
|
# sh ./test.sh [binary] # default ./solo
|
|
#
|
|
# Each test runs the agent in its own directory, because the session file
|
|
# (.solo_sess.json) lives in the current directory. Exits non-zero if any test
|
|
# failed, with the reason on stderr.
|
|
set -u
|
|
|
|
SOLO=${1:-./solo}
|
|
# Canonical, like /proc/self/exe inside the agent, which names its own path to the
|
|
# model so a subagent can be re-launched.
|
|
SOLO=$(perl -MCwd -e 'print Cwd::abs_path($ARGV[0]) // $ARGV[0]' "$SOLO" 2>/dev/null) || SOLO=${1:-./solo}
|
|
[ -x "$SOLO" ] || { echo "test: no such executable: $SOLO" >&2; exit 1; }
|
|
command -v perl >/dev/null 2>&1 || { echo "test: perl is required" >&2; exit 1; }
|
|
|
|
WORK=$(mktemp -d)
|
|
# Long enough that the environment block lands where the allocator dislikes it,
|
|
# short enough to stay a legal path component.
|
|
LONGDIR=/tmp/$(perl -e 'print "d" x 200')
|
|
trap 'rm -rf "$WORK" "$LONGDIR"' EXIT INT TERM
|
|
PASS=0
|
|
FAIL=0
|
|
|
|
# ok EXIT-CODE NAME [DETAIL] -- prints pass, or the detail on stderr.
|
|
ok() {
|
|
if [ "$1" = 0 ]; then
|
|
PASS=$((PASS + 1))
|
|
printf 'PASS %s\n' "$2"
|
|
else
|
|
FAIL=$((FAIL + 1))
|
|
printf 'FAIL %s: %s\n' "$2" "${3:-}" >&2
|
|
fi
|
|
}
|
|
|
|
# solo_in DIR ENV... < lines
|
|
solo_in() {
|
|
_dir=$1; shift
|
|
mkdir -p "$_dir"
|
|
( cd "$_dir" && env "$@" "$SOLO" )
|
|
}
|
|
|
|
# solo_args DIR ARG... -- the one-shot mode: no stdin, no REPL
|
|
solo_args() {
|
|
_dir=$1; shift
|
|
mkdir -p "$_dir"
|
|
( cd "$_dir" && "$SOLO" "$@" < /dev/null )
|
|
}
|
|
|
|
echo "# $SOLO"
|
|
|
|
# --- a setting taken from the environment must not be freed by the agent -------
|
|
# These slots used to point straight at getenv memory, so setstr() called free()
|
|
# on it and the process died inside the slash command that changed it.
|
|
printf '/endpoint http://x.example\n/quit\n' | \
|
|
solo_in "$WORK/envcrash" SOLO_ENDPOINT="http://$(perl -e 'print "a" x 900').example.com/api" \
|
|
>/dev/null 2>&1
|
|
ok $? 'SOLO_ENDPOINT then /endpoint does not crash'
|
|
|
|
printf '/model other/model\n/quit\n' | \
|
|
solo_in "$WORK/envmodel" SOLO_MODEL="vendor/$(perl -e 'print "m" x 500')" >/dev/null 2>&1
|
|
ok $? 'SOLO_MODEL then /model does not crash'
|
|
|
|
printf '/reasoning_effort low\n/quit\n' | \
|
|
solo_in "$WORK/envreason" SOLO_REASON="$(perl -e 'print "r" x 700')" >/dev/null 2>&1
|
|
ok $? 'SOLO_REASON then /reasoning_effort does not crash'
|
|
|
|
# TMP is freed at exit, so this one used to die without a slash command at all.
|
|
printf '/quit\n' | solo_in "$WORK/envtmp" SOLO_TMPDIR="$LONGDIR" >/dev/null 2>&1
|
|
ok $? 'SOLO_TMPDIR then /quit does not crash'
|
|
|
|
# --- an integer parameter must stay an integer in the request ------------------
|
|
# Every number went through json_object_new_double, so /max_tokens 4096 reached
|
|
# the endpoint as 4096.0.
|
|
D=$WORK/params
|
|
printf '/max_tokens 4096\n/seed 3\n/top_p 0.95\n/quit\n' | solo_in "$D" >/dev/null 2>&1
|
|
perl -0777 -ne 'exit(/"max_tokens":4096,/ ? 0 : 1)' "$D/.solo_sess.json"
|
|
ok $? '/max_tokens 4096 is sent as an integer'
|
|
perl -0777 -ne 'exit(/"seed":3,/ ? 0 : 1)' "$D/.solo_sess.json"
|
|
ok $? '/seed 3 is sent as an integer'
|
|
perl -0777 -ne 'exit(/"top_p":0\.9/ ? 0 : 1)' "$D/.solo_sess.json"
|
|
ok $? '/top_p 0.95 is sent as a number'
|
|
perl -0777 -ne 'exit(/"foo"/ ? 1 : 0)' "$D/.solo_sess.json"
|
|
ok $? 'a string parameter is still a string'
|
|
|
|
# --- the timeout is the timeout, not one second less --------------------------
|
|
# The old deadline was time(0) + TO, and time() counts whole seconds, so a
|
|
# command that happened to start late in a second was killed up to a second
|
|
# early. Stagger the start and require that no run dies before the limit.
|
|
D=$WORK/timing
|
|
MIN=99
|
|
for frac in 0 0.25 0.5 0.75; do
|
|
out=$( { sleep "$frac"
|
|
printf '!sh -c "date +%%s.%%N; sleep 6"\n!date +%%s.%%N\n/quit\n'; } | \
|
|
solo_in "$D" SOLO_TIMEOUT=3 2>/dev/null )
|
|
a=$(printf '%s\n' "$out" | grep -o '^[0-9][0-9]*\.[0-9]*' | head -1)
|
|
b=$(printf '%s\n' "$out" | grep -o '^[0-9][0-9]*\.[0-9]*' | tail -1)
|
|
d=$(perl -e 'printf "%.2f", $ARGV[1] - $ARGV[0]' "$a" "$b" 2>/dev/null || echo 0)
|
|
MIN=$(perl -e 'printf "%.2f", $ARGV[1] < $ARGV[0] ? $ARGV[1] : $ARGV[0]' "$MIN" "$d")
|
|
done
|
|
[ "$(perl -e 'print $ARGV[0] >= 2.8 ? 1 : 0' "$MIN")" = 1 ]
|
|
ok $? "SOLO_TIMEOUT=3 never fires early" "a run died after ${MIN}s"
|
|
|
|
# --- SOLO_TIMEOUT below 1 means no limit, not "kill instantly" ----------------
|
|
printf '!echo hi\n/quit\n' | solo_in "$WORK/nolimit" SOLO_TIMEOUT=0 2>&1 | grep -q '^hi$'
|
|
ok $? 'SOLO_TIMEOUT=0 runs commands instead of killing them at once'
|
|
printf '!echo hi\n/quit\n' | solo_in "$WORK/neglimit" SOLO_TIMEOUT=-5 2>&1 | grep -q '^hi$'
|
|
ok $? 'a negative SOLO_TIMEOUT is not an instant timeout'
|
|
|
|
# --- a timed out command leaves nothing behind --------------------------------
|
|
# The child calls setpgid(0,0) and the parent kills the group, so the whole
|
|
# group has to be gone once the turn ends.
|
|
D=$WORK/group
|
|
printf '!sh -c "sleep 41 & sleep 41"\n/quit\n' | solo_in "$D" SOLO_TIMEOUT=1 >/dev/null 2>&1
|
|
sleep 1
|
|
LEFT=$(ps -o args= 2>/dev/null | grep -c '^sleep 41$')
|
|
[ "$LEFT" -eq 0 ]
|
|
ok $? 'a timed out command takes its process group with it' "$LEFT process(es) left"
|
|
|
|
# --- the tool description must quote the real timeout -------------------------
|
|
D=$WORK/desc
|
|
printf '/quit\n' | solo_in "$D" SOLO_TIMEOUT=42 >/dev/null 2>&1
|
|
perl -0777 -ne 'exit(/after 42s is killed/ ? 0 : 1)' "$D/.solo_sess.json"
|
|
ok $? 'the bash tool description quotes SOLO_TIMEOUT'
|
|
|
|
# --- a prompt as an argument runs one task and exits --------------------------
|
|
# The banner moved to stderr, so stdout holds the answer alone and a one-shot run
|
|
# can be piped into another program.
|
|
D=$WORK/oneshot
|
|
solo_args "$D" '!echo hello' >"$WORK/out" 2>"$WORK/err"
|
|
ok $? 'a prompt argument runs the task and exits'
|
|
grep -q '^hello$' "$WORK/out"
|
|
ok $? 'the answer arrives on stdout'
|
|
grep -q 'Agent Solo ready' "$WORK/err"
|
|
ok $? 'the ready banner goes to stderr in one-shot mode'
|
|
if grep -q 'Agent Solo ready' "$WORK/out"; then rc=1; else rc=0; fi
|
|
ok "$rc" 'stdout carries no banner in one-shot mode'
|
|
# Anything left on stdin is ignored: the one-shot run never opens the REPL.
|
|
( cd "$D" && printf '!echo from-stdin\n/quit\n' | "$SOLO" '!echo hello' ) >>"$WORK/out" 2>>"$WORK/err"
|
|
if grep -q 'from-stdin' "$WORK/out"; then rc=1; else rc=0; fi
|
|
ok "$rc" 'a one-shot run never reads stdin'
|
|
|
|
solo_args "$WORK/onestatus" '!sh -c "exit 7"' >/dev/null 2>&1
|
|
[ $? -eq 7 ]
|
|
ok $? 'a ! task hands its own exit status back'
|
|
|
|
solo_args "$WORK/joined" '!echo' one two >"$WORK/joined.out" 2>/dev/null
|
|
grep -q '^one two$' "$WORK/joined.out"
|
|
ok $? 'every word becomes part of one task, quoting optional'
|
|
|
|
# An empty or whitespace-only argument is no task at all, so the REPL opens.
|
|
mkdir -p "$WORK/blank"
|
|
( cd "$WORK/blank" && printf '/quit\n' | "$SOLO" ' ' ) >"$WORK/blank.out" 2>/dev/null
|
|
grep -q 'Agent Solo ready' "$WORK/blank.out"
|
|
ok $? 'a whitespace-only argument still opens the REPL'
|
|
|
|
# --- the usage line belongs to the last turn, not to every turn ----------------
|
|
# The token stats used to be printed after every turn, so they landed in the
|
|
# middle of a tool run and chopped up whatever was piped out of a one-shot run.
|
|
cat >"$WORK/mock.pl" <<'MOCK_EOF'
|
|
# A chat endpoint on the loopback: the first turn asks for a tool, the second one
|
|
# answers. Prints the port it got, then serves until it is killed.
|
|
use IO::Socket::INET;
|
|
my $srv = IO::Socket::INET->new(LocalAddr => '127.0.0.1', LocalPort => 0,
|
|
Listen => 5, ReuseAddr => 1) or exit 1;
|
|
$| = 1;
|
|
print $srv->sockport, "\n";
|
|
my $n = 0;
|
|
while(my $c = $srv->accept) {
|
|
$c->autoflush(1);
|
|
my $req = '';
|
|
while(my $l = <$c>) { last if $l =~ /^\r?\n$/; $req .= $l; }
|
|
print $c "HTTP/1.0 100 Continue\r\n\r\n" if $req =~ /^Expect:/mi;
|
|
my ($len) = $req =~ /Content-Length:\s*(\d+)/i;
|
|
my $body = '';
|
|
read($c, $body, $len) if $len;
|
|
$n++;
|
|
my $call = '{"role":"assistant","content":"","tool_calls":[{"id":"c1","type":"function",'
|
|
. '"function":{"name":"bash","arguments":"{\"command\":\"echo tool-ran\"}"}}]}';
|
|
my $msg = $n == 1 ? $call : '{"role":"assistant","content":"The answer is 42."}';
|
|
my $out = '{"model":"mock/free","usage":{"total_tokens":7,"prompt_tokens":5,"completion_tokens":2},'
|
|
. '"choices":[{"message":' . $msg . '}]}';
|
|
print $c "HTTP/1.0 200 OK\r\nContent-Type: application/json\r\nContent-Length: "
|
|
. length($out) . "\r\n\r\n" . $out;
|
|
close $c;
|
|
}
|
|
MOCK_EOF
|
|
perl "$WORK/mock.pl" >"$WORK/port" 2>/dev/null &
|
|
MOCK=$!
|
|
i=0
|
|
while [ ! -s "$WORK/port" ] && [ "$i" -lt 50 ]; do sleep 0.1; i=$((i + 1)); done
|
|
PORT=$(cat "$WORK/port" 2>/dev/null)
|
|
mkdir -p "$WORK/tokens"
|
|
( cd "$WORK/tokens" && SOLO_ENDPOINT="http://127.0.0.1:$PORT/api" "$SOLO" 'do it' ) \
|
|
>"$WORK/tokens.out" 2>"$WORK/tokens.err"
|
|
kill "$MOCK" 2>/dev/null
|
|
[ "$(grep -c 'tokens:' "$WORK/tokens.out")" -eq 1 ]
|
|
ok $? 'the token stats are printed once, on the last turn' \
|
|
"$(grep -c 'tokens:' "$WORK/tokens.out") line(s)"
|
|
awk '/tool-ran/{t=NR} /tokens:/{s=NR} END{exit !(t && s && s > t)}' "$WORK/tokens.out"
|
|
ok $? 'the tool run is not chopped up by stats' "$(tail -1 "$WORK/tokens.out")"
|
|
grep -q 'The answer is 42\.$' "$WORK/tokens.out"
|
|
ok $? 'the answer of the last turn reaches stdout'
|
|
tail -1 "$WORK/tokens.out" | grep -q '^--- \[mock/free\] tokens:'
|
|
ok $? 'the stats line is the last thing on stdout'
|
|
|
|
# --- the model has to know how to delegate to a subagent ----------------------
|
|
D=$WORK/subagent
|
|
printf '/quit\n' | solo_in "$D" SOLO_ENDPOINT=http://e.example/api/// >/dev/null 2>&1
|
|
perl -0777 -ne 'exit(/split into subagents/ ? 0 : 1)' "$D/.solo_sess.json"
|
|
ok $? 'the system prompt tells the model it can spawn subagents'
|
|
tr -d '\\' <"$D/.solo_sess.json" >"$D/plain" # json-c escapes every slash
|
|
grep -qF "SOLO_ENDPOINT=http://e.example/api/ $SOLO 'the subtask, stated in full'" "$D/plain"
|
|
ok $? 'the subagent line names the endpoint and Solo own path'
|
|
if grep -q 'api///' "$D/.solo_sess.json"; then rc=1; else rc=0; fi
|
|
ok "$rc" 'a trailing slash on SOLO_ENDPOINT is stripped'
|
|
|
|
printf '%d passed, %d failed\n' "$PASS" "$FAIL"
|
|
[ "$FAIL" -eq 0 ]
|