toolcall fixes
This commit is contained in:
Luxferre
+455
@@ -3604,3 +3604,458 @@ func TestWebSearchParameterMappingAndSuppression(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHunyuanReasoningAndToolCallingStreaming(t *testing.T) {
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if strings.HasSuffix(r.URL.Path, "/info") {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"named_endpoints": map[string]interface{}{
|
||||
"/chat": map[string]interface{}{
|
||||
"parameters": []map[string]interface{}{
|
||||
{"parameter_name": "message", "component": "Api"},
|
||||
{"parameter_name": "system_prompt", "component": "Api"},
|
||||
{"parameter_name": "history", "component": "Api"},
|
||||
{"parameter_name": "think_level", "component": "Api"},
|
||||
{"parameter_name": "temperature", "component": "Api"},
|
||||
{"parameter_name": "max_tokens", "component": "Api"},
|
||||
{"parameter_name": "top_p", "component": "Api"},
|
||||
{"parameter_name": "preserved_thinking", "component": "Api"},
|
||||
{"parameter_name": "functions_json_str", "component": "Api"},
|
||||
},
|
||||
"code_snippets": map[string]interface{}{
|
||||
"bash": "curl -X POST http://localhost:7860/gradio_api/call/chat",
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
if strings.HasSuffix(r.URL.Path, "/call/chat") && r.Method == http.MethodPost {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"event_id": "hy3-evt-stream-tools",
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
if strings.Contains(r.URL.Path, "/call/chat/hy3-evt-stream-tools") && r.Method == http.MethodGet {
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
w.Header().Set("Cache-Control", "no-cache")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
flusher, _ := w.(http.Flusher)
|
||||
|
||||
// Event 1: Reasoning part 1
|
||||
fmt.Fprint(w, "event: generating\ndata: [[\"\", \"Step 1 reasoning.\", null]]\n\n")
|
||||
if flusher != nil {
|
||||
flusher.Flush()
|
||||
}
|
||||
|
||||
// Event 2: Reasoning part 2
|
||||
fmt.Fprint(w, "event: generating\ndata: [[\"\", \"Step 1 reasoning. Step 2 reasoning.\", null]]\n\n")
|
||||
if flusher != nil {
|
||||
flusher.Flush()
|
||||
}
|
||||
|
||||
// Event 3: Tool call chunk 1 with partial arguments
|
||||
fmt.Fprint(w, "event: generating\ndata: [[\"\", \"Step 1 reasoning. Step 2 reasoning.\", [{\"id\":\"call_xyz\",\"type\":\"function\",\"function\":{\"name\":\"calculator\",\"arguments\":\"{\\\"expr\\\":\\\"2\"}}]]]\n\n")
|
||||
if flusher != nil {
|
||||
flusher.Flush()
|
||||
}
|
||||
|
||||
// Event 4: Tool call chunk 2 with remaining arguments
|
||||
fmt.Fprint(w, "event: complete\ndata: [[\"\", \"Step 1 reasoning. Step 2 reasoning.\", [{\"id\":\"call_xyz\",\"type\":\"function\",\"function\":{\"name\":\"calculator\",\"arguments\":\"{\\\"expr\\\":\\\"2+2\\\"}\"}}]]]\n\n")
|
||||
if flusher != nil {
|
||||
flusher.Flush()
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
http.NotFound(w, r)
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
gw := NewGradioGateway(ts.URL, "", 10*time.Second)
|
||||
|
||||
reqStream := ChatCompletionRequest{
|
||||
Model: "hy3",
|
||||
Stream: true,
|
||||
Messages: []ChatMessage{
|
||||
{Role: "user", Content: "Calculate 2+2"},
|
||||
},
|
||||
Tools: []Tool{
|
||||
{Type: "function", Function: map[string]interface{}{"name": "calculator"}},
|
||||
},
|
||||
}
|
||||
httpReq := httptest.NewRequest("POST", "/v1/chat/completions", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
err := gw.ExecuteChatCompletion(rec, httpReq, reqStream)
|
||||
if err != nil {
|
||||
t.Fatalf("ExecuteChatCompletion failed: %v", err)
|
||||
}
|
||||
|
||||
lines := strings.Split(rec.Body.String(), "\n")
|
||||
var receivedDeltas []StreamDelta
|
||||
var finishReasons []string
|
||||
var foundDone bool
|
||||
|
||||
for _, line := range lines {
|
||||
line = strings.TrimSpace(line)
|
||||
if strings.HasPrefix(line, "data: ") {
|
||||
payload := strings.TrimPrefix(line, "data: ")
|
||||
if payload == "[DONE]" {
|
||||
foundDone = true
|
||||
continue
|
||||
}
|
||||
var streamResp StreamResponse
|
||||
if err := json.Unmarshal([]byte(payload), &streamResp); err == nil && len(streamResp.Choices) > 0 {
|
||||
choice := streamResp.Choices[0]
|
||||
receivedDeltas = append(receivedDeltas, choice.Delta)
|
||||
if choice.FinishReason != nil {
|
||||
finishReasons = append(finishReasons, *choice.FinishReason)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !foundDone {
|
||||
t.Errorf("expected 'data: [DONE]' in SSE stream")
|
||||
}
|
||||
|
||||
if len(receivedDeltas) < 1 || receivedDeltas[0].Role != "assistant" {
|
||||
t.Errorf("expected first delta to set role='assistant', got %+v", receivedDeltas[0])
|
||||
}
|
||||
|
||||
var combinedReasoning, combinedContent string
|
||||
var toolCallDeltas []ToolCall
|
||||
for _, d := range receivedDeltas {
|
||||
combinedReasoning += d.ReasoningContent
|
||||
combinedContent += d.Content
|
||||
if len(d.ToolCalls) > 0 {
|
||||
toolCallDeltas = append(toolCallDeltas, d.ToolCalls...)
|
||||
}
|
||||
}
|
||||
|
||||
if combinedReasoning != "Step 1 reasoning. Step 2 reasoning." {
|
||||
t.Errorf("combined reasoning=%q, want 'Step 1 reasoning. Step 2 reasoning.'", combinedReasoning)
|
||||
}
|
||||
if combinedContent != "" {
|
||||
t.Errorf("expected no content emitted during tool calling, got %q", combinedContent)
|
||||
}
|
||||
|
||||
if len(toolCallDeltas) != 2 {
|
||||
t.Fatalf("expected exactly 2 tool call deltas, got %d: %+v", len(toolCallDeltas), toolCallDeltas)
|
||||
}
|
||||
|
||||
// First delta chunk: has index, id, type, name, initial arguments
|
||||
firstTC := toolCallDeltas[0]
|
||||
if firstTC.Index == nil || *firstTC.Index != 0 {
|
||||
t.Errorf("first delta index expected 0, got %v", firstTC.Index)
|
||||
}
|
||||
if firstTC.ID != "call_xyz" || firstTC.Type != "function" || firstTC.Function.Name != "calculator" {
|
||||
t.Errorf("first delta expected full metadata, got %+v", firstTC)
|
||||
}
|
||||
if firstTC.Function.Arguments != `{"expr":"2` {
|
||||
t.Errorf("first delta arguments=%q, want %q", firstTC.Function.Arguments, `{"expr":"2`)
|
||||
}
|
||||
|
||||
// Second delta chunk: MUST have empty ID, empty Type, empty Name, and ONLY argument delta "+2\"}"
|
||||
secondTC := toolCallDeltas[1]
|
||||
if secondTC.Index == nil || *secondTC.Index != 0 {
|
||||
t.Errorf("second delta index expected 0, got %v", secondTC.Index)
|
||||
}
|
||||
if secondTC.ID != "" || secondTC.Type != "" || secondTC.Function.Name != "" {
|
||||
t.Errorf("second delta MUST NOT resend ID/Type/Name per OpenAI spec: %+v", secondTC)
|
||||
}
|
||||
if secondTC.Function.Arguments != `+2"}` {
|
||||
t.Errorf("second delta arguments=%q, want %q", secondTC.Function.Arguments, `+2"}`)
|
||||
}
|
||||
|
||||
if len(finishReasons) != 1 || finishReasons[0] != "tool_calls" {
|
||||
t.Errorf("finish_reasons=%v, want ['tool_calls']", finishReasons)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHunyuanToolCallingNonStreaming(t *testing.T) {
|
||||
var receivedPayload map[string]interface{}
|
||||
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if strings.HasSuffix(r.URL.Path, "/info") {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"named_endpoints": map[string]interface{}{
|
||||
"/chat": map[string]interface{}{
|
||||
"parameters": []map[string]interface{}{
|
||||
{"parameter_name": "message", "component": "Api"},
|
||||
{"parameter_name": "system_prompt", "component": "Api"},
|
||||
{"parameter_name": "history", "component": "Api"},
|
||||
{"parameter_name": "think_level", "component": "Api"},
|
||||
{"parameter_name": "temperature", "component": "Api"},
|
||||
{"parameter_name": "max_tokens", "component": "Api"},
|
||||
{"parameter_name": "top_p", "component": "Api"},
|
||||
{"parameter_name": "preserved_thinking", "component": "Api"},
|
||||
{"parameter_name": "functions_json_str", "component": "Api"},
|
||||
},
|
||||
"code_snippets": map[string]interface{}{
|
||||
"bash": "curl -X POST http://localhost:7860/gradio_api/call/chat",
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
if strings.HasSuffix(r.URL.Path, "/call/chat") && r.Method == http.MethodPost {
|
||||
json.NewDecoder(r.Body).Decode(&receivedPayload)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"event_id": "hy3-tool-nonstream",
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
if strings.Contains(r.URL.Path, "/call/chat/hy3-tool-nonstream") && r.Method == http.MethodGet {
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
fmt.Fprint(w, "event: complete\ndata: [[\"\", \"I should call the search tool.\", [{\"id\":\"call_abc\",\"type\":\"function\",\"function\":{\"name\":\"search\",\"arguments\":\"{\\\"query\\\":\\\"golang\\\"}\"}}]]]\n\n")
|
||||
return
|
||||
}
|
||||
|
||||
http.NotFound(w, r)
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
gw := NewGradioGateway(ts.URL, "", 10*time.Second)
|
||||
|
||||
reqNonStream := ChatCompletionRequest{
|
||||
Model: "hy3",
|
||||
Stream: false,
|
||||
Messages: []ChatMessage{
|
||||
{Role: "user", Content: "Search for golang news"},
|
||||
},
|
||||
Tools: []Tool{
|
||||
{Type: "function", Function: map[string]interface{}{"name": "search", "description": "Search web"}},
|
||||
},
|
||||
}
|
||||
|
||||
httpReq := httptest.NewRequest("POST", "/v1/chat/completions", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
err := gw.ExecuteChatCompletion(rec, httpReq, reqNonStream)
|
||||
if err != nil {
|
||||
t.Fatalf("ExecuteChatCompletion non-streaming failed: %v", err)
|
||||
}
|
||||
|
||||
// Verify functions_json_str passed in data[8]
|
||||
dataArr, _ := receivedPayload["data"].([]interface{})
|
||||
if len(dataArr) < 9 || dataArr[8] == nil || dataArr[8] == "" {
|
||||
t.Errorf("expected functions_json_str in data[8], got %v", dataArr)
|
||||
}
|
||||
|
||||
var resp ChatCompletionResponse
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("failed to decode response: %v", err)
|
||||
}
|
||||
|
||||
if len(resp.Choices) != 1 {
|
||||
t.Fatalf("expected 1 choice, got %d", len(resp.Choices))
|
||||
}
|
||||
choice := resp.Choices[0]
|
||||
if choice.FinishReason != "tool_calls" {
|
||||
t.Errorf("finish_reason=%q, want 'tool_calls'", choice.FinishReason)
|
||||
}
|
||||
if choice.Message.Content != nil {
|
||||
t.Errorf("content expected nil, got %v", choice.Message.Content)
|
||||
}
|
||||
if choice.Message.ReasoningContent != "I should call the search tool." {
|
||||
t.Errorf("reasoning_content=%q, want 'I should call the search tool.'", choice.Message.ReasoningContent)
|
||||
}
|
||||
if len(choice.Message.ToolCalls) != 1 {
|
||||
t.Fatalf("expected 1 tool call, got %d", len(choice.Message.ToolCalls))
|
||||
}
|
||||
tc := choice.Message.ToolCalls[0]
|
||||
if tc.ID != "call_abc" || tc.Function.Name != "search" || tc.Function.Arguments != `{"query":"golang"}` {
|
||||
t.Errorf("unexpected tool call: %+v", tc)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHunyuanMultiTurnToolResultHistory(t *testing.T) {
|
||||
gw := &GradioGateway{}
|
||||
|
||||
disc := NewDefaultSpaceDiscovery("https://tencent-hy3.hf.space")
|
||||
disc.IsHunyuan3 = true
|
||||
disc.TotalInputs = 9
|
||||
disc.MessageIndex = 0
|
||||
disc.SystemIndex = 1
|
||||
disc.HistoryIndex = 2
|
||||
disc.HistoryFormat = "messages"
|
||||
disc.ThinkLevelIndex = 3
|
||||
disc.TempIndex = 4
|
||||
disc.MaxTokensIndex = 5
|
||||
disc.TopPIndex = 6
|
||||
disc.PreservedThinkingIndex = 7
|
||||
disc.FunctionsJSONIndex = 8
|
||||
|
||||
req := ChatCompletionRequest{
|
||||
Model: "hy3",
|
||||
Messages: []ChatMessage{
|
||||
{Role: "user", Content: "What is the weather in Tokyo?"},
|
||||
{
|
||||
Role: "assistant",
|
||||
ToolCalls: []ToolCall{
|
||||
{ID: "call_weather_1", Type: "function", Function: ToolCallFunction{Name: "get_weather", Arguments: `{"location":"Tokyo"}`}},
|
||||
},
|
||||
},
|
||||
{
|
||||
Role: "tool",
|
||||
Name: "get_weather",
|
||||
ToolCallID: "call_weather_1",
|
||||
Content: `{"temperature": 20, "condition": "Sunny"}`,
|
||||
},
|
||||
},
|
||||
Tools: []Tool{
|
||||
{Type: "function", Function: map[string]interface{}{"name": "get_weather"}},
|
||||
},
|
||||
}
|
||||
|
||||
data, err := gw.BuildGradioPayload(disc, req)
|
||||
if err != nil {
|
||||
t.Fatalf("BuildGradioPayload failed: %v", err)
|
||||
}
|
||||
|
||||
// 1. Message input (slot 0) should be clean "Tool result for get_weather: ..." without "Query:" or "Please answer..."
|
||||
msgStr, ok := data[0].(string)
|
||||
if !ok {
|
||||
t.Fatalf("expected string message in data[0], got %T", data[0])
|
||||
}
|
||||
expectedMsg := `Tool result for get_weather: {"temperature": 20, "condition": "Sunny"}`
|
||||
if msgStr != expectedMsg {
|
||||
t.Errorf("message=%q, want %q", msgStr, expectedMsg)
|
||||
}
|
||||
|
||||
// 2. History input (slot 2) should contain user turn and assistant tool_call turn
|
||||
histArr, ok := data[2].([]map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatalf("expected []map[string]interface{} history in data[2], got %T", data[2])
|
||||
}
|
||||
if len(histArr) != 2 {
|
||||
t.Fatalf("expected 2 turns in history, got %d: %+v", len(histArr), histArr)
|
||||
}
|
||||
if histArr[0]["role"] != "user" || histArr[0]["content"] != "What is the weather in Tokyo?" {
|
||||
t.Errorf("unexpected turn 0: %+v", histArr[0])
|
||||
}
|
||||
if histArr[1]["role"] != "assistant" {
|
||||
t.Errorf("unexpected turn 1 role: %+v", histArr[1])
|
||||
}
|
||||
tcArr, ok := histArr[1]["tool_calls"].([]ToolCall)
|
||||
if !ok || len(tcArr) != 1 || tcArr[0].Function.Name != "get_weather" {
|
||||
t.Errorf("unexpected turn 1 tool_calls: %+v", histArr[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueueToolCallingStreamingParity(t *testing.T) {
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/info" {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"named_endpoints": map[string]interface{}{
|
||||
"/predict": map[string]interface{}{
|
||||
"parameters": []map[string]interface{}{
|
||||
{"parameter_name": "message", "component": "Api"},
|
||||
{"parameter_name": "tools", "component": "Api"},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
if strings.HasSuffix(r.URL.Path, "/queue/join") {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{"event_id": "queue-evt-1"})
|
||||
return
|
||||
}
|
||||
|
||||
if strings.HasSuffix(r.URL.Path, "/queue/data") {
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
flusher, _ := w.(http.Flusher)
|
||||
|
||||
// Chunk 1: Initial tool call with partial argument
|
||||
fmt.Fprint(w, "data: {\"msg\":\"process_generating\",\"output\":{\"data\":[[\"\", \"\", [{\"id\":\"tc_q1\",\"type\":\"function\",\"function\":{\"name\":\"fetch_data\",\"arguments\":\"{\\\"id\\\": 1\"}}]]]}}\n\n")
|
||||
if flusher != nil {
|
||||
flusher.Flush()
|
||||
}
|
||||
|
||||
// Chunk 2: Completed tool call argument
|
||||
fmt.Fprint(w, "data: {\"msg\":\"process_generating\",\"output\":{\"data\":[[\"\", \"\", [{\"id\":\"tc_q1\",\"type\":\"function\",\"function\":{\"name\":\"fetch_data\",\"arguments\":\"{\\\"id\\\": 123}\"}}]]]}}\n\n")
|
||||
if flusher != nil {
|
||||
flusher.Flush()
|
||||
}
|
||||
|
||||
// Chunk 3: Completed
|
||||
fmt.Fprint(w, "data: {\"msg\":\"process_completed\",\"output\":{\"data\":[[\"\", \"\", [{\"id\":\"tc_q1\",\"type\":\"function\",\"function\":{\"name\":\"fetch_data\",\"arguments\":\"{\\\"id\\\": 123}\"}}]]]}}\n\n")
|
||||
if flusher != nil {
|
||||
flusher.Flush()
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
http.NotFound(w, r)
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
gw := NewGradioGateway(ts.URL, "", 10*time.Second)
|
||||
|
||||
req := ChatCompletionRequest{
|
||||
Model: "test-queue-tools",
|
||||
Stream: true,
|
||||
Messages: []ChatMessage{
|
||||
{Role: "user", Content: "Fetch ID 123"},
|
||||
},
|
||||
Tools: []Tool{
|
||||
{Type: "function", Function: map[string]interface{}{"name": "fetch_data"}},
|
||||
},
|
||||
}
|
||||
|
||||
httpReq := httptest.NewRequest("POST", "/v1/chat/completions", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
err := gw.ExecuteChatCompletion(rec, httpReq, req)
|
||||
if err != nil {
|
||||
t.Fatalf("ExecuteChatCompletion queue streaming failed: %v", err)
|
||||
}
|
||||
|
||||
lines := strings.Split(rec.Body.String(), "\n")
|
||||
var toolCallDeltas []ToolCall
|
||||
var finishReasons []string
|
||||
|
||||
for _, line := range lines {
|
||||
line = strings.TrimSpace(line)
|
||||
if strings.HasPrefix(line, "data: ") && line != "data: [DONE]" {
|
||||
var chunk StreamResponse
|
||||
if err := json.Unmarshal([]byte(strings.TrimPrefix(line, "data: ")), &chunk); err == nil && len(chunk.Choices) > 0 {
|
||||
if len(chunk.Choices[0].Delta.ToolCalls) > 0 {
|
||||
toolCallDeltas = append(toolCallDeltas, chunk.Choices[0].Delta.ToolCalls...)
|
||||
}
|
||||
if chunk.Choices[0].FinishReason != nil {
|
||||
finishReasons = append(finishReasons, *chunk.Choices[0].FinishReason)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(toolCallDeltas) != 2 {
|
||||
t.Fatalf("expected 2 tool call deltas in queue stream, got %d: %+v", len(toolCallDeltas), toolCallDeltas)
|
||||
}
|
||||
if toolCallDeltas[0].ID != "tc_q1" || toolCallDeltas[0].Function.Name != "fetch_data" {
|
||||
t.Errorf("unexpected first delta: %+v", toolCallDeltas[0])
|
||||
}
|
||||
if toolCallDeltas[1].ID != "" || toolCallDeltas[1].Function.Name != "" || toolCallDeltas[1].Function.Arguments != `23}` {
|
||||
t.Errorf("second delta must only have argument delta '23}', got %+v", toolCallDeltas[1])
|
||||
}
|
||||
if len(finishReasons) != 1 || finishReasons[0] != "tool_calls" {
|
||||
t.Errorf("expected finish_reason 'tool_calls', got %v", finishReasons)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user