luxferre/gr2gw
1
0
Fork 0
Code Actions Projects Activity

toolcall fixes

Browse Source
This commit is contained in:
Luxferre
2026-09-08 18:20:06 +03:00
parent c731b57dfc
commit f2b16174fa
2 changed files with 738 additions and 198 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gr2gw.go
+283 -198
View File
@@ -1808,6 +1808,10 @@ func finalizeOutput(frame GradioOutputFrame) (finalContent interface{}, reasonin
toolCalls = frame.ToolCalls
hasTools := len(toolCalls) > 0
if hasTools && strings.TrimSpace(cleanText) == "" {
return nil, reasoning, toolCalls, "tool_calls"
}
if reasoning == "" {
cleanText, reasoning = ExtractThinking(cleanText)
}
@@ -3738,7 +3742,7 @@ func (g *GradioGateway) GetDiscovery(spaceURL, userAgent string) *SpaceDiscovery
func (g *GradioGateway) BuildGradioPayload(disc *SpaceDiscovery, req ChatCompletionRequest) ([]interface{}, error) {
var transformed []ChatMessage
var toolInstruction string
if disc.IsHunyuan3 && disc.FunctionsJSONIndex != -1 {
if (disc.IsHunyuan3 || disc.FunctionsJSONIndex != -1) && disc.FunctionsJSONIndex != -1 {
for _, msg := range req.Messages {
transformed = append(transformed, ChatMessage{
Role: msg.Role,
@@ -3847,7 +3851,7 @@ func (g *GradioGateway) BuildGradioPayload(disc *SpaceDiscovery, req ChatComplet
if toolName == "" {
toolName = lastMsg.ToolCallID
}
if disc.IsHunyuan3 {
if disc.IsHunyuan3 || disc.FunctionsJSONIndex != -1 {
if toolName != "" {
lastUserMessage = fmt.Sprintf("Tool result for %s: %s", toolName, lastContent)
} else {
@@ -4296,8 +4300,9 @@ func extractGradioDiffDelta(v []interface{}) (contentDelta string, reasoningDelt
// ParseGradioStreamOutput extracts structured content, reasoning, and tool calls from Gradio output
func ParseGradioStreamOutput(rawJSON string) (frame GradioOutputFrame) {
isNativeTuple := false
defer func() {
if frame.OK && len(frame.ToolCalls) == 0 && frame.Content != "" {
if !isNativeTuple && frame.OK && len(frame.ToolCalls) == 0 && frame.Content != "" {
tcs, clean, has := DetectToolCalls(frame.Content)
if has && len(tcs) > 0 {
frame.ToolCalls = tcs
@@ -4332,23 +4337,22 @@ func ParseGradioStreamOutput(rawJSON string) (frame GradioOutputFrame) {
// 1. Check if v[0] is an inner slice with len >= 2 (e.g. Hy3: [[content, reasoning, tool_calls, history]])
if inner, ok := v[0].([]interface{}); ok && len(inner) >= 2 {
s0, ok0 := inner[0].(string)
s1, ok1 := inner[1].(string)
if ok0 || ok1 {
frame.Content = s0
frame.Reasoning = s1
if len(inner) >= 3 && inner[2] != nil {
b, err := json.Marshal(inner[2])
if err == nil {
var tcs []ToolCall
if json.Unmarshal(b, &tcs) == nil && len(tcs) > 0 {
frame.ToolCalls = tcs
}
isNativeTuple = true
s0, _ := inner[0].(string)
s1, _ := inner[1].(string)
frame.Content = s0
frame.Reasoning = s1
if len(inner) >= 3 && inner[2] != nil {
b, err := json.Marshal(inner[2])
if err == nil {
var tcs []ToolCall
if json.Unmarshal(b, &tcs) == nil && len(tcs) > 0 {
frame.ToolCalls = tcs
}
}
frame.OK = true
return frame
}
frame.OK = true
return frame
}
// 2. Check if any element of v is a Chatbot message list or Chatbot pair list
@@ -4866,14 +4870,17 @@ func (g *GradioGateway) executeQueueCompletion(w http.ResponseWriter, r *http.Re
flusher, _ := w.(http.Flusher)
streamer := NewStreamer(w, flusher, completionID, createdTime, modelName)
isNativeToolCalling := disc.IsHunyuan3 || disc.FunctionsJSONIndex >= 0
thinkFilter := NewStreamThinkingFilter()
toolFilter := NewStreamToolCallFilter()
reader := bufio.NewReader(streamResp.Body)
var prevContent string
var prevReasoning string
prevToolArgs := make(map[int]string)
nativeReasoningSeen := disc.IsHunyuan3
lastToolCallArgs := make(map[string]string)
emittedToolCallIDs := make(map[string]bool)
nativeReasoningSeen := disc.IsHunyuan3 || disc.ThinkLevelIndex >= 0 || disc.PreservedThinkingIndex >= 0
nativeToolCallsSeen := false
var streamErr error
@@ -4912,84 +4919,124 @@ func (g *GradioGateway) executeQueueCompletion(w http.ResponseWriter, r *http.Re
dataBytes, _ := json.Marshal(qMsg.Output["data"])
frame := ParseGradioStreamOutput(string(dataBytes))
if frame.OK {
// 1. Native reasoning handling
if frame.Reasoning != "" || nativeReasoningSeen {
nativeReasoningSeen = true
var deltaReasoning string
if frame.IsDelta {
deltaReasoning = frame.Reasoning
prevReasoning += deltaReasoning
} else {
if strings.HasPrefix(frame.Reasoning, prevReasoning) {
deltaReasoning = frame.Reasoning[len(prevReasoning):]
} else if prevReasoning == "" {
deltaReasoning = frame.Reasoning
} else {
deltaReasoning = frame.Reasoning
}
if isNativeToolCalling {
// 1. Native reasoning
if len(frame.Reasoning) > len(prevReasoning) {
deltaReasoning := frame.Reasoning[len(prevReasoning):]
prevReasoning = frame.Reasoning
}
if deltaReasoning != "" {
streamer.Reasoning(deltaReasoning)
} else if frame.IsDelta && frame.Reasoning != "" {
streamer.Reasoning(frame.Reasoning)
}
}
// 2. Native tool call handling
if len(frame.ToolCalls) > 0 || nativeToolCallsSeen {
nativeToolCallsSeen = true
for i, tc := range frame.ToolCalls {
prevArgs := prevToolArgs[i]
fullArgs := tc.Function.Arguments
if len(fullArgs) > len(prevArgs) && strings.HasPrefix(fullArgs, prevArgs) {
deltaArgs := fullArgs[len(prevArgs):]
iCopy := i
streamer.ToolCallDelta(ToolCall{
Index: &iCopy,
ID: tc.ID,
Type: tc.Type,
Function: ToolCallFunction{
Name: tc.Function.Name,
Arguments: deltaArgs,
},
})
prevToolArgs[i] = fullArgs
} else if prevArgs == "" {
iCopy := i
streamer.ToolCallDelta(ToolCall{
Index: &iCopy,
ID: tc.ID,
Type: tc.Type,
Function: ToolCallFunction{
Name: tc.Function.Name,
Arguments: fullArgs,
},
})
prevToolArgs[i] = fullArgs
// 2. Native tool calls
if len(frame.ToolCalls) > 0 {
for idx, t := range frame.ToolCalls {
key := t.ID
if key == "" {
key = fmt.Sprintf("idx_%d", idx)
}
prevArgs := lastToolCallArgs[key]
currArgs := t.Function.Arguments
idxCopy := idx
if !emittedToolCallIDs[key] {
emittedToolCallIDs[key] = true
lastToolCallArgs[key] = currArgs
tCopy := ToolCall{
Index: &idxCopy,
ID: t.ID,
Type: "function",
Function: ToolCallFunction{Name: t.Function.Name, Arguments: currArgs},
}
streamer.ToolCallDelta(tCopy)
} else if len(currArgs) > len(prevArgs) {
argDelta := currArgs[len(prevArgs):]
lastToolCallArgs[key] = currArgs
tCopy := ToolCall{
Index: &idxCopy,
Function: ToolCallFunction{Arguments: argDelta},
}
streamer.ToolCallDelta(tCopy)
}
}
}
}
// 3. Content handling
currentText := frame.Content
var delta string
if frame.IsDelta {
delta = frame.Content
prevContent += delta
// 3. Native content
if len(frame.Content) > len(prevContent) {
cDelta := frame.Content[len(prevContent):]
prevContent = frame.Content
streamer.Content(cDelta)
} else if frame.IsDelta && frame.Content != "" {
streamer.Content(frame.Content)
}
} else {
if strings.HasPrefix(currentText, prevContent) {
delta = currentText[len(prevContent):]
} else if prevContent == "" {
delta = currentText
} else {
delta = currentText
// Non-native (prompt-augmented) tool calling
if frame.Reasoning != "" || nativeReasoningSeen {
nativeReasoningSeen = true
var deltaReasoning string
if frame.IsDelta {
deltaReasoning = frame.Reasoning
prevReasoning += deltaReasoning
} else {
if len(frame.Reasoning) > len(prevReasoning) {
deltaReasoning = frame.Reasoning[len(prevReasoning):]
}
prevReasoning = frame.Reasoning
}
if deltaReasoning != "" {
streamer.Reasoning(deltaReasoning)
}
}
prevContent = currentText
}
if delta != "" {
if nativeReasoningSeen || nativeToolCallsSeen {
if nativeReasoningSeen && nativeToolCallsSeen {
streamer.Content(delta)
if len(frame.ToolCalls) > 0 {
nativeToolCallsSeen = true
for idx, t := range frame.ToolCalls {
key := t.ID
if key == "" {
key = fmt.Sprintf("idx_%d", idx)
}
prevArgs := lastToolCallArgs[key]
currArgs := t.Function.Arguments
idxCopy := idx
if !emittedToolCallIDs[key] {
emittedToolCallIDs[key] = true
lastToolCallArgs[key] = currArgs
tCopy := ToolCall{
Index: &idxCopy,
ID: t.ID,
Type: "function",
Function: ToolCallFunction{Name: t.Function.Name, Arguments: currArgs},
}
streamer.ToolCallDelta(tCopy)
} else if len(currArgs) > len(prevArgs) {
argDelta := currArgs[len(prevArgs):]
lastToolCallArgs[key] = currArgs
tCopy := ToolCall{
Index: &idxCopy,
Function: ToolCallFunction{Arguments: argDelta},
}
streamer.ToolCallDelta(tCopy)
}
}
}
currentText := frame.Content
var delta string
if frame.IsDelta {
delta = frame.Content
prevContent += delta
} else {
if len(currentText) > len(prevContent) {
delta = currentText[len(prevContent):]
}
prevContent = currentText
}
if delta != "" {
if nativeToolCallsSeen {
if strings.TrimSpace(delta) != "" {
streamer.Content(delta)
}
} else if nativeReasoningSeen {
toolFilter.Feed(delta, func(cleanChunk string) {
if cleanChunk != "" {
@@ -5000,29 +5047,19 @@ func (g *GradioGateway) executeQueueCompletion(w http.ResponseWriter, r *http.Re
})
} else {
thinkFilter.Feed(delta, func(contentChunk string) {
if contentChunk != "" {
streamer.Content(contentChunk)
}
toolFilter.Feed(contentChunk, func(cleanChunk string) {
if cleanChunk != "" {
streamer.Content(cleanChunk)
}
}, func(tc ToolCall) {
streamer.ToolCallDelta(tc)
})
}, func(reasoningChunk string) {
if reasoningChunk != "" {
streamer.Reasoning(reasoningChunk)
}
})
}
} else {
thinkFilter.Feed(delta, func(contentChunk string) {
toolFilter.Feed(contentChunk, func(cleanChunk string) {
if cleanChunk != "" {
streamer.Content(cleanChunk)
}
}, func(tc ToolCall) {
streamer.ToolCallDelta(tc)
})
}, func(reasoningChunk string) {
if reasoningChunk != "" {
streamer.Reasoning(reasoningChunk)
}
})
}
}
}
@@ -5038,7 +5075,18 @@ func (g *GradioGateway) executeQueueCompletion(w http.ResponseWriter, r *http.Re
return streamErr
}
// Flush remaining tokens in filters if used
if isNativeToolCalling {
if len(emittedToolCallIDs) > 0 {
streamer.Finish("tool_calls")
} else {
streamer.Finish("stop")
}
streamer.Done()
disc.Protocol = "queue"
return nil
}
// Flush remaining tokens in filters if used for non-native
if !nativeReasoningSeen {
thinkFilter.Flush(func(contentChunk string) {
if !nativeToolCallsSeen {
@@ -5070,7 +5118,7 @@ func (g *GradioGateway) executeQueueCompletion(w http.ResponseWriter, r *http.Re
}
finishReason := "stop"
if nativeToolCallsSeen || toolFilter.emittedCall {
if len(emittedToolCallIDs) > 0 || nativeToolCallsSeen || toolFilter.emittedCall {
finishReason = "tool_calls"
}
@@ -5372,14 +5420,17 @@ func (g *GradioGateway) ExecuteChatCompletion(w http.ResponseWriter, r *http.Req
flusher, _ := w.(http.Flusher)
streamer := NewStreamer(w, flusher, completionID, createdTime, modelName)
isNativeToolCalling := disc.IsHunyuan3 || disc.FunctionsJSONIndex >= 0
thinkFilter := NewStreamThinkingFilter()
toolFilter := NewStreamToolCallFilter()
reader := bufio.NewReader(streamResp.Body)
var prevContent string
var prevReasoning string
prevToolArgs := make(map[int]string)
nativeReasoningSeen := disc.IsHunyuan3
lastToolCallArgs := make(map[string]string)
emittedToolCallIDs := make(map[string]bool)
nativeReasoningSeen := disc.IsHunyuan3 || disc.ThinkLevelIndex >= 0 || disc.PreservedThinkingIndex >= 0
nativeToolCallsSeen := false
currentEvent := ""
var streamErr error
@@ -5427,89 +5478,124 @@ func (g *GradioGateway) ExecuteChatCompletion(w http.ResponseWriter, r *http.Req
frame := ParseGradioStreamOutput(dataStr)
if frame.OK {
// 1. Native reasoning handling
if frame.Reasoning != "" || nativeReasoningSeen {
nativeReasoningSeen = true
var deltaReasoning string
if frame.IsDelta {
deltaReasoning = frame.Reasoning
prevReasoning += deltaReasoning
} else {
if strings.HasPrefix(frame.Reasoning, prevReasoning) {
deltaReasoning = frame.Reasoning[len(prevReasoning):]
} else if prevReasoning == "" {
deltaReasoning = frame.Reasoning
} else {
deltaReasoning = frame.Reasoning
}
if isNativeToolCalling {
// 1. Native reasoning
if len(frame.Reasoning) > len(prevReasoning) {
deltaReasoning := frame.Reasoning[len(prevReasoning):]
prevReasoning = frame.Reasoning
}
if deltaReasoning != "" {
streamer.Reasoning(deltaReasoning)
} else if frame.IsDelta && frame.Reasoning != "" {
streamer.Reasoning(frame.Reasoning)
}
}
// 2. Native tool calls handling
if len(frame.ToolCalls) > 0 {
nativeToolCallsSeen = true
for idx, tc := range frame.ToolCalls {
prevArgs, started := prevToolArgs[idx]
currArgs := tc.Function.Arguments
idxCopy := idx
if !started {
tcDelta := ToolCall{
Index: &idxCopy,
ID: tc.ID,
Type: tc.Type,
Function: ToolCallFunction{
Name: tc.Function.Name,
Arguments: currArgs,
},
// 2. Native tool calls
if len(frame.ToolCalls) > 0 {
for idx, t := range frame.ToolCalls {
key := t.ID
if key == "" {
key = fmt.Sprintf("idx_%d", idx)
}
streamer.ToolCallDelta(tcDelta)
prevToolArgs[idx] = currArgs
} else if len(currArgs) > len(prevArgs) {
var argDelta string
if strings.HasPrefix(currArgs, prevArgs) {
argDelta = currArgs[len(prevArgs):]
} else {
argDelta = currArgs[len(prevArgs):]
}
if argDelta != "" {
tcDelta := ToolCall{
Index: &idxCopy,
Function: ToolCallFunction{
Arguments: argDelta,
},
prevArgs := lastToolCallArgs[key]
currArgs := t.Function.Arguments
idxCopy := idx
if !emittedToolCallIDs[key] {
emittedToolCallIDs[key] = true
lastToolCallArgs[key] = currArgs
tCopy := ToolCall{
Index: &idxCopy,
ID: t.ID,
Type: "function",
Function: ToolCallFunction{Name: t.Function.Name, Arguments: currArgs},
}
streamer.ToolCallDelta(tcDelta)
streamer.ToolCallDelta(tCopy)
} else if len(currArgs) > len(prevArgs) {
argDelta := currArgs[len(prevArgs):]
lastToolCallArgs[key] = currArgs
tCopy := ToolCall{
Index: &idxCopy,
Function: ToolCallFunction{Arguments: argDelta},
}
streamer.ToolCallDelta(tCopy)
}
prevToolArgs[idx] = currArgs
}
}
}
// 3. Content handling
currentText := frame.Content
var delta string
if frame.IsDelta {
delta = currentText
prevContent += delta
// 3. Native content
if len(frame.Content) > len(prevContent) {
cDelta := frame.Content[len(prevContent):]
prevContent = frame.Content
streamer.Content(cDelta)
} else if frame.IsDelta && frame.Content != "" {
streamer.Content(frame.Content)
}
} else {
if strings.HasPrefix(currentText, prevContent) {
delta = currentText[len(prevContent):]
} else if prevContent == "" {
delta = currentText
} else {
delta = currentText
// Non-native (prompt-augmented) tool calling
if frame.Reasoning != "" || nativeReasoningSeen {
nativeReasoningSeen = true
var deltaReasoning string
if frame.IsDelta {
deltaReasoning = frame.Reasoning
prevReasoning += deltaReasoning
} else {
if len(frame.Reasoning) > len(prevReasoning) {
deltaReasoning = frame.Reasoning[len(prevReasoning):]
}
prevReasoning = frame.Reasoning
}
if deltaReasoning != "" {
streamer.Reasoning(deltaReasoning)
}
}
prevContent = currentText
}
if delta != "" {
if nativeReasoningSeen || nativeToolCallsSeen {
if nativeReasoningSeen && nativeToolCallsSeen {
streamer.Content(delta)
if len(frame.ToolCalls) > 0 {
nativeToolCallsSeen = true
for idx, t := range frame.ToolCalls {
key := t.ID
if key == "" {
key = fmt.Sprintf("idx_%d", idx)
}
prevArgs := lastToolCallArgs[key]
currArgs := t.Function.Arguments
idxCopy := idx
if !emittedToolCallIDs[key] {
emittedToolCallIDs[key] = true
lastToolCallArgs[key] = currArgs
tCopy := ToolCall{
Index: &idxCopy,
ID: t.ID,
Type: "function",
Function: ToolCallFunction{Name: t.Function.Name, Arguments: currArgs},
}
streamer.ToolCallDelta(tCopy)
} else if len(currArgs) > len(prevArgs) {
argDelta := currArgs[len(prevArgs):]
lastToolCallArgs[key] = currArgs
tCopy := ToolCall{
Index: &idxCopy,
Function: ToolCallFunction{Arguments: argDelta},
}
streamer.ToolCallDelta(tCopy)
}
}
}
currentText := frame.Content
var delta string
if frame.IsDelta {
delta = currentText
prevContent += delta
} else {
if len(currentText) > len(prevContent) {
delta = currentText[len(prevContent):]
}
prevContent = currentText
}
if delta != "" {
if nativeToolCallsSeen {
if strings.TrimSpace(delta) != "" {
streamer.Content(delta)
}
} else if nativeReasoningSeen {
toolFilter.Feed(delta, func(cleanChunk string) {
if cleanChunk != "" {
@@ -5520,29 +5606,19 @@ func (g *GradioGateway) ExecuteChatCompletion(w http.ResponseWriter, r *http.Req
})
} else {
thinkFilter.Feed(delta, func(contentChunk string) {
if contentChunk != "" {
streamer.Content(contentChunk)
}
toolFilter.Feed(contentChunk, func(cleanChunk string) {
if cleanChunk != "" {
streamer.Content(cleanChunk)
}
}, func(tc ToolCall) {
streamer.ToolCallDelta(tc)
})
}, func(reasoningChunk string) {
if reasoningChunk != "" {
streamer.Reasoning(reasoningChunk)
}
})
}
} else {
thinkFilter.Feed(delta, func(contentChunk string) {
toolFilter.Feed(contentChunk, func(cleanChunk string) {
if cleanChunk != "" {
streamer.Content(cleanChunk)
}
}, func(tc ToolCall) {
streamer.ToolCallDelta(tc)
})
}, func(reasoningChunk string) {
if reasoningChunk != "" {
streamer.Reasoning(reasoningChunk)
}
})
}
}
}
@@ -5557,7 +5633,17 @@ func (g *GradioGateway) ExecuteChatCompletion(w http.ResponseWriter, r *http.Req
return nil
}
// Flush remaining tokens in filters if used
if isNativeToolCalling {
if len(emittedToolCallIDs) > 0 {
streamer.Finish("tool_calls")
} else {
streamer.Finish("stop")
}
streamer.Done()
return nil
}
// Flush remaining tokens in filters if used for non-native
if !nativeReasoningSeen {
thinkFilter.Flush(func(contentChunk string) {
if !nativeToolCallsSeen {
@@ -5593,13 +5679,12 @@ func (g *GradioGateway) ExecuteChatCompletion(w http.ResponseWriter, r *http.Req
return g.executeQueueCompletion(w, r, disc, gradioData, req, completionID, createdTime, modelName, effUA)
}
if nativeToolCallsSeen || toolFilter.emittedCall {
if len(emittedToolCallIDs) > 0 || nativeToolCallsSeen || toolFilter.emittedCall {
streamer.Finish("tool_calls")
} else {
streamer.Finish("stop")
}
streamer.Done()
return nil
}
gr2gw_test.go
+455
View File
@@ -3604,3 +3604,458 @@ func TestWebSearchParameterMappingAndSuppression(t *testing.T) {
}
}
func TestHunyuanReasoningAndToolCallingStreaming(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if strings.HasSuffix(r.URL.Path, "/info") {
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(map[string]interface{}{
"named_endpoints": map[string]interface{}{
"/chat": map[string]interface{}{
"parameters": []map[string]interface{}{
{"parameter_name": "message", "component": "Api"},
{"parameter_name": "system_prompt", "component": "Api"},
{"parameter_name": "history", "component": "Api"},
{"parameter_name": "think_level", "component": "Api"},
{"parameter_name": "temperature", "component": "Api"},
{"parameter_name": "max_tokens", "component": "Api"},
{"parameter_name": "top_p", "component": "Api"},
{"parameter_name": "preserved_thinking", "component": "Api"},
{"parameter_name": "functions_json_str", "component": "Api"},
},
"code_snippets": map[string]interface{}{
"bash": "curl -X POST http://localhost:7860/gradio_api/call/chat",
},
},
},
})
return
}
if strings.HasSuffix(r.URL.Path, "/call/chat") && r.Method == http.MethodPost {
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(map[string]interface{}{
"event_id": "hy3-evt-stream-tools",
})
return
}
if strings.Contains(r.URL.Path, "/call/chat/hy3-evt-stream-tools") && r.Method == http.MethodGet {
w.Header().Set("Content-Type", "text/event-stream")
w.Header().Set("Cache-Control", "no-cache")
w.WriteHeader(http.StatusOK)
flusher, _ := w.(http.Flusher)
// Event 1: Reasoning part 1
fmt.Fprint(w, "event: generating\ndata: [[\"\", \"Step 1 reasoning.\", null]]\n\n")
if flusher != nil {
flusher.Flush()
}
// Event 2: Reasoning part 2
fmt.Fprint(w, "event: generating\ndata: [[\"\", \"Step 1 reasoning. Step 2 reasoning.\", null]]\n\n")
if flusher != nil {
flusher.Flush()
}
// Event 3: Tool call chunk 1 with partial arguments
fmt.Fprint(w, "event: generating\ndata: [[\"\", \"Step 1 reasoning. Step 2 reasoning.\", [{\"id\":\"call_xyz\",\"type\":\"function\",\"function\":{\"name\":\"calculator\",\"arguments\":\"{\\\"expr\\\":\\\"2\"}}]]]\n\n")
if flusher != nil {
flusher.Flush()
}
// Event 4: Tool call chunk 2 with remaining arguments
fmt.Fprint(w, "event: complete\ndata: [[\"\", \"Step 1 reasoning. Step 2 reasoning.\", [{\"id\":\"call_xyz\",\"type\":\"function\",\"function\":{\"name\":\"calculator\",\"arguments\":\"{\\\"expr\\\":\\\"2+2\\\"}\"}}]]]\n\n")
if flusher != nil {
flusher.Flush()
}
return
}
http.NotFound(w, r)
}))
defer ts.Close()
gw := NewGradioGateway(ts.URL, "", 10*time.Second)
reqStream := ChatCompletionRequest{
Model: "hy3",
Stream: true,
Messages: []ChatMessage{
{Role: "user", Content: "Calculate 2+2"},
},
Tools: []Tool{
{Type: "function", Function: map[string]interface{}{"name": "calculator"}},
},
}
httpReq := httptest.NewRequest("POST", "/v1/chat/completions", nil)
rec := httptest.NewRecorder()
err := gw.ExecuteChatCompletion(rec, httpReq, reqStream)
if err != nil {
t.Fatalf("ExecuteChatCompletion failed: %v", err)
}
lines := strings.Split(rec.Body.String(), "\n")
var receivedDeltas []StreamDelta
var finishReasons []string
var foundDone bool
for _, line := range lines {
line = strings.TrimSpace(line)
if strings.HasPrefix(line, "data: ") {
payload := strings.TrimPrefix(line, "data: ")
if payload == "[DONE]" {
foundDone = true
continue
}
var streamResp StreamResponse
if err := json.Unmarshal([]byte(payload), &streamResp); err == nil && len(streamResp.Choices) > 0 {
choice := streamResp.Choices[0]
receivedDeltas = append(receivedDeltas, choice.Delta)
if choice.FinishReason != nil {
finishReasons = append(finishReasons, *choice.FinishReason)
}
}
}
}
if !foundDone {
t.Errorf("expected 'data: [DONE]' in SSE stream")
}
if len(receivedDeltas) < 1 || receivedDeltas[0].Role != "assistant" {
t.Errorf("expected first delta to set role='assistant', got %+v", receivedDeltas[0])
}
var combinedReasoning, combinedContent string
var toolCallDeltas []ToolCall
for _, d := range receivedDeltas {
combinedReasoning += d.ReasoningContent
combinedContent += d.Content
if len(d.ToolCalls) > 0 {
toolCallDeltas = append(toolCallDeltas, d.ToolCalls...)
}
}
if combinedReasoning != "Step 1 reasoning. Step 2 reasoning." {
t.Errorf("combined reasoning=%q, want 'Step 1 reasoning. Step 2 reasoning.'", combinedReasoning)
}
if combinedContent != "" {
t.Errorf("expected no content emitted during tool calling, got %q", combinedContent)
}
if len(toolCallDeltas) != 2 {
t.Fatalf("expected exactly 2 tool call deltas, got %d: %+v", len(toolCallDeltas), toolCallDeltas)
}
// First delta chunk: has index, id, type, name, initial arguments
firstTC := toolCallDeltas[0]
if firstTC.Index == nil || *firstTC.Index != 0 {
t.Errorf("first delta index expected 0, got %v", firstTC.Index)
}
if firstTC.ID != "call_xyz" || firstTC.Type != "function" || firstTC.Function.Name != "calculator" {
t.Errorf("first delta expected full metadata, got %+v", firstTC)
}
if firstTC.Function.Arguments != `{"expr":"2` {
t.Errorf("first delta arguments=%q, want %q", firstTC.Function.Arguments, `{"expr":"2`)
}
// Second delta chunk: MUST have empty ID, empty Type, empty Name, and ONLY argument delta "+2\"}"
secondTC := toolCallDeltas[1]
if secondTC.Index == nil || *secondTC.Index != 0 {
t.Errorf("second delta index expected 0, got %v", secondTC.Index)
}
if secondTC.ID != "" || secondTC.Type != "" || secondTC.Function.Name != "" {
t.Errorf("second delta MUST NOT resend ID/Type/Name per OpenAI spec: %+v", secondTC)
}
if secondTC.Function.Arguments != `+2"}` {
t.Errorf("second delta arguments=%q, want %q", secondTC.Function.Arguments, `+2"}`)
}
if len(finishReasons) != 1 || finishReasons[0] != "tool_calls" {
t.Errorf("finish_reasons=%v, want ['tool_calls']", finishReasons)
}
}
func TestHunyuanToolCallingNonStreaming(t *testing.T) {
var receivedPayload map[string]interface{}
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if strings.HasSuffix(r.URL.Path, "/info") {
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(map[string]interface{}{
"named_endpoints": map[string]interface{}{
"/chat": map[string]interface{}{
"parameters": []map[string]interface{}{
{"parameter_name": "message", "component": "Api"},
{"parameter_name": "system_prompt", "component": "Api"},
{"parameter_name": "history", "component": "Api"},
{"parameter_name": "think_level", "component": "Api"},
{"parameter_name": "temperature", "component": "Api"},
{"parameter_name": "max_tokens", "component": "Api"},
{"parameter_name": "top_p", "component": "Api"},
{"parameter_name": "preserved_thinking", "component": "Api"},
{"parameter_name": "functions_json_str", "component": "Api"},
},
"code_snippets": map[string]interface{}{
"bash": "curl -X POST http://localhost:7860/gradio_api/call/chat",
},
},
},
})
return
}
if strings.HasSuffix(r.URL.Path, "/call/chat") && r.Method == http.MethodPost {
json.NewDecoder(r.Body).Decode(&receivedPayload)
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(map[string]interface{}{
"event_id": "hy3-tool-nonstream",
})
return
}
if strings.Contains(r.URL.Path, "/call/chat/hy3-tool-nonstream") && r.Method == http.MethodGet {
w.Header().Set("Content-Type", "text/event-stream")
fmt.Fprint(w, "event: complete\ndata: [[\"\", \"I should call the search tool.\", [{\"id\":\"call_abc\",\"type\":\"function\",\"function\":{\"name\":\"search\",\"arguments\":\"{\\\"query\\\":\\\"golang\\\"}\"}}]]]\n\n")
return
}
http.NotFound(w, r)
}))
defer ts.Close()
gw := NewGradioGateway(ts.URL, "", 10*time.Second)
reqNonStream := ChatCompletionRequest{
Model: "hy3",
Stream: false,
Messages: []ChatMessage{
{Role: "user", Content: "Search for golang news"},
},
Tools: []Tool{
{Type: "function", Function: map[string]interface{}{"name": "search", "description": "Search web"}},
},
}
httpReq := httptest.NewRequest("POST", "/v1/chat/completions", nil)
rec := httptest.NewRecorder()
err := gw.ExecuteChatCompletion(rec, httpReq, reqNonStream)
if err != nil {
t.Fatalf("ExecuteChatCompletion non-streaming failed: %v", err)
}
// Verify functions_json_str passed in data[8]
dataArr, _ := receivedPayload["data"].([]interface{})
if len(dataArr) < 9 || dataArr[8] == nil || dataArr[8] == "" {
t.Errorf("expected functions_json_str in data[8], got %v", dataArr)
}
var resp ChatCompletionResponse
if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {
t.Fatalf("failed to decode response: %v", err)
}
if len(resp.Choices) != 1 {
t.Fatalf("expected 1 choice, got %d", len(resp.Choices))
}
choice := resp.Choices[0]
if choice.FinishReason != "tool_calls" {
t.Errorf("finish_reason=%q, want 'tool_calls'", choice.FinishReason)
}
if choice.Message.Content != nil {
t.Errorf("content expected nil, got %v", choice.Message.Content)
}
if choice.Message.ReasoningContent != "I should call the search tool." {
t.Errorf("reasoning_content=%q, want 'I should call the search tool.'", choice.Message.ReasoningContent)
}
if len(choice.Message.ToolCalls) != 1 {
t.Fatalf("expected 1 tool call, got %d", len(choice.Message.ToolCalls))
}
tc := choice.Message.ToolCalls[0]
if tc.ID != "call_abc" || tc.Function.Name != "search" || tc.Function.Arguments != `{"query":"golang"}` {
t.Errorf("unexpected tool call: %+v", tc)
}
}
func TestHunyuanMultiTurnToolResultHistory(t *testing.T) {
gw := &GradioGateway{}
disc := NewDefaultSpaceDiscovery("https://tencent-hy3.hf.space")
disc.IsHunyuan3 = true
disc.TotalInputs = 9
disc.MessageIndex = 0
disc.SystemIndex = 1
disc.HistoryIndex = 2
disc.HistoryFormat = "messages"
disc.ThinkLevelIndex = 3
disc.TempIndex = 4
disc.MaxTokensIndex = 5
disc.TopPIndex = 6
disc.PreservedThinkingIndex = 7
disc.FunctionsJSONIndex = 8
req := ChatCompletionRequest{
Model: "hy3",
Messages: []ChatMessage{
{Role: "user", Content: "What is the weather in Tokyo?"},
{
Role: "assistant",
ToolCalls: []ToolCall{
{ID: "call_weather_1", Type: "function", Function: ToolCallFunction{Name: "get_weather", Arguments: `{"location":"Tokyo"}`}},
},
},
{
Role: "tool",
Name: "get_weather",
ToolCallID: "call_weather_1",
Content: `{"temperature": 20, "condition": "Sunny"}`,
},
},
Tools: []Tool{
{Type: "function", Function: map[string]interface{}{"name": "get_weather"}},
},
}
data, err := gw.BuildGradioPayload(disc, req)
if err != nil {
t.Fatalf("BuildGradioPayload failed: %v", err)
}
// 1. Message input (slot 0) should be clean "Tool result for get_weather: ..." without "Query:" or "Please answer..."
msgStr, ok := data[0].(string)
if !ok {
t.Fatalf("expected string message in data[0], got %T", data[0])
}
expectedMsg := `Tool result for get_weather: {"temperature": 20, "condition": "Sunny"}`
if msgStr != expectedMsg {
t.Errorf("message=%q, want %q", msgStr, expectedMsg)
}
// 2. History input (slot 2) should contain user turn and assistant tool_call turn
histArr, ok := data[2].([]map[string]interface{})
if !ok {
t.Fatalf("expected []map[string]interface{} history in data[2], got %T", data[2])
}
if len(histArr) != 2 {
t.Fatalf("expected 2 turns in history, got %d: %+v", len(histArr), histArr)
}
if histArr[0]["role"] != "user" || histArr[0]["content"] != "What is the weather in Tokyo?" {
t.Errorf("unexpected turn 0: %+v", histArr[0])
}
if histArr[1]["role"] != "assistant" {
t.Errorf("unexpected turn 1 role: %+v", histArr[1])
}
tcArr, ok := histArr[1]["tool_calls"].([]ToolCall)
if !ok || len(tcArr) != 1 || tcArr[0].Function.Name != "get_weather" {
t.Errorf("unexpected turn 1 tool_calls: %+v", histArr[1])
}
}
func TestQueueToolCallingStreamingParity(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/info" {
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(map[string]interface{}{
"named_endpoints": map[string]interface{}{
"/predict": map[string]interface{}{
"parameters": []map[string]interface{}{
{"parameter_name": "message", "component": "Api"},
{"parameter_name": "tools", "component": "Api"},
},
},
},
})
return
}
if strings.HasSuffix(r.URL.Path, "/queue/join") {
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(map[string]interface{}{"event_id": "queue-evt-1"})
return
}
if strings.HasSuffix(r.URL.Path, "/queue/data") {
w.Header().Set("Content-Type", "text/event-stream")
flusher, _ := w.(http.Flusher)
// Chunk 1: Initial tool call with partial argument
fmt.Fprint(w, "data: {\"msg\":\"process_generating\",\"output\":{\"data\":[[\"\", \"\", [{\"id\":\"tc_q1\",\"type\":\"function\",\"function\":{\"name\":\"fetch_data\",\"arguments\":\"{\\\"id\\\": 1\"}}]]]}}\n\n")
if flusher != nil {
flusher.Flush()
}
// Chunk 2: Completed tool call argument
fmt.Fprint(w, "data: {\"msg\":\"process_generating\",\"output\":{\"data\":[[\"\", \"\", [{\"id\":\"tc_q1\",\"type\":\"function\",\"function\":{\"name\":\"fetch_data\",\"arguments\":\"{\\\"id\\\": 123}\"}}]]]}}\n\n")
if flusher != nil {
flusher.Flush()
}
// Chunk 3: Completed
fmt.Fprint(w, "data: {\"msg\":\"process_completed\",\"output\":{\"data\":[[\"\", \"\", [{\"id\":\"tc_q1\",\"type\":\"function\",\"function\":{\"name\":\"fetch_data\",\"arguments\":\"{\\\"id\\\": 123}\"}}]]]}}\n\n")
if flusher != nil {
flusher.Flush()
}
return
}
http.NotFound(w, r)
}))
defer ts.Close()
gw := NewGradioGateway(ts.URL, "", 10*time.Second)
req := ChatCompletionRequest{
Model: "test-queue-tools",
Stream: true,
Messages: []ChatMessage{
{Role: "user", Content: "Fetch ID 123"},
},
Tools: []Tool{
{Type: "function", Function: map[string]interface{}{"name": "fetch_data"}},
},
}
httpReq := httptest.NewRequest("POST", "/v1/chat/completions", nil)
rec := httptest.NewRecorder()
err := gw.ExecuteChatCompletion(rec, httpReq, req)
if err != nil {
t.Fatalf("ExecuteChatCompletion queue streaming failed: %v", err)
}
lines := strings.Split(rec.Body.String(), "\n")
var toolCallDeltas []ToolCall
var finishReasons []string
for _, line := range lines {
line = strings.TrimSpace(line)
if strings.HasPrefix(line, "data: ") && line != "data: [DONE]" {
var chunk StreamResponse
if err := json.Unmarshal([]byte(strings.TrimPrefix(line, "data: ")), &chunk); err == nil && len(chunk.Choices) > 0 {
if len(chunk.Choices[0].Delta.ToolCalls) > 0 {
toolCallDeltas = append(toolCallDeltas, chunk.Choices[0].Delta.ToolCalls...)
}
if chunk.Choices[0].FinishReason != nil {
finishReasons = append(finishReasons, *chunk.Choices[0].FinishReason)
}
}
}
}
if len(toolCallDeltas) != 2 {
t.Fatalf("expected 2 tool call deltas in queue stream, got %d: %+v", len(toolCallDeltas), toolCallDeltas)
}
if toolCallDeltas[0].ID != "tc_q1" || toolCallDeltas[0].Function.Name != "fetch_data" {
t.Errorf("unexpected first delta: %+v", toolCallDeltas[0])
}
if toolCallDeltas[1].ID != "" || toolCallDeltas[1].Function.Name != "" || toolCallDeltas[1].Function.Arguments != `23}` {
t.Errorf("second delta must only have argument delta '23}', got %+v", toolCallDeltas[1])
}
if len(finishReasons) != 1 || finishReasons[0] != "tool_calls" {
t.Errorf("expected finish_reason 'tool_calls', got %v", finishReasons)
}
}