diff --git a/gr2gw.go b/gr2gw.go index 5a26c09..1f90363 100644 --- a/gr2gw.go +++ b/gr2gw.go @@ -1808,6 +1808,10 @@ func finalizeOutput(frame GradioOutputFrame) (finalContent interface{}, reasonin toolCalls = frame.ToolCalls hasTools := len(toolCalls) > 0 + if hasTools && strings.TrimSpace(cleanText) == "" { + return nil, reasoning, toolCalls, "tool_calls" + } + if reasoning == "" { cleanText, reasoning = ExtractThinking(cleanText) } @@ -3738,7 +3742,7 @@ func (g *GradioGateway) GetDiscovery(spaceURL, userAgent string) *SpaceDiscovery func (g *GradioGateway) BuildGradioPayload(disc *SpaceDiscovery, req ChatCompletionRequest) ([]interface{}, error) { var transformed []ChatMessage var toolInstruction string - if disc.IsHunyuan3 && disc.FunctionsJSONIndex != -1 { + if (disc.IsHunyuan3 || disc.FunctionsJSONIndex != -1) && disc.FunctionsJSONIndex != -1 { for _, msg := range req.Messages { transformed = append(transformed, ChatMessage{ Role: msg.Role, @@ -3847,7 +3851,7 @@ func (g *GradioGateway) BuildGradioPayload(disc *SpaceDiscovery, req ChatComplet if toolName == "" { toolName = lastMsg.ToolCallID } - if disc.IsHunyuan3 { + if disc.IsHunyuan3 || disc.FunctionsJSONIndex != -1 { if toolName != "" { lastUserMessage = fmt.Sprintf("Tool result for %s: %s", toolName, lastContent) } else { @@ -4296,8 +4300,9 @@ func extractGradioDiffDelta(v []interface{}) (contentDelta string, reasoningDelt // ParseGradioStreamOutput extracts structured content, reasoning, and tool calls from Gradio output func ParseGradioStreamOutput(rawJSON string) (frame GradioOutputFrame) { + isNativeTuple := false defer func() { - if frame.OK && len(frame.ToolCalls) == 0 && frame.Content != "" { + if !isNativeTuple && frame.OK && len(frame.ToolCalls) == 0 && frame.Content != "" { tcs, clean, has := DetectToolCalls(frame.Content) if has && len(tcs) > 0 { frame.ToolCalls = tcs @@ -4332,23 +4337,22 @@ func ParseGradioStreamOutput(rawJSON string) (frame GradioOutputFrame) { // 1. Check if v[0] is an inner slice with len >= 2 (e.g. Hy3: [[content, reasoning, tool_calls, history]]) if inner, ok := v[0].([]interface{}); ok && len(inner) >= 2 { - s0, ok0 := inner[0].(string) - s1, ok1 := inner[1].(string) - if ok0 || ok1 { - frame.Content = s0 - frame.Reasoning = s1 - if len(inner) >= 3 && inner[2] != nil { - b, err := json.Marshal(inner[2]) - if err == nil { - var tcs []ToolCall - if json.Unmarshal(b, &tcs) == nil && len(tcs) > 0 { - frame.ToolCalls = tcs - } + isNativeTuple = true + s0, _ := inner[0].(string) + s1, _ := inner[1].(string) + frame.Content = s0 + frame.Reasoning = s1 + if len(inner) >= 3 && inner[2] != nil { + b, err := json.Marshal(inner[2]) + if err == nil { + var tcs []ToolCall + if json.Unmarshal(b, &tcs) == nil && len(tcs) > 0 { + frame.ToolCalls = tcs } } - frame.OK = true - return frame } + frame.OK = true + return frame } // 2. Check if any element of v is a Chatbot message list or Chatbot pair list @@ -4866,14 +4870,17 @@ func (g *GradioGateway) executeQueueCompletion(w http.ResponseWriter, r *http.Re flusher, _ := w.(http.Flusher) streamer := NewStreamer(w, flusher, completionID, createdTime, modelName) + isNativeToolCalling := disc.IsHunyuan3 || disc.FunctionsJSONIndex >= 0 + thinkFilter := NewStreamThinkingFilter() toolFilter := NewStreamToolCallFilter() reader := bufio.NewReader(streamResp.Body) var prevContent string var prevReasoning string - prevToolArgs := make(map[int]string) - nativeReasoningSeen := disc.IsHunyuan3 + lastToolCallArgs := make(map[string]string) + emittedToolCallIDs := make(map[string]bool) + nativeReasoningSeen := disc.IsHunyuan3 || disc.ThinkLevelIndex >= 0 || disc.PreservedThinkingIndex >= 0 nativeToolCallsSeen := false var streamErr error @@ -4912,84 +4919,124 @@ func (g *GradioGateway) executeQueueCompletion(w http.ResponseWriter, r *http.Re dataBytes, _ := json.Marshal(qMsg.Output["data"]) frame := ParseGradioStreamOutput(string(dataBytes)) if frame.OK { - // 1. Native reasoning handling - if frame.Reasoning != "" || nativeReasoningSeen { - nativeReasoningSeen = true - var deltaReasoning string - if frame.IsDelta { - deltaReasoning = frame.Reasoning - prevReasoning += deltaReasoning - } else { - if strings.HasPrefix(frame.Reasoning, prevReasoning) { - deltaReasoning = frame.Reasoning[len(prevReasoning):] - } else if prevReasoning == "" { - deltaReasoning = frame.Reasoning - } else { - deltaReasoning = frame.Reasoning - } + if isNativeToolCalling { + // 1. Native reasoning + if len(frame.Reasoning) > len(prevReasoning) { + deltaReasoning := frame.Reasoning[len(prevReasoning):] prevReasoning = frame.Reasoning - } - if deltaReasoning != "" { streamer.Reasoning(deltaReasoning) + } else if frame.IsDelta && frame.Reasoning != "" { + streamer.Reasoning(frame.Reasoning) } - } - // 2. Native tool call handling - if len(frame.ToolCalls) > 0 || nativeToolCallsSeen { - nativeToolCallsSeen = true - for i, tc := range frame.ToolCalls { - prevArgs := prevToolArgs[i] - fullArgs := tc.Function.Arguments - if len(fullArgs) > len(prevArgs) && strings.HasPrefix(fullArgs, prevArgs) { - deltaArgs := fullArgs[len(prevArgs):] - iCopy := i - streamer.ToolCallDelta(ToolCall{ - Index: &iCopy, - ID: tc.ID, - Type: tc.Type, - Function: ToolCallFunction{ - Name: tc.Function.Name, - Arguments: deltaArgs, - }, - }) - prevToolArgs[i] = fullArgs - } else if prevArgs == "" { - iCopy := i - streamer.ToolCallDelta(ToolCall{ - Index: &iCopy, - ID: tc.ID, - Type: tc.Type, - Function: ToolCallFunction{ - Name: tc.Function.Name, - Arguments: fullArgs, - }, - }) - prevToolArgs[i] = fullArgs + // 2. Native tool calls + if len(frame.ToolCalls) > 0 { + for idx, t := range frame.ToolCalls { + key := t.ID + if key == "" { + key = fmt.Sprintf("idx_%d", idx) + } + prevArgs := lastToolCallArgs[key] + currArgs := t.Function.Arguments + idxCopy := idx + if !emittedToolCallIDs[key] { + emittedToolCallIDs[key] = true + lastToolCallArgs[key] = currArgs + tCopy := ToolCall{ + Index: &idxCopy, + ID: t.ID, + Type: "function", + Function: ToolCallFunction{Name: t.Function.Name, Arguments: currArgs}, + } + streamer.ToolCallDelta(tCopy) + } else if len(currArgs) > len(prevArgs) { + argDelta := currArgs[len(prevArgs):] + lastToolCallArgs[key] = currArgs + tCopy := ToolCall{ + Index: &idxCopy, + Function: ToolCallFunction{Arguments: argDelta}, + } + streamer.ToolCallDelta(tCopy) + } } } - } - // 3. Content handling - currentText := frame.Content - var delta string - if frame.IsDelta { - delta = frame.Content - prevContent += delta + // 3. Native content + if len(frame.Content) > len(prevContent) { + cDelta := frame.Content[len(prevContent):] + prevContent = frame.Content + streamer.Content(cDelta) + } else if frame.IsDelta && frame.Content != "" { + streamer.Content(frame.Content) + } } else { - if strings.HasPrefix(currentText, prevContent) { - delta = currentText[len(prevContent):] - } else if prevContent == "" { - delta = currentText - } else { - delta = currentText + // Non-native (prompt-augmented) tool calling + if frame.Reasoning != "" || nativeReasoningSeen { + nativeReasoningSeen = true + var deltaReasoning string + if frame.IsDelta { + deltaReasoning = frame.Reasoning + prevReasoning += deltaReasoning + } else { + if len(frame.Reasoning) > len(prevReasoning) { + deltaReasoning = frame.Reasoning[len(prevReasoning):] + } + prevReasoning = frame.Reasoning + } + if deltaReasoning != "" { + streamer.Reasoning(deltaReasoning) + } } - prevContent = currentText - } - if delta != "" { - if nativeReasoningSeen || nativeToolCallsSeen { - if nativeReasoningSeen && nativeToolCallsSeen { - streamer.Content(delta) + if len(frame.ToolCalls) > 0 { + nativeToolCallsSeen = true + for idx, t := range frame.ToolCalls { + key := t.ID + if key == "" { + key = fmt.Sprintf("idx_%d", idx) + } + prevArgs := lastToolCallArgs[key] + currArgs := t.Function.Arguments + idxCopy := idx + if !emittedToolCallIDs[key] { + emittedToolCallIDs[key] = true + lastToolCallArgs[key] = currArgs + tCopy := ToolCall{ + Index: &idxCopy, + ID: t.ID, + Type: "function", + Function: ToolCallFunction{Name: t.Function.Name, Arguments: currArgs}, + } + streamer.ToolCallDelta(tCopy) + } else if len(currArgs) > len(prevArgs) { + argDelta := currArgs[len(prevArgs):] + lastToolCallArgs[key] = currArgs + tCopy := ToolCall{ + Index: &idxCopy, + Function: ToolCallFunction{Arguments: argDelta}, + } + streamer.ToolCallDelta(tCopy) + } + } + } + + currentText := frame.Content + var delta string + if frame.IsDelta { + delta = frame.Content + prevContent += delta + } else { + if len(currentText) > len(prevContent) { + delta = currentText[len(prevContent):] + } + prevContent = currentText + } + + if delta != "" { + if nativeToolCallsSeen { + if strings.TrimSpace(delta) != "" { + streamer.Content(delta) + } } else if nativeReasoningSeen { toolFilter.Feed(delta, func(cleanChunk string) { if cleanChunk != "" { @@ -5000,29 +5047,19 @@ func (g *GradioGateway) executeQueueCompletion(w http.ResponseWriter, r *http.Re }) } else { thinkFilter.Feed(delta, func(contentChunk string) { - if contentChunk != "" { - streamer.Content(contentChunk) - } + toolFilter.Feed(contentChunk, func(cleanChunk string) { + if cleanChunk != "" { + streamer.Content(cleanChunk) + } + }, func(tc ToolCall) { + streamer.ToolCallDelta(tc) + }) }, func(reasoningChunk string) { if reasoningChunk != "" { streamer.Reasoning(reasoningChunk) } }) } - } else { - thinkFilter.Feed(delta, func(contentChunk string) { - toolFilter.Feed(contentChunk, func(cleanChunk string) { - if cleanChunk != "" { - streamer.Content(cleanChunk) - } - }, func(tc ToolCall) { - streamer.ToolCallDelta(tc) - }) - }, func(reasoningChunk string) { - if reasoningChunk != "" { - streamer.Reasoning(reasoningChunk) - } - }) } } } @@ -5038,7 +5075,18 @@ func (g *GradioGateway) executeQueueCompletion(w http.ResponseWriter, r *http.Re return streamErr } - // Flush remaining tokens in filters if used + if isNativeToolCalling { + if len(emittedToolCallIDs) > 0 { + streamer.Finish("tool_calls") + } else { + streamer.Finish("stop") + } + streamer.Done() + disc.Protocol = "queue" + return nil + } + + // Flush remaining tokens in filters if used for non-native if !nativeReasoningSeen { thinkFilter.Flush(func(contentChunk string) { if !nativeToolCallsSeen { @@ -5070,7 +5118,7 @@ func (g *GradioGateway) executeQueueCompletion(w http.ResponseWriter, r *http.Re } finishReason := "stop" - if nativeToolCallsSeen || toolFilter.emittedCall { + if len(emittedToolCallIDs) > 0 || nativeToolCallsSeen || toolFilter.emittedCall { finishReason = "tool_calls" } @@ -5372,14 +5420,17 @@ func (g *GradioGateway) ExecuteChatCompletion(w http.ResponseWriter, r *http.Req flusher, _ := w.(http.Flusher) streamer := NewStreamer(w, flusher, completionID, createdTime, modelName) + isNativeToolCalling := disc.IsHunyuan3 || disc.FunctionsJSONIndex >= 0 + thinkFilter := NewStreamThinkingFilter() toolFilter := NewStreamToolCallFilter() reader := bufio.NewReader(streamResp.Body) var prevContent string var prevReasoning string - prevToolArgs := make(map[int]string) - nativeReasoningSeen := disc.IsHunyuan3 + lastToolCallArgs := make(map[string]string) + emittedToolCallIDs := make(map[string]bool) + nativeReasoningSeen := disc.IsHunyuan3 || disc.ThinkLevelIndex >= 0 || disc.PreservedThinkingIndex >= 0 nativeToolCallsSeen := false currentEvent := "" var streamErr error @@ -5427,89 +5478,124 @@ func (g *GradioGateway) ExecuteChatCompletion(w http.ResponseWriter, r *http.Req frame := ParseGradioStreamOutput(dataStr) if frame.OK { - // 1. Native reasoning handling - if frame.Reasoning != "" || nativeReasoningSeen { - nativeReasoningSeen = true - var deltaReasoning string - if frame.IsDelta { - deltaReasoning = frame.Reasoning - prevReasoning += deltaReasoning - } else { - if strings.HasPrefix(frame.Reasoning, prevReasoning) { - deltaReasoning = frame.Reasoning[len(prevReasoning):] - } else if prevReasoning == "" { - deltaReasoning = frame.Reasoning - } else { - deltaReasoning = frame.Reasoning - } + if isNativeToolCalling { + // 1. Native reasoning + if len(frame.Reasoning) > len(prevReasoning) { + deltaReasoning := frame.Reasoning[len(prevReasoning):] prevReasoning = frame.Reasoning - } - if deltaReasoning != "" { streamer.Reasoning(deltaReasoning) + } else if frame.IsDelta && frame.Reasoning != "" { + streamer.Reasoning(frame.Reasoning) } - } - // 2. Native tool calls handling - if len(frame.ToolCalls) > 0 { - nativeToolCallsSeen = true - for idx, tc := range frame.ToolCalls { - prevArgs, started := prevToolArgs[idx] - currArgs := tc.Function.Arguments - idxCopy := idx - if !started { - tcDelta := ToolCall{ - Index: &idxCopy, - ID: tc.ID, - Type: tc.Type, - Function: ToolCallFunction{ - Name: tc.Function.Name, - Arguments: currArgs, - }, + // 2. Native tool calls + if len(frame.ToolCalls) > 0 { + for idx, t := range frame.ToolCalls { + key := t.ID + if key == "" { + key = fmt.Sprintf("idx_%d", idx) } - streamer.ToolCallDelta(tcDelta) - prevToolArgs[idx] = currArgs - } else if len(currArgs) > len(prevArgs) { - var argDelta string - if strings.HasPrefix(currArgs, prevArgs) { - argDelta = currArgs[len(prevArgs):] - } else { - argDelta = currArgs[len(prevArgs):] - } - if argDelta != "" { - tcDelta := ToolCall{ - Index: &idxCopy, - Function: ToolCallFunction{ - Arguments: argDelta, - }, + prevArgs := lastToolCallArgs[key] + currArgs := t.Function.Arguments + idxCopy := idx + if !emittedToolCallIDs[key] { + emittedToolCallIDs[key] = true + lastToolCallArgs[key] = currArgs + tCopy := ToolCall{ + Index: &idxCopy, + ID: t.ID, + Type: "function", + Function: ToolCallFunction{Name: t.Function.Name, Arguments: currArgs}, } - streamer.ToolCallDelta(tcDelta) + streamer.ToolCallDelta(tCopy) + } else if len(currArgs) > len(prevArgs) { + argDelta := currArgs[len(prevArgs):] + lastToolCallArgs[key] = currArgs + tCopy := ToolCall{ + Index: &idxCopy, + Function: ToolCallFunction{Arguments: argDelta}, + } + streamer.ToolCallDelta(tCopy) } - prevToolArgs[idx] = currArgs } } - } - // 3. Content handling - currentText := frame.Content - var delta string - if frame.IsDelta { - delta = currentText - prevContent += delta + // 3. Native content + if len(frame.Content) > len(prevContent) { + cDelta := frame.Content[len(prevContent):] + prevContent = frame.Content + streamer.Content(cDelta) + } else if frame.IsDelta && frame.Content != "" { + streamer.Content(frame.Content) + } } else { - if strings.HasPrefix(currentText, prevContent) { - delta = currentText[len(prevContent):] - } else if prevContent == "" { - delta = currentText - } else { - delta = currentText + // Non-native (prompt-augmented) tool calling + if frame.Reasoning != "" || nativeReasoningSeen { + nativeReasoningSeen = true + var deltaReasoning string + if frame.IsDelta { + deltaReasoning = frame.Reasoning + prevReasoning += deltaReasoning + } else { + if len(frame.Reasoning) > len(prevReasoning) { + deltaReasoning = frame.Reasoning[len(prevReasoning):] + } + prevReasoning = frame.Reasoning + } + if deltaReasoning != "" { + streamer.Reasoning(deltaReasoning) + } } - prevContent = currentText - } - if delta != "" { - if nativeReasoningSeen || nativeToolCallsSeen { - if nativeReasoningSeen && nativeToolCallsSeen { - streamer.Content(delta) + if len(frame.ToolCalls) > 0 { + nativeToolCallsSeen = true + for idx, t := range frame.ToolCalls { + key := t.ID + if key == "" { + key = fmt.Sprintf("idx_%d", idx) + } + prevArgs := lastToolCallArgs[key] + currArgs := t.Function.Arguments + idxCopy := idx + if !emittedToolCallIDs[key] { + emittedToolCallIDs[key] = true + lastToolCallArgs[key] = currArgs + tCopy := ToolCall{ + Index: &idxCopy, + ID: t.ID, + Type: "function", + Function: ToolCallFunction{Name: t.Function.Name, Arguments: currArgs}, + } + streamer.ToolCallDelta(tCopy) + } else if len(currArgs) > len(prevArgs) { + argDelta := currArgs[len(prevArgs):] + lastToolCallArgs[key] = currArgs + tCopy := ToolCall{ + Index: &idxCopy, + Function: ToolCallFunction{Arguments: argDelta}, + } + streamer.ToolCallDelta(tCopy) + } + } + } + + currentText := frame.Content + var delta string + if frame.IsDelta { + delta = currentText + prevContent += delta + } else { + if len(currentText) > len(prevContent) { + delta = currentText[len(prevContent):] + } + prevContent = currentText + } + + if delta != "" { + if nativeToolCallsSeen { + if strings.TrimSpace(delta) != "" { + streamer.Content(delta) + } } else if nativeReasoningSeen { toolFilter.Feed(delta, func(cleanChunk string) { if cleanChunk != "" { @@ -5520,29 +5606,19 @@ func (g *GradioGateway) ExecuteChatCompletion(w http.ResponseWriter, r *http.Req }) } else { thinkFilter.Feed(delta, func(contentChunk string) { - if contentChunk != "" { - streamer.Content(contentChunk) - } + toolFilter.Feed(contentChunk, func(cleanChunk string) { + if cleanChunk != "" { + streamer.Content(cleanChunk) + } + }, func(tc ToolCall) { + streamer.ToolCallDelta(tc) + }) }, func(reasoningChunk string) { if reasoningChunk != "" { streamer.Reasoning(reasoningChunk) } }) } - } else { - thinkFilter.Feed(delta, func(contentChunk string) { - toolFilter.Feed(contentChunk, func(cleanChunk string) { - if cleanChunk != "" { - streamer.Content(cleanChunk) - } - }, func(tc ToolCall) { - streamer.ToolCallDelta(tc) - }) - }, func(reasoningChunk string) { - if reasoningChunk != "" { - streamer.Reasoning(reasoningChunk) - } - }) } } } @@ -5557,7 +5633,17 @@ func (g *GradioGateway) ExecuteChatCompletion(w http.ResponseWriter, r *http.Req return nil } - // Flush remaining tokens in filters if used + if isNativeToolCalling { + if len(emittedToolCallIDs) > 0 { + streamer.Finish("tool_calls") + } else { + streamer.Finish("stop") + } + streamer.Done() + return nil + } + + // Flush remaining tokens in filters if used for non-native if !nativeReasoningSeen { thinkFilter.Flush(func(contentChunk string) { if !nativeToolCallsSeen { @@ -5593,13 +5679,12 @@ func (g *GradioGateway) ExecuteChatCompletion(w http.ResponseWriter, r *http.Req return g.executeQueueCompletion(w, r, disc, gradioData, req, completionID, createdTime, modelName, effUA) } - if nativeToolCallsSeen || toolFilter.emittedCall { + if len(emittedToolCallIDs) > 0 || nativeToolCallsSeen || toolFilter.emittedCall { streamer.Finish("tool_calls") } else { streamer.Finish("stop") } streamer.Done() - return nil } diff --git a/gr2gw_test.go b/gr2gw_test.go index 55bcdcf..07bd274 100644 --- a/gr2gw_test.go +++ b/gr2gw_test.go @@ -3604,3 +3604,458 @@ func TestWebSearchParameterMappingAndSuppression(t *testing.T) { } } +func TestHunyuanReasoningAndToolCallingStreaming(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/info") { + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(map[string]interface{}{ + "named_endpoints": map[string]interface{}{ + "/chat": map[string]interface{}{ + "parameters": []map[string]interface{}{ + {"parameter_name": "message", "component": "Api"}, + {"parameter_name": "system_prompt", "component": "Api"}, + {"parameter_name": "history", "component": "Api"}, + {"parameter_name": "think_level", "component": "Api"}, + {"parameter_name": "temperature", "component": "Api"}, + {"parameter_name": "max_tokens", "component": "Api"}, + {"parameter_name": "top_p", "component": "Api"}, + {"parameter_name": "preserved_thinking", "component": "Api"}, + {"parameter_name": "functions_json_str", "component": "Api"}, + }, + "code_snippets": map[string]interface{}{ + "bash": "curl -X POST http://localhost:7860/gradio_api/call/chat", + }, + }, + }, + }) + return + } + + if strings.HasSuffix(r.URL.Path, "/call/chat") && r.Method == http.MethodPost { + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(map[string]interface{}{ + "event_id": "hy3-evt-stream-tools", + }) + return + } + + if strings.Contains(r.URL.Path, "/call/chat/hy3-evt-stream-tools") && r.Method == http.MethodGet { + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Cache-Control", "no-cache") + w.WriteHeader(http.StatusOK) + flusher, _ := w.(http.Flusher) + + // Event 1: Reasoning part 1 + fmt.Fprint(w, "event: generating\ndata: [[\"\", \"Step 1 reasoning.\", null]]\n\n") + if flusher != nil { + flusher.Flush() + } + + // Event 2: Reasoning part 2 + fmt.Fprint(w, "event: generating\ndata: [[\"\", \"Step 1 reasoning. Step 2 reasoning.\", null]]\n\n") + if flusher != nil { + flusher.Flush() + } + + // Event 3: Tool call chunk 1 with partial arguments + fmt.Fprint(w, "event: generating\ndata: [[\"\", \"Step 1 reasoning. Step 2 reasoning.\", [{\"id\":\"call_xyz\",\"type\":\"function\",\"function\":{\"name\":\"calculator\",\"arguments\":\"{\\\"expr\\\":\\\"2\"}}]]]\n\n") + if flusher != nil { + flusher.Flush() + } + + // Event 4: Tool call chunk 2 with remaining arguments + fmt.Fprint(w, "event: complete\ndata: [[\"\", \"Step 1 reasoning. Step 2 reasoning.\", [{\"id\":\"call_xyz\",\"type\":\"function\",\"function\":{\"name\":\"calculator\",\"arguments\":\"{\\\"expr\\\":\\\"2+2\\\"}\"}}]]]\n\n") + if flusher != nil { + flusher.Flush() + } + return + } + + http.NotFound(w, r) + })) + defer ts.Close() + + gw := NewGradioGateway(ts.URL, "", 10*time.Second) + + reqStream := ChatCompletionRequest{ + Model: "hy3", + Stream: true, + Messages: []ChatMessage{ + {Role: "user", Content: "Calculate 2+2"}, + }, + Tools: []Tool{ + {Type: "function", Function: map[string]interface{}{"name": "calculator"}}, + }, + } + httpReq := httptest.NewRequest("POST", "/v1/chat/completions", nil) + rec := httptest.NewRecorder() + + err := gw.ExecuteChatCompletion(rec, httpReq, reqStream) + if err != nil { + t.Fatalf("ExecuteChatCompletion failed: %v", err) + } + + lines := strings.Split(rec.Body.String(), "\n") + var receivedDeltas []StreamDelta + var finishReasons []string + var foundDone bool + + for _, line := range lines { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "data: ") { + payload := strings.TrimPrefix(line, "data: ") + if payload == "[DONE]" { + foundDone = true + continue + } + var streamResp StreamResponse + if err := json.Unmarshal([]byte(payload), &streamResp); err == nil && len(streamResp.Choices) > 0 { + choice := streamResp.Choices[0] + receivedDeltas = append(receivedDeltas, choice.Delta) + if choice.FinishReason != nil { + finishReasons = append(finishReasons, *choice.FinishReason) + } + } + } + } + + if !foundDone { + t.Errorf("expected 'data: [DONE]' in SSE stream") + } + + if len(receivedDeltas) < 1 || receivedDeltas[0].Role != "assistant" { + t.Errorf("expected first delta to set role='assistant', got %+v", receivedDeltas[0]) + } + + var combinedReasoning, combinedContent string + var toolCallDeltas []ToolCall + for _, d := range receivedDeltas { + combinedReasoning += d.ReasoningContent + combinedContent += d.Content + if len(d.ToolCalls) > 0 { + toolCallDeltas = append(toolCallDeltas, d.ToolCalls...) + } + } + + if combinedReasoning != "Step 1 reasoning. Step 2 reasoning." { + t.Errorf("combined reasoning=%q, want 'Step 1 reasoning. Step 2 reasoning.'", combinedReasoning) + } + if combinedContent != "" { + t.Errorf("expected no content emitted during tool calling, got %q", combinedContent) + } + + if len(toolCallDeltas) != 2 { + t.Fatalf("expected exactly 2 tool call deltas, got %d: %+v", len(toolCallDeltas), toolCallDeltas) + } + + // First delta chunk: has index, id, type, name, initial arguments + firstTC := toolCallDeltas[0] + if firstTC.Index == nil || *firstTC.Index != 0 { + t.Errorf("first delta index expected 0, got %v", firstTC.Index) + } + if firstTC.ID != "call_xyz" || firstTC.Type != "function" || firstTC.Function.Name != "calculator" { + t.Errorf("first delta expected full metadata, got %+v", firstTC) + } + if firstTC.Function.Arguments != `{"expr":"2` { + t.Errorf("first delta arguments=%q, want %q", firstTC.Function.Arguments, `{"expr":"2`) + } + + // Second delta chunk: MUST have empty ID, empty Type, empty Name, and ONLY argument delta "+2\"}" + secondTC := toolCallDeltas[1] + if secondTC.Index == nil || *secondTC.Index != 0 { + t.Errorf("second delta index expected 0, got %v", secondTC.Index) + } + if secondTC.ID != "" || secondTC.Type != "" || secondTC.Function.Name != "" { + t.Errorf("second delta MUST NOT resend ID/Type/Name per OpenAI spec: %+v", secondTC) + } + if secondTC.Function.Arguments != `+2"}` { + t.Errorf("second delta arguments=%q, want %q", secondTC.Function.Arguments, `+2"}`) + } + + if len(finishReasons) != 1 || finishReasons[0] != "tool_calls" { + t.Errorf("finish_reasons=%v, want ['tool_calls']", finishReasons) + } +} + +func TestHunyuanToolCallingNonStreaming(t *testing.T) { + var receivedPayload map[string]interface{} + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/info") { + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(map[string]interface{}{ + "named_endpoints": map[string]interface{}{ + "/chat": map[string]interface{}{ + "parameters": []map[string]interface{}{ + {"parameter_name": "message", "component": "Api"}, + {"parameter_name": "system_prompt", "component": "Api"}, + {"parameter_name": "history", "component": "Api"}, + {"parameter_name": "think_level", "component": "Api"}, + {"parameter_name": "temperature", "component": "Api"}, + {"parameter_name": "max_tokens", "component": "Api"}, + {"parameter_name": "top_p", "component": "Api"}, + {"parameter_name": "preserved_thinking", "component": "Api"}, + {"parameter_name": "functions_json_str", "component": "Api"}, + }, + "code_snippets": map[string]interface{}{ + "bash": "curl -X POST http://localhost:7860/gradio_api/call/chat", + }, + }, + }, + }) + return + } + + if strings.HasSuffix(r.URL.Path, "/call/chat") && r.Method == http.MethodPost { + json.NewDecoder(r.Body).Decode(&receivedPayload) + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(map[string]interface{}{ + "event_id": "hy3-tool-nonstream", + }) + return + } + + if strings.Contains(r.URL.Path, "/call/chat/hy3-tool-nonstream") && r.Method == http.MethodGet { + w.Header().Set("Content-Type", "text/event-stream") + fmt.Fprint(w, "event: complete\ndata: [[\"\", \"I should call the search tool.\", [{\"id\":\"call_abc\",\"type\":\"function\",\"function\":{\"name\":\"search\",\"arguments\":\"{\\\"query\\\":\\\"golang\\\"}\"}}]]]\n\n") + return + } + + http.NotFound(w, r) + })) + defer ts.Close() + + gw := NewGradioGateway(ts.URL, "", 10*time.Second) + + reqNonStream := ChatCompletionRequest{ + Model: "hy3", + Stream: false, + Messages: []ChatMessage{ + {Role: "user", Content: "Search for golang news"}, + }, + Tools: []Tool{ + {Type: "function", Function: map[string]interface{}{"name": "search", "description": "Search web"}}, + }, + } + + httpReq := httptest.NewRequest("POST", "/v1/chat/completions", nil) + rec := httptest.NewRecorder() + + err := gw.ExecuteChatCompletion(rec, httpReq, reqNonStream) + if err != nil { + t.Fatalf("ExecuteChatCompletion non-streaming failed: %v", err) + } + + // Verify functions_json_str passed in data[8] + dataArr, _ := receivedPayload["data"].([]interface{}) + if len(dataArr) < 9 || dataArr[8] == nil || dataArr[8] == "" { + t.Errorf("expected functions_json_str in data[8], got %v", dataArr) + } + + var resp ChatCompletionResponse + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("failed to decode response: %v", err) + } + + if len(resp.Choices) != 1 { + t.Fatalf("expected 1 choice, got %d", len(resp.Choices)) + } + choice := resp.Choices[0] + if choice.FinishReason != "tool_calls" { + t.Errorf("finish_reason=%q, want 'tool_calls'", choice.FinishReason) + } + if choice.Message.Content != nil { + t.Errorf("content expected nil, got %v", choice.Message.Content) + } + if choice.Message.ReasoningContent != "I should call the search tool." { + t.Errorf("reasoning_content=%q, want 'I should call the search tool.'", choice.Message.ReasoningContent) + } + if len(choice.Message.ToolCalls) != 1 { + t.Fatalf("expected 1 tool call, got %d", len(choice.Message.ToolCalls)) + } + tc := choice.Message.ToolCalls[0] + if tc.ID != "call_abc" || tc.Function.Name != "search" || tc.Function.Arguments != `{"query":"golang"}` { + t.Errorf("unexpected tool call: %+v", tc) + } +} + +func TestHunyuanMultiTurnToolResultHistory(t *testing.T) { + gw := &GradioGateway{} + + disc := NewDefaultSpaceDiscovery("https://tencent-hy3.hf.space") + disc.IsHunyuan3 = true + disc.TotalInputs = 9 + disc.MessageIndex = 0 + disc.SystemIndex = 1 + disc.HistoryIndex = 2 + disc.HistoryFormat = "messages" + disc.ThinkLevelIndex = 3 + disc.TempIndex = 4 + disc.MaxTokensIndex = 5 + disc.TopPIndex = 6 + disc.PreservedThinkingIndex = 7 + disc.FunctionsJSONIndex = 8 + + req := ChatCompletionRequest{ + Model: "hy3", + Messages: []ChatMessage{ + {Role: "user", Content: "What is the weather in Tokyo?"}, + { + Role: "assistant", + ToolCalls: []ToolCall{ + {ID: "call_weather_1", Type: "function", Function: ToolCallFunction{Name: "get_weather", Arguments: `{"location":"Tokyo"}`}}, + }, + }, + { + Role: "tool", + Name: "get_weather", + ToolCallID: "call_weather_1", + Content: `{"temperature": 20, "condition": "Sunny"}`, + }, + }, + Tools: []Tool{ + {Type: "function", Function: map[string]interface{}{"name": "get_weather"}}, + }, + } + + data, err := gw.BuildGradioPayload(disc, req) + if err != nil { + t.Fatalf("BuildGradioPayload failed: %v", err) + } + + // 1. Message input (slot 0) should be clean "Tool result for get_weather: ..." without "Query:" or "Please answer..." + msgStr, ok := data[0].(string) + if !ok { + t.Fatalf("expected string message in data[0], got %T", data[0]) + } + expectedMsg := `Tool result for get_weather: {"temperature": 20, "condition": "Sunny"}` + if msgStr != expectedMsg { + t.Errorf("message=%q, want %q", msgStr, expectedMsg) + } + + // 2. History input (slot 2) should contain user turn and assistant tool_call turn + histArr, ok := data[2].([]map[string]interface{}) + if !ok { + t.Fatalf("expected []map[string]interface{} history in data[2], got %T", data[2]) + } + if len(histArr) != 2 { + t.Fatalf("expected 2 turns in history, got %d: %+v", len(histArr), histArr) + } + if histArr[0]["role"] != "user" || histArr[0]["content"] != "What is the weather in Tokyo?" { + t.Errorf("unexpected turn 0: %+v", histArr[0]) + } + if histArr[1]["role"] != "assistant" { + t.Errorf("unexpected turn 1 role: %+v", histArr[1]) + } + tcArr, ok := histArr[1]["tool_calls"].([]ToolCall) + if !ok || len(tcArr) != 1 || tcArr[0].Function.Name != "get_weather" { + t.Errorf("unexpected turn 1 tool_calls: %+v", histArr[1]) + } +} + +func TestQueueToolCallingStreamingParity(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/info" { + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(map[string]interface{}{ + "named_endpoints": map[string]interface{}{ + "/predict": map[string]interface{}{ + "parameters": []map[string]interface{}{ + {"parameter_name": "message", "component": "Api"}, + {"parameter_name": "tools", "component": "Api"}, + }, + }, + }, + }) + return + } + + if strings.HasSuffix(r.URL.Path, "/queue/join") { + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(map[string]interface{}{"event_id": "queue-evt-1"}) + return + } + + if strings.HasSuffix(r.URL.Path, "/queue/data") { + w.Header().Set("Content-Type", "text/event-stream") + flusher, _ := w.(http.Flusher) + + // Chunk 1: Initial tool call with partial argument + fmt.Fprint(w, "data: {\"msg\":\"process_generating\",\"output\":{\"data\":[[\"\", \"\", [{\"id\":\"tc_q1\",\"type\":\"function\",\"function\":{\"name\":\"fetch_data\",\"arguments\":\"{\\\"id\\\": 1\"}}]]]}}\n\n") + if flusher != nil { + flusher.Flush() + } + + // Chunk 2: Completed tool call argument + fmt.Fprint(w, "data: {\"msg\":\"process_generating\",\"output\":{\"data\":[[\"\", \"\", [{\"id\":\"tc_q1\",\"type\":\"function\",\"function\":{\"name\":\"fetch_data\",\"arguments\":\"{\\\"id\\\": 123}\"}}]]]}}\n\n") + if flusher != nil { + flusher.Flush() + } + + // Chunk 3: Completed + fmt.Fprint(w, "data: {\"msg\":\"process_completed\",\"output\":{\"data\":[[\"\", \"\", [{\"id\":\"tc_q1\",\"type\":\"function\",\"function\":{\"name\":\"fetch_data\",\"arguments\":\"{\\\"id\\\": 123}\"}}]]]}}\n\n") + if flusher != nil { + flusher.Flush() + } + return + } + + http.NotFound(w, r) + })) + defer ts.Close() + + gw := NewGradioGateway(ts.URL, "", 10*time.Second) + + req := ChatCompletionRequest{ + Model: "test-queue-tools", + Stream: true, + Messages: []ChatMessage{ + {Role: "user", Content: "Fetch ID 123"}, + }, + Tools: []Tool{ + {Type: "function", Function: map[string]interface{}{"name": "fetch_data"}}, + }, + } + + httpReq := httptest.NewRequest("POST", "/v1/chat/completions", nil) + rec := httptest.NewRecorder() + + err := gw.ExecuteChatCompletion(rec, httpReq, req) + if err != nil { + t.Fatalf("ExecuteChatCompletion queue streaming failed: %v", err) + } + + lines := strings.Split(rec.Body.String(), "\n") + var toolCallDeltas []ToolCall + var finishReasons []string + + for _, line := range lines { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "data: ") && line != "data: [DONE]" { + var chunk StreamResponse + if err := json.Unmarshal([]byte(strings.TrimPrefix(line, "data: ")), &chunk); err == nil && len(chunk.Choices) > 0 { + if len(chunk.Choices[0].Delta.ToolCalls) > 0 { + toolCallDeltas = append(toolCallDeltas, chunk.Choices[0].Delta.ToolCalls...) + } + if chunk.Choices[0].FinishReason != nil { + finishReasons = append(finishReasons, *chunk.Choices[0].FinishReason) + } + } + } + } + + if len(toolCallDeltas) != 2 { + t.Fatalf("expected 2 tool call deltas in queue stream, got %d: %+v", len(toolCallDeltas), toolCallDeltas) + } + if toolCallDeltas[0].ID != "tc_q1" || toolCallDeltas[0].Function.Name != "fetch_data" { + t.Errorf("unexpected first delta: %+v", toolCallDeltas[0]) + } + if toolCallDeltas[1].ID != "" || toolCallDeltas[1].Function.Name != "" || toolCallDeltas[1].Function.Arguments != `23}` { + t.Errorf("second delta must only have argument delta '23}', got %+v", toolCallDeltas[1]) + } + if len(finishReasons) != 1 || finishReasons[0] != "tool_calls" { + t.Errorf("expected finish_reason 'tool_calls', got %v", finishReasons) + } +} + +