luxferre/gr2gw
1
0
Fork 0
Code Actions Projects Activity

toolcall fixes

Browse Source
This commit is contained in:
Luxferre
2026-09-08 18:20:06 +03:00
parent c731b57dfc
commit f2b16174fa
2 changed files with 738 additions and 198 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gr2gw.go
+283 -198
View File
@@ -1808,6 +1808,10 @@ func finalizeOutput(frame GradioOutputFrame) (finalContent interface{}, reasonin
toolCalls = frame.ToolCalls
hasTools := len(toolCalls) > 0
if hasTools && strings.TrimSpace(cleanText) == "" {
return nil, reasoning, toolCalls, "tool_calls"
}
if reasoning == "" {
cleanText, reasoning = ExtractThinking(cleanText)
}
@@ -3738,7 +3742,7 @@ func (g *GradioGateway) GetDiscovery(spaceURL, userAgent string) *SpaceDiscovery
func (g *GradioGateway) BuildGradioPayload(disc *SpaceDiscovery, req ChatCompletionRequest) ([]interface{}, error) {
var transformed []ChatMessage
var toolInstruction string
if disc.IsHunyuan3 && disc.FunctionsJSONIndex != -1 {
if (disc.IsHunyuan3 || disc.FunctionsJSONIndex != -1) && disc.FunctionsJSONIndex != -1 {
for _, msg := range req.Messages {
transformed = append(transformed, ChatMessage{
Role: msg.Role,
@@ -3847,7 +3851,7 @@ func (g *GradioGateway) BuildGradioPayload(disc *SpaceDiscovery, req ChatComplet
if toolName == "" {
toolName = lastMsg.ToolCallID
}
if disc.IsHunyuan3 {
if disc.IsHunyuan3 || disc.FunctionsJSONIndex != -1 {
if toolName != "" {
lastUserMessage = fmt.Sprintf("Tool result for %s: %s", toolName, lastContent)
} else {
@@ -4296,8 +4300,9 @@ func extractGradioDiffDelta(v []interface{}) (contentDelta string, reasoningDelt
// ParseGradioStreamOutput extracts structured content, reasoning, and tool calls from Gradio output
func ParseGradioStreamOutput(rawJSON string) (frame GradioOutputFrame) {
isNativeTuple := false
defer func() {
if frame.OK && len(frame.ToolCalls) == 0 && frame.Content != "" {
if !isNativeTuple && frame.OK && len(frame.ToolCalls) == 0 && frame.Content != "" {
tcs, clean, has := DetectToolCalls(frame.Content)
if has && len(tcs) > 0 {
frame.ToolCalls = tcs
@@ -4332,23 +4337,22 @@ func ParseGradioStreamOutput(rawJSON string) (frame GradioOutputFrame) {
// 1. Check if v[0] is an inner slice with len >= 2 (e.g. Hy3: [[content, reasoning, tool_calls, history]])
if inner, ok := v[0].([]interface{}); ok && len(inner) >= 2 {
s0, ok0 := inner[0].(string)
s1, ok1 := inner[1].(string)
if ok0 || ok1 {
frame.Content = s0
frame.Reasoning = s1
if len(inner) >= 3 && inner[2] != nil {
b, err := json.Marshal(inner[2])
if err == nil {
var tcs []ToolCall
if json.Unmarshal(b, &tcs) == nil && len(tcs) > 0 {
frame.ToolCalls = tcs
}
isNativeTuple = true
s0, _ := inner[0].(string)
s1, _ := inner[1].(string)
frame.Content = s0
frame.Reasoning = s1
if len(inner) >= 3 && inner[2] != nil {
b, err := json.Marshal(inner[2])
if err == nil {
var tcs []ToolCall
if json.Unmarshal(b, &tcs) == nil && len(tcs) > 0 {
frame.ToolCalls = tcs
}
}
frame.OK = true
return frame
}
frame.OK = true
return frame
}
// 2. Check if any element of v is a Chatbot message list or Chatbot pair list
@@ -4866,14 +4870,17 @@ func (g *GradioGateway) executeQueueCompletion(w http.ResponseWriter, r *http.Re
flusher, _ := w.(http.Flusher)
streamer := NewStreamer(w, flusher, completionID, createdTime, modelName)
isNativeToolCalling := disc.IsHunyuan3 || disc.FunctionsJSONIndex >= 0
thinkFilter := NewStreamThinkingFilter()
toolFilter := NewStreamToolCallFilter()
reader := bufio.NewReader(streamResp.Body)
var prevContent string
var prevReasoning string
prevToolArgs := make(map[int]string)
nativeReasoningSeen := disc.IsHunyuan3
lastToolCallArgs := make(map[string]string)
emittedToolCallIDs := make(map[string]bool)
nativeReasoningSeen := disc.IsHunyuan3 || disc.ThinkLevelIndex >= 0 || disc.PreservedThinkingIndex >= 0
nativeToolCallsSeen := false
var streamErr error
@@ -4912,84 +4919,124 @@ func (g *GradioGateway) executeQueueCompletion(w http.ResponseWriter, r *http.Re
dataBytes, _ := json.Marshal(qMsg.Output["data"])
frame := ParseGradioStreamOutput(string(dataBytes))
if frame.OK {
// 1. Native reasoning handling
if frame.Reasoning != "" || nativeReasoningSeen {
nativeReasoningSeen = true
var deltaReasoning string
if frame.IsDelta {
deltaReasoning = frame.Reasoning
prevReasoning += deltaReasoning
} else {
if strings.HasPrefix(frame.Reasoning, prevReasoning) {
deltaReasoning = frame.Reasoning[len(prevReasoning):]
} else if prevReasoning == "" {
deltaReasoning = frame.Reasoning
} else {
deltaReasoning = frame.Reasoning
}
if isNativeToolCalling {
// 1. Native reasoning
if len(frame.Reasoning) > len(prevReasoning) {
deltaReasoning := frame.Reasoning[len(prevReasoning):]
prevReasoning = frame.Reasoning
}
if deltaReasoning != "" {
streamer.Reasoning(deltaReasoning)
} else if frame.IsDelta && frame.Reasoning != "" {
streamer.Reasoning(frame.Reasoning)
}
}
// 2. Native tool call handling
if len(frame.ToolCalls) > 0 || nativeToolCallsSeen {
nativeToolCallsSeen = true
for i, tc := range frame.ToolCalls {
prevArgs := prevToolArgs[i]
fullArgs := tc.Function.Arguments
if len(fullArgs) > len(prevArgs) && strings.HasPrefix(fullArgs, prevArgs) {
deltaArgs := fullArgs[len(prevArgs):]
iCopy := i
streamer.ToolCallDelta(ToolCall{
Index: &iCopy,
ID: tc.ID,
Type: tc.Type,
Function: ToolCallFunction{
Name: tc.Function.Name,
Arguments: deltaArgs,
},
})
prevToolArgs[i] = fullArgs
} else if prevArgs == "" {
iCopy := i
streamer.ToolCallDelta(ToolCall{
Index: &iCopy,
ID: tc.ID,
Type: tc.Type,
Function: ToolCallFunction{
Name: tc.Function.Name,
Arguments: fullArgs,
},
})
prevToolArgs[i] = fullArgs
// 2. Native tool calls
if len(frame.ToolCalls) > 0 {
for idx, t := range frame.ToolCalls {
key := t.ID
if key == "" {
key = fmt.Sprintf("idx_%d", idx)
}
prevArgs := lastToolCallArgs[key]
currArgs := t.Function.Arguments
idxCopy := idx
if !emittedToolCallIDs[key] {
emittedToolCallIDs[key] = true
lastToolCallArgs[key] = currArgs
tCopy := ToolCall{
Index: &idxCopy,
ID: t.ID,
Type: "function",
Function: ToolCallFunction{Name: t.Function.Name, Arguments: currArgs},
}
streamer.ToolCallDelta(tCopy)
} else if len(currArgs) > len(prevArgs) {
argDelta := currArgs[len(prevArgs):]
lastToolCallArgs[key] = currArgs
tCopy := ToolCall{
Index: &idxCopy,
Function: ToolCallFunction{Arguments: argDelta},
}
streamer.ToolCallDelta(tCopy)
}
}
}
}
// 3. Content handling
currentText := frame.Content
var delta string
if frame.IsDelta {
delta = frame.Content
prevContent += delta
// 3. Native content
if len(frame.Content) > len(prevContent) {
cDelta := frame.Content[len(prevContent):]
prevContent = frame.Content
streamer.Content(cDelta)
} else if frame.IsDelta && frame.Content != "" {
streamer.Content(frame.Content)
}
} else {
if strings.HasPrefix(currentText, prevContent) {
delta = currentText[len(prevContent):]
} else if prevContent == "" {
delta = currentText
} else {
delta = currentText
// Non-native (prompt-augmented) tool calling
if frame.Reasoning != "" || nativeReasoningSeen {
nativeReasoningSeen = true
var deltaReasoning string
if frame.IsDelta {
deltaReasoning = frame.Reasoning
prevReasoning += deltaReasoning
} else {
if len(frame.Reasoning) > len(prevReasoning) {
deltaReasoning = frame.Reasoning[len(prevReasoning):]
}
prevReasoning = frame.Reasoning
}
if deltaReasoning != "" {
streamer.Reasoning(deltaReasoning)
}
}
prevContent = currentText
}
if delta != "" {
if nativeReasoningSeen || nativeToolCallsSeen {
if nativeReasoningSeen && nativeToolCallsSeen {
streamer.Content(delta)
if len(frame.ToolCalls) > 0 {
nativeToolCallsSeen = true
for idx, t := range frame.ToolCalls {
key := t.ID
if key == "" {
key = fmt.Sprintf("idx_%d", idx)
}
prevArgs := lastToolCallArgs[key]
currArgs := t.Function.Arguments
idxCopy := idx
if !emittedToolCallIDs[key] {
emittedToolCallIDs[key] = true
lastToolCallArgs[key] = currArgs
tCopy := ToolCall{
Index: &idxCopy,
ID: t.ID,
Type: "function",
Function: ToolCallFunction{Name: t.Function.Name, Arguments: currArgs},
}
streamer.ToolCallDelta(tCopy)
} else if len(currArgs) > len(prevArgs) {
argDelta := currArgs[len(prevArgs):]
lastToolCallArgs[key] = currArgs
tCopy := ToolCall{
Index: &idxCopy,
Function: ToolCallFunction{Arguments: argDelta},
}
streamer.ToolCallDelta(tCopy)
}
}
}
currentText := frame.Content
var delta string
if frame.IsDelta {
delta = frame.Content
prevContent += delta
} else {
if len(currentText) > len(prevContent) {
delta = currentText[len(prevContent):]
}
prevContent = currentText
}
if delta != "" {
if nativeToolCallsSeen {
if strings.TrimSpace(delta) != "" {
streamer.Content(delta)
}
} else if nativeReasoningSeen {
toolFilter.Feed(delta, func(cleanChunk string) {
if cleanChunk != "" {
@@ -5000,29 +5047,19 @@ func (g *GradioGateway) executeQueueCompletion(w http.ResponseWriter, r *http.Re
})
} else {
thinkFilter.Feed(delta, func(contentChunk string) {
if contentChunk != "" {
streamer.Content(contentChunk)
}
toolFilter.Feed(contentChunk, func(cleanChunk string) {
if cleanChunk != "" {
streamer.Content(cleanChunk)
}
}, func(tc ToolCall) {
streamer.ToolCallDelta(tc)
})
}, func(reasoningChunk string) {
if reasoningChunk != "" {
streamer.Reasoning(reasoningChunk)
}
})
}
} else {
thinkFilter.Feed(delta, func(contentChunk string) {
toolFilter.Feed(contentChunk, func(cleanChunk string) {
if cleanChunk != "" {
streamer.Content(cleanChunk)
}
}, func(tc ToolCall) {
streamer.ToolCallDelta(tc)
})
}, func(reasoningChunk string) {
if reasoningChunk != "" {
streamer.Reasoning(reasoningChunk)
}
})
}
}
}
@@ -5038,7 +5075,18 @@ func (g *GradioGateway) executeQueueCompletion(w http.ResponseWriter, r *http.Re
return streamErr
}
// Flush remaining tokens in filters if used
if isNativeToolCalling {
if len(emittedToolCallIDs) > 0 {
streamer.Finish("tool_calls")
} else {
streamer.Finish("stop")
}
streamer.Done()
disc.Protocol = "queue"
return nil
}
// Flush remaining tokens in filters if used for non-native
if !nativeReasoningSeen {
thinkFilter.Flush(func(contentChunk string) {
if !nativeToolCallsSeen {
@@ -5070,7 +5118,7 @@ func (g *GradioGateway) executeQueueCompletion(w http.ResponseWriter, r *http.Re
}
finishReason := "stop"
if nativeToolCallsSeen || toolFilter.emittedCall {
if len(emittedToolCallIDs) > 0 || nativeToolCallsSeen || toolFilter.emittedCall {
finishReason = "tool_calls"
}
@@ -5372,14 +5420,17 @@ func (g *GradioGateway) ExecuteChatCompletion(w http.ResponseWriter, r *http.Req
flusher, _ := w.(http.Flusher)
streamer := NewStreamer(w, flusher, completionID, createdTime, modelName)
isNativeToolCalling := disc.IsHunyuan3 || disc.FunctionsJSONIndex >= 0
thinkFilter := NewStreamThinkingFilter()
toolFilter := NewStreamToolCallFilter()
reader := bufio.NewReader(streamResp.Body)
var prevContent string
var prevReasoning string
prevToolArgs := make(map[int]string)
nativeReasoningSeen := disc.IsHunyuan3
lastToolCallArgs := make(map[string]string)
emittedToolCallIDs := make(map[string]bool)
nativeReasoningSeen := disc.IsHunyuan3 || disc.ThinkLevelIndex >= 0 || disc.PreservedThinkingIndex >= 0
nativeToolCallsSeen := false
currentEvent := ""
var streamErr error
@@ -5427,89 +5478,124 @@ func (g *GradioGateway) ExecuteChatCompletion(w http.ResponseWriter, r *http.Req
frame := ParseGradioStreamOutput(dataStr)
if frame.OK {
// 1. Native reasoning handling
if frame.Reasoning != "" || nativeReasoningSeen {
nativeReasoningSeen = true
var deltaReasoning string
if frame.IsDelta {
deltaReasoning = frame.Reasoning
prevReasoning += deltaReasoning
} else {
if strings.HasPrefix(frame.Reasoning, prevReasoning) {
deltaReasoning = frame.Reasoning[len(prevReasoning):]
} else if prevReasoning == "" {
deltaReasoning = frame.Reasoning
} else {
deltaReasoning = frame.Reasoning
}
if isNativeToolCalling {
// 1. Native reasoning
if len(frame.Reasoning) > len(prevReasoning) {
deltaReasoning := frame.Reasoning[len(prevReasoning):]
prevReasoning = frame.Reasoning
}
if deltaReasoning != "" {
streamer.Reasoning(deltaReasoning)
} else if frame.IsDelta && frame.Reasoning != "" {
streamer.Reasoning(frame.Reasoning)
}
}
// 2. Native tool calls handling
if len(frame.ToolCalls) > 0 {
nativeToolCallsSeen = true
for idx, tc := range frame.ToolCalls {
prevArgs, started := prevToolArgs[idx]
currArgs := tc.Function.Arguments
idxCopy := idx
if !started {
tcDelta := ToolCall{
Index: &idxCopy,
ID: tc.ID,
Type: tc.Type,
Function: ToolCallFunction{
Name: tc.Function.Name,
Arguments: currArgs,
},
// 2. Native tool calls
if len(frame.ToolCalls) > 0 {
for idx, t := range frame.ToolCalls {
key := t.ID
if key == "" {
key = fmt.Sprintf("idx_%d", idx)
}
streamer.ToolCallDelta(tcDelta)
prevToolArgs[idx] = currArgs
} else if len(currArgs) > len(prevArgs) {
var argDelta string
if strings.HasPrefix(currArgs, prevArgs) {
argDelta = currArgs[len(prevArgs):]
} else {
argDelta = currArgs[len(prevArgs):]
}
if argDelta != "" {
tcDelta := ToolCall{
Index: &idxCopy,
Function: ToolCallFunction{
Arguments: argDelta,
},
prevArgs := lastToolCallArgs[key]
currArgs := t.Function.Arguments
idxCopy := idx
if !emittedToolCallIDs[key] {
emittedToolCallIDs[key] = true
lastToolCallArgs[key] = currArgs
tCopy := ToolCall{
Index: &idxCopy,
ID: t.ID,
Type: "function",
Function: ToolCallFunction{Name: t.Function.Name, Arguments: currArgs},
}
streamer.ToolCallDelta(tcDelta)
streamer.ToolCallDelta(tCopy)
} else if len(currArgs) > len(prevArgs) {
argDelta := currArgs[len(prevArgs):]
lastToolCallArgs[key] = currArgs
tCopy := ToolCall{
Index: &idxCopy,
Function: ToolCallFunction{Arguments: argDelta},
}
streamer.ToolCallDelta(tCopy)
}
prevToolArgs[idx] = currArgs
}
}
}
// 3. Content handling
currentText := frame.Content
var delta string
if frame.IsDelta {
delta = currentText
prevContent += delta
// 3. Native content
if len(frame.Content) > len(prevContent) {
cDelta := frame.Content[len(prevContent):]
prevContent = frame.Content
streamer.Content(cDelta)
} else if frame.IsDelta && frame.Content != "" {
streamer.Content(frame.Content)
}
} else {
if strings.HasPrefix(currentText, prevContent) {
delta = currentText[len(prevContent):]
} else if prevContent == "" {
delta = currentText
} else {
delta = currentText
// Non-native (prompt-augmented) tool calling
if frame.Reasoning != "" || nativeReasoningSeen {
nativeReasoningSeen = true
var deltaReasoning string
if frame.IsDelta {
deltaReasoning = frame.Reasoning
prevReasoning += deltaReasoning
} else {
if len(frame.Reasoning) > len(prevReasoning) {
deltaReasoning = frame.Reasoning[len(prevReasoning):]
}
prevReasoning = frame.Reasoning
}
if deltaReasoning != "" {
streamer.Reasoning(deltaReasoning)
}
}
prevContent = currentText
}
if delta != "" {
if nativeReasoningSeen || nativeToolCallsSeen {
if nativeReasoningSeen && nativeToolCallsSeen {
streamer.Content(delta)
if len(frame.ToolCalls) > 0 {
nativeToolCallsSeen = true
for idx, t := range frame.ToolCalls {
key := t.ID
if key == "" {
key = fmt.Sprintf("idx_%d", idx)
}
prevArgs := lastToolCallArgs[key]
currArgs := t.Function.Arguments
idxCopy := idx
if !emittedToolCallIDs[key] {
emittedToolCallIDs[key] = true
lastToolCallArgs[key] = currArgs
tCopy := ToolCall{
Index: &idxCopy,
ID: t.ID,
Type: "function",
Function: ToolCallFunction{Name: t.Function.Name, Arguments: currArgs},
}
streamer.ToolCallDelta(tCopy)
} else if len(currArgs) > len(prevArgs) {
argDelta := currArgs[len(prevArgs):]
lastToolCallArgs[key] = currArgs
tCopy := ToolCall{
Index: &idxCopy,
Function: ToolCallFunction{Arguments: argDelta},
}
streamer.ToolCallDelta(tCopy)
}
}
}
currentText := frame.Content
var delta string
if frame.IsDelta {
delta = currentText
prevContent += delta
} else {
if len(currentText) > len(prevContent) {
delta = currentText[len(prevContent):]
}
prevContent = currentText
}
if delta != "" {
if nativeToolCallsSeen {
if strings.TrimSpace(delta) != "" {
streamer.Content(delta)
}
} else if nativeReasoningSeen {
toolFilter.Feed(delta, func(cleanChunk string) {
if cleanChunk != "" {
@@ -5520,29 +5606,19 @@ func (g *GradioGateway) ExecuteChatCompletion(w http.ResponseWriter, r *http.Req
})
} else {
thinkFilter.Feed(delta, func(contentChunk string) {
if contentChunk != "" {
streamer.Content(contentChunk)
}
toolFilter.Feed(contentChunk, func(cleanChunk string) {
if cleanChunk != "" {
streamer.Content(cleanChunk)
}
}, func(tc ToolCall) {
streamer.ToolCallDelta(tc)
})
}, func(reasoningChunk string) {
if reasoningChunk != "" {
streamer.Reasoning(reasoningChunk)
}
})
}
} else {
thinkFilter.Feed(delta, func(contentChunk string) {
toolFilter.Feed(contentChunk, func(cleanChunk string) {
if cleanChunk != "" {
streamer.Content(cleanChunk)
}
}, func(tc ToolCall) {
streamer.ToolCallDelta(tc)
})
}, func(reasoningChunk string) {
if reasoningChunk != "" {
streamer.Reasoning(reasoningChunk)
}
})
}
}
}
@@ -5557,7 +5633,17 @@ func (g *GradioGateway) ExecuteChatCompletion(w http.ResponseWriter, r *http.Req
return nil
}
// Flush remaining tokens in filters if used
if isNativeToolCalling {
if len(emittedToolCallIDs) > 0 {
streamer.Finish("tool_calls")
} else {
streamer.Finish("stop")
}
streamer.Done()
return nil
}
// Flush remaining tokens in filters if used for non-native
if !nativeReasoningSeen {
thinkFilter.Flush(func(contentChunk string) {
if !nativeToolCallsSeen {
@@ -5593,13 +5679,12 @@ func (g *GradioGateway) ExecuteChatCompletion(w http.ResponseWriter, r *http.Req
return g.executeQueueCompletion(w, r, disc, gradioData, req, completionID, createdTime, modelName, effUA)
}
if nativeToolCallsSeen || toolFilter.emittedCall {
if len(emittedToolCallIDs) > 0 || nativeToolCallsSeen || toolFilter.emittedCall {
streamer.Finish("tool_calls")
} else {
streamer.Finish("stop")
}
streamer.Done()
return nil
}