feat(hy3): add native support for tencent hunyuan 3 gradio space
This commit is contained in:
@@ -47,7 +47,7 @@ type ModelsResponse struct {
|
||||
}
|
||||
|
||||
type ToolCallFunction struct {
|
||||
Name string `json:"name"`
|
||||
Name string `json:"name,omitempty"`
|
||||
Arguments string `json:"arguments"`
|
||||
}
|
||||
|
||||
@@ -1214,15 +1214,19 @@ type SpaceDiscovery struct {
|
||||
CleanEndpoint string // e.g. "chat_fn" or "chat"
|
||||
Protocol string // "call", "queue", "predict"
|
||||
TotalInputs int
|
||||
ParamMappings []SpaceParamMapping
|
||||
HistoryIndex int // -1 if none
|
||||
MessageIndex int // index for user message text
|
||||
SystemIndex int // -1 if none
|
||||
TempIndex int // -1 if none
|
||||
MaxTokensIndex int // -1 if none
|
||||
TopPIndex int // -1 if none
|
||||
HistoryFormat string // "messages", "pairs", "none"
|
||||
LastDiscovered time.Time
|
||||
ParamMappings []SpaceParamMapping
|
||||
HistoryIndex int // -1 if none
|
||||
MessageIndex int // index for user message text
|
||||
SystemIndex int // -1 if none
|
||||
TempIndex int // -1 if none
|
||||
MaxTokensIndex int // -1 if none
|
||||
TopPIndex int // -1 if none
|
||||
ThinkLevelIndex int // -1 if none
|
||||
FunctionsJSONIndex int // -1 if none
|
||||
PreservedThinkingIndex int // -1 if none
|
||||
IsHunyuan3 bool
|
||||
HistoryFormat string // "messages", "pairs", "none"
|
||||
LastDiscovered time.Time
|
||||
}
|
||||
|
||||
func (d *SpaceDiscovery) GetModelList() []ModelItem {
|
||||
@@ -1279,14 +1283,17 @@ func InspectSpace(client *http.Client, rawURL, userAgent string) (*SpaceDiscover
|
||||
CleanEndpoint: "chat_fn",
|
||||
Protocol: "call",
|
||||
TotalInputs: 1,
|
||||
HistoryIndex: -1,
|
||||
MessageIndex: 0,
|
||||
SystemIndex: -1,
|
||||
TempIndex: -1,
|
||||
MaxTokensIndex: -1,
|
||||
TopPIndex: -1,
|
||||
HistoryFormat: "messages",
|
||||
LastDiscovered: time.Now(),
|
||||
HistoryIndex: -1,
|
||||
MessageIndex: 0,
|
||||
SystemIndex: -1,
|
||||
TempIndex: -1,
|
||||
MaxTokensIndex: -1,
|
||||
TopPIndex: -1,
|
||||
ThinkLevelIndex: -1,
|
||||
FunctionsJSONIndex: -1,
|
||||
PreservedThinkingIndex: -1,
|
||||
HistoryFormat: "messages",
|
||||
LastDiscovered: time.Now(),
|
||||
}
|
||||
|
||||
// 1. Try fetching /gradio_api/info or /info
|
||||
@@ -1535,6 +1542,12 @@ func InspectSpace(client *http.Client, rawURL, userAgent string) (*SpaceDiscover
|
||||
discovery.HistoryIndex = idx
|
||||
} else if strings.Contains(pName, "system") {
|
||||
discovery.SystemIndex = idx
|
||||
} else if strings.Contains(pName, "think_level") || strings.Contains(pName, "thinking_level") {
|
||||
discovery.ThinkLevelIndex = idx
|
||||
} else if strings.Contains(pName, "functions") || strings.Contains(pName, "tools") {
|
||||
discovery.FunctionsJSONIndex = idx
|
||||
} else if strings.Contains(pName, "preserved") {
|
||||
discovery.PreservedThinkingIndex = idx
|
||||
} else if strings.Contains(pName, "temp") {
|
||||
discovery.TempIndex = idx
|
||||
} else if strings.Contains(pName, "token") {
|
||||
@@ -1545,6 +1558,14 @@ func InspectSpace(client *http.Client, rawURL, userAgent string) (*SpaceDiscover
|
||||
}
|
||||
}
|
||||
|
||||
if discovery.FunctionsJSONIndex != -1 || discovery.ThinkLevelIndex != -1 || strings.Contains(cleanURL, "hy3") || strings.Contains(cleanURL, "hunyuan") {
|
||||
discovery.IsHunyuan3 = true
|
||||
discovery.Models = append(discovery.Models, "hy3", "hunyuan3", "tencent/Hy3")
|
||||
if discovery.PrimaryModel == "gradio-chat" || discovery.PrimaryModel == "" {
|
||||
discovery.PrimaryModel = "hy3"
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure total inputs is at least 1
|
||||
if discovery.TotalInputs < 1 {
|
||||
discovery.TotalInputs = 1
|
||||
@@ -1659,7 +1680,21 @@ func (g *GradioGateway) GetDiscovery(spaceURL, userAgent string) *SpaceDiscovery
|
||||
|
||||
// BuildGradioPayload packages OpenAI messages and parameters into the target Gradio input array.
|
||||
func (g *GradioGateway) BuildGradioPayload(disc *SpaceDiscovery, req ChatCompletionRequest) ([]interface{}, error) {
|
||||
transformed, _, _ := TransformMessages(req)
|
||||
var transformed []ChatMessage
|
||||
if disc.IsHunyuan3 && disc.FunctionsJSONIndex != -1 {
|
||||
for _, msg := range req.Messages {
|
||||
transformed = append(transformed, ChatMessage{
|
||||
Role: msg.Role,
|
||||
Content: msg.GetContentString(),
|
||||
ReasoningContent: msg.ReasoningContent,
|
||||
ToolCalls: msg.ToolCalls,
|
||||
ToolCallID: msg.ToolCallID,
|
||||
Name: msg.Name,
|
||||
})
|
||||
}
|
||||
} else {
|
||||
transformed, _, _ = TransformMessages(req)
|
||||
}
|
||||
|
||||
var systemPromptStr string
|
||||
var historyArray []map[string]interface{}
|
||||
@@ -1678,12 +1713,72 @@ func (g *GradioGateway) BuildGradioPayload(disc *SpaceDiscovery, req ChatComplet
|
||||
if len(nonSystem) > 0 {
|
||||
for i := 0; i < len(nonSystem)-1; i++ {
|
||||
m := nonSystem[i]
|
||||
historyArray = append(historyArray, map[string]interface{}{
|
||||
"role": m.Role,
|
||||
"content": m.GetContentString(),
|
||||
})
|
||||
cStr := m.GetContentString()
|
||||
item := map[string]interface{}{"role": m.Role}
|
||||
switch m.Role {
|
||||
case "assistant":
|
||||
if cStr != "" {
|
||||
item["content"] = cStr
|
||||
} else {
|
||||
item["content"] = nil
|
||||
}
|
||||
if m.ReasoningContent != "" {
|
||||
item["reasoning_content"] = m.ReasoningContent
|
||||
}
|
||||
if len(m.ToolCalls) > 0 {
|
||||
item["tool_calls"] = m.ToolCalls
|
||||
}
|
||||
case "tool", "function":
|
||||
item["role"] = "tool"
|
||||
item["content"] = cStr
|
||||
toolID := m.ToolCallID
|
||||
if toolID == "" {
|
||||
toolID = m.Name
|
||||
}
|
||||
if toolID != "" {
|
||||
item["tool_call_id"] = toolID
|
||||
}
|
||||
if m.Name != "" {
|
||||
item["name"] = m.Name
|
||||
}
|
||||
default:
|
||||
item["content"] = cStr
|
||||
}
|
||||
historyArray = append(historyArray, item)
|
||||
}
|
||||
|
||||
lastMsg := nonSystem[len(nonSystem)-1]
|
||||
lastContent := lastMsg.GetContentString()
|
||||
if lastMsg.Role == "tool" || lastMsg.Role == "function" {
|
||||
toolName := lastMsg.Name
|
||||
if toolName == "" {
|
||||
toolName = lastMsg.ToolCallID
|
||||
}
|
||||
if disc.IsHunyuan3 {
|
||||
toolItem := map[string]interface{}{
|
||||
"role": "tool",
|
||||
"content": lastContent,
|
||||
}
|
||||
if lastMsg.ToolCallID != "" {
|
||||
toolItem["tool_call_id"] = lastMsg.ToolCallID
|
||||
} else if toolName != "" {
|
||||
toolItem["tool_call_id"] = toolName
|
||||
}
|
||||
if lastMsg.Name != "" {
|
||||
toolItem["name"] = lastMsg.Name
|
||||
}
|
||||
historyArray = append(historyArray, toolItem)
|
||||
lastUserMessage = "Please proceed based on the tool results."
|
||||
} else {
|
||||
if toolName != "" {
|
||||
lastUserMessage = fmt.Sprintf("Tool result for %s: %s", toolName, lastContent)
|
||||
} else {
|
||||
lastUserMessage = lastContent
|
||||
}
|
||||
}
|
||||
} else {
|
||||
lastUserMessage = lastContent
|
||||
}
|
||||
lastUserMessage = nonSystem[len(nonSystem)-1].GetContentString()
|
||||
} else if systemPromptStr != "" {
|
||||
lastUserMessage = systemPromptStr
|
||||
}
|
||||
@@ -1750,9 +1845,29 @@ func (g *GradioGateway) BuildGradioPayload(disc *SpaceDiscovery, req ChatComplet
|
||||
data[disc.SystemIndex] = systemPromptStr
|
||||
}
|
||||
|
||||
if disc.ThinkLevelIndex >= 0 && disc.ThinkLevelIndex < len(data) {
|
||||
thinkLevel := "high"
|
||||
if req.ReasoningEffort != "" {
|
||||
effort := strings.ToLower(req.ReasoningEffort)
|
||||
switch effort {
|
||||
case "none", "off", "no_think", "0":
|
||||
thinkLevel = "no_think"
|
||||
case "low", "1":
|
||||
thinkLevel = "low"
|
||||
case "medium", "high", "2", "3":
|
||||
thinkLevel = "high"
|
||||
default:
|
||||
thinkLevel = effort
|
||||
}
|
||||
}
|
||||
data[disc.ThinkLevelIndex] = thinkLevel
|
||||
}
|
||||
|
||||
if disc.TempIndex >= 0 && disc.TempIndex < len(data) {
|
||||
if req.Temperature != nil {
|
||||
data[disc.TempIndex] = *req.Temperature
|
||||
} else if disc.IsHunyuan3 {
|
||||
data[disc.TempIndex] = nil
|
||||
} else {
|
||||
data[disc.TempIndex] = 0.7
|
||||
}
|
||||
@@ -1765,64 +1880,188 @@ func (g *GradioGateway) BuildGradioPayload(disc *SpaceDiscovery, req ChatComplet
|
||||
if disc.TopPIndex >= 0 && disc.TopPIndex < len(data) {
|
||||
if req.TopP != nil {
|
||||
data[disc.TopPIndex] = *req.TopP
|
||||
} else if disc.IsHunyuan3 {
|
||||
data[disc.TopPIndex] = 0
|
||||
} else {
|
||||
data[disc.TopPIndex] = 1.0
|
||||
}
|
||||
}
|
||||
|
||||
if disc.FunctionsJSONIndex >= 0 && disc.FunctionsJSONIndex < len(data) {
|
||||
functionsJSONStr := ""
|
||||
if len(req.Tools) > 0 {
|
||||
b, err := json.Marshal(req.Tools)
|
||||
if err == nil {
|
||||
functionsJSONStr = string(b)
|
||||
}
|
||||
}
|
||||
data[disc.FunctionsJSONIndex] = functionsJSONStr
|
||||
}
|
||||
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// ExtractTextFromGradioOutput extracts the assistant text string from Gradio output chunks
|
||||
func ExtractTextFromGradioOutput(rawJSON string) (string, bool) {
|
||||
// GradioOutputFrame holds parsed elements from a Gradio SSE output chunk
|
||||
type GradioOutputFrame struct {
|
||||
Content string
|
||||
Reasoning string
|
||||
ToolCalls []ToolCall
|
||||
OK bool
|
||||
}
|
||||
|
||||
// ParseGradioStreamOutput extracts structured content, reasoning, and tool calls from Gradio output
|
||||
func ParseGradioStreamOutput(rawJSON string) GradioOutputFrame {
|
||||
var frame GradioOutputFrame
|
||||
var val interface{}
|
||||
if err := json.Unmarshal([]byte(rawJSON), &val); err != nil {
|
||||
return "", false
|
||||
return frame
|
||||
}
|
||||
|
||||
switch v := val.(type) {
|
||||
case string:
|
||||
return v, true
|
||||
frame.Content = v
|
||||
frame.OK = true
|
||||
return frame
|
||||
|
||||
case []interface{}:
|
||||
if len(v) == 0 {
|
||||
return "", false
|
||||
return frame
|
||||
}
|
||||
// Check if first element is string
|
||||
if s, ok := v[0].(string); ok {
|
||||
return s, true
|
||||
}
|
||||
// Check if it's a list of messages: [{"role":..., "content":...}]
|
||||
if len(v) > 0 {
|
||||
lastItem := v[len(v)-1]
|
||||
if m, ok := lastItem.(map[string]interface{}); ok {
|
||||
if c, ok := m["content"].(string); ok {
|
||||
return c, true
|
||||
}
|
||||
if parts, ok := m["content"].([]interface{}); ok && len(parts) > 0 {
|
||||
for _, p := range parts {
|
||||
if pm, ok := p.(map[string]interface{}); ok {
|
||||
if t, ok := pm["text"].(string); ok {
|
||||
return t, true
|
||||
|
||||
// Check if v[0] is an inner slice (e.g. Hy3: [[content, reasoning, tool_calls, history]])
|
||||
if inner, ok := v[0].([]interface{}); ok {
|
||||
if len(inner) >= 2 {
|
||||
s0, ok0 := inner[0].(string)
|
||||
s1, ok1 := inner[1].(string)
|
||||
if ok0 && ok1 {
|
||||
frame.Content = s0
|
||||
frame.Reasoning = s1
|
||||
if len(inner) >= 3 {
|
||||
if tcSlice, ok := inner[2].([]interface{}); ok && len(tcSlice) > 0 {
|
||||
b, err := json.Marshal(tcSlice)
|
||||
if err == nil {
|
||||
var tcs []ToolCall
|
||||
if err := json.Unmarshal(b, &tcs); err == nil {
|
||||
frame.ToolCalls = tcs
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
frame.OK = true
|
||||
return frame
|
||||
}
|
||||
|
||||
// Check if inner is a chat pair: ["user msg", "assistant msg"]
|
||||
if len(inner) == 2 {
|
||||
if aStr, ok := inner[1].(string); ok {
|
||||
frame.Content = aStr
|
||||
frame.OK = true
|
||||
return frame
|
||||
}
|
||||
}
|
||||
}
|
||||
// Check if it's pairs: [[u1, a1], [u2, a2]]
|
||||
if pair, ok := lastItem.([]interface{}); ok && len(pair) >= 2 {
|
||||
if aStr, ok := pair[1].(string); ok {
|
||||
return aStr, true
|
||||
|
||||
// Check if inner is a list of chat message maps: [{"role":..., "content":...}, ...]
|
||||
// or list of pairs: [["u", "a"], ...]
|
||||
if len(inner) > 0 {
|
||||
lastItem := inner[len(inner)-1]
|
||||
if m, ok := lastItem.(map[string]interface{}); ok {
|
||||
if c, ok := m["content"].(string); ok {
|
||||
frame.Content = c
|
||||
frame.OK = true
|
||||
}
|
||||
if r, ok := m["reasoning_content"].(string); ok {
|
||||
frame.Reasoning = r
|
||||
}
|
||||
if tcsRaw, ok := m["tool_calls"].([]interface{}); ok && len(tcsRaw) > 0 {
|
||||
b, err := json.Marshal(tcsRaw)
|
||||
if err == nil {
|
||||
var tcs []ToolCall
|
||||
if err := json.Unmarshal(b, &tcs); err == nil {
|
||||
frame.ToolCalls = tcs
|
||||
}
|
||||
}
|
||||
}
|
||||
if frame.OK {
|
||||
return frame
|
||||
}
|
||||
} else if pair, ok := lastItem.([]interface{}); ok && len(pair) >= 2 {
|
||||
if aStr, ok := pair[1].(string); ok {
|
||||
frame.Content = aStr
|
||||
frame.OK = true
|
||||
return frame
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check if v[0] is string (standard single output e.g. ["content", null])
|
||||
if s, ok := v[0].(string); ok {
|
||||
frame.Content = s
|
||||
frame.OK = true
|
||||
return frame
|
||||
}
|
||||
|
||||
// Check if v is a flat list of messages: [{"role": "assistant", ...}]
|
||||
lastItem := v[len(v)-1]
|
||||
if m, ok := lastItem.(map[string]interface{}); ok {
|
||||
if c, ok := m["content"].(string); ok {
|
||||
frame.Content = c
|
||||
frame.OK = true
|
||||
}
|
||||
if r, ok := m["reasoning_content"].(string); ok {
|
||||
frame.Reasoning = r
|
||||
}
|
||||
if tcsRaw, ok := m["tool_calls"].([]interface{}); ok && len(tcsRaw) > 0 {
|
||||
b, err := json.Marshal(tcsRaw)
|
||||
if err == nil {
|
||||
var tcs []ToolCall
|
||||
if err := json.Unmarshal(b, &tcs); err == nil {
|
||||
frame.ToolCalls = tcs
|
||||
}
|
||||
}
|
||||
}
|
||||
if frame.OK {
|
||||
return frame
|
||||
}
|
||||
}
|
||||
|
||||
case map[string]interface{}:
|
||||
for _, key := range []string{"text", "content", "response", "data", "value"} {
|
||||
if s, ok := v[key].(string); ok {
|
||||
return s, true
|
||||
frame.Content = s
|
||||
frame.OK = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if r, ok := v["reasoning_content"].(string); ok {
|
||||
frame.Reasoning = r
|
||||
} else if r, ok := v["reasoning"].(string); ok {
|
||||
frame.Reasoning = r
|
||||
}
|
||||
if tcsRaw, ok := v["tool_calls"].([]interface{}); ok && len(tcsRaw) > 0 {
|
||||
b, err := json.Marshal(tcsRaw)
|
||||
if err == nil {
|
||||
var tcs []ToolCall
|
||||
if err := json.Unmarshal(b, &tcs); err == nil {
|
||||
frame.ToolCalls = tcs
|
||||
}
|
||||
}
|
||||
}
|
||||
if frame.OK {
|
||||
return frame
|
||||
}
|
||||
}
|
||||
|
||||
return frame
|
||||
}
|
||||
|
||||
// ExtractTextFromGradioOutput extracts the assistant text string from Gradio output chunks (compatibility wrapper)
|
||||
func ExtractTextFromGradioOutput(rawJSON string) (string, bool) {
|
||||
frame := ParseGradioStreamOutput(rawJSON)
|
||||
if frame.OK {
|
||||
return frame.Content, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
@@ -1917,7 +2156,7 @@ func (g *GradioGateway) ExecuteChatCompletion(w http.ResponseWriter, r *http.Req
|
||||
// 3. Handle Non-Streaming vs Streaming
|
||||
if !req.Stream {
|
||||
reader := bufio.NewReader(streamResp.Body)
|
||||
var latestFullText string
|
||||
var latestFrame GradioOutputFrame
|
||||
currentEvent := ""
|
||||
|
||||
for {
|
||||
@@ -1937,8 +2176,8 @@ func (g *GradioGateway) ExecuteChatCompletion(w http.ResponseWriter, r *http.Req
|
||||
if currentEvent == "error" {
|
||||
return fmt.Errorf("gradio stream error: %s", dataStr)
|
||||
}
|
||||
if txt, ok := ExtractTextFromGradioOutput(dataStr); ok {
|
||||
latestFullText = txt
|
||||
if frame := ParseGradioStreamOutput(dataStr); frame.OK {
|
||||
latestFrame = frame
|
||||
}
|
||||
if currentEvent == "complete" {
|
||||
break
|
||||
@@ -1946,14 +2185,23 @@ func (g *GradioGateway) ExecuteChatCompletion(w http.ResponseWriter, r *http.Req
|
||||
}
|
||||
}
|
||||
|
||||
cleanText, reasoning := ExtractThinking(latestFullText)
|
||||
toolCalls, remainingText, hasTools := DetectToolCalls(cleanText)
|
||||
cleanText := latestFrame.Content
|
||||
reasoning := latestFrame.Reasoning
|
||||
toolCalls := latestFrame.ToolCalls
|
||||
hasTools := len(toolCalls) > 0
|
||||
|
||||
if reasoning == "" {
|
||||
cleanText, reasoning = ExtractThinking(cleanText)
|
||||
}
|
||||
if !hasTools {
|
||||
toolCalls, cleanText, hasTools = DetectToolCalls(cleanText)
|
||||
}
|
||||
|
||||
finishReason := "stop"
|
||||
var finalContent interface{} = remainingText
|
||||
var finalContent interface{} = cleanText
|
||||
if hasTools && len(toolCalls) > 0 {
|
||||
finishReason = "tool_calls"
|
||||
if strings.TrimSpace(remainingText) == "" {
|
||||
if strings.TrimSpace(cleanText) == "" {
|
||||
finalContent = nil
|
||||
}
|
||||
}
|
||||
@@ -1976,7 +2224,11 @@ func (g *GradioGateway) ExecuteChatCompletion(w http.ResponseWriter, r *http.Req
|
||||
toolFilter := NewStreamToolCallFilter()
|
||||
|
||||
reader := bufio.NewReader(streamResp.Body)
|
||||
var prevText string
|
||||
var prevContent string
|
||||
var prevReasoning string
|
||||
prevToolArgs := make(map[int]string)
|
||||
nativeReasoningSeen := false
|
||||
nativeToolCallsSeen := false
|
||||
currentEvent := ""
|
||||
|
||||
for {
|
||||
@@ -1997,31 +2249,115 @@ func (g *GradioGateway) ExecuteChatCompletion(w http.ResponseWriter, r *http.Req
|
||||
break
|
||||
}
|
||||
|
||||
if currentText, ok := ExtractTextFromGradioOutput(dataStr); ok {
|
||||
frame := ParseGradioStreamOutput(dataStr)
|
||||
if frame.OK {
|
||||
// 1. Native reasoning handling
|
||||
if frame.Reasoning != "" || nativeReasoningSeen {
|
||||
nativeReasoningSeen = true
|
||||
var deltaReasoning string
|
||||
if strings.HasPrefix(frame.Reasoning, prevReasoning) {
|
||||
deltaReasoning = frame.Reasoning[len(prevReasoning):]
|
||||
} else if prevReasoning == "" {
|
||||
deltaReasoning = frame.Reasoning
|
||||
} else {
|
||||
deltaReasoning = frame.Reasoning
|
||||
}
|
||||
prevReasoning = frame.Reasoning
|
||||
if deltaReasoning != "" {
|
||||
streamer.Reasoning(deltaReasoning)
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Native tool calls handling
|
||||
if len(frame.ToolCalls) > 0 {
|
||||
nativeToolCallsSeen = true
|
||||
for idx, tc := range frame.ToolCalls {
|
||||
prevArgs, started := prevToolArgs[idx]
|
||||
currArgs := tc.Function.Arguments
|
||||
idxCopy := idx
|
||||
if !started {
|
||||
tcDelta := ToolCall{
|
||||
Index: &idxCopy,
|
||||
ID: tc.ID,
|
||||
Type: tc.Type,
|
||||
Function: ToolCallFunction{
|
||||
Name: tc.Function.Name,
|
||||
Arguments: currArgs,
|
||||
},
|
||||
}
|
||||
streamer.ToolCallDelta(tcDelta)
|
||||
prevToolArgs[idx] = currArgs
|
||||
} else if len(currArgs) > len(prevArgs) {
|
||||
var argDelta string
|
||||
if strings.HasPrefix(currArgs, prevArgs) {
|
||||
argDelta = currArgs[len(prevArgs):]
|
||||
} else {
|
||||
argDelta = currArgs[len(prevArgs):]
|
||||
}
|
||||
if argDelta != "" {
|
||||
tcDelta := ToolCall{
|
||||
Index: &idxCopy,
|
||||
Function: ToolCallFunction{
|
||||
Arguments: argDelta,
|
||||
},
|
||||
}
|
||||
streamer.ToolCallDelta(tcDelta)
|
||||
}
|
||||
prevToolArgs[idx] = currArgs
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Content handling
|
||||
currentText := frame.Content
|
||||
var delta string
|
||||
if strings.HasPrefix(currentText, prevText) {
|
||||
delta = currentText[len(prevText):]
|
||||
} else if prevText == "" {
|
||||
if strings.HasPrefix(currentText, prevContent) {
|
||||
delta = currentText[len(prevContent):]
|
||||
} else if prevContent == "" {
|
||||
delta = currentText
|
||||
} else {
|
||||
delta = currentText
|
||||
}
|
||||
prevText = currentText
|
||||
prevContent = currentText
|
||||
|
||||
if delta != "" {
|
||||
thinkFilter.Feed(delta, func(contentChunk string) {
|
||||
toolFilter.Feed(contentChunk, func(cleanChunk string) {
|
||||
if cleanChunk != "" {
|
||||
streamer.Content(cleanChunk)
|
||||
}
|
||||
}, func(tc ToolCall) {
|
||||
streamer.ToolCallDelta(tc)
|
||||
})
|
||||
}, func(reasoningChunk string) {
|
||||
if reasoningChunk != "" {
|
||||
streamer.Reasoning(reasoningChunk)
|
||||
if nativeReasoningSeen || nativeToolCallsSeen {
|
||||
if nativeReasoningSeen && nativeToolCallsSeen {
|
||||
streamer.Content(delta)
|
||||
} else if nativeReasoningSeen {
|
||||
toolFilter.Feed(delta, func(cleanChunk string) {
|
||||
if cleanChunk != "" {
|
||||
streamer.Content(cleanChunk)
|
||||
}
|
||||
}, func(tc ToolCall) {
|
||||
streamer.ToolCallDelta(tc)
|
||||
})
|
||||
} else {
|
||||
thinkFilter.Feed(delta, func(contentChunk string) {
|
||||
if contentChunk != "" {
|
||||
streamer.Content(contentChunk)
|
||||
}
|
||||
}, func(reasoningChunk string) {
|
||||
if reasoningChunk != "" {
|
||||
streamer.Reasoning(reasoningChunk)
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
} else {
|
||||
thinkFilter.Feed(delta, func(contentChunk string) {
|
||||
toolFilter.Feed(contentChunk, func(cleanChunk string) {
|
||||
if cleanChunk != "" {
|
||||
streamer.Content(cleanChunk)
|
||||
}
|
||||
}, func(tc ToolCall) {
|
||||
streamer.ToolCallDelta(tc)
|
||||
})
|
||||
}, func(reasoningChunk string) {
|
||||
if reasoningChunk != "" {
|
||||
streamer.Reasoning(reasoningChunk)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2031,30 +2367,38 @@ func (g *GradioGateway) ExecuteChatCompletion(w http.ResponseWriter, r *http.Req
|
||||
}
|
||||
}
|
||||
|
||||
// Flush remaining tokens in filters
|
||||
thinkFilter.Flush(func(contentChunk string) {
|
||||
toolFilter.Feed(contentChunk, func(cleanChunk string) {
|
||||
// Flush remaining tokens in filters if used
|
||||
if !nativeReasoningSeen {
|
||||
thinkFilter.Flush(func(contentChunk string) {
|
||||
if !nativeToolCallsSeen {
|
||||
toolFilter.Feed(contentChunk, func(cleanChunk string) {
|
||||
if cleanChunk != "" {
|
||||
streamer.Content(cleanChunk)
|
||||
}
|
||||
}, func(tc ToolCall) {
|
||||
streamer.ToolCallDelta(tc)
|
||||
})
|
||||
} else if contentChunk != "" {
|
||||
streamer.Content(contentChunk)
|
||||
}
|
||||
}, func(reasoningChunk string) {
|
||||
if reasoningChunk != "" {
|
||||
streamer.Reasoning(reasoningChunk)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
if !nativeToolCallsSeen {
|
||||
toolFilter.Flush(func(cleanChunk string) {
|
||||
if cleanChunk != "" {
|
||||
streamer.Content(cleanChunk)
|
||||
}
|
||||
}, func(tc ToolCall) {
|
||||
streamer.ToolCallDelta(tc)
|
||||
})
|
||||
}, func(reasoningChunk string) {
|
||||
if reasoningChunk != "" {
|
||||
streamer.Reasoning(reasoningChunk)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
toolFilter.Flush(func(cleanChunk string) {
|
||||
if cleanChunk != "" {
|
||||
streamer.Content(cleanChunk)
|
||||
}
|
||||
}, func(tc ToolCall) {
|
||||
streamer.ToolCallDelta(tc)
|
||||
})
|
||||
|
||||
if toolFilter.emittedCall {
|
||||
if nativeToolCallsSeen || toolFilter.emittedCall {
|
||||
streamer.Finish("tool_calls")
|
||||
} else {
|
||||
streamer.Finish("stop")
|
||||
|
||||
Reference in New Issue
Block a user