feat(hy3): add native support for tencent hunyuan 3 gradio space
This commit is contained in:
Luxferre
@@ -10,19 +10,25 @@ Default demo space: `https://ghost2513-openai-gpt-oss-120b.hf.space`
|
||||
|
||||
- **Zero External Dependencies**: Pure Go standard library (`net/http`, `encoding/json`, `bufio`, etc.).
|
||||
- **Automatic Space Introspection**: Dynamically queries `/gradio_api/info`, `/config`, and Hugging Face space metadata to discover models, endpoints, and input parameter mappings.
|
||||
- **Native Tencent Hunyuan 3 (`tencent-hy3`) Support**:
|
||||
- Full native zero-degradation handling for official spaces like `https://tencent-hy3.hf.space`.
|
||||
- Maps `functions_json_str` natively without polluting the system prompt.
|
||||
- Preserves multi-turn reasoning content and tool call history in standard OpenAI message schemas.
|
||||
- Maps `reasoning_effort` (`no_think`, `low`, `high`) directly to `think_level`.
|
||||
- **Universal Multi-turn Handling**:
|
||||
- Automatically formats conversation history into structured inputs when the space supports them.
|
||||
- Transparently composes multi-turn dialogue (`System`, `User`, `Assistant`) into single prompt inputs when the space only accepts a single message textbox.
|
||||
- Automatically pads hidden/State inputs (e.g. Gradio State components) to prevent backend argument count mismatches.
|
||||
- **Real-Time Streaming & Accumulation Filter**:
|
||||
- Automatically computes token deltas from cumulative or incremental Gradio SSE output streams.
|
||||
- Automatically computes token deltas from cumulative or incremental Gradio SSE output streams (including 2D Hy3 frames `[[content, reasoning, tool_calls, history]]`).
|
||||
- Emits standards-compliant `chat.completion.chunk` SSE events in real time.
|
||||
- **Thinking & Reasoning Token Separation**:
|
||||
- Detects `<think>...</think>` tags in real time.
|
||||
- Streams native reasoning chunks as `delta.reasoning_content` in real time.
|
||||
- Detects `<think>...</think>` tags in real time as fallback for standard spaces.
|
||||
- Separates reasoning into `delta.reasoning_content` (streaming) and `message.reasoning_content` (non-streaming).
|
||||
- Keeps `content` clean without tag leakage.
|
||||
- **Full Tool Calling & Function Interception**:
|
||||
- Formats schemas into system prompts with strict function calling instructions.
|
||||
- Formats schemas into native `functions_json_str` (Hy3) or system prompts (standard spaces).
|
||||
- **`StreamToolCallFilter`**: Stateful sliding-window filter that prevents `<tool_call>` tags from leaking into `delta.content`. Emits structured OpenAI `delta.tool_calls` chunks and sets `finish_reason: "tool_calls"`.
|
||||
- Seamlessly maintains multi-turn context when tool results are submitted back via `role: "tool"`.
|
||||
- **Built-in SOCKS5 Proxy Client**:
|
||||
|
||||
@@ -47,7 +47,7 @@ type ModelsResponse struct {
|
||||
}
|
||||
|
||||
type ToolCallFunction struct {
|
||||
Name string `json:"name"`
|
||||
Name string `json:"name,omitempty"`
|
||||
Arguments string `json:"arguments"`
|
||||
}
|
||||
|
||||
@@ -1221,6 +1221,10 @@ type SpaceDiscovery struct {
|
||||
TempIndex int // -1 if none
|
||||
MaxTokensIndex int // -1 if none
|
||||
TopPIndex int // -1 if none
|
||||
ThinkLevelIndex int // -1 if none
|
||||
FunctionsJSONIndex int // -1 if none
|
||||
PreservedThinkingIndex int // -1 if none
|
||||
IsHunyuan3 bool
|
||||
HistoryFormat string // "messages", "pairs", "none"
|
||||
LastDiscovered time.Time
|
||||
}
|
||||
@@ -1285,6 +1289,9 @@ func InspectSpace(client *http.Client, rawURL, userAgent string) (*SpaceDiscover
|
||||
TempIndex: -1,
|
||||
MaxTokensIndex: -1,
|
||||
TopPIndex: -1,
|
||||
ThinkLevelIndex: -1,
|
||||
FunctionsJSONIndex: -1,
|
||||
PreservedThinkingIndex: -1,
|
||||
HistoryFormat: "messages",
|
||||
LastDiscovered: time.Now(),
|
||||
}
|
||||
@@ -1535,6 +1542,12 @@ func InspectSpace(client *http.Client, rawURL, userAgent string) (*SpaceDiscover
|
||||
discovery.HistoryIndex = idx
|
||||
} else if strings.Contains(pName, "system") {
|
||||
discovery.SystemIndex = idx
|
||||
} else if strings.Contains(pName, "think_level") || strings.Contains(pName, "thinking_level") {
|
||||
discovery.ThinkLevelIndex = idx
|
||||
} else if strings.Contains(pName, "functions") || strings.Contains(pName, "tools") {
|
||||
discovery.FunctionsJSONIndex = idx
|
||||
} else if strings.Contains(pName, "preserved") {
|
||||
discovery.PreservedThinkingIndex = idx
|
||||
} else if strings.Contains(pName, "temp") {
|
||||
discovery.TempIndex = idx
|
||||
} else if strings.Contains(pName, "token") {
|
||||
@@ -1545,6 +1558,14 @@ func InspectSpace(client *http.Client, rawURL, userAgent string) (*SpaceDiscover
|
||||
}
|
||||
}
|
||||
|
||||
if discovery.FunctionsJSONIndex != -1 || discovery.ThinkLevelIndex != -1 || strings.Contains(cleanURL, "hy3") || strings.Contains(cleanURL, "hunyuan") {
|
||||
discovery.IsHunyuan3 = true
|
||||
discovery.Models = append(discovery.Models, "hy3", "hunyuan3", "tencent/Hy3")
|
||||
if discovery.PrimaryModel == "gradio-chat" || discovery.PrimaryModel == "" {
|
||||
discovery.PrimaryModel = "hy3"
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure total inputs is at least 1
|
||||
if discovery.TotalInputs < 1 {
|
||||
discovery.TotalInputs = 1
|
||||
@@ -1659,7 +1680,21 @@ func (g *GradioGateway) GetDiscovery(spaceURL, userAgent string) *SpaceDiscovery
|
||||
|
||||
// BuildGradioPayload packages OpenAI messages and parameters into the target Gradio input array.
|
||||
func (g *GradioGateway) BuildGradioPayload(disc *SpaceDiscovery, req ChatCompletionRequest) ([]interface{}, error) {
|
||||
transformed, _, _ := TransformMessages(req)
|
||||
var transformed []ChatMessage
|
||||
if disc.IsHunyuan3 && disc.FunctionsJSONIndex != -1 {
|
||||
for _, msg := range req.Messages {
|
||||
transformed = append(transformed, ChatMessage{
|
||||
Role: msg.Role,
|
||||
Content: msg.GetContentString(),
|
||||
ReasoningContent: msg.ReasoningContent,
|
||||
ToolCalls: msg.ToolCalls,
|
||||
ToolCallID: msg.ToolCallID,
|
||||
Name: msg.Name,
|
||||
})
|
||||
}
|
||||
} else {
|
||||
transformed, _, _ = TransformMessages(req)
|
||||
}
|
||||
|
||||
var systemPromptStr string
|
||||
var historyArray []map[string]interface{}
|
||||
@@ -1678,12 +1713,72 @@ func (g *GradioGateway) BuildGradioPayload(disc *SpaceDiscovery, req ChatComplet
|
||||
if len(nonSystem) > 0 {
|
||||
for i := 0; i < len(nonSystem)-1; i++ {
|
||||
m := nonSystem[i]
|
||||
historyArray = append(historyArray, map[string]interface{}{
|
||||
"role": m.Role,
|
||||
"content": m.GetContentString(),
|
||||
})
|
||||
cStr := m.GetContentString()
|
||||
item := map[string]interface{}{"role": m.Role}
|
||||
switch m.Role {
|
||||
case "assistant":
|
||||
if cStr != "" {
|
||||
item["content"] = cStr
|
||||
} else {
|
||||
item["content"] = nil
|
||||
}
|
||||
if m.ReasoningContent != "" {
|
||||
item["reasoning_content"] = m.ReasoningContent
|
||||
}
|
||||
if len(m.ToolCalls) > 0 {
|
||||
item["tool_calls"] = m.ToolCalls
|
||||
}
|
||||
case "tool", "function":
|
||||
item["role"] = "tool"
|
||||
item["content"] = cStr
|
||||
toolID := m.ToolCallID
|
||||
if toolID == "" {
|
||||
toolID = m.Name
|
||||
}
|
||||
if toolID != "" {
|
||||
item["tool_call_id"] = toolID
|
||||
}
|
||||
if m.Name != "" {
|
||||
item["name"] = m.Name
|
||||
}
|
||||
default:
|
||||
item["content"] = cStr
|
||||
}
|
||||
historyArray = append(historyArray, item)
|
||||
}
|
||||
|
||||
lastMsg := nonSystem[len(nonSystem)-1]
|
||||
lastContent := lastMsg.GetContentString()
|
||||
if lastMsg.Role == "tool" || lastMsg.Role == "function" {
|
||||
toolName := lastMsg.Name
|
||||
if toolName == "" {
|
||||
toolName = lastMsg.ToolCallID
|
||||
}
|
||||
if disc.IsHunyuan3 {
|
||||
toolItem := map[string]interface{}{
|
||||
"role": "tool",
|
||||
"content": lastContent,
|
||||
}
|
||||
if lastMsg.ToolCallID != "" {
|
||||
toolItem["tool_call_id"] = lastMsg.ToolCallID
|
||||
} else if toolName != "" {
|
||||
toolItem["tool_call_id"] = toolName
|
||||
}
|
||||
if lastMsg.Name != "" {
|
||||
toolItem["name"] = lastMsg.Name
|
||||
}
|
||||
historyArray = append(historyArray, toolItem)
|
||||
lastUserMessage = "Please proceed based on the tool results."
|
||||
} else {
|
||||
if toolName != "" {
|
||||
lastUserMessage = fmt.Sprintf("Tool result for %s: %s", toolName, lastContent)
|
||||
} else {
|
||||
lastUserMessage = lastContent
|
||||
}
|
||||
}
|
||||
} else {
|
||||
lastUserMessage = lastContent
|
||||
}
|
||||
lastUserMessage = nonSystem[len(nonSystem)-1].GetContentString()
|
||||
} else if systemPromptStr != "" {
|
||||
lastUserMessage = systemPromptStr
|
||||
}
|
||||
@@ -1750,9 +1845,29 @@ func (g *GradioGateway) BuildGradioPayload(disc *SpaceDiscovery, req ChatComplet
|
||||
data[disc.SystemIndex] = systemPromptStr
|
||||
}
|
||||
|
||||
if disc.ThinkLevelIndex >= 0 && disc.ThinkLevelIndex < len(data) {
|
||||
thinkLevel := "high"
|
||||
if req.ReasoningEffort != "" {
|
||||
effort := strings.ToLower(req.ReasoningEffort)
|
||||
switch effort {
|
||||
case "none", "off", "no_think", "0":
|
||||
thinkLevel = "no_think"
|
||||
case "low", "1":
|
||||
thinkLevel = "low"
|
||||
case "medium", "high", "2", "3":
|
||||
thinkLevel = "high"
|
||||
default:
|
||||
thinkLevel = effort
|
||||
}
|
||||
}
|
||||
data[disc.ThinkLevelIndex] = thinkLevel
|
||||
}
|
||||
|
||||
if disc.TempIndex >= 0 && disc.TempIndex < len(data) {
|
||||
if req.Temperature != nil {
|
||||
data[disc.TempIndex] = *req.Temperature
|
||||
} else if disc.IsHunyuan3 {
|
||||
data[disc.TempIndex] = nil
|
||||
} else {
|
||||
data[disc.TempIndex] = 0.7
|
||||
}
|
||||
@@ -1765,64 +1880,188 @@ func (g *GradioGateway) BuildGradioPayload(disc *SpaceDiscovery, req ChatComplet
|
||||
if disc.TopPIndex >= 0 && disc.TopPIndex < len(data) {
|
||||
if req.TopP != nil {
|
||||
data[disc.TopPIndex] = *req.TopP
|
||||
} else if disc.IsHunyuan3 {
|
||||
data[disc.TopPIndex] = 0
|
||||
} else {
|
||||
data[disc.TopPIndex] = 1.0
|
||||
}
|
||||
}
|
||||
|
||||
if disc.FunctionsJSONIndex >= 0 && disc.FunctionsJSONIndex < len(data) {
|
||||
functionsJSONStr := ""
|
||||
if len(req.Tools) > 0 {
|
||||
b, err := json.Marshal(req.Tools)
|
||||
if err == nil {
|
||||
functionsJSONStr = string(b)
|
||||
}
|
||||
}
|
||||
data[disc.FunctionsJSONIndex] = functionsJSONStr
|
||||
}
|
||||
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// ExtractTextFromGradioOutput extracts the assistant text string from Gradio output chunks
|
||||
func ExtractTextFromGradioOutput(rawJSON string) (string, bool) {
|
||||
// GradioOutputFrame holds parsed elements from a Gradio SSE output chunk
|
||||
type GradioOutputFrame struct {
|
||||
Content string
|
||||
Reasoning string
|
||||
ToolCalls []ToolCall
|
||||
OK bool
|
||||
}
|
||||
|
||||
// ParseGradioStreamOutput extracts structured content, reasoning, and tool calls from Gradio output
|
||||
func ParseGradioStreamOutput(rawJSON string) GradioOutputFrame {
|
||||
var frame GradioOutputFrame
|
||||
var val interface{}
|
||||
if err := json.Unmarshal([]byte(rawJSON), &val); err != nil {
|
||||
return "", false
|
||||
return frame
|
||||
}
|
||||
|
||||
switch v := val.(type) {
|
||||
case string:
|
||||
return v, true
|
||||
frame.Content = v
|
||||
frame.OK = true
|
||||
return frame
|
||||
|
||||
case []interface{}:
|
||||
if len(v) == 0 {
|
||||
return "", false
|
||||
return frame
|
||||
}
|
||||
// Check if first element is string
|
||||
if s, ok := v[0].(string); ok {
|
||||
return s, true
|
||||
}
|
||||
// Check if it's a list of messages: [{"role":..., "content":...}]
|
||||
if len(v) > 0 {
|
||||
lastItem := v[len(v)-1]
|
||||
if m, ok := lastItem.(map[string]interface{}); ok {
|
||||
if c, ok := m["content"].(string); ok {
|
||||
return c, true
|
||||
}
|
||||
if parts, ok := m["content"].([]interface{}); ok && len(parts) > 0 {
|
||||
for _, p := range parts {
|
||||
if pm, ok := p.(map[string]interface{}); ok {
|
||||
if t, ok := pm["text"].(string); ok {
|
||||
return t, true
|
||||
|
||||
// Check if v[0] is an inner slice (e.g. Hy3: [[content, reasoning, tool_calls, history]])
|
||||
if inner, ok := v[0].([]interface{}); ok {
|
||||
if len(inner) >= 2 {
|
||||
s0, ok0 := inner[0].(string)
|
||||
s1, ok1 := inner[1].(string)
|
||||
if ok0 && ok1 {
|
||||
frame.Content = s0
|
||||
frame.Reasoning = s1
|
||||
if len(inner) >= 3 {
|
||||
if tcSlice, ok := inner[2].([]interface{}); ok && len(tcSlice) > 0 {
|
||||
b, err := json.Marshal(tcSlice)
|
||||
if err == nil {
|
||||
var tcs []ToolCall
|
||||
if err := json.Unmarshal(b, &tcs); err == nil {
|
||||
frame.ToolCalls = tcs
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
frame.OK = true
|
||||
return frame
|
||||
}
|
||||
// Check if it's pairs: [[u1, a1], [u2, a2]]
|
||||
if pair, ok := lastItem.([]interface{}); ok && len(pair) >= 2 {
|
||||
if aStr, ok := pair[1].(string); ok {
|
||||
return aStr, true
|
||||
}
|
||||
}
|
||||
}
|
||||
case map[string]interface{}:
|
||||
for _, key := range []string{"text", "content", "response", "data", "value"} {
|
||||
if s, ok := v[key].(string); ok {
|
||||
return s, true
|
||||
|
||||
// Check if inner is a chat pair: ["user msg", "assistant msg"]
|
||||
if len(inner) == 2 {
|
||||
if aStr, ok := inner[1].(string); ok {
|
||||
frame.Content = aStr
|
||||
frame.OK = true
|
||||
return frame
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check if inner is a list of chat message maps: [{"role":..., "content":...}, ...]
|
||||
// or list of pairs: [["u", "a"], ...]
|
||||
if len(inner) > 0 {
|
||||
lastItem := inner[len(inner)-1]
|
||||
if m, ok := lastItem.(map[string]interface{}); ok {
|
||||
if c, ok := m["content"].(string); ok {
|
||||
frame.Content = c
|
||||
frame.OK = true
|
||||
}
|
||||
if r, ok := m["reasoning_content"].(string); ok {
|
||||
frame.Reasoning = r
|
||||
}
|
||||
if tcsRaw, ok := m["tool_calls"].([]interface{}); ok && len(tcsRaw) > 0 {
|
||||
b, err := json.Marshal(tcsRaw)
|
||||
if err == nil {
|
||||
var tcs []ToolCall
|
||||
if err := json.Unmarshal(b, &tcs); err == nil {
|
||||
frame.ToolCalls = tcs
|
||||
}
|
||||
}
|
||||
}
|
||||
if frame.OK {
|
||||
return frame
|
||||
}
|
||||
} else if pair, ok := lastItem.([]interface{}); ok && len(pair) >= 2 {
|
||||
if aStr, ok := pair[1].(string); ok {
|
||||
frame.Content = aStr
|
||||
frame.OK = true
|
||||
return frame
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check if v[0] is string (standard single output e.g. ["content", null])
|
||||
if s, ok := v[0].(string); ok {
|
||||
frame.Content = s
|
||||
frame.OK = true
|
||||
return frame
|
||||
}
|
||||
|
||||
// Check if v is a flat list of messages: [{"role": "assistant", ...}]
|
||||
lastItem := v[len(v)-1]
|
||||
if m, ok := lastItem.(map[string]interface{}); ok {
|
||||
if c, ok := m["content"].(string); ok {
|
||||
frame.Content = c
|
||||
frame.OK = true
|
||||
}
|
||||
if r, ok := m["reasoning_content"].(string); ok {
|
||||
frame.Reasoning = r
|
||||
}
|
||||
if tcsRaw, ok := m["tool_calls"].([]interface{}); ok && len(tcsRaw) > 0 {
|
||||
b, err := json.Marshal(tcsRaw)
|
||||
if err == nil {
|
||||
var tcs []ToolCall
|
||||
if err := json.Unmarshal(b, &tcs); err == nil {
|
||||
frame.ToolCalls = tcs
|
||||
}
|
||||
}
|
||||
}
|
||||
if frame.OK {
|
||||
return frame
|
||||
}
|
||||
}
|
||||
|
||||
case map[string]interface{}:
|
||||
for _, key := range []string{"text", "content", "response", "data", "value"} {
|
||||
if s, ok := v[key].(string); ok {
|
||||
frame.Content = s
|
||||
frame.OK = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if r, ok := v["reasoning_content"].(string); ok {
|
||||
frame.Reasoning = r
|
||||
} else if r, ok := v["reasoning"].(string); ok {
|
||||
frame.Reasoning = r
|
||||
}
|
||||
if tcsRaw, ok := v["tool_calls"].([]interface{}); ok && len(tcsRaw) > 0 {
|
||||
b, err := json.Marshal(tcsRaw)
|
||||
if err == nil {
|
||||
var tcs []ToolCall
|
||||
if err := json.Unmarshal(b, &tcs); err == nil {
|
||||
frame.ToolCalls = tcs
|
||||
}
|
||||
}
|
||||
}
|
||||
if frame.OK {
|
||||
return frame
|
||||
}
|
||||
}
|
||||
|
||||
return frame
|
||||
}
|
||||
|
||||
// ExtractTextFromGradioOutput extracts the assistant text string from Gradio output chunks (compatibility wrapper)
|
||||
func ExtractTextFromGradioOutput(rawJSON string) (string, bool) {
|
||||
frame := ParseGradioStreamOutput(rawJSON)
|
||||
if frame.OK {
|
||||
return frame.Content, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
@@ -1917,7 +2156,7 @@ func (g *GradioGateway) ExecuteChatCompletion(w http.ResponseWriter, r *http.Req
|
||||
// 3. Handle Non-Streaming vs Streaming
|
||||
if !req.Stream {
|
||||
reader := bufio.NewReader(streamResp.Body)
|
||||
var latestFullText string
|
||||
var latestFrame GradioOutputFrame
|
||||
currentEvent := ""
|
||||
|
||||
for {
|
||||
@@ -1937,8 +2176,8 @@ func (g *GradioGateway) ExecuteChatCompletion(w http.ResponseWriter, r *http.Req
|
||||
if currentEvent == "error" {
|
||||
return fmt.Errorf("gradio stream error: %s", dataStr)
|
||||
}
|
||||
if txt, ok := ExtractTextFromGradioOutput(dataStr); ok {
|
||||
latestFullText = txt
|
||||
if frame := ParseGradioStreamOutput(dataStr); frame.OK {
|
||||
latestFrame = frame
|
||||
}
|
||||
if currentEvent == "complete" {
|
||||
break
|
||||
@@ -1946,14 +2185,23 @@ func (g *GradioGateway) ExecuteChatCompletion(w http.ResponseWriter, r *http.Req
|
||||
}
|
||||
}
|
||||
|
||||
cleanText, reasoning := ExtractThinking(latestFullText)
|
||||
toolCalls, remainingText, hasTools := DetectToolCalls(cleanText)
|
||||
cleanText := latestFrame.Content
|
||||
reasoning := latestFrame.Reasoning
|
||||
toolCalls := latestFrame.ToolCalls
|
||||
hasTools := len(toolCalls) > 0
|
||||
|
||||
if reasoning == "" {
|
||||
cleanText, reasoning = ExtractThinking(cleanText)
|
||||
}
|
||||
if !hasTools {
|
||||
toolCalls, cleanText, hasTools = DetectToolCalls(cleanText)
|
||||
}
|
||||
|
||||
finishReason := "stop"
|
||||
var finalContent interface{} = remainingText
|
||||
var finalContent interface{} = cleanText
|
||||
if hasTools && len(toolCalls) > 0 {
|
||||
finishReason = "tool_calls"
|
||||
if strings.TrimSpace(remainingText) == "" {
|
||||
if strings.TrimSpace(cleanText) == "" {
|
||||
finalContent = nil
|
||||
}
|
||||
}
|
||||
@@ -1976,7 +2224,11 @@ func (g *GradioGateway) ExecuteChatCompletion(w http.ResponseWriter, r *http.Req
|
||||
toolFilter := NewStreamToolCallFilter()
|
||||
|
||||
reader := bufio.NewReader(streamResp.Body)
|
||||
var prevText string
|
||||
var prevContent string
|
||||
var prevReasoning string
|
||||
prevToolArgs := make(map[int]string)
|
||||
nativeReasoningSeen := false
|
||||
nativeToolCallsSeen := false
|
||||
currentEvent := ""
|
||||
|
||||
for {
|
||||
@@ -1997,18 +2249,101 @@ func (g *GradioGateway) ExecuteChatCompletion(w http.ResponseWriter, r *http.Req
|
||||
break
|
||||
}
|
||||
|
||||
if currentText, ok := ExtractTextFromGradioOutput(dataStr); ok {
|
||||
frame := ParseGradioStreamOutput(dataStr)
|
||||
if frame.OK {
|
||||
// 1. Native reasoning handling
|
||||
if frame.Reasoning != "" || nativeReasoningSeen {
|
||||
nativeReasoningSeen = true
|
||||
var deltaReasoning string
|
||||
if strings.HasPrefix(frame.Reasoning, prevReasoning) {
|
||||
deltaReasoning = frame.Reasoning[len(prevReasoning):]
|
||||
} else if prevReasoning == "" {
|
||||
deltaReasoning = frame.Reasoning
|
||||
} else {
|
||||
deltaReasoning = frame.Reasoning
|
||||
}
|
||||
prevReasoning = frame.Reasoning
|
||||
if deltaReasoning != "" {
|
||||
streamer.Reasoning(deltaReasoning)
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Native tool calls handling
|
||||
if len(frame.ToolCalls) > 0 {
|
||||
nativeToolCallsSeen = true
|
||||
for idx, tc := range frame.ToolCalls {
|
||||
prevArgs, started := prevToolArgs[idx]
|
||||
currArgs := tc.Function.Arguments
|
||||
idxCopy := idx
|
||||
if !started {
|
||||
tcDelta := ToolCall{
|
||||
Index: &idxCopy,
|
||||
ID: tc.ID,
|
||||
Type: tc.Type,
|
||||
Function: ToolCallFunction{
|
||||
Name: tc.Function.Name,
|
||||
Arguments: currArgs,
|
||||
},
|
||||
}
|
||||
streamer.ToolCallDelta(tcDelta)
|
||||
prevToolArgs[idx] = currArgs
|
||||
} else if len(currArgs) > len(prevArgs) {
|
||||
var argDelta string
|
||||
if strings.HasPrefix(currArgs, prevArgs) {
|
||||
argDelta = currArgs[len(prevArgs):]
|
||||
} else {
|
||||
argDelta = currArgs[len(prevArgs):]
|
||||
}
|
||||
if argDelta != "" {
|
||||
tcDelta := ToolCall{
|
||||
Index: &idxCopy,
|
||||
Function: ToolCallFunction{
|
||||
Arguments: argDelta,
|
||||
},
|
||||
}
|
||||
streamer.ToolCallDelta(tcDelta)
|
||||
}
|
||||
prevToolArgs[idx] = currArgs
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Content handling
|
||||
currentText := frame.Content
|
||||
var delta string
|
||||
if strings.HasPrefix(currentText, prevText) {
|
||||
delta = currentText[len(prevText):]
|
||||
} else if prevText == "" {
|
||||
if strings.HasPrefix(currentText, prevContent) {
|
||||
delta = currentText[len(prevContent):]
|
||||
} else if prevContent == "" {
|
||||
delta = currentText
|
||||
} else {
|
||||
delta = currentText
|
||||
}
|
||||
prevText = currentText
|
||||
prevContent = currentText
|
||||
|
||||
if delta != "" {
|
||||
if nativeReasoningSeen || nativeToolCallsSeen {
|
||||
if nativeReasoningSeen && nativeToolCallsSeen {
|
||||
streamer.Content(delta)
|
||||
} else if nativeReasoningSeen {
|
||||
toolFilter.Feed(delta, func(cleanChunk string) {
|
||||
if cleanChunk != "" {
|
||||
streamer.Content(cleanChunk)
|
||||
}
|
||||
}, func(tc ToolCall) {
|
||||
streamer.ToolCallDelta(tc)
|
||||
})
|
||||
} else {
|
||||
thinkFilter.Feed(delta, func(contentChunk string) {
|
||||
if contentChunk != "" {
|
||||
streamer.Content(contentChunk)
|
||||
}
|
||||
}, func(reasoningChunk string) {
|
||||
if reasoningChunk != "" {
|
||||
streamer.Reasoning(reasoningChunk)
|
||||
}
|
||||
})
|
||||
}
|
||||
} else {
|
||||
thinkFilter.Feed(delta, func(contentChunk string) {
|
||||
toolFilter.Feed(contentChunk, func(cleanChunk string) {
|
||||
if cleanChunk != "" {
|
||||
@@ -2024,6 +2359,7 @@ func (g *GradioGateway) ExecuteChatCompletion(w http.ResponseWriter, r *http.Req
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if currentEvent == "complete" {
|
||||
break
|
||||
@@ -2031,8 +2367,10 @@ func (g *GradioGateway) ExecuteChatCompletion(w http.ResponseWriter, r *http.Req
|
||||
}
|
||||
}
|
||||
|
||||
// Flush remaining tokens in filters
|
||||
// Flush remaining tokens in filters if used
|
||||
if !nativeReasoningSeen {
|
||||
thinkFilter.Flush(func(contentChunk string) {
|
||||
if !nativeToolCallsSeen {
|
||||
toolFilter.Feed(contentChunk, func(cleanChunk string) {
|
||||
if cleanChunk != "" {
|
||||
streamer.Content(cleanChunk)
|
||||
@@ -2040,12 +2378,17 @@ func (g *GradioGateway) ExecuteChatCompletion(w http.ResponseWriter, r *http.Req
|
||||
}, func(tc ToolCall) {
|
||||
streamer.ToolCallDelta(tc)
|
||||
})
|
||||
} else if contentChunk != "" {
|
||||
streamer.Content(contentChunk)
|
||||
}
|
||||
}, func(reasoningChunk string) {
|
||||
if reasoningChunk != "" {
|
||||
streamer.Reasoning(reasoningChunk)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
if !nativeToolCallsSeen {
|
||||
toolFilter.Flush(func(cleanChunk string) {
|
||||
if cleanChunk != "" {
|
||||
streamer.Content(cleanChunk)
|
||||
@@ -2053,8 +2396,9 @@ func (g *GradioGateway) ExecuteChatCompletion(w http.ResponseWriter, r *http.Req
|
||||
}, func(tc ToolCall) {
|
||||
streamer.ToolCallDelta(tc)
|
||||
})
|
||||
}
|
||||
|
||||
if toolFilter.emittedCall {
|
||||
if nativeToolCallsSeen || toolFilter.emittedCall {
|
||||
streamer.Finish("tool_calls")
|
||||
} else {
|
||||
streamer.Finish("stop")
|
||||
|
||||
+245
@@ -247,3 +247,248 @@ func TestMockGradioServerCompletion(t *testing.T) {
|
||||
t.Errorf("expected stream output to contain delta tokens, got:\n%s", streamOutput)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseGradioStreamOutput(t *testing.T) {
|
||||
// 1. Standard 1D Gradio array
|
||||
frame1 := ParseGradioStreamOutput(`["Hello from 1D", null]`)
|
||||
if !frame1.OK || frame1.Content != "Hello from 1D" || frame1.Reasoning != "" || len(frame1.ToolCalls) != 0 {
|
||||
t.Errorf("unexpected frame1: %+v", frame1)
|
||||
}
|
||||
|
||||
// 2. Hy3 2D array frame with reasoning
|
||||
hy3Raw := `[["Hello answer", "Let me think deeply...", [], [{"role": "user", "content": "hi"}]]]`
|
||||
frame2 := ParseGradioStreamOutput(hy3Raw)
|
||||
if !frame2.OK || frame2.Content != "Hello answer" || frame2.Reasoning != "Let me think deeply..." || len(frame2.ToolCalls) != 0 {
|
||||
t.Errorf("unexpected frame2: %+v", frame2)
|
||||
}
|
||||
|
||||
// 3. Hy3 2D array frame with tool calls
|
||||
hy3ToolRaw := `[["", "Calling weather tool", [{"id": "call_abc", "type": "function", "function": {"name": "get_weather", "arguments": "{\"city\": \"Tokyo\"}"}}], []]]`
|
||||
frame3 := ParseGradioStreamOutput(hy3ToolRaw)
|
||||
if !frame3.OK || frame3.Content != "" || frame3.Reasoning != "Calling weather tool" || len(frame3.ToolCalls) != 1 {
|
||||
t.Fatalf("unexpected frame3: %+v", frame3)
|
||||
}
|
||||
if frame3.ToolCalls[0].ID != "call_abc" || frame3.ToolCalls[0].Function.Name != "get_weather" {
|
||||
t.Errorf("unexpected tool call in frame3: %+v", frame3.ToolCalls[0])
|
||||
}
|
||||
|
||||
// 4. Chat pairs
|
||||
pairRaw := `[[["user prompt", "assistant answer"]]]`
|
||||
frame4 := ParseGradioStreamOutput(pairRaw)
|
||||
if !frame4.OK || frame4.Content != "assistant answer" {
|
||||
t.Errorf("unexpected frame4: %+v", frame4)
|
||||
}
|
||||
|
||||
// 5. Messages array
|
||||
msgRaw := `[[{"role": "assistant", "content": "msg answer", "reasoning_content": "msg think"}]]`
|
||||
frame5 := ParseGradioStreamOutput(msgRaw)
|
||||
if !frame5.OK || frame5.Content != "msg answer" || frame5.Reasoning != "msg think" {
|
||||
t.Errorf("unexpected frame5: %+v", frame5)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHunyuan3BuildPayload(t *testing.T) {
|
||||
gw := &GradioGateway{}
|
||||
disc := &SpaceDiscovery{
|
||||
TotalInputs: 9,
|
||||
MessageIndex: 0,
|
||||
SystemIndex: 1,
|
||||
HistoryIndex: 2,
|
||||
ThinkLevelIndex: 3,
|
||||
TempIndex: 4,
|
||||
MaxTokensIndex: 5,
|
||||
TopPIndex: 6,
|
||||
FunctionsJSONIndex: 8,
|
||||
IsHunyuan3: true,
|
||||
}
|
||||
|
||||
temp := 0.2
|
||||
req := ChatCompletionRequest{
|
||||
Model: "hy3",
|
||||
ReasoningEffort: "low",
|
||||
Temperature: &temp,
|
||||
Tools: []Tool{
|
||||
{
|
||||
Type: "function",
|
||||
Function: map[string]interface{}{
|
||||
"name": "calc",
|
||||
},
|
||||
},
|
||||
},
|
||||
Messages: []ChatMessage{
|
||||
{Role: "system", Content: "Be helpful"},
|
||||
{Role: "user", Content: "2+2"},
|
||||
{
|
||||
Role: "assistant",
|
||||
ReasoningContent: "Thinking...",
|
||||
ToolCalls: []ToolCall{
|
||||
{ID: "c1", Type: "function", Function: ToolCallFunction{Name: "calc", Arguments: `{"expr":"2+2"}`}},
|
||||
},
|
||||
},
|
||||
{Role: "tool", ToolCallID: "c1", Content: "4"},
|
||||
},
|
||||
}
|
||||
|
||||
data, err := gw.BuildGradioPayload(disc, req)
|
||||
if err != nil {
|
||||
t.Fatalf("BuildGradioPayload failed: %v", err)
|
||||
}
|
||||
|
||||
if len(data) != 9 {
|
||||
t.Fatalf("expected 9 payload items, got %d", len(data))
|
||||
}
|
||||
|
||||
// Message parameter (0): should be prompt continuation since last was tool
|
||||
if msg, ok := data[0].(string); !ok || msg != "Please proceed based on the tool results." {
|
||||
t.Errorf("expected continuation prompt, got %v", data[0])
|
||||
}
|
||||
|
||||
// System parameter (1)
|
||||
if sys, ok := data[1].(string); !ok || sys != "Be helpful" {
|
||||
t.Errorf("expected 'Be helpful', got %v", data[1])
|
||||
}
|
||||
|
||||
// History parameter (2): should contain all messages including the tool turn
|
||||
hist, ok := data[2].([]map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatalf("expected history slice of maps, got %T", data[2])
|
||||
}
|
||||
if len(hist) != 3 {
|
||||
t.Fatalf("expected 3 history items (user, assistant, tool), got %d", len(hist))
|
||||
}
|
||||
if hist[2]["role"] != "tool" || hist[2]["content"] != "4" || hist[2]["tool_call_id"] != "c1" {
|
||||
t.Errorf("unexpected tool history entry: %+v", hist[2])
|
||||
}
|
||||
|
||||
// ThinkLevel parameter (3)
|
||||
if data[3] != "low" {
|
||||
t.Errorf("expected think_level 'low', got %v", data[3])
|
||||
}
|
||||
|
||||
// Temp parameter (4)
|
||||
if data[4] != 0.2 {
|
||||
t.Errorf("expected temp 0.2, got %v", data[4])
|
||||
}
|
||||
|
||||
// FunctionsJSON parameter (8)
|
||||
fnStr, ok := data[8].(string)
|
||||
if !ok || !strings.Contains(fnStr, "calc") {
|
||||
t.Errorf("expected functions_json_str to contain 'calc', got %v", data[8])
|
||||
}
|
||||
}
|
||||
|
||||
func TestHunyuan3MockServerCompletion(t *testing.T) {
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/gradio_api/info" {
|
||||
resp := GradioAPIInfoResponse{
|
||||
NamedEndpoints: map[string]GradioEndpointInfo{
|
||||
"/chat": {
|
||||
Parameters: []GradioParamInfo{
|
||||
{ParameterName: "message"},
|
||||
{ParameterName: "system_prompt"},
|
||||
{ParameterName: "history"},
|
||||
{ParameterName: "think_level"},
|
||||
{ParameterName: "temperature"},
|
||||
{ParameterName: "max_tokens"},
|
||||
{ParameterName: "top_p"},
|
||||
{ParameterName: "preserved_thinking"},
|
||||
{ParameterName: "functions_json_str"},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(resp)
|
||||
return
|
||||
}
|
||||
|
||||
if r.URL.Path == "/gradio_api/call/chat" {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(GradioJoinResponse{EventID: "evt_hy3"})
|
||||
return
|
||||
}
|
||||
|
||||
if r.URL.Path == "/gradio_api/call/chat/evt_hy3" {
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
flusher, ok := w.(http.Flusher)
|
||||
if !ok {
|
||||
t.Fatal("expected flusher")
|
||||
}
|
||||
// Frame 1: Reasoning delta
|
||||
fmt.Fprintf(w, "event: generating\ndata: [[\"\", \"Reasoning part 1 \", [], []]]\n\n")
|
||||
flusher.Flush()
|
||||
// Frame 2: Tool call initiated
|
||||
fmt.Fprintf(w, "event: generating\ndata: [[\"\", \"Reasoning part 1 and 2\", [{\"id\": \"call_hy3\", \"type\": \"function\", \"function\": {\"name\": \"search\", \"arguments\": \"{\\\"q\\\": \\\"tencent\\\"}\"}}], []]]\n\n")
|
||||
flusher.Flush()
|
||||
// Frame 3: Completion
|
||||
fmt.Fprintf(w, "event: complete\ndata: [[\"\", \"Reasoning part 1 and 2\", [{\"id\": \"call_hy3\", \"type\": \"function\", \"function\": {\"name\": \"search\", \"arguments\": \"{\\\"q\\\": \\\"tencent\\\"}\"}}], []]]\n\n")
|
||||
flusher.Flush()
|
||||
return
|
||||
}
|
||||
|
||||
http.NotFound(w, r)
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
gw := NewGradioGateway(ts.URL, "", 10*time.Second)
|
||||
|
||||
// 1. Non-streaming tool call test
|
||||
reqBody := ChatCompletionRequest{
|
||||
Model: "hy3",
|
||||
Messages: []ChatMessage{
|
||||
{Role: "user", Content: "search for tencent"},
|
||||
},
|
||||
Stream: false,
|
||||
}
|
||||
b, _ := json.Marshal(reqBody)
|
||||
httpReq := httptest.NewRequest("POST", "/v1/chat/completions", bytes.NewBuffer(b))
|
||||
httpReq.Header.Set("Content-Type", "application/json")
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
err := gw.ExecuteChatCompletion(rec, httpReq, reqBody)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected completion error: %v", err)
|
||||
}
|
||||
|
||||
var resp ChatCompletionResponse
|
||||
if err := json.NewDecoder(rec.Body).Decode(&resp); err != nil {
|
||||
t.Fatalf("failed to decode response: %v", err)
|
||||
}
|
||||
|
||||
if resp.Choices[0].FinishReason != "tool_calls" {
|
||||
t.Errorf("expected finish_reason 'tool_calls', got %q", resp.Choices[0].FinishReason)
|
||||
}
|
||||
if resp.Choices[0].Message.ReasoningContent != "Reasoning part 1 and 2" {
|
||||
t.Errorf("expected native reasoning, got %q", resp.Choices[0].Message.ReasoningContent)
|
||||
}
|
||||
if len(resp.Choices[0].Message.ToolCalls) != 1 {
|
||||
t.Fatalf("expected 1 tool call, got %d", len(resp.Choices[0].Message.ToolCalls))
|
||||
}
|
||||
if resp.Choices[0].Message.ToolCalls[0].Function.Name != "search" {
|
||||
t.Errorf("expected function 'search', got %q", resp.Choices[0].Message.ToolCalls[0].Function.Name)
|
||||
}
|
||||
|
||||
// 2. Streaming tool call test
|
||||
reqBodyStream := reqBody
|
||||
reqBodyStream.Stream = true
|
||||
recStream := httptest.NewRecorder()
|
||||
err = gw.ExecuteChatCompletion(recStream, httpReq, reqBodyStream)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected streaming error: %v", err)
|
||||
}
|
||||
|
||||
streamOut := recStream.Body.String()
|
||||
if !strings.Contains(streamOut, "reasoning_content") {
|
||||
t.Errorf("expected stream to contain reasoning_content, got:\n%s", streamOut)
|
||||
}
|
||||
if !strings.Contains(streamOut, "tool_calls") {
|
||||
t.Errorf("expected stream to contain tool_calls, got:\n%s", streamOut)
|
||||
}
|
||||
if !strings.Contains(streamOut, "call_hy3") {
|
||||
t.Errorf("expected stream to contain tool call ID call_hy3, got:\n%s", streamOut)
|
||||
}
|
||||
if !strings.Contains(streamOut, "\"finish_reason\":\"tool_calls\"") {
|
||||
t.Errorf("expected stream finish_reason tool_calls, got:\n%s", streamOut)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user