Restore hygate parity for Hunyuan3 and fix thinking separation
This commit is contained in:
@@ -3215,16 +3215,19 @@ func InspectSpace(client *http.Client, rawURL, userAgent string) (*SpaceDiscover
|
|||||||
|
|
||||||
// Select protocol
|
// Select protocol
|
||||||
isCallV2 := false
|
isCallV2 := false
|
||||||
if strings.HasPrefix(discovery.GradioVersion, "6.") {
|
hasSnippet := false
|
||||||
isCallV2 = true
|
|
||||||
}
|
|
||||||
if bestEndpointInfo != nil && bestEndpointInfo.CodeSnippets != nil {
|
if bestEndpointInfo != nil && bestEndpointInfo.CodeSnippets != nil {
|
||||||
if bashSnippet, ok := bestEndpointInfo.CodeSnippets["bash"].(string); ok {
|
if bashSnippet, ok := bestEndpointInfo.CodeSnippets["bash"].(string); ok && bashSnippet != "" {
|
||||||
|
hasSnippet = true
|
||||||
if strings.Contains(bashSnippet, "/call/v2/") {
|
if strings.Contains(bashSnippet, "/call/v2/") {
|
||||||
isCallV2 = true
|
isCallV2 = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if !hasSnippet && strings.HasPrefix(discovery.GradioVersion, "6.") {
|
||||||
|
isCallV2 = true
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
if strings.HasPrefix(discovery.GradioVersion, "3.") {
|
if strings.HasPrefix(discovery.GradioVersion, "3.") {
|
||||||
discovery.Protocol = "predict"
|
discovery.Protocol = "predict"
|
||||||
@@ -3299,11 +3302,12 @@ func InspectSpace(client *http.Client, rawURL, userAgent string) (*SpaceDiscover
|
|||||||
if strings.Contains(pName, "function") || strings.Contains(pName, "tool") || strings.Contains(pLabel, "tool") || strings.Contains(pLabel, "function") || strings.Contains(cLabel, "tool") || strings.Contains(cLabel, "function") {
|
if strings.Contains(pName, "function") || strings.Contains(pName, "tool") || strings.Contains(pLabel, "tool") || strings.Contains(pLabel, "function") || strings.Contains(cLabel, "tool") || strings.Contains(cLabel, "function") {
|
||||||
mapping.ParamType = "tools"
|
mapping.ParamType = "tools"
|
||||||
discovery.FunctionsJSONIndex = idx
|
discovery.FunctionsJSONIndex = idx
|
||||||
|
} else if strings.Contains(pName, "preserved") || strings.Contains(cLabel, "preserved") {
|
||||||
|
mapping.ParamType = "preserved_thinking"
|
||||||
|
discovery.PreservedThinkingIndex = idx
|
||||||
} else if strings.Contains(pName, "think_level") || strings.Contains(pName, "thinking") || strings.Contains(pLabel, "think") || strings.Contains(cLabel, "think") {
|
} else if strings.Contains(pName, "think_level") || strings.Contains(pName, "thinking") || strings.Contains(pLabel, "think") || strings.Contains(cLabel, "think") {
|
||||||
mapping.ParamType = "think_level"
|
mapping.ParamType = "think_level"
|
||||||
discovery.ThinkLevelIndex = idx
|
discovery.ThinkLevelIndex = idx
|
||||||
} else if strings.Contains(pName, "preserved") || strings.Contains(cLabel, "preserved") {
|
|
||||||
discovery.PreservedThinkingIndex = idx
|
|
||||||
} else if strings.Contains(pName, "system") || strings.Contains(pLabel, "system") || strings.Contains(cLabel, "system") || strings.Contains(cLabel, "instruction") {
|
} else if strings.Contains(pName, "system") || strings.Contains(pLabel, "system") || strings.Contains(cLabel, "system") || strings.Contains(cLabel, "instruction") {
|
||||||
mapping.ParamType = "system_prompt"
|
mapping.ParamType = "system_prompt"
|
||||||
discovery.SystemIndex = idx
|
discovery.SystemIndex = idx
|
||||||
@@ -3532,6 +3536,9 @@ func InspectSpace(client *http.Client, rawURL, userAgent string) (*SpaceDiscover
|
|||||||
if discovery.PrimaryModel == "gradio-chat" || discovery.PrimaryModel == "" {
|
if discovery.PrimaryModel == "gradio-chat" || discovery.PrimaryModel == "" {
|
||||||
discovery.PrimaryModel = "hy3"
|
discovery.PrimaryModel = "hy3"
|
||||||
}
|
}
|
||||||
|
if discovery.Protocol == "call_v2" {
|
||||||
|
discovery.Protocol = "call"
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 9. Tool Calling Support Picture Resolution
|
// 9. Tool Calling Support Picture Resolution
|
||||||
@@ -3991,6 +3998,10 @@ func (g *GradioGateway) BuildGradioPayload(disc *SpaceDiscovery, req ChatComplet
|
|||||||
data[disc.ThinkLevelIndex] = thinkLevel
|
data[disc.ThinkLevelIndex] = thinkLevel
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if disc.PreservedThinkingIndex >= 0 && disc.PreservedThinkingIndex != disc.MessageIndex && disc.PreservedThinkingIndex != disc.HistoryIndex && disc.PreservedThinkingIndex != disc.SystemIndex && disc.PreservedThinkingIndex < len(data) {
|
||||||
|
data[disc.PreservedThinkingIndex] = nil
|
||||||
|
}
|
||||||
|
|
||||||
if disc.TempIndex >= 0 && disc.TempIndex != disc.MessageIndex && disc.TempIndex != disc.HistoryIndex && disc.TempIndex < len(data) {
|
if disc.TempIndex >= 0 && disc.TempIndex != disc.MessageIndex && disc.TempIndex != disc.HistoryIndex && disc.TempIndex < len(data) {
|
||||||
if req.Temperature != nil {
|
if req.Temperature != nil {
|
||||||
data[disc.TempIndex] = *req.Temperature
|
data[disc.TempIndex] = *req.Temperature
|
||||||
@@ -4046,39 +4057,136 @@ type GradioOutputFrame struct {
|
|||||||
OK bool
|
OK bool
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func toInt(v interface{}) (int, bool) {
|
||||||
|
switch n := v.(type) {
|
||||||
|
case int:
|
||||||
|
return n, true
|
||||||
|
case int64:
|
||||||
|
return int(n), true
|
||||||
|
case float64:
|
||||||
|
return int(n), true
|
||||||
|
case json.Number:
|
||||||
|
i, err := n.Int64()
|
||||||
|
return int(i), err == nil
|
||||||
|
default:
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func isReasoningPath(path interface{}) bool {
|
||||||
|
switch p := path.(type) {
|
||||||
|
case []interface{}:
|
||||||
|
for i, elem := range p {
|
||||||
|
if s, ok := elem.(string); ok {
|
||||||
|
sLow := strings.ToLower(s)
|
||||||
|
if strings.Contains(sLow, "reason") || strings.Contains(sLow, "thought") || strings.Contains(sLow, "think") {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if num, ok := toInt(elem); ok {
|
||||||
|
if len(p) == 1 && num == 1 {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if len(p) == 2 && i == 1 && num == 1 {
|
||||||
|
if p0, ok0 := toInt(p[0]); ok0 && p0 == 0 {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case string:
|
||||||
|
sLow := strings.ToLower(p)
|
||||||
|
if strings.Contains(sLow, "reason") || strings.Contains(sLow, "thought") || strings.Contains(sLow, "think") {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func isContentPath(path interface{}) bool {
|
||||||
|
switch p := path.(type) {
|
||||||
|
case []interface{}:
|
||||||
|
if len(p) == 0 {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
for _, elem := range p {
|
||||||
|
if s, ok := elem.(string); ok {
|
||||||
|
sLow := strings.ToLower(s)
|
||||||
|
if strings.Contains(sLow, "content") || strings.Contains(sLow, "text") || strings.Contains(sLow, "value") {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if num, ok := toInt(p[0]); ok && num == 0 {
|
||||||
|
if len(p) == 1 {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if len(p) >= 2 {
|
||||||
|
if p1, ok1 := toInt(p[1]); ok1 && p1 == 0 {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if s1, ok1 := p[1].(string); ok1 && (strings.Contains(s1, "content") || strings.Contains(s1, "text")) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case string:
|
||||||
|
sLow := strings.ToLower(p)
|
||||||
|
if strings.Contains(sLow, "content") || strings.Contains(sLow, "text") || strings.Contains(sLow, "value") {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
// extractGradioDiffDelta extracts delta string from Gradio 6 streaming diff operations
|
// extractGradioDiffDelta extracts delta string from Gradio 6 streaming diff operations
|
||||||
func extractGradioDiffDelta(v []interface{}) (string, bool) {
|
func extractGradioDiffDelta(v []interface{}) (contentDelta string, reasoningDelta string, ok bool) {
|
||||||
if len(v) == 0 {
|
var diffItems [][]interface{}
|
||||||
return "", false
|
var findOps func(items []interface{})
|
||||||
}
|
findOps = func(items []interface{}) {
|
||||||
if len(v) >= 3 {
|
if len(items) >= 3 {
|
||||||
if op, ok := v[0].(string); ok && (op == "append" || op == "add") {
|
if op, ok := items[0].(string); ok && (op == "append" || op == "add") {
|
||||||
if delta, ok := v[2].(string); ok {
|
diffItems = append(diffItems, items)
|
||||||
return delta, true
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, it := range items {
|
||||||
|
if sub, ok := it.([]interface{}); ok {
|
||||||
|
findOps(sub)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for _, item := range v {
|
findOps(v)
|
||||||
if innerList, ok := item.([]interface{}); ok && len(innerList) > 0 {
|
|
||||||
for _, opItem := range innerList {
|
if len(diffItems) == 0 {
|
||||||
if opArr, ok := opItem.([]interface{}); ok && len(opArr) >= 3 {
|
return "", "", false
|
||||||
if op, ok := opArr[0].(string); ok && (op == "append" || op == "add") {
|
}
|
||||||
if delta, ok := opArr[2].(string); ok {
|
|
||||||
return delta, true
|
hasMatch := false
|
||||||
}
|
for _, item := range diffItems {
|
||||||
}
|
delta, isStr := item[2].(string)
|
||||||
}
|
if !isStr || delta == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
path := item[1]
|
||||||
|
if isReasoningPath(path) {
|
||||||
|
if reasoningDelta == "" {
|
||||||
|
reasoningDelta = delta
|
||||||
|
hasMatch = true
|
||||||
}
|
}
|
||||||
if len(innerList) >= 3 {
|
} else if isContentPath(path) {
|
||||||
if op, ok := innerList[0].(string); ok && (op == "append" || op == "add") {
|
if contentDelta == "" {
|
||||||
if delta, ok := innerList[2].(string); ok {
|
contentDelta = delta
|
||||||
return delta, true
|
hasMatch = true
|
||||||
}
|
}
|
||||||
}
|
} else {
|
||||||
|
if contentDelta == "" {
|
||||||
|
contentDelta = delta
|
||||||
|
hasMatch = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return "", false
|
return contentDelta, reasoningDelta, hasMatch
|
||||||
}
|
}
|
||||||
|
|
||||||
// ParseGradioStreamOutput extracts structured content, reasoning, and tool calls from Gradio output
|
// ParseGradioStreamOutput extracts structured content, reasoning, and tool calls from Gradio output
|
||||||
@@ -4109,30 +4217,33 @@ func ParseGradioStreamOutput(rawJSON string) (frame GradioOutputFrame) {
|
|||||||
return frame
|
return frame
|
||||||
}
|
}
|
||||||
|
|
||||||
if delta, ok := extractGradioDiffDelta(v); ok {
|
if cDelta, rDelta, ok := extractGradioDiffDelta(v); ok {
|
||||||
frame.Content = delta
|
frame.Content = cDelta
|
||||||
|
frame.Reasoning = rDelta
|
||||||
frame.IsDelta = true
|
frame.IsDelta = true
|
||||||
frame.OK = true
|
frame.OK = true
|
||||||
return frame
|
return frame
|
||||||
}
|
}
|
||||||
|
|
||||||
// 1. Check if v[0] is an inner slice with len >= 3 (e.g. Hy3: [[content, reasoning, tool_calls, history]])
|
// 1. Check if v[0] is an inner slice with len >= 2 (e.g. Hy3: [[content, reasoning, tool_calls, history]])
|
||||||
if inner, ok := v[0].([]interface{}); ok && len(inner) >= 3 {
|
if inner, ok := v[0].([]interface{}); ok && len(inner) >= 2 {
|
||||||
s0, _ := inner[0].(string)
|
s0, ok0 := inner[0].(string)
|
||||||
s1, _ := inner[1].(string)
|
s1, ok1 := inner[1].(string)
|
||||||
frame.Content = s0
|
if ok0 || ok1 {
|
||||||
frame.Reasoning = s1
|
frame.Content = s0
|
||||||
if inner[2] != nil {
|
frame.Reasoning = s1
|
||||||
b, err := json.Marshal(inner[2])
|
if len(inner) >= 3 && inner[2] != nil {
|
||||||
if err == nil {
|
b, err := json.Marshal(inner[2])
|
||||||
var tcs []ToolCall
|
if err == nil {
|
||||||
if json.Unmarshal(b, &tcs) == nil && len(tcs) > 0 {
|
var tcs []ToolCall
|
||||||
frame.ToolCalls = tcs
|
if json.Unmarshal(b, &tcs) == nil && len(tcs) > 0 {
|
||||||
|
frame.ToolCalls = tcs
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
frame.OK = true
|
||||||
|
return frame
|
||||||
}
|
}
|
||||||
frame.OK = true
|
|
||||||
return frame
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// 2. Check if any element of v is a Chatbot message list or Chatbot pair list
|
// 2. Check if any element of v is a Chatbot message list or Chatbot pair list
|
||||||
@@ -4270,22 +4381,6 @@ func ParseGradioStreamOutput(rawJSON string) (frame GradioOutputFrame) {
|
|||||||
return frame
|
return frame
|
||||||
}
|
}
|
||||||
|
|
||||||
// 4. Check if v[0] is an inner pair [content, reasoning]
|
|
||||||
if inner, ok := v[0].([]interface{}); ok && len(inner) == 2 {
|
|
||||||
s0, ok0 := inner[0].(string)
|
|
||||||
s1, ok1 := inner[1].(string)
|
|
||||||
if ok0 && ok1 {
|
|
||||||
frame.Content = s0
|
|
||||||
frame.Reasoning = s1
|
|
||||||
frame.OK = true
|
|
||||||
return frame
|
|
||||||
}
|
|
||||||
if ok1 {
|
|
||||||
frame.Content = s1
|
|
||||||
frame.OK = true
|
|
||||||
return frame
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 5. Check if v[0] is a non-empty string or single output
|
// 5. Check if v[0] is a non-empty string or single output
|
||||||
if s, ok := v[0].(string); ok && (s != "" || len(v) == 1) {
|
if s, ok := v[0].(string); ok && (s != "" || len(v) == 1) {
|
||||||
@@ -4618,15 +4713,16 @@ func (g *GradioGateway) executeQueueCompletion(w http.ResponseWriter, r *http.Re
|
|||||||
if frame.OK {
|
if frame.OK {
|
||||||
if frame.IsDelta {
|
if frame.IsDelta {
|
||||||
latestFrame.Content += frame.Content
|
latestFrame.Content += frame.Content
|
||||||
|
latestFrame.Reasoning += frame.Reasoning
|
||||||
} else {
|
} else {
|
||||||
if len(frame.ToolCalls) > 0 {
|
if len(frame.ToolCalls) > 0 {
|
||||||
latestFrame.Content = frame.Content
|
latestFrame.Content = frame.Content
|
||||||
} else if frame.Content != "" || latestFrame.Content == "" {
|
} else if frame.Content != "" || latestFrame.Content == "" {
|
||||||
latestFrame.Content = frame.Content
|
latestFrame.Content = frame.Content
|
||||||
}
|
}
|
||||||
}
|
if frame.Reasoning != "" || latestFrame.Reasoning == "" {
|
||||||
if frame.Reasoning != "" || latestFrame.Reasoning == "" {
|
latestFrame.Reasoning = frame.Reasoning
|
||||||
latestFrame.Reasoning = frame.Reasoning
|
}
|
||||||
}
|
}
|
||||||
if len(frame.ToolCalls) > 0 {
|
if len(frame.ToolCalls) > 0 {
|
||||||
latestFrame.ToolCalls = frame.ToolCalls
|
latestFrame.ToolCalls = frame.ToolCalls
|
||||||
@@ -4668,7 +4764,7 @@ func (g *GradioGateway) executeQueueCompletion(w http.ResponseWriter, r *http.Re
|
|||||||
var prevContent string
|
var prevContent string
|
||||||
var prevReasoning string
|
var prevReasoning string
|
||||||
prevToolArgs := make(map[int]string)
|
prevToolArgs := make(map[int]string)
|
||||||
nativeReasoningSeen := false
|
nativeReasoningSeen := disc.IsHunyuan3
|
||||||
nativeToolCallsSeen := false
|
nativeToolCallsSeen := false
|
||||||
var streamErr error
|
var streamErr error
|
||||||
|
|
||||||
@@ -4710,14 +4806,23 @@ func (g *GradioGateway) executeQueueCompletion(w http.ResponseWriter, r *http.Re
|
|||||||
// 1. Native reasoning handling
|
// 1. Native reasoning handling
|
||||||
if frame.Reasoning != "" || nativeReasoningSeen {
|
if frame.Reasoning != "" || nativeReasoningSeen {
|
||||||
nativeReasoningSeen = true
|
nativeReasoningSeen = true
|
||||||
if len(frame.Reasoning) > len(prevReasoning) && strings.HasPrefix(frame.Reasoning, prevReasoning) {
|
var deltaReasoning string
|
||||||
delta := frame.Reasoning[len(prevReasoning):]
|
if frame.IsDelta {
|
||||||
streamer.Reasoning(delta)
|
deltaReasoning = frame.Reasoning
|
||||||
prevReasoning = frame.Reasoning
|
prevReasoning += deltaReasoning
|
||||||
} else if prevReasoning == "" && frame.Reasoning != "" {
|
} else {
|
||||||
streamer.Reasoning(frame.Reasoning)
|
if strings.HasPrefix(frame.Reasoning, prevReasoning) {
|
||||||
|
deltaReasoning = frame.Reasoning[len(prevReasoning):]
|
||||||
|
} else if prevReasoning == "" {
|
||||||
|
deltaReasoning = frame.Reasoning
|
||||||
|
} else {
|
||||||
|
deltaReasoning = frame.Reasoning
|
||||||
|
}
|
||||||
prevReasoning = frame.Reasoning
|
prevReasoning = frame.Reasoning
|
||||||
}
|
}
|
||||||
|
if deltaReasoning != "" {
|
||||||
|
streamer.Reasoning(deltaReasoning)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 2. Native tool call handling
|
// 2. Native tool call handling
|
||||||
@@ -4929,8 +5034,10 @@ func (g *GradioGateway) ExecuteChatCompletion(w http.ResponseWriter, r *http.Req
|
|||||||
return json.Marshal(v2Payload)
|
return json.Marshal(v2Payload)
|
||||||
}
|
}
|
||||||
payloadMap := map[string]interface{}{
|
payloadMap := map[string]interface{}{
|
||||||
"data": gradioData,
|
"data": gradioData,
|
||||||
"session_hash": GenerateUUID(),
|
}
|
||||||
|
if !disc.IsHunyuan3 {
|
||||||
|
payloadMap["session_hash"] = GenerateUUID()
|
||||||
}
|
}
|
||||||
return json.Marshal(payloadMap)
|
return json.Marshal(payloadMap)
|
||||||
}
|
}
|
||||||
@@ -5115,13 +5222,18 @@ func (g *GradioGateway) ExecuteChatCompletion(w http.ResponseWriter, r *http.Req
|
|||||||
return fmt.Errorf("upstream Gradio error: %s", errMsg)
|
return fmt.Errorf("upstream Gradio error: %s", errMsg)
|
||||||
}
|
}
|
||||||
if frame := ParseGradioStreamOutput(dataStr); frame.OK {
|
if frame := ParseGradioStreamOutput(dataStr); frame.OK {
|
||||||
if len(frame.ToolCalls) > 0 {
|
if frame.IsDelta {
|
||||||
latestFrame.Content = frame.Content
|
latestFrame.Content += frame.Content
|
||||||
} else if frame.Content != "" || latestFrame.Content == "" {
|
latestFrame.Reasoning += frame.Reasoning
|
||||||
latestFrame.Content = frame.Content
|
} else {
|
||||||
}
|
if len(frame.ToolCalls) > 0 {
|
||||||
if frame.Reasoning != "" || latestFrame.Reasoning == "" {
|
latestFrame.Content = frame.Content
|
||||||
latestFrame.Reasoning = frame.Reasoning
|
} else if frame.Content != "" || latestFrame.Content == "" {
|
||||||
|
latestFrame.Content = frame.Content
|
||||||
|
}
|
||||||
|
if frame.Reasoning != "" || latestFrame.Reasoning == "" {
|
||||||
|
latestFrame.Reasoning = frame.Reasoning
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if len(frame.ToolCalls) > 0 {
|
if len(frame.ToolCalls) > 0 {
|
||||||
latestFrame.ToolCalls = frame.ToolCalls
|
latestFrame.ToolCalls = frame.ToolCalls
|
||||||
@@ -5161,7 +5273,7 @@ func (g *GradioGateway) ExecuteChatCompletion(w http.ResponseWriter, r *http.Req
|
|||||||
var prevContent string
|
var prevContent string
|
||||||
var prevReasoning string
|
var prevReasoning string
|
||||||
prevToolArgs := make(map[int]string)
|
prevToolArgs := make(map[int]string)
|
||||||
nativeReasoningSeen := false
|
nativeReasoningSeen := disc.IsHunyuan3
|
||||||
nativeToolCallsSeen := false
|
nativeToolCallsSeen := false
|
||||||
currentEvent := ""
|
currentEvent := ""
|
||||||
var streamErr error
|
var streamErr error
|
||||||
@@ -5213,14 +5325,19 @@ func (g *GradioGateway) ExecuteChatCompletion(w http.ResponseWriter, r *http.Req
|
|||||||
if frame.Reasoning != "" || nativeReasoningSeen {
|
if frame.Reasoning != "" || nativeReasoningSeen {
|
||||||
nativeReasoningSeen = true
|
nativeReasoningSeen = true
|
||||||
var deltaReasoning string
|
var deltaReasoning string
|
||||||
if strings.HasPrefix(frame.Reasoning, prevReasoning) {
|
if frame.IsDelta {
|
||||||
deltaReasoning = frame.Reasoning[len(prevReasoning):]
|
|
||||||
} else if prevReasoning == "" {
|
|
||||||
deltaReasoning = frame.Reasoning
|
deltaReasoning = frame.Reasoning
|
||||||
|
prevReasoning += deltaReasoning
|
||||||
} else {
|
} else {
|
||||||
deltaReasoning = frame.Reasoning
|
if strings.HasPrefix(frame.Reasoning, prevReasoning) {
|
||||||
|
deltaReasoning = frame.Reasoning[len(prevReasoning):]
|
||||||
|
} else if prevReasoning == "" {
|
||||||
|
deltaReasoning = frame.Reasoning
|
||||||
|
} else {
|
||||||
|
deltaReasoning = frame.Reasoning
|
||||||
|
}
|
||||||
|
prevReasoning = frame.Reasoning
|
||||||
}
|
}
|
||||||
prevReasoning = frame.Reasoning
|
|
||||||
if deltaReasoning != "" {
|
if deltaReasoning != "" {
|
||||||
streamer.Reasoning(deltaReasoning)
|
streamer.Reasoning(deltaReasoning)
|
||||||
}
|
}
|
||||||
@@ -5269,14 +5386,19 @@ func (g *GradioGateway) ExecuteChatCompletion(w http.ResponseWriter, r *http.Req
|
|||||||
// 3. Content handling
|
// 3. Content handling
|
||||||
currentText := frame.Content
|
currentText := frame.Content
|
||||||
var delta string
|
var delta string
|
||||||
if strings.HasPrefix(currentText, prevContent) {
|
if frame.IsDelta {
|
||||||
delta = currentText[len(prevContent):]
|
|
||||||
} else if prevContent == "" {
|
|
||||||
delta = currentText
|
delta = currentText
|
||||||
|
prevContent += delta
|
||||||
} else {
|
} else {
|
||||||
delta = currentText
|
if strings.HasPrefix(currentText, prevContent) {
|
||||||
|
delta = currentText[len(prevContent):]
|
||||||
|
} else if prevContent == "" {
|
||||||
|
delta = currentText
|
||||||
|
} else {
|
||||||
|
delta = currentText
|
||||||
|
}
|
||||||
|
prevContent = currentText
|
||||||
}
|
}
|
||||||
prevContent = currentText
|
|
||||||
|
|
||||||
if delta != "" {
|
if delta != "" {
|
||||||
if nativeReasoningSeen || nativeToolCallsSeen {
|
if nativeReasoningSeen || nativeToolCallsSeen {
|
||||||
|
|||||||
+284
@@ -2957,3 +2957,287 @@ func TestMultiTurnToolResponseNotWrappedInQuery(t *testing.T) {
|
|||||||
t.Errorf("message input should contain tool response, got:\n%s", msgStr)
|
t.Errorf("message input should contain tool response, got:\n%s", msgStr)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestParseGradioStreamOutputHunyuanTuple(t *testing.T) {
|
||||||
|
// Test Hunyuan 3 4-element tuple during reasoning phase: [content, reasoning, tools, history]
|
||||||
|
raw1 := `[["", "Thinking through the query...", [], [{"role": "user", "content": "hi"}]]]`
|
||||||
|
f1 := ParseGradioStreamOutput(raw1)
|
||||||
|
if !f1.OK {
|
||||||
|
t.Fatalf("expected f1.OK to be true")
|
||||||
|
}
|
||||||
|
if f1.Content != "" {
|
||||||
|
t.Errorf("expected empty content during reasoning phase, got %q", f1.Content)
|
||||||
|
}
|
||||||
|
if f1.Reasoning != "Thinking through the query..." {
|
||||||
|
t.Errorf("expected reasoning 'Thinking through the query...', got %q", f1.Reasoning)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test Hunyuan 3 completion with both content and reasoning
|
||||||
|
raw2 := `[["Hello! How can I help?", "Thinking through the query...", [], [{"role": "user", "content": "hi"}]]]`
|
||||||
|
f2 := ParseGradioStreamOutput(raw2)
|
||||||
|
if !f2.OK {
|
||||||
|
t.Fatalf("expected f2.OK to be true")
|
||||||
|
}
|
||||||
|
if f2.Content != "Hello! How can I help?" {
|
||||||
|
t.Errorf("expected content 'Hello! How can I help?', got %q", f2.Content)
|
||||||
|
}
|
||||||
|
if f2.Reasoning != "Thinking through the query..." {
|
||||||
|
t.Errorf("expected reasoning 'Thinking through the query...', got %q", f2.Reasoning)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test 2-element tuple [content, reasoning]
|
||||||
|
raw3 := `[["", "Still thinking..."]]`
|
||||||
|
f3 := ParseGradioStreamOutput(raw3)
|
||||||
|
if !f3.OK {
|
||||||
|
t.Fatalf("expected f3.OK to be true")
|
||||||
|
}
|
||||||
|
if f3.Content != "" {
|
||||||
|
t.Errorf("expected empty content, got %q", f3.Content)
|
||||||
|
}
|
||||||
|
if f3.Reasoning != "Still thinking..." {
|
||||||
|
t.Errorf("expected reasoning 'Still thinking...', got %q", f3.Reasoning)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGradioDiffDeltaReasoningContentSeparation(t *testing.T) {
|
||||||
|
// Diff appending to reasoning at path [1]
|
||||||
|
raw1 := `[[["append", [1], "reasoning delta "], ["append", [3, 1, "reasoning_content"], "reasoning delta "]]]`
|
||||||
|
f1 := ParseGradioStreamOutput(raw1)
|
||||||
|
if !f1.OK || !f1.IsDelta {
|
||||||
|
t.Fatalf("expected f1 to be valid delta frame")
|
||||||
|
}
|
||||||
|
if f1.Content != "" {
|
||||||
|
t.Errorf("expected empty content, got %q", f1.Content)
|
||||||
|
}
|
||||||
|
if f1.Reasoning != "reasoning delta " {
|
||||||
|
t.Errorf("expected reasoning delta 'reasoning delta ', got %q", f1.Reasoning)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Diff appending to content at path [0]
|
||||||
|
raw2 := `[[["append", [0], "content delta "], ["append", [3, 1, "content"], "content delta "]]]`
|
||||||
|
f2 := ParseGradioStreamOutput(raw2)
|
||||||
|
if !f2.OK || !f2.IsDelta {
|
||||||
|
t.Fatalf("expected f2 to be valid delta frame")
|
||||||
|
}
|
||||||
|
if f2.Content != "content delta " {
|
||||||
|
t.Errorf("expected content delta 'content delta ', got %q", f2.Content)
|
||||||
|
}
|
||||||
|
if f2.Reasoning != "" {
|
||||||
|
t.Errorf("expected empty reasoning, got %q", f2.Reasoning)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHunyuan3CallAndStreamingNoInterleaving(t *testing.T) {
|
||||||
|
var callPayloadReceived map[string]interface{}
|
||||||
|
|
||||||
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.URL.Path == "/gradio_api/info" {
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||||
|
"named_endpoints": map[string]interface{}{
|
||||||
|
"/chat": map[string]interface{}{
|
||||||
|
"parameters": []map[string]interface{}{
|
||||||
|
{"parameter_name": "message", "component": "Api"},
|
||||||
|
{"parameter_name": "system_prompt", "component": "Api"},
|
||||||
|
{"parameter_name": "history", "component": "Api"},
|
||||||
|
{"parameter_name": "think_level", "component": "Api"},
|
||||||
|
{"parameter_name": "temperature", "component": "Api"},
|
||||||
|
{"parameter_name": "max_tokens", "component": "Api"},
|
||||||
|
{"parameter_name": "top_p", "component": "Api"},
|
||||||
|
{"parameter_name": "preserved_thinking", "component": "Api"},
|
||||||
|
{"parameter_name": "functions_json_str", "component": "Api"},
|
||||||
|
},
|
||||||
|
"code_snippets": map[string]interface{}{
|
||||||
|
"bash": "curl -X POST http://localhost:7860/gradio_api/call/chat -s -H \"Content-Type: application/json\" -d '{\"data\": [\"...\", \"\", null, \"high\", null, 0, 0, null, \"\"]}' | awk -F'\"' '{ print $4}' | read EVENT_ID; curl -N http://localhost:7860/gradio_api/call/chat/$EVENT_ID",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if r.URL.Path == "/config" {
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||||
|
"version": "6.12.0",
|
||||||
|
"components": []map[string]interface{}{
|
||||||
|
{"id": 30, "type": "api", "props": map[string]interface{}{"label": "1st"}},
|
||||||
|
{"id": 31, "type": "api", "props": map[string]interface{}{"label": "2nd"}},
|
||||||
|
{"id": 32, "type": "api", "props": map[string]interface{}{"label": "3rd"}},
|
||||||
|
{"id": 33, "type": "api", "props": map[string]interface{}{"label": "4th"}},
|
||||||
|
{"id": 34, "type": "api", "props": map[string]interface{}{"label": "5th"}},
|
||||||
|
{"id": 35, "type": "api", "props": map[string]interface{}{"label": "6th"}},
|
||||||
|
{"id": 36, "type": "api", "props": map[string]interface{}{"label": "7th"}},
|
||||||
|
{"id": 37, "type": "api", "props": map[string]interface{}{"label": "8th"}},
|
||||||
|
{"id": 38, "type": "api", "props": map[string]interface{}{"label": "9th"}},
|
||||||
|
{"id": 39, "type": "api", "props": map[string]interface{}{"label": "out"}},
|
||||||
|
},
|
||||||
|
"dependencies": []map[string]interface{}{
|
||||||
|
{
|
||||||
|
"id": 8,
|
||||||
|
"api_name": "chat",
|
||||||
|
"inputs": []int{30, 31, 32, 33, 34, 35, 36, 37, 38},
|
||||||
|
"outputs": []int{39},
|
||||||
|
"types": map[string]interface{}{"generator": true},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if r.URL.Path == "/gradio_api/call/chat" {
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&callPayloadReceived); err != nil {
|
||||||
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||||
|
"event_id": "hy3-event-999",
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if r.URL.Path == "/gradio_api/call/chat/hy3-event-999" {
|
||||||
|
w.Header().Set("Content-Type", "text/event-stream")
|
||||||
|
w.Header().Set("Cache-Control", "no-cache")
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
flusher, _ := w.(http.Flusher)
|
||||||
|
|
||||||
|
// Event 1: Thinking chunk 1
|
||||||
|
fmt.Fprintf(w, "event: generating\ndata: [[\"\", \"Thinking about the answer.\", [], []]]\n\n")
|
||||||
|
if flusher != nil {
|
||||||
|
flusher.Flush()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Event 2: Thinking chunk 2
|
||||||
|
fmt.Fprintf(w, "event: generating\ndata: [[\"\", \"Thinking about the answer. Planning response.\", [], []]]\n\n")
|
||||||
|
if flusher != nil {
|
||||||
|
flusher.Flush()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Event 3: Content start
|
||||||
|
fmt.Fprintf(w, "event: generating\ndata: [[\"Hello \", \"Thinking about the answer. Planning response.\", [], []]]\n\n")
|
||||||
|
if flusher != nil {
|
||||||
|
flusher.Flush()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Event 4: Content complete
|
||||||
|
fmt.Fprintf(w, "event: complete\ndata: [[\"Hello world!\", \"Thinking about the answer. Planning response.\", [], []]]\n\n")
|
||||||
|
if flusher != nil {
|
||||||
|
flusher.Flush()
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
http.NotFound(w, r)
|
||||||
|
}))
|
||||||
|
defer ts.Close()
|
||||||
|
|
||||||
|
gw := NewGradioGateway(ts.URL, "", 10*time.Second)
|
||||||
|
|
||||||
|
// 1. Verify Space Discovery for Hunyuan 3 space
|
||||||
|
disc := gw.GetDiscovery(ts.URL, "test-ua")
|
||||||
|
if !disc.IsHunyuan3 {
|
||||||
|
t.Errorf("expected IsHunyuan3 to be true")
|
||||||
|
}
|
||||||
|
if disc.Protocol != "call" {
|
||||||
|
t.Errorf("expected Protocol 'call', got %q", disc.Protocol)
|
||||||
|
}
|
||||||
|
if disc.ThinkLevelIndex != 3 {
|
||||||
|
t.Errorf("expected ThinkLevelIndex 3, got %d", disc.ThinkLevelIndex)
|
||||||
|
}
|
||||||
|
if disc.PreservedThinkingIndex != 7 {
|
||||||
|
t.Errorf("expected PreservedThinkingIndex 7, got %d", disc.PreservedThinkingIndex)
|
||||||
|
}
|
||||||
|
if disc.FunctionsJSONIndex != 8 {
|
||||||
|
t.Errorf("expected FunctionsJSONIndex 8, got %d", disc.FunctionsJSONIndex)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Verify Streaming Request: NO interleaving between reasoning and content
|
||||||
|
reqStream := ChatCompletionRequest{
|
||||||
|
Model: "hy3",
|
||||||
|
Stream: true,
|
||||||
|
Messages: []ChatMessage{
|
||||||
|
{Role: "user", Content: "Hello"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
httpReqStream := httptest.NewRequest("POST", "/v1/chat/completions", nil)
|
||||||
|
recStream := httptest.NewRecorder()
|
||||||
|
|
||||||
|
err := gw.ExecuteChatCompletion(recStream, httpReqStream, reqStream)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ExecuteChatCompletion streaming failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify call payload has no session_hash for Hunyuan 3
|
||||||
|
if _, hasHash := callPayloadReceived["session_hash"]; hasHash {
|
||||||
|
t.Errorf("session_hash should NOT be present in call payload for Hunyuan 3")
|
||||||
|
}
|
||||||
|
|
||||||
|
var streamedReasoning strings.Builder
|
||||||
|
var streamedContent strings.Builder
|
||||||
|
|
||||||
|
lines := strings.Split(recStream.Body.String(), "\n")
|
||||||
|
for _, line := range lines {
|
||||||
|
line = strings.TrimSpace(line)
|
||||||
|
if strings.HasPrefix(line, "data: ") && line != "data: [DONE]" {
|
||||||
|
var chunk StreamResponse
|
||||||
|
if err := json.Unmarshal([]byte(strings.TrimPrefix(line, "data: ")), &chunk); err == nil {
|
||||||
|
if len(chunk.Choices) > 0 {
|
||||||
|
delta := chunk.Choices[0].Delta
|
||||||
|
if delta.ReasoningContent != "" {
|
||||||
|
if delta.Content != "" {
|
||||||
|
t.Errorf("interleaving detected: chunk has both reasoning and content: %+v", delta)
|
||||||
|
}
|
||||||
|
streamedReasoning.WriteString(delta.ReasoningContent)
|
||||||
|
}
|
||||||
|
if delta.Content != "" {
|
||||||
|
if strings.Contains(delta.Content, "Thinking") {
|
||||||
|
t.Errorf("leakage detected: content chunk contains reasoning: %q", delta.Content)
|
||||||
|
}
|
||||||
|
streamedContent.WriteString(delta.Content)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if streamedReasoning.String() != "Thinking about the answer. Planning response." {
|
||||||
|
t.Errorf("expected full reasoning 'Thinking about the answer. Planning response.', got %q", streamedReasoning.String())
|
||||||
|
}
|
||||||
|
if streamedContent.String() != "Hello world!" {
|
||||||
|
t.Errorf("expected full content 'Hello world!', got %q", streamedContent.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. Verify Non-Streaming Request: separate content and reasoning_content
|
||||||
|
reqNonStream := ChatCompletionRequest{
|
||||||
|
Model: "hy3",
|
||||||
|
Stream: false,
|
||||||
|
Messages: []ChatMessage{
|
||||||
|
{Role: "user", Content: "Hello"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
httpReqNonStream := httptest.NewRequest("POST", "/v1/chat/completions", nil)
|
||||||
|
recNonStream := httptest.NewRecorder()
|
||||||
|
|
||||||
|
err = gw.ExecuteChatCompletion(recNonStream, httpReqNonStream, reqNonStream)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ExecuteChatCompletion non-streaming failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var nonStreamResp ChatCompletionResponse
|
||||||
|
if err := json.NewDecoder(recNonStream.Body).Decode(&nonStreamResp); err != nil {
|
||||||
|
t.Fatalf("failed to decode non-stream response: %v", err)
|
||||||
|
}
|
||||||
|
if len(nonStreamResp.Choices) == 0 {
|
||||||
|
t.Fatalf("expected at least 1 choice")
|
||||||
|
}
|
||||||
|
choice := nonStreamResp.Choices[0]
|
||||||
|
if choice.Message.ReasoningContent != "Thinking about the answer. Planning response." {
|
||||||
|
t.Errorf("expected reasoning_content 'Thinking about the answer. Planning response.', got %q", choice.Message.ReasoningContent)
|
||||||
|
}
|
||||||
|
if choice.Message.GetContentString() != "Hello world!" {
|
||||||
|
t.Errorf("expected content 'Hello world!', got %q", choice.Message.GetContentString())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user