5757 lines
184 KiB
Go
5757 lines
184 KiB
Go
// gr2gw: Universal Gradio to OpenAI LLM proxy gateway in Go
|
|
// Created by Luxferre in 2026, released into the public domain
|
|
|
|
package main
|
|
|
|
import (
|
|
"bufio"
|
|
"bytes"
|
|
"context"
|
|
"crypto/rand"
|
|
"encoding/json"
|
|
"flag"
|
|
"fmt"
|
|
"io"
|
|
"log"
|
|
"net"
|
|
"net/http"
|
|
"net/url"
|
|
"os"
|
|
"regexp"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
var (
|
|
DefaultSpaceURL = "https://tencent-hy3.hf.space"
|
|
DefaultUserAgent = "Mozilla/5.0 (X11; Linux x86_64; rv:153.0) Gecko/20100101 Firefox/153.0"
|
|
ConfiguredUserAgent string
|
|
ConfiguredModelName string
|
|
)
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// OpenAI API data structures
|
|
// ---------------------------------------------------------------------------
|
|
|
|
type ModelItem struct {
|
|
ID string `json:"id"`
|
|
Object string `json:"object"`
|
|
Created int64 `json:"created"`
|
|
OwnedBy string `json:"owned_by"`
|
|
}
|
|
|
|
type ModelsResponse struct {
|
|
Object string `json:"object"`
|
|
Data []ModelItem `json:"data"`
|
|
}
|
|
|
|
type ToolCallFunction struct {
|
|
Name string `json:"name,omitempty"`
|
|
Arguments string `json:"arguments"`
|
|
}
|
|
|
|
type ToolCall struct {
|
|
Index *int `json:"index,omitempty"`
|
|
ID string `json:"id,omitempty"`
|
|
Type string `json:"type,omitempty"`
|
|
Function ToolCallFunction `json:"function"`
|
|
}
|
|
|
|
type Tool struct {
|
|
Type string `json:"type"`
|
|
Function interface{} `json:"function"`
|
|
}
|
|
|
|
type ChatMessage struct {
|
|
Role string `json:"role"`
|
|
Content interface{} `json:"content"`
|
|
ReasoningContent string `json:"reasoning_content,omitempty"`
|
|
ToolCalls []ToolCall `json:"tool_calls,omitempty"`
|
|
ToolCallID string `json:"tool_call_id,omitempty"`
|
|
Name string `json:"name,omitempty"`
|
|
}
|
|
|
|
func extractContentString(val interface{}) string {
|
|
if val == nil {
|
|
return ""
|
|
}
|
|
if str, ok := val.(string); ok {
|
|
return str
|
|
}
|
|
if parts, ok := val.([]interface{}); ok {
|
|
var sb strings.Builder
|
|
for _, p := range parts {
|
|
if str, ok := p.(string); ok {
|
|
sb.WriteString(str)
|
|
} else if itemMap, ok := p.(map[string]interface{}); ok {
|
|
if textVal, ok := itemMap["text"].(string); ok {
|
|
sb.WriteString(textVal)
|
|
} else if valStr, ok := itemMap["value"].(string); ok {
|
|
sb.WriteString(valStr)
|
|
}
|
|
}
|
|
}
|
|
return sb.String()
|
|
}
|
|
if itemMap, ok := val.(map[string]interface{}); ok {
|
|
if textVal, ok := itemMap["text"].(string); ok {
|
|
return textVal
|
|
} else if valStr, ok := itemMap["value"].(string); ok {
|
|
return valStr
|
|
}
|
|
}
|
|
b, err := json.Marshal(val)
|
|
if err == nil {
|
|
return string(b)
|
|
}
|
|
return fmt.Sprintf("%v", val)
|
|
}
|
|
|
|
func (m *ChatMessage) GetContentString() string {
|
|
return extractContentString(m.Content)
|
|
}
|
|
|
|
type ChatCompletionRequest struct {
|
|
Model string `json:"model"`
|
|
Messages []ChatMessage `json:"messages"`
|
|
Tools []Tool `json:"tools,omitempty"`
|
|
ToolChoice interface{} `json:"tool_choice,omitempty"`
|
|
Stream bool `json:"stream"`
|
|
MaxTokens int `json:"max_tokens"`
|
|
MaxCompletionTokens int `json:"max_completion_tokens"`
|
|
Temperature *float64 `json:"temperature,omitempty"`
|
|
TopP *float64 `json:"top_p,omitempty"`
|
|
ReasoningEffort string `json:"reasoning_effort,omitempty"`
|
|
}
|
|
|
|
type ChatCompletionResponseChoice struct {
|
|
Index int `json:"index"`
|
|
Message ChatMessage `json:"message"`
|
|
FinishReason string `json:"finish_reason"`
|
|
}
|
|
|
|
type Usage struct {
|
|
PromptTokens int `json:"prompt_tokens"`
|
|
CompletionTokens int `json:"completion_tokens"`
|
|
TotalTokens int `json:"total_tokens"`
|
|
}
|
|
|
|
type ChatCompletionResponse struct {
|
|
ID string `json:"id"`
|
|
Object string `json:"object"`
|
|
Created int64 `json:"created"`
|
|
Model string `json:"model"`
|
|
Choices []ChatCompletionResponseChoice `json:"choices"`
|
|
Usage Usage `json:"usage"`
|
|
}
|
|
|
|
type StreamDelta struct {
|
|
Role string `json:"role,omitempty"`
|
|
Content string `json:"content,omitempty"`
|
|
ReasoningContent string `json:"reasoning_content,omitempty"`
|
|
ToolCalls []ToolCall `json:"tool_calls,omitempty"`
|
|
}
|
|
|
|
type StreamChoice struct {
|
|
Index int `json:"index"`
|
|
Delta StreamDelta `json:"delta"`
|
|
FinishReason *string `json:"finish_reason,omitempty"`
|
|
}
|
|
|
|
type StreamResponse struct {
|
|
ID string `json:"id"`
|
|
Object string `json:"object"`
|
|
Created int64 `json:"created"`
|
|
Model string `json:"model"`
|
|
Choices []StreamChoice `json:"choices"`
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// SOCKS5 proxy client (RFC 1928 / RFC 1929)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
type SOCKS5Config struct {
|
|
Address string
|
|
Username string
|
|
Password string
|
|
}
|
|
|
|
func ParseSOCKS5URL(proxyURL string) (*SOCKS5Config, error) {
|
|
cleanURL := strings.TrimSpace(proxyURL)
|
|
if cleanURL == "" {
|
|
return nil, nil
|
|
}
|
|
if strings.HasPrefix(cleanURL, "socks5://") {
|
|
cleanURL = strings.TrimPrefix(cleanURL, "socks5://")
|
|
} else if strings.HasPrefix(cleanURL, "socks5h://") {
|
|
cleanURL = strings.TrimPrefix(cleanURL, "socks5h://")
|
|
}
|
|
|
|
cfg := &SOCKS5Config{}
|
|
if atIdx := strings.LastIndex(cleanURL, "@"); atIdx != -1 {
|
|
userPass := cleanURL[:atIdx]
|
|
cfg.Address = cleanURL[atIdx+1:]
|
|
if colonIdx := strings.Index(userPass, ":"); colonIdx != -1 {
|
|
cfg.Username = userPass[:colonIdx]
|
|
cfg.Password = userPass[colonIdx+1:]
|
|
} else {
|
|
cfg.Username = userPass
|
|
}
|
|
} else {
|
|
cfg.Address = cleanURL
|
|
}
|
|
|
|
if !strings.Contains(cfg.Address, ":") {
|
|
cfg.Address = cfg.Address + ":1080"
|
|
}
|
|
return cfg, nil
|
|
}
|
|
|
|
func DialSOCKS5(ctx context.Context, proxyURL, targetAddr string) (net.Conn, error) {
|
|
cfg, err := ParseSOCKS5URL(proxyURL)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("invalid socks5 proxy configuration: %w", err)
|
|
}
|
|
if cfg == nil {
|
|
var d net.Dialer
|
|
return d.DialContext(ctx, "tcp", targetAddr)
|
|
}
|
|
|
|
var d net.Dialer
|
|
conn, err := d.DialContext(ctx, "tcp", cfg.Address)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to connect to socks5 proxy at %s: %w", cfg.Address, err)
|
|
}
|
|
|
|
deadline, ok := ctx.Deadline()
|
|
if !ok {
|
|
deadline = time.Now().Add(30 * time.Second)
|
|
}
|
|
conn.SetDeadline(deadline)
|
|
defer conn.SetDeadline(time.Time{})
|
|
|
|
// 1. Negotiation Greeting (RFC 1928)
|
|
var greeting []byte
|
|
if cfg.Username != "" {
|
|
greeting = []byte{0x05, 0x02, 0x00, 0x02}
|
|
} else {
|
|
greeting = []byte{0x05, 0x01, 0x00}
|
|
}
|
|
|
|
if _, err := conn.Write(greeting); err != nil {
|
|
conn.Close()
|
|
return nil, fmt.Errorf("failed to write socks5 greeting: %w", err)
|
|
}
|
|
|
|
resp := make([]byte, 2)
|
|
if _, err := io.ReadFull(conn, resp); err != nil {
|
|
conn.Close()
|
|
return nil, fmt.Errorf("failed to read socks5 greeting response: %w", err)
|
|
}
|
|
|
|
if resp[0] != 0x05 {
|
|
conn.Close()
|
|
return nil, fmt.Errorf("unsupported socks version: 0x%02x", resp[0])
|
|
}
|
|
|
|
// 2. Authentication if required (RFC 1929)
|
|
if resp[1] == 0x02 {
|
|
if cfg.Username == "" {
|
|
conn.Close()
|
|
return nil, fmt.Errorf("socks5 proxy requires authentication, but no credentials provided")
|
|
}
|
|
uLen := byte(len(cfg.Username))
|
|
pLen := byte(len(cfg.Password))
|
|
authReq := []byte{0x01, uLen}
|
|
authReq = append(authReq, []byte(cfg.Username)...)
|
|
authReq = append(authReq, pLen)
|
|
authReq = append(authReq, []byte(cfg.Password)...)
|
|
|
|
if _, err := conn.Write(authReq); err != nil {
|
|
conn.Close()
|
|
return nil, fmt.Errorf("failed to send socks5 authentication: %w", err)
|
|
}
|
|
|
|
authResp := make([]byte, 2)
|
|
if _, err := io.ReadFull(conn, authResp); err != nil {
|
|
conn.Close()
|
|
return nil, fmt.Errorf("failed to read socks5 auth response: %w", err)
|
|
}
|
|
if authResp[1] != 0x00 {
|
|
conn.Close()
|
|
return nil, fmt.Errorf("socks5 authentication failed with status 0x%02x", authResp[1])
|
|
}
|
|
} else if resp[1] != 0x00 {
|
|
conn.Close()
|
|
return nil, fmt.Errorf("socks5 proxy rejected authentication methods: 0x%02x", resp[1])
|
|
}
|
|
|
|
// 3. Connection Request (CONNECT command)
|
|
host, portStr, err := net.SplitHostPort(targetAddr)
|
|
if err != nil {
|
|
conn.Close()
|
|
return nil, fmt.Errorf("invalid target address %s: %w", targetAddr, err)
|
|
}
|
|
|
|
port, err := strconv.Atoi(portStr)
|
|
if err != nil || port < 1 || port > 65535 {
|
|
conn.Close()
|
|
return nil, fmt.Errorf("invalid port in target address %s", targetAddr)
|
|
}
|
|
|
|
reqBuf := []byte{0x05, 0x01, 0x00}
|
|
ip := net.ParseIP(host)
|
|
if ip4 := ip.To4(); ip4 != nil {
|
|
reqBuf = append(reqBuf, 0x01)
|
|
reqBuf = append(reqBuf, ip4...)
|
|
} else if ip6 := ip.To16(); ip6 != nil {
|
|
reqBuf = append(reqBuf, 0x04)
|
|
reqBuf = append(reqBuf, ip6...)
|
|
} else {
|
|
reqBuf = append(reqBuf, 0x03, byte(len(host)))
|
|
reqBuf = append(reqBuf, []byte(host)...)
|
|
}
|
|
reqBuf = append(reqBuf, byte(port>>8), byte(port&0xFF))
|
|
|
|
if _, err := conn.Write(reqBuf); err != nil {
|
|
conn.Close()
|
|
return nil, fmt.Errorf("failed to send socks5 connect request: %w", err)
|
|
}
|
|
|
|
// 4. Connection Response
|
|
respHdr := make([]byte, 4)
|
|
if _, err := io.ReadFull(conn, respHdr); err != nil {
|
|
conn.Close()
|
|
return nil, fmt.Errorf("failed to read socks5 connect response: %w", err)
|
|
}
|
|
|
|
if respHdr[1] != 0x00 {
|
|
conn.Close()
|
|
return nil, fmt.Errorf("socks5 connect failed with reply code 0x%02x", respHdr[1])
|
|
}
|
|
|
|
switch respHdr[3] {
|
|
case 0x01:
|
|
bnd := make([]byte, 6)
|
|
io.ReadFull(conn, bnd)
|
|
case 0x03:
|
|
lenBuf := make([]byte, 1)
|
|
io.ReadFull(conn, lenBuf)
|
|
bnd := make([]byte, int(lenBuf[0])+2)
|
|
io.ReadFull(conn, bnd)
|
|
case 0x04:
|
|
bnd := make([]byte, 18)
|
|
io.ReadFull(conn, bnd)
|
|
}
|
|
|
|
return conn, nil
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Helper utilities
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func GenerateUUID() string {
|
|
var b [16]byte
|
|
_, err := rand.Read(b[:])
|
|
if err != nil {
|
|
return fmt.Sprintf("%d", time.Now().UnixNano())
|
|
}
|
|
b[6] = (b[6] & 0x0f) | 0x40
|
|
b[8] = (b[8] & 0x3f) | 0x80
|
|
return fmt.Sprintf("%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:])
|
|
}
|
|
|
|
func GenerateSessionHash() string {
|
|
const chars = "abcdefghijklmnopqrstuvwxyz0123456789"
|
|
var b [12]byte
|
|
rand.Read(b[:])
|
|
var sb strings.Builder
|
|
for _, v := range b {
|
|
sb.WriteByte(chars[int(v)%len(chars)])
|
|
}
|
|
return sb.String()
|
|
}
|
|
|
|
func FibonacciDelay(attempt int) time.Duration {
|
|
if attempt <= 0 {
|
|
return 1 * time.Second
|
|
}
|
|
a, b := 1, 1
|
|
for i := 1; i < attempt; i++ {
|
|
a, b = b, a+b
|
|
}
|
|
return time.Duration(a) * time.Second
|
|
}
|
|
|
|
func DoWithFibonacciRetry(client *http.Client, makeReq func() (*http.Request, error), maxRetries int) (*http.Response, error) {
|
|
var lastErr error
|
|
for attempt := 1; attempt <= maxRetries; attempt++ {
|
|
req, err := makeReq()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
resp, err := client.Do(req)
|
|
if err == nil && resp.StatusCode == http.StatusOK {
|
|
return resp, nil
|
|
}
|
|
|
|
if resp != nil {
|
|
respBody, _ := io.ReadAll(resp.Body)
|
|
resp.Body.Close()
|
|
lastErr = fmt.Errorf("HTTP status %d: %s", resp.StatusCode, string(respBody))
|
|
if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusMethodNotAllowed {
|
|
break
|
|
}
|
|
if _, ok := extractFailedGeneration(string(respBody)); ok {
|
|
break
|
|
}
|
|
} else {
|
|
lastErr = err
|
|
}
|
|
|
|
if attempt < maxRetries {
|
|
delay := FibonacciDelay(attempt)
|
|
time.Sleep(delay)
|
|
}
|
|
}
|
|
return nil, fmt.Errorf("request failed after %d retries: %v", maxRetries, lastErr)
|
|
}
|
|
|
|
func EnableCORS(w http.ResponseWriter) {
|
|
w.Header().Set("Access-Control-Allow-Origin", "*")
|
|
w.Header().Set("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
|
|
w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Authorization, api-key, X-User-Agent, X-Gradio-Space, X-Space-URL")
|
|
}
|
|
|
|
func ResolveMaxTokens(req ChatCompletionRequest) int {
|
|
mt := req.MaxTokens
|
|
if mt == 0 && req.MaxCompletionTokens > 0 {
|
|
mt = req.MaxCompletionTokens
|
|
}
|
|
if mt <= 0 {
|
|
mt = 131072
|
|
}
|
|
return mt
|
|
}
|
|
|
|
func EffectiveUserAgent(r *http.Request) string {
|
|
if r != nil {
|
|
if c := r.Header.Get("X-User-Agent"); c != "" {
|
|
return c
|
|
}
|
|
}
|
|
if ConfiguredUserAgent != "" {
|
|
return ConfiguredUserAgent
|
|
}
|
|
return DefaultUserAgent
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Tool and message processing
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func BuildToolInstruction(tools []Tool, toolChoice interface{}) string {
|
|
if len(tools) == 0 {
|
|
return ""
|
|
}
|
|
if tcStr, ok := toolChoice.(string); ok && tcStr == "none" {
|
|
return ""
|
|
}
|
|
|
|
var cleanToolDefs []map[string]interface{}
|
|
sampleFnName := "function_name"
|
|
sampleArgs := `{"param1": "value1"}`
|
|
|
|
for _, t := range tools {
|
|
if fnMap, ok := t.Function.(map[string]interface{}); ok {
|
|
cleanToolDefs = append(cleanToolDefs, fnMap)
|
|
if sampleFnName == "function_name" {
|
|
if n, ok := fnMap["name"].(string); ok && n != "" {
|
|
sampleFnName = n
|
|
if params, ok := fnMap["parameters"].(map[string]interface{}); ok {
|
|
if props, ok := params["properties"].(map[string]interface{}); ok {
|
|
sampleArgMap := make(map[string]interface{})
|
|
for propName := range props {
|
|
sampleArgMap[propName] = "value"
|
|
break
|
|
}
|
|
if len(sampleArgMap) > 0 {
|
|
if b, err := json.Marshal(sampleArgMap); err == nil {
|
|
sampleArgs = string(b)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
cleanToolDefs = append(cleanToolDefs, map[string]interface{}{
|
|
"type": t.Type,
|
|
"function": t.Function,
|
|
})
|
|
}
|
|
}
|
|
|
|
toolsBytes, _ := json.MarshalIndent(cleanToolDefs, "", " ")
|
|
|
|
directive := "If a tool is relevant, emit the tool call in <tool_call> tags with JSON content. If no tools are relevant, answer the query directly."
|
|
if tcStr, ok := toolChoice.(string); ok && tcStr == "required" {
|
|
directive = "You MUST call one of the available tools for this query and emit the tool call in <tool_call> tags with JSON content."
|
|
} else if tcMap, ok := toolChoice.(map[string]interface{}); ok {
|
|
if fnMap, ok := tcMap["function"].(map[string]interface{}); ok {
|
|
if fnName, ok := fnMap["name"].(string); ok && fnName != "" {
|
|
directive = fmt.Sprintf("You MUST call the %s tool for this query and emit the tool call in <tool_call> tags with JSON content.", fnName)
|
|
}
|
|
}
|
|
}
|
|
|
|
return fmt.Sprintf(`You are an API router and assistant. Convert the user query into the appropriate tool call using the available tools.
|
|
Available Tools:
|
|
%s
|
|
|
|
Syntax:
|
|
<tool_call>
|
|
{"name": "%s", "arguments": %s}
|
|
</tool_call>
|
|
|
|
When a tool result is returned in <tool_response>, formulate your answer based on it.
|
|
%s`, string(toolsBytes), sampleFnName, sampleArgs, directive)
|
|
}
|
|
|
|
func TransformMessages(req ChatCompletionRequest) (processed []ChatMessage, toolInstruction string, hasSystem bool) {
|
|
// 1. Build lookup map from tool_call_id to function name across all assistant messages
|
|
toolIDToName := make(map[string]string)
|
|
for _, msg := range req.Messages {
|
|
for _, tc := range msg.ToolCalls {
|
|
if tc.ID != "" && tc.Function.Name != "" {
|
|
toolIDToName[tc.ID] = tc.Function.Name
|
|
}
|
|
}
|
|
}
|
|
|
|
toolInstruction = BuildToolInstruction(req.Tools, req.ToolChoice)
|
|
|
|
// 2. Process and coalesce messages preserving turn parity
|
|
var staged []ChatMessage
|
|
for i := 0; i < len(req.Messages); i++ {
|
|
msg := req.Messages[i]
|
|
contentStr := msg.GetContentString()
|
|
|
|
switch msg.Role {
|
|
case "system":
|
|
hasSystem = true
|
|
staged = append(staged, ChatMessage{Role: "system", Content: contentStr})
|
|
|
|
case "assistant":
|
|
var sb strings.Builder
|
|
if contentStr != "" {
|
|
sb.WriteString(contentStr)
|
|
}
|
|
for _, tc := range msg.ToolCalls {
|
|
if sb.Len() > 0 {
|
|
sb.WriteString("\n")
|
|
}
|
|
args := tc.Function.Arguments
|
|
if strings.TrimSpace(args) == "" {
|
|
args = "{}"
|
|
}
|
|
sb.WriteString(fmt.Sprintf("<tool_call>\n{\"name\": %q, \"arguments\": %s}\n</tool_call>", tc.Function.Name, args))
|
|
}
|
|
staged = append(staged, ChatMessage{
|
|
Role: "assistant",
|
|
Content: sb.String(),
|
|
ReasoningContent: msg.ReasoningContent,
|
|
ToolCalls: msg.ToolCalls,
|
|
})
|
|
|
|
case "tool", "function":
|
|
// Gather consecutive tool returns into a coalesced turn
|
|
var toolResponses []string
|
|
j := i
|
|
for j < len(req.Messages) && (req.Messages[j].Role == "tool" || req.Messages[j].Role == "function") {
|
|
tMsg := req.Messages[j]
|
|
tContent := tMsg.GetContentString()
|
|
tName := tMsg.Name
|
|
if tName == "" && tMsg.ToolCallID != "" {
|
|
if mapped, ok := toolIDToName[tMsg.ToolCallID]; ok {
|
|
tName = mapped
|
|
} else {
|
|
tName = tMsg.ToolCallID
|
|
}
|
|
}
|
|
var contentJSON []byte
|
|
if json.Valid([]byte(tContent)) {
|
|
contentJSON = []byte(tContent)
|
|
} else {
|
|
contentJSON, _ = json.Marshal(tContent)
|
|
}
|
|
toolResponses = append(toolResponses, fmt.Sprintf("<tool_response>\n{\"name\": %q, \"content\": %s}\n</tool_response>", tName, string(contentJSON)))
|
|
j++
|
|
}
|
|
i = j - 1 // advance loop
|
|
|
|
promptSuffix := "Please answer the user's request based on the tool result."
|
|
if len(toolResponses) > 1 {
|
|
promptSuffix = "Please answer the user's request based on the tool results."
|
|
}
|
|
coalesced := strings.Join(toolResponses, "\n") + "\n\n" + promptSuffix
|
|
staged = append(staged, ChatMessage{
|
|
Role: "user",
|
|
Content: coalesced,
|
|
})
|
|
|
|
default: // "user" or other roles
|
|
staged = append(staged, ChatMessage{Role: msg.Role, Content: contentStr})
|
|
}
|
|
}
|
|
|
|
// 3. Inject tool instructions into system prompt
|
|
if toolInstruction != "" {
|
|
if hasSystem {
|
|
systemInjected := false
|
|
for i, m := range staged {
|
|
if m.Role == "system" {
|
|
staged[i].Content = m.GetContentString() + "\n\n" + strings.TrimSpace(toolInstruction)
|
|
systemInjected = true
|
|
break
|
|
}
|
|
}
|
|
if !systemInjected {
|
|
staged = append([]ChatMessage{
|
|
{Role: "system", Content: strings.TrimSpace(toolInstruction)},
|
|
}, staged...)
|
|
}
|
|
} else {
|
|
staged = append([]ChatMessage{
|
|
{Role: "system", Content: strings.TrimSpace(toolInstruction)},
|
|
}, staged...)
|
|
hasSystem = true
|
|
}
|
|
}
|
|
|
|
return staged, toolInstruction, hasSystem
|
|
}
|
|
|
|
func cleanJSONBlock(input string) string {
|
|
s := strings.TrimSpace(input)
|
|
if strings.HasPrefix(s, "```") {
|
|
lines := strings.Split(s, "\n")
|
|
if len(lines) >= 2 {
|
|
if strings.HasPrefix(lines[len(lines)-1], "```") {
|
|
lines = lines[1 : len(lines)-1]
|
|
} else {
|
|
lines = lines[1:]
|
|
}
|
|
s = strings.TrimSpace(strings.Join(lines, "\n"))
|
|
}
|
|
}
|
|
return s
|
|
}
|
|
|
|
func sanitizeJSONValue(v interface{}) interface{} {
|
|
switch val := v.(type) {
|
|
case string:
|
|
return strings.TrimSpace(val)
|
|
case map[string]interface{}:
|
|
res := make(map[string]interface{})
|
|
for k, item := range val {
|
|
res[k] = sanitizeJSONValue(item)
|
|
}
|
|
return res
|
|
case []interface{}:
|
|
res := make([]interface{}, len(val))
|
|
for i, item := range val {
|
|
res[i] = sanitizeJSONValue(item)
|
|
}
|
|
return res
|
|
default:
|
|
return v
|
|
}
|
|
}
|
|
|
|
type ToolTagPair struct {
|
|
Start string
|
|
End string
|
|
}
|
|
|
|
var ToolTagPairs = []ToolTagPair{
|
|
{Start: "<tool_call>", End: "</tool_call>"},
|
|
{Start: "<tool_calls>", End: "</tool_calls>"},
|
|
{Start: "<toolCall>", End: "</toolCall>"},
|
|
{Start: "<toolCalls>", End: "</toolCalls>"},
|
|
{Start: "<function_call>", End: "</function_call>"},
|
|
{Start: "<functionCall>", End: "</functionCall>"},
|
|
{Start: "<function_calls>", End: "</function_calls>"},
|
|
{Start: "<functionCalls>", End: "</functionCalls>"},
|
|
{Start: "<invoke>", End: "</invoke>"},
|
|
{Start: "<call>", End: "</call>"},
|
|
{Start: "<command>", End: "</command>"},
|
|
{Start: "<commands>", End: "</commands>"},
|
|
{Start: "<action>", End: "</action>"},
|
|
{Start: "<function>", End: "</function>"},
|
|
{Start: "[TOOL_CALLS]", End: "[/TOOL_CALLS]"},
|
|
{Start: "[TOOL_CALL]", End: "[/TOOL_CALL]"},
|
|
}
|
|
|
|
func getToolStartPrefixes() []string {
|
|
seen := make(map[string]bool)
|
|
var prefixes []string
|
|
for _, pair := range ToolTagPairs {
|
|
for i := 1; i <= len(pair.Start); i++ {
|
|
pref := pair.Start[:i]
|
|
if !seen[pref] {
|
|
seen[pref] = true
|
|
prefixes = append(prefixes, pref)
|
|
}
|
|
}
|
|
}
|
|
extraStarts := []string{
|
|
"<tool_call",
|
|
"<toolCall",
|
|
"<function_call",
|
|
"<functionCall",
|
|
"<invoke",
|
|
"<call:",
|
|
"<call ",
|
|
"<tool_call:",
|
|
"<function=",
|
|
"<command",
|
|
"<action",
|
|
"Action:",
|
|
"Command:",
|
|
}
|
|
for _, tag := range extraStarts {
|
|
for i := 1; i <= len(tag); i++ {
|
|
pref := tag[:i]
|
|
if !seen[pref] {
|
|
seen[pref] = true
|
|
prefixes = append(prefixes, pref)
|
|
}
|
|
}
|
|
}
|
|
return prefixes
|
|
}
|
|
|
|
var toolStartPrefixes = getToolStartPrefixes()
|
|
|
|
func repairPythonJSON(s string) string {
|
|
reTrue := regexp.MustCompile(`:\s*True\b`)
|
|
s = reTrue.ReplaceAllString(s, ": true")
|
|
reFalse := regexp.MustCompile(`:\s*False\b`)
|
|
s = reFalse.ReplaceAllString(s, ": false")
|
|
reNone := regexp.MustCompile(`:\s*None\b`)
|
|
s = reNone.ReplaceAllString(s, ": null")
|
|
reTrueList := regexp.MustCompile(`([,\[])\s*True\b`)
|
|
s = reTrueList.ReplaceAllString(s, "$1 true")
|
|
reFalseList := regexp.MustCompile(`([,\[])\s*False\b`)
|
|
s = reFalseList.ReplaceAllString(s, "$1 false")
|
|
reNoneList := regexp.MustCompile(`([,\[])\s*None\b`)
|
|
s = reNoneList.ReplaceAllString(s, "$1 null")
|
|
|
|
if strings.Contains(s, "'") {
|
|
var sb strings.Builder
|
|
inDouble := false
|
|
inSingle := false
|
|
escaped := false
|
|
for i := 0; i < len(s); i++ {
|
|
c := s[i]
|
|
if escaped {
|
|
if c == '\'' && inSingle {
|
|
sb.WriteByte('\'')
|
|
} else {
|
|
sb.WriteByte('\\')
|
|
sb.WriteByte(c)
|
|
}
|
|
escaped = false
|
|
continue
|
|
}
|
|
if c == '\\' {
|
|
escaped = true
|
|
continue
|
|
}
|
|
if c == '"' && !inSingle {
|
|
inDouble = !inDouble
|
|
sb.WriteByte(c)
|
|
continue
|
|
}
|
|
if c == '\'' && !inDouble {
|
|
inSingle = !inSingle
|
|
sb.WriteByte('"')
|
|
continue
|
|
}
|
|
sb.WriteByte(c)
|
|
}
|
|
if escaped {
|
|
sb.WriteByte('\\')
|
|
}
|
|
s = sb.String()
|
|
}
|
|
return s
|
|
}
|
|
|
|
func parsePythonKwargs(argsStr string) (map[string]interface{}, bool) {
|
|
s := strings.TrimSpace(argsStr)
|
|
if s == "" {
|
|
return make(map[string]interface{}), true
|
|
}
|
|
|
|
res := make(map[string]interface{})
|
|
i := 0
|
|
n := len(s)
|
|
|
|
for i < n {
|
|
for i < n && (s[i] == ' ' || s[i] == '\t' || s[i] == '\r' || s[i] == '\n' || s[i] == ',') {
|
|
i++
|
|
}
|
|
if i >= n {
|
|
break
|
|
}
|
|
|
|
keyStart := i
|
|
for i < n && ((s[i] >= 'a' && s[i] <= 'z') || (s[i] >= 'A' && s[i] <= 'Z') || (s[i] >= '0' && s[i] <= '9') || s[i] == '_') {
|
|
i++
|
|
}
|
|
key := s[keyStart:i]
|
|
if key == "" {
|
|
break
|
|
}
|
|
|
|
for i < n && (s[i] == ' ' || s[i] == '\t') {
|
|
i++
|
|
}
|
|
if i >= n || s[i] != '=' {
|
|
break
|
|
}
|
|
i++
|
|
|
|
for i < n && (s[i] == ' ' || s[i] == '\t') {
|
|
i++
|
|
}
|
|
if i >= n {
|
|
break
|
|
}
|
|
|
|
if s[i] == '"' || s[i] == '\'' {
|
|
quote := s[i]
|
|
i++
|
|
var valBuilder strings.Builder
|
|
escaped := false
|
|
for i < n {
|
|
c := s[i]
|
|
if escaped {
|
|
valBuilder.WriteByte(c)
|
|
escaped = false
|
|
i++
|
|
continue
|
|
}
|
|
if c == '\\' {
|
|
escaped = true
|
|
i++
|
|
continue
|
|
}
|
|
if c == quote {
|
|
break
|
|
}
|
|
valBuilder.WriteByte(c)
|
|
i++
|
|
}
|
|
if i < n && s[i] == quote {
|
|
i++
|
|
}
|
|
res[key] = valBuilder.String()
|
|
} else if s[i] == '{' || s[i] == '[' {
|
|
openChar := s[i]
|
|
closeChar := byte('}')
|
|
if openChar == '[' {
|
|
closeChar = ']'
|
|
}
|
|
valStart := i
|
|
depth := 0
|
|
inStr := false
|
|
var strQuote byte
|
|
escaped := false
|
|
for i < n {
|
|
c := s[i]
|
|
if escaped {
|
|
escaped = false
|
|
i++
|
|
continue
|
|
}
|
|
if c == '\\' {
|
|
escaped = true
|
|
i++
|
|
continue
|
|
}
|
|
if inStr {
|
|
if c == strQuote {
|
|
inStr = false
|
|
}
|
|
i++
|
|
continue
|
|
}
|
|
if c == '"' || c == '\'' {
|
|
inStr = true
|
|
strQuote = c
|
|
i++
|
|
continue
|
|
}
|
|
if c == openChar {
|
|
depth++
|
|
} else if c == closeChar {
|
|
depth--
|
|
if depth == 0 {
|
|
i++
|
|
break
|
|
}
|
|
}
|
|
i++
|
|
}
|
|
subStr := s[valStart:i]
|
|
var subJSON interface{}
|
|
cleanedSub := repairPythonJSON(subStr)
|
|
if json.Unmarshal([]byte(cleanedSub), &subJSON) == nil {
|
|
res[key] = subJSON
|
|
} else {
|
|
res[key] = subStr
|
|
}
|
|
} else {
|
|
valStart := i
|
|
for i < n && s[i] != ',' && s[i] != ')' && s[i] != '\n' {
|
|
i++
|
|
}
|
|
rawVal := strings.TrimSpace(s[valStart:i])
|
|
if rawVal == "True" || rawVal == "true" {
|
|
res[key] = true
|
|
} else if rawVal == "False" || rawVal == "false" {
|
|
res[key] = false
|
|
} else if rawVal == "None" || rawVal == "null" {
|
|
res[key] = nil
|
|
} else if num, err := strconv.ParseFloat(rawVal, 64); err == nil {
|
|
res[key] = num
|
|
} else {
|
|
res[key] = rawVal
|
|
}
|
|
}
|
|
}
|
|
|
|
return res, len(res) > 0
|
|
}
|
|
|
|
func parsePythonFunctionCall(input string) (ToolCall, bool) {
|
|
s := strings.TrimSpace(input)
|
|
if strings.HasPrefix(s, "```") {
|
|
s = cleanJSONBlock(s)
|
|
}
|
|
reCall := regexp.MustCompile(`^(?:tools\.|functions\.)?([a-zA-Z_][a-zA-Z0-9_]*)\s*\(([\s\S]*)\)$`)
|
|
matches := reCall.FindStringSubmatch(s)
|
|
if len(matches) < 3 {
|
|
return ToolCall{}, false
|
|
}
|
|
|
|
fnName := matches[1]
|
|
rawArgs := strings.TrimSpace(matches[2])
|
|
|
|
if rawArgs == "" {
|
|
return ToolCall{
|
|
ID: "call_" + GenerateUUID()[:8],
|
|
Type: "function",
|
|
Function: ToolCallFunction{
|
|
Name: fnName,
|
|
Arguments: "{}",
|
|
},
|
|
}, true
|
|
}
|
|
|
|
if strings.HasPrefix(rawArgs, "{") && strings.HasSuffix(rawArgs, "}") {
|
|
repaired := repairPythonJSON(rawArgs)
|
|
var dummy map[string]interface{}
|
|
if json.Unmarshal([]byte(repaired), &dummy) == nil {
|
|
b, _ := json.Marshal(dummy)
|
|
return ToolCall{
|
|
ID: "call_" + GenerateUUID()[:8],
|
|
Type: "function",
|
|
Function: ToolCallFunction{
|
|
Name: fnName,
|
|
Arguments: string(b),
|
|
},
|
|
}, true
|
|
}
|
|
}
|
|
|
|
argMap, ok := parsePythonKwargs(rawArgs)
|
|
if ok && len(argMap) > 0 {
|
|
b, err := json.Marshal(argMap)
|
|
if err == nil {
|
|
return ToolCall{
|
|
ID: "call_" + GenerateUUID()[:8],
|
|
Type: "function",
|
|
Function: ToolCallFunction{
|
|
Name: fnName,
|
|
Arguments: string(b),
|
|
},
|
|
}, true
|
|
}
|
|
}
|
|
|
|
if (strings.HasPrefix(rawArgs, `"`) && strings.HasSuffix(rawArgs, `"`)) ||
|
|
(strings.HasPrefix(rawArgs, `'`) && strings.HasSuffix(rawArgs, `'`)) {
|
|
val := rawArgs[1 : len(rawArgs)-1]
|
|
b, _ := json.Marshal(map[string]interface{}{"input": val})
|
|
return ToolCall{
|
|
ID: "call_" + GenerateUUID()[:8],
|
|
Type: "function",
|
|
Function: ToolCallFunction{
|
|
Name: fnName,
|
|
Arguments: string(b),
|
|
},
|
|
}, true
|
|
}
|
|
|
|
return ToolCall{}, false
|
|
}
|
|
|
|
func parseMultiplePythonFunctionCalls(input string) ([]ToolCall, bool) {
|
|
s := strings.TrimSpace(input)
|
|
if strings.HasPrefix(s, "[") && strings.HasSuffix(s, "]") {
|
|
s = s[1 : len(s)-1]
|
|
}
|
|
reCalls := regexp.MustCompile(`(?:tools\.|functions\.)?([a-zA-Z_][a-zA-Z0-9_]*)\s*\(([^)]*)\)`)
|
|
matches := reCalls.FindAllString(s, -1)
|
|
if len(matches) == 0 {
|
|
return nil, false
|
|
}
|
|
var calls []ToolCall
|
|
for _, m := range matches {
|
|
if tc, ok := parsePythonFunctionCall(m); ok {
|
|
calls = append(calls, tc)
|
|
}
|
|
}
|
|
if len(calls) > 0 {
|
|
return calls, true
|
|
}
|
|
return nil, false
|
|
}
|
|
|
|
func parseReActToolCall(input string) (ToolCall, bool) {
|
|
reAction := regexp.MustCompile(`(?i)(?:Action|Command):\s*([a-zA-Z0-9_.-]+)\s*\n+(?:Action Input|Arguments|Parameters|Args):\s*(\{[\s\S]*?\}|\[[\s\S]*?\]|[^\n]+)`)
|
|
matches := reAction.FindStringSubmatch(input)
|
|
if len(matches) < 3 {
|
|
return ToolCall{}, false
|
|
}
|
|
|
|
fnName := strings.TrimSpace(matches[1])
|
|
fnName = strings.TrimPrefix(fnName, "tools.")
|
|
fnName = strings.TrimPrefix(fnName, "functions.")
|
|
rawArgs := strings.TrimSpace(matches[2])
|
|
|
|
if strings.HasPrefix(rawArgs, "{") && strings.HasSuffix(rawArgs, "}") {
|
|
repaired := repairPythonJSON(rawArgs)
|
|
var dummy map[string]interface{}
|
|
if json.Unmarshal([]byte(repaired), &dummy) == nil {
|
|
b, _ := json.Marshal(dummy)
|
|
return ToolCall{
|
|
ID: "call_" + GenerateUUID()[:8],
|
|
Type: "function",
|
|
Function: ToolCallFunction{
|
|
Name: fnName,
|
|
Arguments: string(b),
|
|
},
|
|
}, true
|
|
}
|
|
}
|
|
|
|
if argMap, ok := parsePythonKwargs(rawArgs); ok && len(argMap) > 0 {
|
|
b, _ := json.Marshal(argMap)
|
|
return ToolCall{
|
|
ID: "call_" + GenerateUUID()[:8],
|
|
Type: "function",
|
|
Function: ToolCallFunction{
|
|
Name: fnName,
|
|
Arguments: string(b),
|
|
},
|
|
}, true
|
|
}
|
|
|
|
b, _ := json.Marshal(map[string]interface{}{"input": rawArgs})
|
|
return ToolCall{
|
|
ID: "call_" + GenerateUUID()[:8],
|
|
Type: "function",
|
|
Function: ToolCallFunction{
|
|
Name: fnName,
|
|
Arguments: string(b),
|
|
},
|
|
}, true
|
|
}
|
|
|
|
func repairToolCallJSON(input string) (ToolCall, bool) {
|
|
s := strings.TrimSpace(input)
|
|
if !strings.HasPrefix(s, "{") || !strings.HasSuffix(s, "}") {
|
|
return ToolCall{}, false
|
|
}
|
|
reName := regexp.MustCompile(`"(?:name|function|function_name|action|command|call|tool|tool_name)"\s*:\s*"([^"]+)"`)
|
|
matches := reName.FindStringSubmatch(s)
|
|
if len(matches) < 2 {
|
|
return ToolCall{}, false
|
|
}
|
|
fnName := matches[1]
|
|
|
|
reArgsObj := regexp.MustCompile(`"(?:arguments|parameters|params|args|input|action_input|inputs|properties)"\s*:\s*(\{[\s\S]*\})`)
|
|
argMatches := reArgsObj.FindStringSubmatch(s)
|
|
argsStr := "{}"
|
|
if len(argMatches) >= 2 {
|
|
candidate := argMatches[1]
|
|
var dummy map[string]interface{}
|
|
if json.Unmarshal([]byte(candidate), &dummy) == nil {
|
|
argsStr = candidate
|
|
} else if json.Unmarshal([]byte(repairPythonJSON(candidate)), &dummy) == nil {
|
|
b, _ := json.Marshal(dummy)
|
|
argsStr = string(b)
|
|
}
|
|
} else {
|
|
reArgsStr := regexp.MustCompile(`"(?:arguments|parameters|params|args|input|action_input|inputs|properties)"\s*:\s*"((?:\\.|[^"\\])*)"`)
|
|
strMatches := reArgsStr.FindStringSubmatch(s)
|
|
if len(strMatches) >= 2 {
|
|
var unescaped string
|
|
if json.Unmarshal([]byte(`"`+strMatches[1]+`"`), &unescaped) == nil {
|
|
argsStr = unescaped
|
|
}
|
|
}
|
|
}
|
|
|
|
return ToolCall{
|
|
ID: "call_" + GenerateUUID()[:8],
|
|
Type: "function",
|
|
Function: ToolCallFunction{
|
|
Name: fnName,
|
|
Arguments: argsStr,
|
|
},
|
|
}, true
|
|
}
|
|
|
|
func parseSingleToolCall(jsonStr string) (ToolCall, bool) {
|
|
cleaned := cleanJSONBlock(jsonStr)
|
|
var raw map[string]interface{}
|
|
err := json.Unmarshal([]byte(cleaned), &raw)
|
|
if err != nil {
|
|
repaired := repairPythonJSON(cleaned)
|
|
err = json.Unmarshal([]byte(repaired), &raw)
|
|
}
|
|
if err == nil {
|
|
sanitizedRaw, ok := sanitizeJSONValue(raw).(map[string]interface{})
|
|
if !ok {
|
|
sanitizedRaw = raw
|
|
}
|
|
|
|
for _, wrapperKey := range []string{"function", "function_call", "tool_call", "tool", "call", "command", "action"} {
|
|
if fnObj, ok := sanitizedRaw[wrapperKey].(map[string]interface{}); ok {
|
|
nameVal := ""
|
|
for _, nk := range []string{"name", "function", "function_name", "action", "command", "call", "tool", "tool_name"} {
|
|
if n, ok := fnObj[nk].(string); ok && n != "" {
|
|
nameVal = n
|
|
break
|
|
}
|
|
}
|
|
if nameVal != "" {
|
|
argsStr := "{}"
|
|
var argsVal interface{}
|
|
for _, ak := range []string{"arguments", "parameters", "params", "args", "input", "action_input", "inputs", "properties"} {
|
|
if a, hasA := fnObj[ak]; hasA {
|
|
argsVal = a
|
|
break
|
|
}
|
|
}
|
|
if argsVal != nil {
|
|
if s, isStr := argsVal.(string); isStr {
|
|
var innerObj interface{}
|
|
if json.Unmarshal([]byte(s), &innerObj) == nil {
|
|
b, _ := json.Marshal(sanitizeJSONValue(innerObj))
|
|
argsStr = string(b)
|
|
} else if json.Unmarshal([]byte(repairPythonJSON(s)), &innerObj) == nil {
|
|
b, _ := json.Marshal(sanitizeJSONValue(innerObj))
|
|
argsStr = string(b)
|
|
} else {
|
|
argsStr = strings.TrimSpace(s)
|
|
}
|
|
} else {
|
|
b, _ := json.Marshal(sanitizeJSONValue(argsVal))
|
|
argsStr = string(b)
|
|
}
|
|
}
|
|
return ToolCall{
|
|
ID: "call_" + GenerateUUID()[:8],
|
|
Type: "function",
|
|
Function: ToolCallFunction{
|
|
Name: nameVal,
|
|
Arguments: argsStr,
|
|
},
|
|
}, true
|
|
}
|
|
}
|
|
}
|
|
|
|
nameVal := ""
|
|
for _, key := range []string{"name", "function", "function_name", "tool", "tool_name", "action", "command", "call"} {
|
|
if n, ok := sanitizedRaw[key].(string); ok && n != "" {
|
|
nameVal = n
|
|
break
|
|
}
|
|
}
|
|
|
|
if nameVal != "" {
|
|
argsStr := "{}"
|
|
var argsVal interface{}
|
|
for _, key := range []string{"arguments", "parameters", "params", "args", "input", "action_input", "inputs", "properties"} {
|
|
if a, ok := sanitizedRaw[key]; ok {
|
|
argsVal = a
|
|
break
|
|
}
|
|
}
|
|
if argsVal != nil {
|
|
if s, isStr := argsVal.(string); isStr {
|
|
var innerObj interface{}
|
|
if json.Unmarshal([]byte(s), &innerObj) == nil {
|
|
b, _ := json.Marshal(sanitizeJSONValue(innerObj))
|
|
argsStr = string(b)
|
|
} else if json.Unmarshal([]byte(repairPythonJSON(s)), &innerObj) == nil {
|
|
b, _ := json.Marshal(sanitizeJSONValue(innerObj))
|
|
argsStr = string(b)
|
|
} else {
|
|
argsStr = strings.TrimSpace(s)
|
|
}
|
|
} else {
|
|
b, _ := json.Marshal(sanitizeJSONValue(argsVal))
|
|
argsStr = string(b)
|
|
}
|
|
} else {
|
|
argsMap := make(map[string]interface{})
|
|
for k, v := range sanitizedRaw {
|
|
if k != "name" && k != "function" && k != "function_name" && k != "type" && k != "action" && k != "command" && k != "call" && k != "tool" && k != "tool_name" && k != "id" {
|
|
argsMap[k] = v
|
|
}
|
|
}
|
|
if len(argsMap) > 0 {
|
|
b, _ := json.Marshal(sanitizeJSONValue(argsMap))
|
|
argsStr = string(b)
|
|
}
|
|
}
|
|
return ToolCall{
|
|
ID: "call_" + GenerateUUID()[:8],
|
|
Type: "function",
|
|
Function: ToolCallFunction{
|
|
Name: nameVal,
|
|
Arguments: argsStr,
|
|
},
|
|
}, true
|
|
}
|
|
}
|
|
|
|
if tc, ok := repairToolCallJSON(cleaned); ok {
|
|
return tc, true
|
|
}
|
|
if tc, ok := parsePythonFunctionCall(cleaned); ok {
|
|
return tc, true
|
|
}
|
|
return parseReActToolCall(cleaned)
|
|
}
|
|
|
|
func parseMultipleToolCalls(raw string) ([]ToolCall, bool) {
|
|
cleaned := cleanJSONBlock(raw)
|
|
if cleaned == "" {
|
|
return nil, false
|
|
}
|
|
|
|
var rawList []interface{}
|
|
err := json.Unmarshal([]byte(cleaned), &rawList)
|
|
if err != nil {
|
|
err = json.Unmarshal([]byte(repairPythonJSON(cleaned)), &rawList)
|
|
}
|
|
if err == nil {
|
|
var calls []ToolCall
|
|
for _, item := range rawList {
|
|
b, err := json.Marshal(item)
|
|
if err == nil {
|
|
if tc, ok := parseSingleToolCall(string(b)); ok {
|
|
calls = append(calls, tc)
|
|
}
|
|
}
|
|
}
|
|
if len(calls) > 0 {
|
|
return calls, true
|
|
}
|
|
}
|
|
|
|
var rawMap map[string]interface{}
|
|
err = json.Unmarshal([]byte(cleaned), &rawMap)
|
|
if err != nil {
|
|
err = json.Unmarshal([]byte(repairPythonJSON(cleaned)), &rawMap)
|
|
}
|
|
if err == nil {
|
|
for _, listKey := range []string{"tool_calls", "calls", "functions", "actions", "commands"} {
|
|
if subArr, ok := rawMap[listKey].([]interface{}); ok && len(subArr) > 0 {
|
|
var calls []ToolCall
|
|
for _, item := range subArr {
|
|
b, err := json.Marshal(item)
|
|
if err == nil {
|
|
if tc, ok := parseSingleToolCall(string(b)); ok {
|
|
calls = append(calls, tc)
|
|
}
|
|
}
|
|
}
|
|
if len(calls) > 0 {
|
|
return calls, true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if tc, ok := parseSingleToolCall(cleaned); ok {
|
|
return []ToolCall{tc}, true
|
|
}
|
|
|
|
if calls, ok := parseMultiplePythonFunctionCalls(cleaned); ok && len(calls) > 0 {
|
|
return calls, true
|
|
}
|
|
|
|
return nil, false
|
|
}
|
|
|
|
func parseXMLToolCall(block string) ([]ToolCall, bool) {
|
|
inner := strings.TrimSpace(block)
|
|
|
|
var tagFnName string
|
|
reTagAttr := regexp.MustCompile(`^<[a-zA-Z0-9_.:-]+\s+[^>]*?(?:name|function)=["']([^"']+)["'][^>]*>`)
|
|
if m := reTagAttr.FindStringSubmatch(inner); len(m) >= 2 {
|
|
tagFnName = m[1]
|
|
} else {
|
|
reColonTag := regexp.MustCompile(`^<(?:call|tool_call|function_call):([a-zA-Z0-9_-]+)>`)
|
|
if m := reColonTag.FindStringSubmatch(inner); len(m) >= 2 {
|
|
tagFnName = m[1]
|
|
} else {
|
|
reEqTag := regexp.MustCompile(`^<function=([a-zA-Z0-9_-]+)>`)
|
|
if m := reEqTag.FindStringSubmatch(inner); len(m) >= 2 {
|
|
tagFnName = m[1]
|
|
}
|
|
}
|
|
}
|
|
|
|
if tagFnName != "" {
|
|
reOpen := regexp.MustCompile(`^<[^>]+>`)
|
|
reClose := regexp.MustCompile(`</[^>]+>\s*$`)
|
|
innerStripped := reOpen.ReplaceAllString(inner, "")
|
|
innerStripped = reClose.ReplaceAllString(innerStripped, "")
|
|
innerStripped = cleanJSONBlock(innerStripped)
|
|
|
|
reParam := regexp.MustCompile(`<(?:parameter|param)\s+name=["']([^"']+)["']>([\s\S]*?)</(?:parameter|param)>`)
|
|
paramMatches := reParam.FindAllStringSubmatch(innerStripped, -1)
|
|
if len(paramMatches) > 0 {
|
|
paramMap := make(map[string]interface{})
|
|
for _, pm := range paramMatches {
|
|
k := pm[1]
|
|
v := strings.TrimSpace(pm[2])
|
|
if num, err := strconv.ParseFloat(v, 64); err == nil && !strings.HasPrefix(v, "0") {
|
|
paramMap[k] = num
|
|
} else if v == "true" {
|
|
paramMap[k] = true
|
|
} else if v == "false" {
|
|
paramMap[k] = false
|
|
} else {
|
|
paramMap[k] = v
|
|
}
|
|
}
|
|
b, _ := json.Marshal(paramMap)
|
|
return []ToolCall{{
|
|
ID: "call_" + GenerateUUID()[:8],
|
|
Type: "function",
|
|
Function: ToolCallFunction{
|
|
Name: tagFnName,
|
|
Arguments: string(b),
|
|
},
|
|
}}, true
|
|
}
|
|
|
|
var argsStr string
|
|
for _, argKey := range []string{"arguments", "parameters", "args", "input", "action_input"} {
|
|
openTag := "<" + argKey + ">"
|
|
closeTag := "</" + argKey + ">"
|
|
if strings.Contains(innerStripped, openTag) && strings.Contains(innerStripped, closeTag) {
|
|
aStart := strings.Index(innerStripped, openTag) + len(openTag)
|
|
aEnd := strings.Index(innerStripped, closeTag)
|
|
if aStart < aEnd {
|
|
argsStr = strings.TrimSpace(innerStripped[aStart:aEnd])
|
|
break
|
|
}
|
|
}
|
|
}
|
|
if argsStr == "" {
|
|
argsStr = innerStripped
|
|
}
|
|
|
|
if strings.HasPrefix(argsStr, "{") && strings.HasSuffix(argsStr, "}") {
|
|
repaired := repairPythonJSON(argsStr)
|
|
var dummy map[string]interface{}
|
|
if json.Unmarshal([]byte(repaired), &dummy) == nil {
|
|
b, _ := json.Marshal(dummy)
|
|
return []ToolCall{{
|
|
ID: "call_" + GenerateUUID()[:8],
|
|
Type: "function",
|
|
Function: ToolCallFunction{
|
|
Name: tagFnName,
|
|
Arguments: string(b),
|
|
},
|
|
}}, true
|
|
}
|
|
}
|
|
|
|
if argsStr != "" && !json.Valid([]byte(argsStr)) {
|
|
reTagOpen := regexp.MustCompile(`<([a-zA-Z0-9_-]+)>`)
|
|
openMatches := reTagOpen.FindAllStringSubmatchIndex(argsStr, -1)
|
|
if len(openMatches) > 0 {
|
|
xmlMap := make(map[string]interface{})
|
|
for _, match := range openMatches {
|
|
tagName := argsStr[match[2]:match[3]]
|
|
closeTag := "</" + tagName + ">"
|
|
closeIdx := strings.Index(argsStr[match[1]:], closeTag)
|
|
if closeIdx != -1 {
|
|
v := strings.TrimSpace(argsStr[match[1] : match[1]+closeIdx])
|
|
if num, err := strconv.ParseFloat(v, 64); err == nil && !strings.HasPrefix(v, "0") {
|
|
xmlMap[tagName] = num
|
|
} else if v == "true" {
|
|
xmlMap[tagName] = true
|
|
} else if v == "false" {
|
|
xmlMap[tagName] = false
|
|
} else {
|
|
xmlMap[tagName] = v
|
|
}
|
|
}
|
|
}
|
|
if len(xmlMap) > 0 {
|
|
if b, err := json.Marshal(xmlMap); err == nil {
|
|
return []ToolCall{{
|
|
ID: "call_" + GenerateUUID()[:8],
|
|
Type: "function",
|
|
Function: ToolCallFunction{
|
|
Name: tagFnName,
|
|
Arguments: string(b),
|
|
},
|
|
}}, true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if argMap, ok := parsePythonKwargs(argsStr); ok && len(argMap) > 0 {
|
|
b, _ := json.Marshal(argMap)
|
|
return []ToolCall{{
|
|
ID: "call_" + GenerateUUID()[:8],
|
|
Type: "function",
|
|
Function: ToolCallFunction{
|
|
Name: tagFnName,
|
|
Arguments: string(b),
|
|
},
|
|
}}, true
|
|
}
|
|
|
|
if strings.TrimSpace(argsStr) == "" {
|
|
argsStr = "{}"
|
|
}
|
|
return []ToolCall{{
|
|
ID: "call_" + GenerateUUID()[:8],
|
|
Type: "function",
|
|
Function: ToolCallFunction{
|
|
Name: tagFnName,
|
|
Arguments: argsStr,
|
|
},
|
|
}}, true
|
|
}
|
|
|
|
for _, pair := range ToolTagPairs {
|
|
inner = strings.ReplaceAll(inner, pair.Start, "")
|
|
inner = strings.ReplaceAll(inner, pair.End, "")
|
|
}
|
|
inner = cleanJSONBlock(inner)
|
|
|
|
if calls, ok := parseMultipleToolCalls(inner); ok && len(calls) > 0 {
|
|
return calls, true
|
|
}
|
|
|
|
var fnName string
|
|
for _, tagKey := range []string{"name", "function", "action", "command"} {
|
|
openTag := "<" + tagKey + ">"
|
|
closeTag := "</" + tagKey + ">"
|
|
if strings.Contains(inner, openTag) && strings.Contains(inner, closeTag) {
|
|
nStart := strings.Index(inner, openTag) + len(openTag)
|
|
nEnd := strings.Index(inner, closeTag)
|
|
if nStart < nEnd {
|
|
fnName = strings.TrimSpace(inner[nStart:nEnd])
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
var argsStr string
|
|
for _, argKey := range []string{"arguments", "parameters", "args", "input", "action_input"} {
|
|
openTag := "<" + argKey + ">"
|
|
closeTag := "</" + argKey + ">"
|
|
if strings.Contains(inner, openTag) && strings.Contains(inner, closeTag) {
|
|
aStart := strings.Index(inner, openTag) + len(openTag)
|
|
aEnd := strings.Index(inner, closeTag)
|
|
if aStart < aEnd {
|
|
argsStr = strings.TrimSpace(inner[aStart:aEnd])
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
if argsStr != "" && !json.Valid([]byte(argsStr)) {
|
|
reTagOpen := regexp.MustCompile(`<([a-zA-Z0-9_-]+)>`)
|
|
openMatches := reTagOpen.FindAllStringSubmatchIndex(argsStr, -1)
|
|
if len(openMatches) > 0 {
|
|
xmlMap := make(map[string]interface{})
|
|
for _, match := range openMatches {
|
|
tagName := argsStr[match[2]:match[3]]
|
|
closeTag := "</" + tagName + ">"
|
|
closeIdx := strings.Index(argsStr[match[1]:], closeTag)
|
|
if closeIdx != -1 {
|
|
v := strings.TrimSpace(argsStr[match[1] : match[1]+closeIdx])
|
|
if num, err := strconv.ParseFloat(v, 64); err == nil && !strings.HasPrefix(v, "0") {
|
|
xmlMap[tagName] = num
|
|
} else if v == "true" {
|
|
xmlMap[tagName] = true
|
|
} else if v == "false" {
|
|
xmlMap[tagName] = false
|
|
} else {
|
|
xmlMap[tagName] = v
|
|
}
|
|
}
|
|
}
|
|
if len(xmlMap) > 0 {
|
|
if b, err := json.Marshal(xmlMap); err == nil {
|
|
argsStr = string(b)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if fnName != "" {
|
|
if argsStr == "" {
|
|
argsStr = "{}"
|
|
}
|
|
return []ToolCall{{
|
|
ID: "call_" + GenerateUUID()[:8],
|
|
Type: "function",
|
|
Function: ToolCallFunction{
|
|
Name: fnName,
|
|
Arguments: argsStr,
|
|
},
|
|
}}, true
|
|
}
|
|
|
|
return nil, false
|
|
}
|
|
|
|
func scrubToolMarkers(text string) string {
|
|
s := text
|
|
for _, pair := range ToolTagPairs {
|
|
s = strings.ReplaceAll(s, pair.Start, "")
|
|
s = strings.ReplaceAll(s, pair.End, "")
|
|
}
|
|
reTagsWithAttrs := regexp.MustCompile(`(?i)</?(?:tool_calls?|toolCalls?|function_calls?|functionCalls?|invoke|call|command|commands|action|function)(?:\s+[^>]*)?>`)
|
|
s = reTagsWithAttrs.ReplaceAllString(s, "")
|
|
|
|
reColonTags := regexp.MustCompile(`(?i)</?(?:call|tool_call|function_call):[a-zA-Z0-9_-]+(?:\s+[^>]*)?>`)
|
|
s = reColonTags.ReplaceAllString(s, "")
|
|
|
|
reEqTags := regexp.MustCompile(`(?i)</?function=[a-zA-Z0-9_-]+>`)
|
|
s = reEqTags.ReplaceAllString(s, "")
|
|
|
|
reBracketTags := regexp.MustCompile(`(?i)\[/?TOOL_CALLS?\]`)
|
|
s = reBracketTags.ReplaceAllString(s, "")
|
|
|
|
reTags := regexp.MustCompile(`(?s)<(?:name|function|action|command|call)>[^<]*</(?:name|function|action|command|call)>`)
|
|
s = reTags.ReplaceAllString(s, "")
|
|
reArgTags := regexp.MustCompile(`(?s)<(?:arguments|parameters|args|input|action_input|inputs)>[\s\S]*?</(?:arguments|parameters|args|input|action_input|inputs)>`)
|
|
s = reArgTags.ReplaceAllString(s, "")
|
|
reParamTags := regexp.MustCompile(`(?s)<(?:parameter|param)(?:\s+[^>]*)?>[\s\S]*?</(?:parameter|param)>`)
|
|
s = reParamTags.ReplaceAllString(s, "")
|
|
|
|
reReAct := regexp.MustCompile(`(?i)(?:Action|Command):\s*[a-zA-Z0-9_.-]+`)
|
|
s = reReAct.ReplaceAllString(s, "")
|
|
reReActInput := regexp.MustCompile(`(?i)(?:Action Input|Arguments|Parameters|Args):\s*`)
|
|
s = reReActInput.ReplaceAllString(s, "")
|
|
|
|
extraTags := []string{
|
|
"<name>", "</name>",
|
|
"<arguments>", "</arguments>",
|
|
"<parameters>", "</parameters>",
|
|
"<argument>", "</argument>",
|
|
"<parameter>", "</parameter>",
|
|
"<param>", "</param>",
|
|
"<action_input>", "</action_input>",
|
|
"<command>", "</command>",
|
|
}
|
|
for _, tag := range extraTags {
|
|
s = strings.ReplaceAll(s, tag, "")
|
|
}
|
|
|
|
reEmptyFences := regexp.MustCompile("(?s)```(?:xml|json|ya?ml)?\\s*```")
|
|
s = reEmptyFences.ReplaceAllString(s, "")
|
|
|
|
reCitationDisclaimer := regexp.MustCompile(`(?i)\*?Web evidence was retrieved[^\n*]*\.\*?`)
|
|
s = reCitationDisclaimer.ReplaceAllString(s, "")
|
|
|
|
trimmed := strings.TrimSpace(s)
|
|
if trimmed == "```xml" || trimmed == "```json" || trimmed == "```" {
|
|
return ""
|
|
}
|
|
if strings.Trim(trimmed, "`\r\n\t ") == "" {
|
|
return ""
|
|
}
|
|
|
|
if strings.Count(s, "```") == 1 {
|
|
reLeadingFence := regexp.MustCompile("^\\s*```(?:xml|json|ya?ml)?\\s*\\n?")
|
|
reTrailingFence := regexp.MustCompile("\\n?\\s*```(?:xml|json|ya?ml)?\\s*$")
|
|
s = reLeadingFence.ReplaceAllString(s, "")
|
|
s = reTrailingFence.ReplaceAllString(s, "")
|
|
}
|
|
|
|
return strings.TrimSpace(s)
|
|
}
|
|
|
|
func ExtractToolCallBlocks(content string) (blocks []string, preCallText string, remaining string) {
|
|
remaining = content
|
|
var firstPreamble string
|
|
firstBlockFound := false
|
|
|
|
reDynamicOpen := regexp.MustCompile(`(?i)<(?:tool_calls?|toolCalls?|function_calls?|functionCalls?|invoke|call|command|commands|action|function)(?:\s+[^>]*)?>|<(?:call|tool_call|function_call):[a-zA-Z0-9_-]+(?:\s+[^>]*)?>|<function=[a-zA-Z0-9_-]+>|\[TOOL_CALLS?\]`)
|
|
|
|
for {
|
|
loc := reDynamicOpen.FindStringIndex(remaining)
|
|
if loc == nil {
|
|
break
|
|
}
|
|
sIdx := loc[0]
|
|
openTag := remaining[loc[0]:loc[1]]
|
|
rest := remaining[loc[1]:]
|
|
|
|
var closeTagPattern string
|
|
if strings.HasPrefix(strings.ToLower(openTag), "[tool_call") {
|
|
closeTagPattern = `(?i)\[/TOOL_CALLS?\]`
|
|
} else if strings.HasPrefix(openTag, "<function=") {
|
|
closeTagPattern = `(?i)</function>`
|
|
} else if strings.HasPrefix(openTag, "<call:") || strings.HasPrefix(openTag, "<tool_call:") || strings.HasPrefix(openTag, "<function_call:") {
|
|
colonIdx := strings.Index(openTag, ":")
|
|
gtIdx := strings.Index(openTag, ">")
|
|
tagSub := openTag[1:gtIdx]
|
|
prefix := openTag[1:colonIdx]
|
|
closeTagPattern = fmt.Sprintf(`(?i)</%s>|</%s>`, regexp.QuoteMeta(tagSub), regexp.QuoteMeta(prefix))
|
|
} else {
|
|
reTagName := regexp.MustCompile(`^<([a-zA-Z0-9_]+)`)
|
|
if m := reTagName.FindStringSubmatch(openTag); len(m) >= 2 {
|
|
tagName := m[1]
|
|
closeTagPattern = fmt.Sprintf(`(?i)</%s>`, regexp.QuoteMeta(tagName))
|
|
} else {
|
|
closeTagPattern = `(?i)</[a-zA-Z0-9_]+>`
|
|
}
|
|
}
|
|
|
|
reClose := regexp.MustCompile(closeTagPattern)
|
|
closeLoc := reClose.FindStringIndex(rest)
|
|
|
|
var blockText string
|
|
if closeLoc != nil {
|
|
blockEndPos := loc[1] + closeLoc[1]
|
|
blockText = remaining[sIdx:blockEndPos]
|
|
|
|
before := remaining[:sIdx]
|
|
after := remaining[blockEndPos:]
|
|
|
|
reFenceOpen := regexp.MustCompile("(?s)\\n?\\s*```(?:xml|json|ya?ml)?\\s*$")
|
|
reFenceClose := regexp.MustCompile("^\\s*```\\s*\\n?")
|
|
if reFenceOpen.MatchString(before) && reFenceClose.MatchString(after) {
|
|
before = reFenceOpen.ReplaceAllString(before, "")
|
|
after = reFenceClose.ReplaceAllString(after, "")
|
|
}
|
|
if !firstBlockFound {
|
|
preClean := reFenceOpen.ReplaceAllString(before, "")
|
|
firstPreamble = strings.TrimRight(preClean, "\r\n ")
|
|
firstBlockFound = true
|
|
}
|
|
remaining = strings.TrimSpace(before + after)
|
|
} else {
|
|
nextLoc := reDynamicOpen.FindStringIndex(rest)
|
|
before := remaining[:sIdx]
|
|
reFenceOpen := regexp.MustCompile("(?s)\\n?\\s*```(?:xml|json|ya?ml)?\\s*$")
|
|
if !firstBlockFound {
|
|
preClean := reFenceOpen.ReplaceAllString(before, "")
|
|
firstPreamble = strings.TrimRight(preClean, "\r\n ")
|
|
firstBlockFound = true
|
|
}
|
|
if nextLoc != nil {
|
|
blockEndPos := loc[1] + nextLoc[0]
|
|
blockText = remaining[sIdx:blockEndPos]
|
|
remaining = strings.TrimSpace(remaining[:sIdx] + remaining[blockEndPos:])
|
|
} else {
|
|
blockText = remaining[sIdx:]
|
|
remaining = strings.TrimSpace(remaining[:sIdx])
|
|
}
|
|
}
|
|
|
|
blocks = append(blocks, blockText)
|
|
}
|
|
|
|
if len(blocks) == 0 {
|
|
reAction := regexp.MustCompile(`(?i)(?:Action|Command):\s*[a-zA-Z0-9_.-]+\s*\n+(?:Action Input|Arguments|Parameters|Args):\s*(?:\{[\s\S]*?\}|\[[\s\S]*?\]|[^\n]+)`)
|
|
if loc := reAction.FindStringIndex(remaining); loc != nil {
|
|
blockText := remaining[loc[0]:loc[1]]
|
|
before := remaining[:loc[0]]
|
|
after := remaining[loc[1]:]
|
|
firstPreamble = strings.TrimSpace(before)
|
|
remaining = strings.TrimSpace(before + " " + after)
|
|
blocks = append(blocks, blockText)
|
|
}
|
|
}
|
|
|
|
return blocks, firstPreamble, remaining
|
|
}
|
|
|
|
func DetectToolCalls(content string) ([]ToolCall, string, bool) {
|
|
blocks, preCallText, _ := ExtractToolCallBlocks(content)
|
|
var calls []ToolCall
|
|
|
|
for _, block := range blocks {
|
|
if tcs, ok := parseXMLToolCall(block); ok {
|
|
calls = append(calls, tcs...)
|
|
}
|
|
}
|
|
|
|
if len(calls) > 0 {
|
|
return calls, scrubToolMarkers(preCallText), true
|
|
}
|
|
|
|
reFenced := regexp.MustCompile("(?s)```(?:json|xml)?\\s*([\\s\\S]*?)\\s*```")
|
|
matches := reFenced.FindAllStringSubmatchIndex(content, -1)
|
|
if len(matches) > 0 {
|
|
var fencedCalls []ToolCall
|
|
firstFenceStart := -1
|
|
for _, loc := range matches {
|
|
fenceInner := strings.TrimSpace(content[loc[2]:loc[3]])
|
|
if tcs, ok := parseMultipleToolCalls(fenceInner); ok && len(tcs) > 0 {
|
|
if firstFenceStart == -1 {
|
|
firstFenceStart = loc[0]
|
|
}
|
|
fencedCalls = append(fencedCalls, tcs...)
|
|
}
|
|
}
|
|
if len(fencedCalls) > 0 {
|
|
pre := ""
|
|
if firstFenceStart > 0 {
|
|
pre = content[:firstFenceStart]
|
|
}
|
|
return fencedCalls, scrubToolMarkers(pre), true
|
|
}
|
|
}
|
|
|
|
trimmed := strings.TrimSpace(content)
|
|
if strings.HasPrefix(trimmed, "{") || strings.HasPrefix(trimmed, "[") {
|
|
if tcs, ok := parseMultipleToolCalls(trimmed); ok && len(tcs) > 0 {
|
|
return tcs, "", true
|
|
}
|
|
}
|
|
|
|
reCallInContent := regexp.MustCompile(`(?m)^(?:tools\.|functions\.)?([a-zA-Z_][a-zA-Z0-9_]*)\s*\(([^)]*)\)\s*$`)
|
|
callLocs := reCallInContent.FindAllStringIndex(content, -1)
|
|
if len(callLocs) > 0 {
|
|
var pyCalls []ToolCall
|
|
firstCallStart := -1
|
|
for _, loc := range callLocs {
|
|
callStr := strings.TrimSpace(content[loc[0]:loc[1]])
|
|
if tc, ok := parsePythonFunctionCall(callStr); ok {
|
|
if firstCallStart == -1 {
|
|
firstCallStart = loc[0]
|
|
}
|
|
pyCalls = append(pyCalls, tc)
|
|
}
|
|
}
|
|
if len(pyCalls) > 0 {
|
|
pre := ""
|
|
if firstCallStart > 0 {
|
|
pre = content[:firstCallStart]
|
|
}
|
|
return pyCalls, scrubToolMarkers(pre), true
|
|
}
|
|
}
|
|
|
|
if tc, ok := parseReActToolCall(content); ok {
|
|
reAction := regexp.MustCompile(`(?i)(?:Action|Command):\s*[a-zA-Z0-9_.-]+`)
|
|
loc := reAction.FindStringIndex(content)
|
|
pre := ""
|
|
if loc != nil && loc[0] > 0 {
|
|
pre = content[:loc[0]]
|
|
}
|
|
return []ToolCall{tc}, scrubToolMarkers(pre), true
|
|
}
|
|
|
|
if tcs, ok := parseMultiplePythonFunctionCalls(trimmed); ok && len(tcs) > 0 {
|
|
return tcs, "", true
|
|
}
|
|
|
|
return nil, content, false
|
|
}
|
|
|
|
func finalizeOutput(frame GradioOutputFrame) (finalContent interface{}, reasoning string, toolCalls []ToolCall, finishReason string) {
|
|
cleanText := frame.Content
|
|
reasoning = frame.Reasoning
|
|
toolCalls = frame.ToolCalls
|
|
hasTools := len(toolCalls) > 0
|
|
|
|
if reasoning == "" {
|
|
cleanText, reasoning = ExtractThinking(cleanText)
|
|
}
|
|
|
|
if !hasTools {
|
|
toolCalls, cleanText, hasTools = DetectToolCalls(cleanText)
|
|
} else {
|
|
if extraCalls, extraClean, extraHas := DetectToolCalls(cleanText); extraHas && len(extraCalls) > 0 {
|
|
for _, ec := range extraCalls {
|
|
duplicate := false
|
|
for _, tc := range toolCalls {
|
|
if tc.Function.Name == ec.Function.Name && tc.Function.Arguments == ec.Function.Arguments {
|
|
duplicate = true
|
|
break
|
|
}
|
|
}
|
|
if !duplicate {
|
|
toolCalls = append(toolCalls, ec)
|
|
}
|
|
}
|
|
cleanText = extraClean
|
|
} else {
|
|
cleanText = scrubToolMarkers(cleanText)
|
|
}
|
|
}
|
|
|
|
finishReason = "stop"
|
|
finalContent = cleanText
|
|
if hasTools && len(toolCalls) > 0 {
|
|
finishReason = "tool_calls"
|
|
cleanText = scrubToolMarkers(cleanText)
|
|
if strings.TrimSpace(cleanText) == "" {
|
|
finalContent = nil
|
|
} else {
|
|
finalContent = strings.TrimSpace(cleanText)
|
|
}
|
|
}
|
|
|
|
return finalContent, reasoning, toolCalls, finishReason
|
|
}
|
|
|
|
func ExtractThinking(content string) (string, string) {
|
|
if strings.Contains(content, "<think>") && strings.Contains(content, "</think>") {
|
|
start := strings.Index(content, "<think>")
|
|
end := strings.Index(content, "</think>")
|
|
if start < end {
|
|
reasoning := content[start+len("<think>") : end]
|
|
rem := content[:start] + content[end+len("</think>"):]
|
|
rem = strings.TrimPrefix(rem, "\n\n")
|
|
rem = strings.TrimPrefix(rem, "\n")
|
|
return rem, reasoning
|
|
}
|
|
}
|
|
return content, ""
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Response framing & streamer
|
|
// ---------------------------------------------------------------------------
|
|
|
|
type FinalOutput struct {
|
|
Content interface{}
|
|
ReasoningContent string
|
|
ToolCalls []ToolCall
|
|
FinishReason string
|
|
}
|
|
|
|
func WriteCompletionResponse(w http.ResponseWriter, completionID string, created int64, model string, out FinalOutput) {
|
|
finish := out.FinishReason
|
|
if finish == "" {
|
|
finish = "stop"
|
|
}
|
|
resp := ChatCompletionResponse{
|
|
ID: completionID,
|
|
Object: "chat.completion",
|
|
Created: created,
|
|
Model: model,
|
|
Choices: []ChatCompletionResponseChoice{
|
|
{
|
|
Index: 0,
|
|
Message: ChatMessage{
|
|
Role: "assistant",
|
|
Content: out.Content,
|
|
ReasoningContent: out.ReasoningContent,
|
|
ToolCalls: out.ToolCalls,
|
|
},
|
|
FinishReason: finish,
|
|
},
|
|
},
|
|
Usage: Usage{
|
|
PromptTokens: 0,
|
|
CompletionTokens: 0,
|
|
TotalTokens: 0,
|
|
},
|
|
}
|
|
w.Header().Set("Content-Type", "application/json")
|
|
json.NewEncoder(w).Encode(resp)
|
|
}
|
|
|
|
func extractFailedGeneration(raw string) (string, bool) {
|
|
var obj map[string]interface{}
|
|
if err := json.Unmarshal([]byte(raw), &obj); err == nil {
|
|
if fg, ok := obj["failed_generation"].(string); ok && fg != "" {
|
|
return strings.TrimSpace(fg), true
|
|
}
|
|
if errVal, ok := obj["error"]; ok {
|
|
if errMap, ok := errVal.(map[string]interface{}); ok {
|
|
if fg, ok := errMap["failed_generation"].(string); ok && fg != "" {
|
|
return strings.TrimSpace(fg), true
|
|
}
|
|
} else if errStr, ok := errVal.(string); ok {
|
|
if fg, ok := extractFailedGeneration(errStr); ok {
|
|
return fg, true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
keyIdx := strings.Index(raw, `"failed_generation"`)
|
|
if keyIdx == -1 {
|
|
keyIdx = strings.Index(raw, `'failed_generation'`)
|
|
}
|
|
if keyIdx == -1 {
|
|
keyIdx = strings.Index(raw, `failed_generation`)
|
|
}
|
|
if keyIdx == -1 {
|
|
return "", false
|
|
}
|
|
|
|
colonIdx := strings.Index(raw[keyIdx:], ":")
|
|
if colonIdx == -1 {
|
|
return "", false
|
|
}
|
|
valStart := keyIdx + colonIdx + 1
|
|
|
|
for valStart < len(raw) && (raw[valStart] == ' ' || raw[valStart] == '\t' || raw[valStart] == '\r' || raw[valStart] == '\n') {
|
|
valStart++
|
|
}
|
|
if valStart >= len(raw) {
|
|
return "", false
|
|
}
|
|
|
|
firstChar := raw[valStart]
|
|
var candidate string
|
|
|
|
if firstChar == '\'' || firstChar == '"' {
|
|
quoteChar := firstChar
|
|
var b strings.Builder
|
|
escaped := false
|
|
for i := valStart + 1; i < len(raw); i++ {
|
|
ch := raw[i]
|
|
if escaped {
|
|
switch ch {
|
|
case 'n':
|
|
b.WriteByte('\n')
|
|
case 'r':
|
|
b.WriteByte('\r')
|
|
case 't':
|
|
b.WriteByte('\t')
|
|
case '\\':
|
|
b.WriteByte('\\')
|
|
case '\'':
|
|
b.WriteByte('\'')
|
|
case '"':
|
|
b.WriteByte('"')
|
|
default:
|
|
b.WriteByte('\\')
|
|
b.WriteByte(ch)
|
|
}
|
|
escaped = false
|
|
} else if ch == '\\' {
|
|
escaped = true
|
|
} else if ch == quoteChar {
|
|
break
|
|
} else {
|
|
b.WriteByte(ch)
|
|
}
|
|
}
|
|
candidate = strings.TrimSpace(b.String())
|
|
} else if firstChar == '{' || firstChar == '[' {
|
|
openChar := firstChar
|
|
closeChar := byte('}')
|
|
if openChar == '[' {
|
|
closeChar = ']'
|
|
}
|
|
depth := 0
|
|
inStr := false
|
|
var strQuote byte
|
|
escaped := false
|
|
endIdx := -1
|
|
|
|
for i := valStart; i < len(raw); i++ {
|
|
ch := raw[i]
|
|
if inStr {
|
|
if escaped {
|
|
escaped = false
|
|
} else if ch == '\\' {
|
|
escaped = true
|
|
} else if ch == strQuote {
|
|
inStr = false
|
|
}
|
|
} else {
|
|
if ch == '"' || ch == '\'' {
|
|
inStr = true
|
|
strQuote = ch
|
|
} else if ch == openChar {
|
|
depth++
|
|
} else if ch == closeChar {
|
|
depth--
|
|
if depth == 0 {
|
|
endIdx = i + 1
|
|
break
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if endIdx != -1 {
|
|
candidate = strings.TrimSpace(raw[valStart:endIdx])
|
|
}
|
|
}
|
|
|
|
if candidate != "" {
|
|
if strings.Contains(candidate, `\"`) {
|
|
candidate = strings.ReplaceAll(candidate, `\"`, `"`)
|
|
}
|
|
if strings.Contains(candidate, `\n`) {
|
|
candidate = strings.ReplaceAll(candidate, `\n`, "\n")
|
|
}
|
|
return candidate, true
|
|
}
|
|
|
|
return "", false
|
|
}
|
|
|
|
func extractGradioErrorMessage(dataStr string) string {
|
|
var strVal string
|
|
if err := json.Unmarshal([]byte(dataStr), &strVal); err == nil && strVal != "" {
|
|
return strVal
|
|
}
|
|
var errObj map[string]interface{}
|
|
if err := json.Unmarshal([]byte(dataStr), &errObj); err == nil {
|
|
if e, ok := errObj["error"].(string); ok && e != "" {
|
|
return e
|
|
}
|
|
if m, ok := errObj["message"].(string); ok && m != "" {
|
|
return m
|
|
}
|
|
if eNull, ok := errObj["error"]; ok && eNull == nil {
|
|
if t, ok := errObj["title"].(string); ok && t != "" {
|
|
return t
|
|
}
|
|
return "internal space error (check Gradio inputs/types)"
|
|
}
|
|
}
|
|
clean := strings.TrimSpace(dataStr)
|
|
if clean == "null" || clean == "" {
|
|
return "upstream Gradio space returned null error (space may have show_error=False or failed input validation)"
|
|
}
|
|
return clean
|
|
}
|
|
|
|
func isProtocolOrInputError(errMsg string) bool {
|
|
lower := strings.ToLower(errMsg)
|
|
if strings.Contains(lower, "unauthorized") ||
|
|
strings.Contains(lower, "forbidden") ||
|
|
strings.Contains(lower, "quota exceeded") ||
|
|
strings.Contains(lower, "payment required") ||
|
|
strings.Contains(lower, "rate limit") {
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
type Streamer struct {
|
|
w http.ResponseWriter
|
|
flusher http.Flusher
|
|
id string
|
|
created int64
|
|
model string
|
|
started bool
|
|
}
|
|
|
|
func NewStreamer(w http.ResponseWriter, flusher http.Flusher, id string, created int64, model string) *Streamer {
|
|
return &Streamer{w: w, flusher: flusher, id: id, created: created, model: model}
|
|
}
|
|
|
|
func (s *Streamer) EnsureStarted() {
|
|
if s.started {
|
|
return
|
|
}
|
|
s.w.Header().Set("Content-Type", "text/event-stream")
|
|
s.w.Header().Set("Cache-Control", "no-cache")
|
|
s.w.Header().Set("Connection", "keep-alive")
|
|
s.started = true
|
|
s.Role()
|
|
}
|
|
|
|
func (s *Streamer) Role() {
|
|
sendStreamDelta(s.w, s.flusher, s.id, s.created, s.model, StreamDelta{Role: "assistant"})
|
|
}
|
|
|
|
func (s *Streamer) Reasoning(text string) {
|
|
s.EnsureStarted()
|
|
sendStreamDelta(s.w, s.flusher, s.id, s.created, s.model, StreamDelta{ReasoningContent: text})
|
|
}
|
|
|
|
func (s *Streamer) Content(text string) {
|
|
s.EnsureStarted()
|
|
sendStreamDelta(s.w, s.flusher, s.id, s.created, s.model, StreamDelta{Content: text})
|
|
}
|
|
|
|
func (s *Streamer) ToolCallDelta(tc ToolCall) {
|
|
s.EnsureStarted()
|
|
sendStreamDelta(s.w, s.flusher, s.id, s.created, s.model, StreamDelta{ToolCalls: []ToolCall{tc}})
|
|
}
|
|
|
|
func (s *Streamer) Finish(reason string) {
|
|
if !s.started {
|
|
return
|
|
}
|
|
sendStreamChunk(s.w, s.flusher, s.id, s.created, s.model, StreamDelta{}, &reason)
|
|
}
|
|
|
|
func (s *Streamer) Done() {
|
|
if !s.started {
|
|
return
|
|
}
|
|
fmt.Fprintf(s.w, "data: [DONE]\n\n")
|
|
if s.flusher != nil {
|
|
s.flusher.Flush()
|
|
}
|
|
}
|
|
|
|
func (s *Streamer) Error(errMsg string) {
|
|
s.EnsureStarted()
|
|
errChunk := map[string]interface{}{
|
|
"error": map[string]interface{}{
|
|
"message": errMsg,
|
|
"type": "upstream_error",
|
|
"code": 502,
|
|
},
|
|
}
|
|
b, _ := json.Marshal(errChunk)
|
|
fmt.Fprintf(s.w, "data: %s\n\n", b)
|
|
if s.flusher != nil {
|
|
s.flusher.Flush()
|
|
}
|
|
}
|
|
|
|
func sendStreamDelta(w http.ResponseWriter, flusher http.Flusher, completionID string, createdTime int64, modelName string, delta StreamDelta) {
|
|
sendStreamChunk(w, flusher, completionID, createdTime, modelName, delta, nil)
|
|
}
|
|
|
|
func sendStreamChunk(w http.ResponseWriter, flusher http.Flusher, completionID string, createdTime int64, modelName string, delta StreamDelta, finishReason *string) {
|
|
chunk := StreamResponse{
|
|
ID: completionID,
|
|
Object: "chat.completion.chunk",
|
|
Created: createdTime,
|
|
Model: modelName,
|
|
Choices: []StreamChoice{
|
|
{
|
|
Index: 0,
|
|
Delta: delta,
|
|
FinishReason: finishReason,
|
|
},
|
|
},
|
|
}
|
|
b, _ := json.Marshal(chunk)
|
|
fmt.Fprintf(w, "data: %s\n\n", b)
|
|
if flusher != nil {
|
|
flusher.Flush()
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Stateful thinking tag filter for streaming
|
|
// ---------------------------------------------------------------------------
|
|
|
|
type StreamThinkingFilter struct {
|
|
inThinking bool
|
|
buf string
|
|
}
|
|
|
|
func NewStreamThinkingFilter() *StreamThinkingFilter {
|
|
return &StreamThinkingFilter{}
|
|
}
|
|
|
|
func hasPrefixOf(target string, prefixes []string) int {
|
|
maxMatch := 0
|
|
for _, p := range prefixes {
|
|
if strings.HasSuffix(target, p) && len(p) > maxMatch {
|
|
maxMatch = len(p)
|
|
}
|
|
}
|
|
return maxMatch
|
|
}
|
|
|
|
func hasSuffixPrefixOf(target string, tag string) int {
|
|
maxMatch := 0
|
|
for i := 1; i < len(tag); i++ {
|
|
p := tag[:i]
|
|
if strings.HasSuffix(target, p) && len(p) > maxMatch {
|
|
maxMatch = len(p)
|
|
}
|
|
}
|
|
return maxMatch
|
|
}
|
|
|
|
func (f *StreamThinkingFilter) Feed(chunk string, onContent func(string), onReasoning func(string)) {
|
|
f.buf += chunk
|
|
thinkStartTag := "<think>"
|
|
thinkEndTag := "</think>"
|
|
|
|
thinkStartPrefixes := []string{"<", "<t", "<th", "<thi", "<thin", "<think"}
|
|
thinkEndPrefixes := []string{"<", "</", "</t", "</th", "</thi", "</thin", "</think"}
|
|
|
|
for len(f.buf) > 0 {
|
|
if !f.inThinking {
|
|
if idx := strings.Index(f.buf, thinkStartTag); idx != -1 {
|
|
before := f.buf[:idx]
|
|
if before != "" {
|
|
onContent(before)
|
|
}
|
|
f.inThinking = true
|
|
f.buf = f.buf[idx+len(thinkStartTag):]
|
|
} else if matchLen := hasPrefixOf(f.buf, thinkStartPrefixes); matchLen > 0 {
|
|
safe := f.buf[:len(f.buf)-matchLen]
|
|
if safe != "" {
|
|
onContent(safe)
|
|
}
|
|
f.buf = f.buf[len(f.buf)-matchLen:]
|
|
break
|
|
} else {
|
|
onContent(f.buf)
|
|
f.buf = ""
|
|
break
|
|
}
|
|
} else {
|
|
if idx := strings.Index(f.buf, thinkEndTag); idx != -1 {
|
|
before := f.buf[:idx]
|
|
if before != "" {
|
|
onReasoning(before)
|
|
}
|
|
f.inThinking = false
|
|
f.buf = f.buf[idx+len(thinkEndTag):]
|
|
f.buf = strings.TrimPrefix(f.buf, "\n\n")
|
|
f.buf = strings.TrimPrefix(f.buf, "\n")
|
|
} else if matchLen := hasPrefixOf(f.buf, thinkEndPrefixes); matchLen > 0 {
|
|
safe := f.buf[:len(f.buf)-matchLen]
|
|
if safe != "" {
|
|
onReasoning(safe)
|
|
}
|
|
f.buf = f.buf[len(f.buf)-matchLen:]
|
|
break
|
|
} else {
|
|
onReasoning(f.buf)
|
|
f.buf = ""
|
|
break
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (f *StreamThinkingFilter) Flush(onContent func(string), onReasoning func(string)) {
|
|
if len(f.buf) > 0 {
|
|
if f.inThinking {
|
|
onReasoning(f.buf)
|
|
} else {
|
|
onContent(f.buf)
|
|
}
|
|
f.buf = ""
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Stateful tool call tag filter for streaming
|
|
// ---------------------------------------------------------------------------
|
|
|
|
var potentialFencePrefixes = []string{
|
|
"```",
|
|
"```x", "```xm", "```xml", "```xml\r", "```xml\n", "```xml\r\n",
|
|
"```j", "```js", "```jso", "```json", "```json\r", "```json\n", "```json\r\n",
|
|
"```\r", "```\n", "```\r\n",
|
|
}
|
|
|
|
type StreamToolCallFilter struct {
|
|
inToolCall bool
|
|
buf string
|
|
toolCallBuf string
|
|
toolIndex int
|
|
emittedCall bool
|
|
activePair ToolTagPair
|
|
activeEndTag string
|
|
}
|
|
|
|
func NewStreamToolCallFilter() *StreamToolCallFilter {
|
|
return &StreamToolCallFilter{}
|
|
}
|
|
|
|
func (f *StreamToolCallFilter) cleanTrailingAfterToolCall() {
|
|
for {
|
|
stripped := false
|
|
for _, pair := range ToolTagPairs {
|
|
trimmed := strings.TrimLeft(f.buf, " \t\r\n")
|
|
if strings.HasPrefix(trimmed, pair.End) {
|
|
idx := strings.Index(f.buf, pair.End)
|
|
f.buf = f.buf[idx+len(pair.End):]
|
|
stripped = true
|
|
break
|
|
}
|
|
}
|
|
if !stripped {
|
|
break
|
|
}
|
|
}
|
|
|
|
reDynamicClose := regexp.MustCompile(`(?i)^\s*</(?:tool_calls?|toolCalls?|function_calls?|functionCalls?|invoke|call|command|commands|action|function)(?:\s+[^>]*)?>|^\s*</(?:call|tool_call|function_call):[a-zA-Z0-9_-]+(?:\s+[^>]*)?>|^\s*\[/TOOL_CALLS?\]`)
|
|
f.buf = reDynamicClose.ReplaceAllString(f.buf, "")
|
|
|
|
reCloseFence := regexp.MustCompile("^\\s*```\\s*\\n?")
|
|
f.buf = reCloseFence.ReplaceAllString(f.buf, "")
|
|
|
|
if f.buf == "\n" || f.buf == "\r\n" || f.buf == "\n\n" {
|
|
f.buf = ""
|
|
}
|
|
}
|
|
|
|
func (f *StreamToolCallFilter) Feed(chunk string, onContent func(string), onToolCall func(ToolCall)) {
|
|
f.buf += chunk
|
|
|
|
for len(f.buf) > 0 {
|
|
if !f.inToolCall {
|
|
if f.emittedCall {
|
|
f.cleanTrailingAfterToolCall()
|
|
if len(f.buf) == 0 {
|
|
break
|
|
}
|
|
}
|
|
|
|
earliestIdx := -1
|
|
var matchedPair ToolTagPair
|
|
|
|
for _, pair := range ToolTagPairs {
|
|
if idx := strings.Index(f.buf, pair.Start); idx != -1 {
|
|
if earliestIdx == -1 || idx < earliestIdx {
|
|
earliestIdx = idx
|
|
matchedPair = pair
|
|
}
|
|
}
|
|
}
|
|
|
|
reDynamicOpen := regexp.MustCompile(`(?i)<(?:tool_calls?|toolCalls?|function_calls?|functionCalls?|invoke|call|command|commands|action|function)(?:\s+[^>]*)?>|<(?:call|tool_call|function_call):[a-zA-Z0-9_-]+(?:\s+[^>]*)?>|<function=[a-zA-Z0-9_-]+>|\[TOOL_CALLS?\]`)
|
|
if loc := reDynamicOpen.FindStringIndex(f.buf); loc != nil {
|
|
if earliestIdx == -1 || loc[0] < earliestIdx {
|
|
earliestIdx = loc[0]
|
|
matchedTag := f.buf[loc[0]:loc[1]]
|
|
var endTag string
|
|
if strings.HasPrefix(strings.ToLower(matchedTag), "[tool_call") {
|
|
endTag = "[/TOOL_CALLS]"
|
|
} else if strings.HasPrefix(matchedTag, "<function=") {
|
|
endTag = "</function>"
|
|
} else if strings.HasPrefix(matchedTag, "<call:") || strings.HasPrefix(matchedTag, "<tool_call:") || strings.HasPrefix(matchedTag, "<function_call:") {
|
|
gtIdx := strings.Index(matchedTag, ">")
|
|
tagSub := matchedTag[1:gtIdx]
|
|
endTag = "</" + tagSub + ">"
|
|
} else {
|
|
reTagName := regexp.MustCompile(`^<([a-zA-Z0-9_]+)`)
|
|
if m := reTagName.FindStringSubmatch(matchedTag); len(m) >= 2 {
|
|
endTag = "</" + m[1] + ">"
|
|
} else {
|
|
endTag = "</tool_call>"
|
|
}
|
|
}
|
|
matchedPair = ToolTagPair{Start: matchedTag, End: endTag}
|
|
}
|
|
}
|
|
|
|
if earliestIdx != -1 {
|
|
before := f.buf[:earliestIdx]
|
|
reTrailingFence := regexp.MustCompile("(?s)\\n?\\s*```(?:xml|json|ya?ml)?\\s*$")
|
|
if reTrailingFence.MatchString(before) {
|
|
before = reTrailingFence.ReplaceAllString(before, "")
|
|
}
|
|
if strings.TrimSpace(before) == "" {
|
|
before = ""
|
|
} else {
|
|
before = strings.TrimRight(before, "\r\n")
|
|
}
|
|
if before != "" && !f.emittedCall {
|
|
onContent(before)
|
|
}
|
|
f.inToolCall = true
|
|
f.activePair = matchedPair
|
|
f.activeEndTag = matchedPair.End
|
|
f.buf = f.buf[earliestIdx+len(matchedPair.Start):]
|
|
} else {
|
|
matchLen := hasPrefixOf(f.buf, toolStartPrefixes)
|
|
fenceLen := hasPrefixOf(f.buf, potentialFencePrefixes)
|
|
holdLen := matchLen
|
|
if fenceLen > holdLen {
|
|
holdLen = fenceLen
|
|
}
|
|
|
|
if holdLen > 0 {
|
|
safe := f.buf[:len(f.buf)-holdLen]
|
|
if matchLen > 0 {
|
|
reTrailingFence := regexp.MustCompile("(?s)\\n?\\s*```(?:xml|json|ya?ml)?\\s*$")
|
|
safe = reTrailingFence.ReplaceAllString(safe, "")
|
|
safe = strings.TrimRight(safe, "\r\n")
|
|
}
|
|
if strings.TrimSpace(safe) != "" && !f.emittedCall {
|
|
onContent(safe)
|
|
}
|
|
f.buf = f.buf[len(f.buf)-holdLen:]
|
|
break
|
|
} else if len(f.buf) < 16 && strings.TrimSpace(f.buf) == "" {
|
|
break
|
|
} else {
|
|
if !f.emittedCall {
|
|
onContent(f.buf)
|
|
}
|
|
f.buf = ""
|
|
break
|
|
}
|
|
}
|
|
} else {
|
|
if idx := strings.Index(f.buf, f.activeEndTag); idx != -1 {
|
|
f.toolCallBuf += f.buf[:idx]
|
|
f.buf = f.buf[idx+len(f.activeEndTag):]
|
|
f.inToolCall = false
|
|
|
|
if tcs, ok := parseMultipleToolCalls(f.toolCallBuf); ok && len(tcs) > 0 {
|
|
for _, tc := range tcs {
|
|
idxCopy := f.toolIndex
|
|
tc.Index = &idxCopy
|
|
f.toolIndex++
|
|
f.emittedCall = true
|
|
onToolCall(tc)
|
|
}
|
|
} else if tcs2, ok2 := parseXMLToolCall(f.activePair.Start + f.toolCallBuf + f.activePair.End); ok2 && len(tcs2) > 0 {
|
|
for _, tc := range tcs2 {
|
|
idxCopy := f.toolIndex
|
|
tc.Index = &idxCopy
|
|
f.toolIndex++
|
|
f.emittedCall = true
|
|
onToolCall(tc)
|
|
}
|
|
} else if tc3, ok3 := parsePythonFunctionCall(f.toolCallBuf); ok3 {
|
|
idxCopy := f.toolIndex
|
|
tc3.Index = &idxCopy
|
|
f.toolIndex++
|
|
f.emittedCall = true
|
|
onToolCall(tc3)
|
|
} else if tc4, ok4 := parseReActToolCall(f.toolCallBuf); ok4 {
|
|
idxCopy := f.toolIndex
|
|
tc4.Index = &idxCopy
|
|
f.toolIndex++
|
|
f.emittedCall = true
|
|
onToolCall(tc4)
|
|
} else if !f.emittedCall {
|
|
onContent(f.activePair.Start + f.toolCallBuf + f.activePair.End)
|
|
}
|
|
f.toolCallBuf = ""
|
|
|
|
if f.emittedCall {
|
|
f.cleanTrailingAfterToolCall()
|
|
}
|
|
} else if matchLen := hasSuffixPrefixOf(f.buf, f.activeEndTag); matchLen > 0 {
|
|
safe := f.buf[:len(f.buf)-matchLen]
|
|
f.toolCallBuf += safe
|
|
f.buf = f.buf[len(f.buf)-matchLen:]
|
|
break
|
|
} else {
|
|
f.toolCallBuf += f.buf
|
|
f.buf = ""
|
|
break
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (f *StreamToolCallFilter) Flush(onContent func(string), onToolCall func(ToolCall)) {
|
|
if f.inToolCall && len(f.toolCallBuf) > 0 {
|
|
if tcs, ok := parseMultipleToolCalls(f.toolCallBuf); ok && len(tcs) > 0 {
|
|
for _, tc := range tcs {
|
|
idxCopy := f.toolIndex
|
|
tc.Index = &idxCopy
|
|
f.toolIndex++
|
|
f.emittedCall = true
|
|
onToolCall(tc)
|
|
}
|
|
} else if tcs2, ok2 := parseXMLToolCall(f.activePair.Start + f.toolCallBuf + f.activePair.End); ok2 && len(tcs2) > 0 {
|
|
for _, tc := range tcs2 {
|
|
idxCopy := f.toolIndex
|
|
tc.Index = &idxCopy
|
|
f.toolIndex++
|
|
f.emittedCall = true
|
|
onToolCall(tc)
|
|
}
|
|
} else if tc3, ok3 := parsePythonFunctionCall(f.toolCallBuf); ok3 {
|
|
idxCopy := f.toolIndex
|
|
tc3.Index = &idxCopy
|
|
f.toolIndex++
|
|
f.emittedCall = true
|
|
onToolCall(tc3)
|
|
} else if tc4, ok4 := parseReActToolCall(f.toolCallBuf); ok4 {
|
|
idxCopy := f.toolIndex
|
|
tc4.Index = &idxCopy
|
|
f.toolIndex++
|
|
f.emittedCall = true
|
|
onToolCall(tc4)
|
|
} else if !f.emittedCall {
|
|
onContent(f.activePair.Start + f.toolCallBuf)
|
|
}
|
|
f.toolCallBuf = ""
|
|
}
|
|
if len(f.buf) > 0 {
|
|
if !f.emittedCall {
|
|
onContent(f.buf)
|
|
}
|
|
f.buf = ""
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Universal Gradio space inspector & metadata discovery
|
|
// ---------------------------------------------------------------------------
|
|
|
|
type GradioParamInfo struct {
|
|
Label string `json:"label"`
|
|
ParameterName string `json:"parameter_name"`
|
|
ParameterDefault interface{} `json:"parameter_default,omitempty"`
|
|
Component string `json:"component"`
|
|
Type interface{} `json:"type,omitempty"`
|
|
PythonType interface{} `json:"python_type,omitempty"`
|
|
}
|
|
|
|
type GradioEndpointInfo struct {
|
|
Parameters []GradioParamInfo `json:"parameters"`
|
|
Returns []GradioParamInfo `json:"returns"`
|
|
APIVisibility string `json:"api_visibility"`
|
|
Description string `json:"description"`
|
|
CodeSnippets map[string]interface{} `json:"code_snippets,omitempty"`
|
|
}
|
|
|
|
type GradioAPIInfoResponse struct {
|
|
NamedEndpoints map[string]GradioEndpointInfo `json:"named_endpoints"`
|
|
UnnamedEndpoints map[string]GradioEndpointInfo `json:"unnamed_endpoints"`
|
|
}
|
|
|
|
type GradioComponent struct {
|
|
ID int `json:"id"`
|
|
Type string `json:"type"`
|
|
Props map[string]interface{} `json:"props"`
|
|
SkipAPI bool `json:"skip_api"`
|
|
}
|
|
|
|
type GradioDependencyTypes struct {
|
|
Generator bool `json:"generator"`
|
|
Cancel bool `json:"cancel"`
|
|
}
|
|
|
|
type GradioDependency struct {
|
|
ID int `json:"id"`
|
|
APIName interface{} `json:"api_name"`
|
|
Inputs []int `json:"inputs"`
|
|
Outputs []int `json:"outputs"`
|
|
Queue interface{} `json:"queue"`
|
|
Types GradioDependencyTypes `json:"types"`
|
|
APIVisibility string `json:"api_visibility"`
|
|
}
|
|
|
|
type GradioConfigResponse struct {
|
|
Version string `json:"version"`
|
|
APIPrefix string `json:"api_prefix"`
|
|
Mode string `json:"mode"`
|
|
Title string `json:"title"`
|
|
Components []GradioComponent `json:"components"`
|
|
Dependencies []GradioDependency `json:"dependencies"`
|
|
}
|
|
|
|
type HFSpaceCardData struct {
|
|
Title string `json:"title"`
|
|
ShortDescription string `json:"short_description"`
|
|
}
|
|
|
|
type HFSpaceInfoResponse struct {
|
|
ID string `json:"id"`
|
|
Models []string `json:"models"`
|
|
CardData HFSpaceCardData `json:"cardData"`
|
|
}
|
|
|
|
type SpaceParamMapping struct {
|
|
InputIndex int `json:"input_index"`
|
|
ComponentID int `json:"component_id"`
|
|
ComponentType string `json:"component_type,omitempty"`
|
|
Label string `json:"label,omitempty"`
|
|
ParamName string `json:"param_name,omitempty"`
|
|
ParamType string `json:"param_type"` // "message", "history", "system_prompt", "temperature", "max_tokens", "top_p", "think_level", "tools", "stream", "state", "web_search", "other"
|
|
DefaultValue interface{} `json:"default_value,omitempty"`
|
|
Choices []string `json:"choices,omitempty"`
|
|
}
|
|
|
|
type SpaceDiscovery struct {
|
|
SpaceURL string `json:"space_url"`
|
|
Title string `json:"title"`
|
|
GradioVersion string `json:"gradio_version"`
|
|
Flavor string `json:"flavor"`
|
|
Protocol string `json:"protocol"` // "call", "queue", "predict"
|
|
APIPrefix string `json:"api_prefix"` // e.g. "/gradio_api" or ""
|
|
Endpoint string `json:"endpoint"` // e.g. "/chat_fn" or "/chat" or "/run/predict"
|
|
CleanEndpoint string `json:"clean_endpoint"` // e.g. "chat_fn" or "chat"
|
|
FnIndex int `json:"fn_index"` // -1 if not applicable
|
|
Models []string `json:"models"`
|
|
PrimaryModel string `json:"primary_model"`
|
|
TotalInputs int `json:"total_inputs"`
|
|
RawTotalInputs int `json:"raw_total_inputs,omitempty"`
|
|
RawDefaultInputs []interface{} `json:"raw_default_inputs,omitempty"`
|
|
ParamMappings []SpaceParamMapping `json:"param_mappings"`
|
|
DefaultInputs []interface{} `json:"default_inputs"`
|
|
MessageIsMultimodal bool `json:"message_is_multimodal"`
|
|
HistoryIndex int `json:"history_index"` // -1 if none
|
|
MessageIndex int `json:"message_index"` // index for user message text
|
|
SystemIndex int `json:"system_index"` // -1 if none
|
|
DefaultSystemPrompt string `json:"default_system_prompt,omitempty"` // default space system prompt if present
|
|
TempIndex int `json:"temp_index"` // -1 if none
|
|
MaxTokensIndex int `json:"max_tokens_index"` // -1 if none
|
|
TopPIndex int `json:"top_p_index"` // -1 if none
|
|
StreamIndex int `json:"stream_index"` // -1 if none
|
|
ThinkLevelIndex int `json:"think_level_index"` // -1 if none
|
|
FunctionsJSONIndex int `json:"functions_json_index"` // -1 if none
|
|
PreservedThinkingIndex int `json:"preserved_thinking_index"` // -1 if none
|
|
WebSearchIndex int `json:"web_search_index"` // -1 if none
|
|
IsHunyuan3 bool `json:"is_hunyuan3"`
|
|
HistoryFormat string `json:"history_format"` // "messages", "pairs", "gradio_messages", "none"
|
|
ToolCallMode string `json:"tool_call_mode"` // "native_slot", "prompt_augmented_system", "prompt_augmented_first_turn", "prompt_augmented_single_prompt"
|
|
LastDiscovered time.Time `json:"last_discovered"`
|
|
}
|
|
|
|
func (d *SpaceDiscovery) Summary() string {
|
|
var sb strings.Builder
|
|
sb.WriteString("================================================================================\n")
|
|
sb.WriteString("Gradio Space Resolution Picture\n")
|
|
sb.WriteString("--------------------------------------------------------------------------------\n")
|
|
sb.WriteString(fmt.Sprintf("Space URL: %s\n", d.SpaceURL))
|
|
if d.Title != "" {
|
|
sb.WriteString(fmt.Sprintf("Title: %s\n", d.Title))
|
|
}
|
|
sb.WriteString(fmt.Sprintf("Gradio Version: %s\n", d.GradioVersion))
|
|
sb.WriteString(fmt.Sprintf("UI Flavor: %s\n", d.Flavor))
|
|
sb.WriteString(fmt.Sprintf("Protocol: %s\n", d.Protocol))
|
|
sb.WriteString(fmt.Sprintf("API Prefix: %s\n", d.APIPrefix))
|
|
sb.WriteString(fmt.Sprintf("Resolved Endpoint: %s\n", d.Endpoint))
|
|
if d.FnIndex >= 0 {
|
|
sb.WriteString(fmt.Sprintf("Function Index: %d\n", d.FnIndex))
|
|
}
|
|
sb.WriteString(fmt.Sprintf("Primary Model: %s\n", d.PrimaryModel))
|
|
if len(d.Models) > 0 {
|
|
sb.WriteString(fmt.Sprintf("Exposed Models: %s\n", strings.Join(d.Models, ", ")))
|
|
}
|
|
sb.WriteString(fmt.Sprintf("History Format: %s\n", d.HistoryFormat))
|
|
sb.WriteString(fmt.Sprintf("Tool Call Support: %s\n", d.ToolCallMode))
|
|
sb.WriteString(fmt.Sprintf("Total Input Slots: %d\n", d.TotalInputs))
|
|
if len(d.ParamMappings) > 0 {
|
|
sb.WriteString("Input Slot Mappings:\n")
|
|
for _, m := range d.ParamMappings {
|
|
desc := m.ComponentType
|
|
if desc == "" {
|
|
desc = "Unknown"
|
|
}
|
|
if m.Label != "" {
|
|
desc += fmt.Sprintf(" (label=%q)", m.Label)
|
|
}
|
|
sb.WriteString(fmt.Sprintf(" [%d] Component ID %-4d %-30s -> %s\n", m.InputIndex, m.ComponentID, desc, m.ParamType))
|
|
}
|
|
}
|
|
sb.WriteString("================================================================================")
|
|
return sb.String()
|
|
}
|
|
|
|
func (d *SpaceDiscovery) GetModelList() []ModelItem {
|
|
now := time.Now().Unix()
|
|
var items []ModelItem
|
|
seen := make(map[string]bool)
|
|
|
|
if ConfiguredModelName != "" {
|
|
seen[ConfiguredModelName] = true
|
|
items = append(items, ModelItem{
|
|
ID: ConfiguredModelName,
|
|
Object: "model",
|
|
Created: now,
|
|
OwnedBy: "gradio",
|
|
})
|
|
}
|
|
|
|
for _, m := range d.Models {
|
|
if m != "" && !seen[m] {
|
|
seen[m] = true
|
|
items = append(items, ModelItem{
|
|
ID: m,
|
|
Object: "model",
|
|
Created: now,
|
|
OwnedBy: "gradio",
|
|
})
|
|
}
|
|
}
|
|
|
|
if d.PrimaryModel != "" && !seen[d.PrimaryModel] {
|
|
seen[d.PrimaryModel] = true
|
|
items = append(items, ModelItem{
|
|
ID: d.PrimaryModel,
|
|
Object: "model",
|
|
Created: now,
|
|
OwnedBy: "gradio",
|
|
})
|
|
}
|
|
|
|
if len(items) == 0 {
|
|
items = append(items, ModelItem{
|
|
ID: "gradio-chat",
|
|
Object: "model",
|
|
Created: now,
|
|
OwnedBy: "gradio",
|
|
})
|
|
}
|
|
|
|
return items
|
|
}
|
|
|
|
func (d *SpaceDiscovery) MatchesModel(requested string) bool {
|
|
if requested == "" {
|
|
return true
|
|
}
|
|
cleanReq := strings.TrimPrefix(requested, "models/")
|
|
cleanReq = strings.TrimPrefix(cleanReq, "openai/")
|
|
cleanReq = strings.ToLower(cleanReq)
|
|
|
|
if ConfiguredModelName != "" && strings.ToLower(ConfiguredModelName) == cleanReq {
|
|
return true
|
|
}
|
|
|
|
for _, m := range d.Models {
|
|
mClean := strings.TrimPrefix(m, "models/")
|
|
mClean = strings.TrimPrefix(mClean, "openai/")
|
|
if strings.ToLower(mClean) == cleanReq || strings.ToLower(m) == cleanReq {
|
|
return true
|
|
}
|
|
}
|
|
|
|
if strings.ToLower(d.PrimaryModel) == cleanReq {
|
|
return true
|
|
}
|
|
|
|
return cleanReq == "default" || cleanReq == "gradio" || cleanReq == "gradio-chat"
|
|
}
|
|
|
|
func NewDefaultSpaceDiscovery(spaceURL string) *SpaceDiscovery {
|
|
cleanURL := strings.TrimRight(spaceURL, "/")
|
|
if cleanURL != "" && !strings.HasPrefix(cleanURL, "http://") && !strings.HasPrefix(cleanURL, "https://") {
|
|
cleanURL = "https://" + cleanURL
|
|
}
|
|
return &SpaceDiscovery{
|
|
SpaceURL: cleanURL,
|
|
GradioVersion: "unknown",
|
|
Flavor: "generic",
|
|
Protocol: "call",
|
|
APIPrefix: "/gradio_api",
|
|
Endpoint: "/chat_fn",
|
|
CleanEndpoint: "chat_fn",
|
|
FnIndex: -1,
|
|
TotalInputs: 1,
|
|
HistoryIndex: -1,
|
|
MessageIndex: 0,
|
|
SystemIndex: -1,
|
|
DefaultSystemPrompt: "",
|
|
TempIndex: -1,
|
|
MaxTokensIndex: -1,
|
|
TopPIndex: -1,
|
|
StreamIndex: -1,
|
|
ThinkLevelIndex: -1,
|
|
FunctionsJSONIndex: -1,
|
|
PreservedThinkingIndex: -1,
|
|
WebSearchIndex: -1,
|
|
HistoryFormat: "messages",
|
|
ToolCallMode: "prompt_augmented_single_prompt",
|
|
LastDiscovered: time.Now(),
|
|
}
|
|
}
|
|
|
|
// DetectGradioFlavor classifies the architecture of the Gradio space.
|
|
func DetectGradioFlavor(configResp GradioConfigResponse, compMap map[int]GradioComponent) string {
|
|
mode := strings.ToLower(strings.TrimSpace(configResp.Mode))
|
|
hasChatbot := false
|
|
hasMultimodal := false
|
|
for _, c := range compMap {
|
|
cType := strings.ToLower(c.Type)
|
|
if cType == "chatbot" {
|
|
hasChatbot = true
|
|
} else if cType == "multimodaltextbox" {
|
|
hasMultimodal = true
|
|
}
|
|
}
|
|
|
|
switch mode {
|
|
case "chat_interface":
|
|
if hasMultimodal {
|
|
return "ChatInterface (Multimodal)"
|
|
}
|
|
return "ChatInterface"
|
|
case "blocks":
|
|
if hasChatbot {
|
|
if hasMultimodal {
|
|
return "Blocks (Multimodal Chat)"
|
|
}
|
|
return "Blocks (Chat)"
|
|
}
|
|
return "Blocks"
|
|
case "interface":
|
|
return "Interface"
|
|
default:
|
|
if mode != "" {
|
|
return mode
|
|
}
|
|
if hasChatbot {
|
|
return "ChatInterface"
|
|
}
|
|
return "Generic"
|
|
}
|
|
}
|
|
|
|
// ScoreCandidateEndpoint computes a heuristic score for an endpoint based on its API name,
|
|
// parameter signatures, and dependency graph topology.
|
|
func ScoreCandidateEndpoint(apiName string, parameters []GradioParamInfo, dep *GradioDependency, compMap map[int]GradioComponent) int {
|
|
score := 0
|
|
cleanName := strings.TrimPrefix(apiName, "/")
|
|
lowerName := strings.ToLower(cleanName)
|
|
|
|
hasInputs := len(parameters) > 0 || (dep != nil && len(dep.Inputs) > 0)
|
|
if !hasInputs {
|
|
return -1000
|
|
}
|
|
|
|
if strings.Contains(lowerName, "clear") || strings.Contains(lowerName, "reset") ||
|
|
strings.Contains(lowerName, "init") || strings.Contains(lowerName, "undo") ||
|
|
strings.Contains(lowerName, "delete") || strings.Contains(lowerName, "remove") ||
|
|
strings.Contains(lowerName, "pop") || strings.Contains(lowerName, "clean") {
|
|
score -= 600
|
|
}
|
|
if strings.Contains(lowerName, "vote") || strings.Contains(lowerName, "like") ||
|
|
strings.Contains(lowerName, "dislike") || strings.Contains(lowerName, "flag") ||
|
|
strings.Contains(lowerName, "feedback") || strings.Contains(lowerName, "report") {
|
|
score -= 500
|
|
}
|
|
if strings.Contains(lowerName, "download") || strings.Contains(lowerName, "export") ||
|
|
strings.Contains(lowerName, "save") || strings.Contains(lowerName, "upload") ||
|
|
strings.Contains(lowerName, "auth") || strings.Contains(lowerName, "login") ||
|
|
strings.Contains(lowerName, "theme") || strings.Contains(lowerName, "token") {
|
|
score -= 400
|
|
}
|
|
if strings.Contains(lowerName, "whisper") || strings.Contains(lowerName, "transcribe") ||
|
|
strings.Contains(lowerName, "tts") || strings.Contains(lowerName, "speech") ||
|
|
strings.Contains(lowerName, "diffusion") || strings.Contains(lowerName, "draw") ||
|
|
strings.Contains(lowerName, "sdxl") || strings.Contains(lowerName, "upscale") {
|
|
score -= 300
|
|
}
|
|
if strings.Contains(lowerName, "lambda") {
|
|
score -= 200
|
|
}
|
|
|
|
// Penalize user-submission helper endpoints (common in Gradio Blocks multi-step chat interfaces
|
|
// where a "user" function simply appends input to history and clears the textbox)
|
|
isUserHandler := lowerName == "user" || strings.HasPrefix(lowerName, "user_") ||
|
|
strings.Contains(lowerName, "add_text") || strings.Contains(lowerName, "add_msg") ||
|
|
strings.Contains(lowerName, "add_message") || strings.Contains(lowerName, "append_to_history")
|
|
if isUserHandler {
|
|
score -= 600
|
|
} else if strings.HasPrefix(lowerName, "user") {
|
|
// Check if name is user followed by digits/underscores (e.g. user2, user3, user_1)
|
|
suffix := strings.TrimPrefix(lowerName, "user")
|
|
isDigitsOrUnder := true
|
|
for _, r := range suffix {
|
|
if (r < '0' || r > '9') && r != '_' {
|
|
isDigitsOrUnder = false
|
|
break
|
|
}
|
|
}
|
|
if isDigitsOrUnder && len(suffix) > 0 {
|
|
score -= 600
|
|
}
|
|
}
|
|
|
|
if lowerName == "chat" || lowerName == "chat_fn" || lowerName == "bot" || lowerName == "bot_fn" {
|
|
score += 150
|
|
} else if strings.Contains(lowerName, "chat") || strings.Contains(lowerName, "conversation") ||
|
|
strings.Contains(lowerName, "dialogue") || strings.Contains(lowerName, "chatbot") ||
|
|
strings.Contains(lowerName, "bot") {
|
|
score += 120
|
|
}
|
|
if lowerName == "predict" || lowerName == "generate" {
|
|
score += 80
|
|
}
|
|
if strings.Contains(lowerName, "respond") || strings.Contains(lowerName, "reply") || strings.Contains(lowerName, "answer") {
|
|
score += 90
|
|
}
|
|
if strings.Contains(lowerName, "generate") || strings.Contains(lowerName, "completion") || strings.Contains(lowerName, "infer") || strings.Contains(lowerName, "predict") {
|
|
score += 70
|
|
}
|
|
if strings.Contains(lowerName, "ask") || strings.Contains(lowerName, "query") || strings.Contains(lowerName, "prompt") || strings.Contains(lowerName, "talk") || strings.Contains(lowerName, "run") {
|
|
score += 50
|
|
}
|
|
if strings.Contains(lowerName, "submit") {
|
|
score += 40
|
|
}
|
|
|
|
for _, p := range parameters {
|
|
pLower := strings.ToLower(p.ParameterName)
|
|
pLabel := strings.ToLower(p.Label)
|
|
pComp := strings.ToLower(p.Component)
|
|
|
|
if pComp == "multimodaltextbox" {
|
|
score += 80
|
|
} else if pComp == "textbox" {
|
|
if strings.Contains(pLower, "message") || strings.Contains(pLabel, "message") ||
|
|
strings.Contains(pLower, "prompt") || strings.Contains(pLabel, "prompt") ||
|
|
strings.Contains(pLower, "query") || strings.Contains(pLabel, "query") ||
|
|
strings.Contains(pLower, "input") || strings.Contains(pLabel, "input") {
|
|
score += 60
|
|
} else {
|
|
score += 30
|
|
}
|
|
} else if pComp == "chatbot" {
|
|
score += 70
|
|
} else if pComp == "state" {
|
|
score += 20
|
|
} else if pComp == "slider" || pComp == "number" {
|
|
if strings.Contains(pLower, "temp") || strings.Contains(pLabel, "temp") ||
|
|
strings.Contains(pLower, "token") || strings.Contains(pLabel, "token") ||
|
|
strings.Contains(pLower, "top_p") || strings.Contains(pLabel, "top_p") {
|
|
score += 20
|
|
}
|
|
}
|
|
if strings.Contains(pLower, "tool") || strings.Contains(pLabel, "tool") ||
|
|
strings.Contains(pLower, "function") || strings.Contains(pLabel, "function") {
|
|
score += 50
|
|
}
|
|
}
|
|
|
|
if dep != nil {
|
|
if dep.Types.Generator {
|
|
score += 150
|
|
}
|
|
// Check if any textbox component in dep.Inputs is also in dep.Outputs (textbox-clearing UI handler)
|
|
for _, inID := range dep.Inputs {
|
|
if comp, exists := compMap[inID]; exists {
|
|
if strings.ToLower(comp.Type) == "textbox" {
|
|
for _, outID := range dep.Outputs {
|
|
if outID == inID {
|
|
score -= 600
|
|
break
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
for _, inID := range dep.Inputs {
|
|
if comp, exists := compMap[inID]; exists {
|
|
cType := strings.ToLower(comp.Type)
|
|
cLabel := ""
|
|
if comp.Props != nil {
|
|
if l, ok := comp.Props["label"].(string); ok {
|
|
cLabel = strings.ToLower(l)
|
|
}
|
|
}
|
|
if cType == "multimodaltextbox" {
|
|
score += 80
|
|
} else if cType == "textbox" {
|
|
if strings.Contains(cLabel, "message") || strings.Contains(cLabel, "prompt") ||
|
|
strings.Contains(cLabel, "query") || strings.Contains(cLabel, "input") {
|
|
score += 60
|
|
} else if strings.Contains(cLabel, "system") || strings.Contains(cLabel, "instruction") {
|
|
score += 30
|
|
} else {
|
|
score += 20
|
|
}
|
|
} else if cType == "chatbot" {
|
|
score += 70
|
|
} else if cType == "state" {
|
|
score += 20
|
|
} else if cType == "slider" || cType == "number" {
|
|
if strings.Contains(cLabel, "temp") || strings.Contains(cLabel, "token") || strings.Contains(cLabel, "top") {
|
|
score += 20
|
|
}
|
|
}
|
|
if strings.Contains(cLabel, "tool") || strings.Contains(cLabel, "function") {
|
|
score += 50
|
|
}
|
|
}
|
|
}
|
|
for _, outID := range dep.Outputs {
|
|
if comp, exists := compMap[outID]; exists {
|
|
cType := strings.ToLower(comp.Type)
|
|
if cType == "chatbot" {
|
|
score += 100
|
|
} else if cType == "textbox" || cType == "markdown" {
|
|
isSharedInput := false
|
|
for _, inID := range dep.Inputs {
|
|
if inID == outID {
|
|
isSharedInput = true
|
|
break
|
|
}
|
|
}
|
|
if !isSharedInput {
|
|
score += 50
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return score
|
|
}
|
|
|
|
// InspectSpace queries Gradio's /gradio_api/info, /config, and HuggingFace Space APIs
|
|
// to build an adaptive schema mapping for any Gradio space.
|
|
func InspectSpace(client *http.Client, rawURL, userAgent string) (*SpaceDiscovery, error) {
|
|
cleanURL := strings.TrimRight(rawURL, "/")
|
|
if !strings.HasPrefix(cleanURL, "http://") && !strings.HasPrefix(cleanURL, "https://") {
|
|
cleanURL = "https://" + cleanURL
|
|
}
|
|
|
|
discovery := NewDefaultSpaceDiscovery(cleanURL)
|
|
|
|
// 1. Try fetching /gradio_api/info or /info
|
|
var infoResp GradioAPIInfoResponse
|
|
infoFetched := false
|
|
|
|
for _, path := range []string{"/gradio_api/info", "/info"} {
|
|
infoURL := cleanURL + path
|
|
req, err := http.NewRequest("GET", infoURL, nil)
|
|
if err == nil {
|
|
req.Header.Set("User-Agent", userAgent)
|
|
resp, err := client.Do(req)
|
|
if err == nil && resp.StatusCode == http.StatusOK {
|
|
if json.NewDecoder(resp.Body).Decode(&infoResp) == nil {
|
|
infoFetched = true
|
|
if path == "/gradio_api/info" {
|
|
discovery.APIPrefix = "/gradio_api"
|
|
} else {
|
|
discovery.APIPrefix = ""
|
|
}
|
|
}
|
|
resp.Body.Close()
|
|
if infoFetched {
|
|
break
|
|
}
|
|
} else if resp != nil {
|
|
resp.Body.Close()
|
|
}
|
|
}
|
|
}
|
|
|
|
// 2. Try fetching /config
|
|
var configResp GradioConfigResponse
|
|
configFetched := false
|
|
|
|
for _, path := range []string{"/config", "/gradio_api/config"} {
|
|
cfgURL := cleanURL + path
|
|
req, err := http.NewRequest("GET", cfgURL, nil)
|
|
if err == nil {
|
|
req.Header.Set("User-Agent", userAgent)
|
|
resp, err := client.Do(req)
|
|
if err == nil && resp.StatusCode == http.StatusOK {
|
|
if json.NewDecoder(resp.Body).Decode(&configResp) == nil {
|
|
configFetched = true
|
|
if configResp.APIPrefix != "" {
|
|
discovery.APIPrefix = configResp.APIPrefix
|
|
}
|
|
if configResp.Title != "" {
|
|
discovery.Title = configResp.Title
|
|
}
|
|
}
|
|
resp.Body.Close()
|
|
if configFetched {
|
|
break
|
|
}
|
|
} else if resp != nil {
|
|
resp.Body.Close()
|
|
}
|
|
}
|
|
}
|
|
|
|
// 3. Inspect Hugging Face Space Metadata if hosted on HF
|
|
parsedURL, _ := url.Parse(cleanURL)
|
|
if parsedURL != nil && (strings.HasSuffix(parsedURL.Host, ".hf.space") || strings.Contains(parsedURL.Host, "huggingface.co")) {
|
|
subdomain := strings.TrimSuffix(parsedURL.Host, ".hf.space")
|
|
var owner, name string
|
|
dashIdx := strings.Index(subdomain, "-")
|
|
if dashIdx != -1 {
|
|
owner = subdomain[:dashIdx]
|
|
name = subdomain[dashIdx+1:]
|
|
}
|
|
|
|
if owner != "" && name != "" {
|
|
hfAPIURL := fmt.Sprintf("https://huggingface.co/api/spaces/%s/%s", owner, name)
|
|
req, err := http.NewRequest("GET", hfAPIURL, nil)
|
|
if err == nil {
|
|
req.Header.Set("User-Agent", userAgent)
|
|
resp, err := client.Do(req)
|
|
if err == nil && resp.StatusCode == http.StatusOK {
|
|
var hfResp HFSpaceInfoResponse
|
|
if json.NewDecoder(resp.Body).Decode(&hfResp) == nil {
|
|
for _, m := range hfResp.Models {
|
|
discovery.Models = append(discovery.Models, m)
|
|
cleanM := strings.TrimPrefix(m, "openai/")
|
|
cleanM = strings.TrimPrefix(cleanM, "models/")
|
|
if cleanM != m {
|
|
discovery.Models = append(discovery.Models, cleanM)
|
|
}
|
|
}
|
|
if hfResp.CardData.Title != "" && discovery.Title == "" {
|
|
discovery.Title = hfResp.CardData.Title
|
|
}
|
|
if len(discovery.Models) > 0 {
|
|
discovery.PrimaryModel = discovery.Models[0]
|
|
}
|
|
}
|
|
resp.Body.Close()
|
|
} else if resp != nil {
|
|
resp.Body.Close()
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Fallback model names if not discovered
|
|
if len(discovery.Models) == 0 {
|
|
if parsedURL != nil && strings.HasSuffix(parsedURL.Host, ".hf.space") {
|
|
sub := strings.TrimSuffix(parsedURL.Host, ".hf.space")
|
|
parts := strings.Split(sub, "-")
|
|
if len(parts) > 1 {
|
|
cleanModel := strings.Join(parts[1:], "-")
|
|
discovery.Models = append(discovery.Models, cleanModel)
|
|
discovery.PrimaryModel = cleanModel
|
|
}
|
|
}
|
|
}
|
|
if discovery.PrimaryModel == "" {
|
|
if len(discovery.Models) > 0 {
|
|
discovery.PrimaryModel = discovery.Models[0]
|
|
} else {
|
|
discovery.PrimaryModel = "gradio-chat"
|
|
discovery.Models = append(discovery.Models, "gradio-chat")
|
|
}
|
|
}
|
|
|
|
// 4. Detect Gradio version and UI flavor
|
|
compMap := make(map[int]GradioComponent)
|
|
if configFetched {
|
|
for _, comp := range configResp.Components {
|
|
compMap[comp.ID] = comp
|
|
}
|
|
if configResp.Version != "" {
|
|
discovery.GradioVersion = configResp.Version
|
|
} else if infoFetched {
|
|
discovery.GradioVersion = "4+ (inferred from /gradio_api/info)"
|
|
}
|
|
discovery.Flavor = DetectGradioFlavor(configResp, compMap)
|
|
} else if infoFetched {
|
|
discovery.GradioVersion = "4+ (inferred from /gradio_api/info)"
|
|
discovery.Flavor = "Generic"
|
|
}
|
|
|
|
// 5. Score and select the best chat completion endpoint
|
|
bestEndpoint := ""
|
|
bestScore := -1000
|
|
var bestEndpointInfo *GradioEndpointInfo
|
|
var bestMatchingDep *GradioDependency
|
|
|
|
// Try named endpoints from /gradio_api/info first
|
|
if infoFetched && len(infoResp.NamedEndpoints) > 0 {
|
|
epNames := make([]string, 0, len(infoResp.NamedEndpoints))
|
|
for epName := range infoResp.NamedEndpoints {
|
|
epNames = append(epNames, epName)
|
|
}
|
|
sort.Strings(epNames)
|
|
for _, epName := range epNames {
|
|
epInfo := infoResp.NamedEndpoints[epName]
|
|
cleanTarget := strings.TrimPrefix(epName, "/")
|
|
var matchingDep *GradioDependency
|
|
if configFetched {
|
|
for _, dep := range configResp.Dependencies {
|
|
depAPIName := ""
|
|
if s, ok := dep.APIName.(string); ok {
|
|
depAPIName = strings.TrimPrefix(s, "/")
|
|
}
|
|
if depAPIName == cleanTarget {
|
|
depCopy := dep
|
|
matchingDep = &depCopy
|
|
break
|
|
}
|
|
}
|
|
}
|
|
score := ScoreCandidateEndpoint(epName, epInfo.Parameters, matchingDep, compMap)
|
|
if score > bestScore {
|
|
bestScore = score
|
|
bestEndpoint = epName
|
|
epCopy := epInfo
|
|
bestEndpointInfo = &epCopy
|
|
bestMatchingDep = matchingDep
|
|
}
|
|
}
|
|
}
|
|
|
|
// Fallback: if no named endpoint from /info, inspect dependencies in config
|
|
if (bestEndpoint == "" || bestScore <= 0) && configFetched && len(configResp.Dependencies) > 0 {
|
|
depScore := -1000
|
|
for _, dep := range configResp.Dependencies {
|
|
apiName := ""
|
|
if s, ok := dep.APIName.(string); ok {
|
|
apiName = s
|
|
}
|
|
score := ScoreCandidateEndpoint(apiName, nil, &dep, compMap)
|
|
if score > depScore {
|
|
depScore = score
|
|
depCopy := dep
|
|
bestMatchingDep = &depCopy
|
|
if apiName != "" {
|
|
bestEndpoint = "/" + strings.TrimPrefix(apiName, "/")
|
|
} else {
|
|
bestEndpoint = fmt.Sprintf("/%d", dep.ID)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if bestMatchingDep != nil {
|
|
discovery.FnIndex = bestMatchingDep.ID
|
|
}
|
|
|
|
// Select protocol
|
|
isCallV2 := false
|
|
if bestEndpointInfo != nil && bestEndpointInfo.CodeSnippets != nil {
|
|
if bashSnippet, ok := bestEndpointInfo.CodeSnippets["bash"].(string); ok && bashSnippet != "" {
|
|
if strings.Contains(bashSnippet, "/call/v2/") {
|
|
isCallV2 = true
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
if strings.HasPrefix(discovery.GradioVersion, "3.") {
|
|
discovery.Protocol = "predict"
|
|
discovery.APIPrefix = ""
|
|
discovery.Endpoint = "/run/predict"
|
|
discovery.CleanEndpoint = "run/predict"
|
|
} else if isCallV2 {
|
|
discovery.Protocol = "call_v2"
|
|
if bestEndpoint != "" {
|
|
discovery.Endpoint = bestEndpoint
|
|
discovery.CleanEndpoint = strings.TrimPrefix(bestEndpoint, "/")
|
|
}
|
|
} else {
|
|
discovery.Protocol = "call"
|
|
if bestEndpoint != "" {
|
|
discovery.Endpoint = bestEndpoint
|
|
discovery.CleanEndpoint = strings.TrimPrefix(bestEndpoint, "/")
|
|
}
|
|
}
|
|
|
|
// 6. Correlate with config.dependencies to determine exact input count & state padding
|
|
if bestMatchingDep != nil {
|
|
discovery.TotalInputs = len(bestMatchingDep.Inputs)
|
|
discovery.RawTotalInputs = len(bestMatchingDep.Inputs)
|
|
discovery.DefaultInputs = make([]interface{}, len(bestMatchingDep.Inputs))
|
|
discovery.ParamMappings = nil
|
|
discovery.MessageIndex = -1
|
|
|
|
for idx, compID := range bestMatchingDep.Inputs {
|
|
mapping := SpaceParamMapping{
|
|
InputIndex: idx,
|
|
ComponentID: compID,
|
|
ParamType: "other",
|
|
}
|
|
|
|
var pName, pLabel, pComp string
|
|
if bestEndpointInfo != nil && idx < len(bestEndpointInfo.Parameters) {
|
|
p := bestEndpointInfo.Parameters[idx]
|
|
pName = strings.ToLower(p.ParameterName)
|
|
pLabel = strings.ToLower(p.Label)
|
|
pComp = strings.ToLower(p.Component)
|
|
if p.ParameterDefault != nil && discovery.DefaultInputs[idx] == nil {
|
|
discovery.DefaultInputs[idx] = p.ParameterDefault
|
|
mapping.DefaultValue = p.ParameterDefault
|
|
}
|
|
mapping.Label = p.ParameterName
|
|
mapping.ParamName = p.ParameterName
|
|
if p.Component != "" {
|
|
mapping.ComponentType = p.Component
|
|
}
|
|
}
|
|
|
|
if comp, exists := compMap[compID]; exists {
|
|
if mapping.ComponentType == "" {
|
|
mapping.ComponentType = comp.Type
|
|
}
|
|
cType := strings.ToLower(comp.Type)
|
|
cLabel := ""
|
|
if comp.Props != nil {
|
|
if l, ok := comp.Props["label"].(string); ok {
|
|
cLabel = strings.ToLower(l)
|
|
if mapping.Label == "" {
|
|
mapping.Label = l
|
|
}
|
|
}
|
|
if val, ok := comp.Props["value"]; ok {
|
|
discovery.DefaultInputs[idx] = val
|
|
mapping.DefaultValue = val
|
|
}
|
|
if chList, ok := comp.Props["choices"].([]interface{}); ok {
|
|
for _, ch := range chList {
|
|
if chStr, ok := ch.(string); ok {
|
|
mapping.Choices = append(mapping.Choices, chStr)
|
|
} else if chPair, ok := ch.([]interface{}); ok && len(chPair) > 0 {
|
|
if chStr, ok := chPair[0].(string); ok {
|
|
mapping.Choices = append(mapping.Choices, chStr)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if bestEndpointInfo != nil && idx < len(bestEndpointInfo.Parameters) && len(mapping.Choices) == 0 {
|
|
p := bestEndpointInfo.Parameters[idx]
|
|
if typeMap, ok := p.Type.(map[string]interface{}); ok {
|
|
if enumArr, ok := typeMap["enum"].([]interface{}); ok {
|
|
for _, e := range enumArr {
|
|
if s, ok := e.(string); ok {
|
|
mapping.Choices = append(mapping.Choices, s)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if strings.Contains(pName, "function") || strings.Contains(pName, "tool") || strings.Contains(pLabel, "tool") || strings.Contains(pLabel, "function") || strings.Contains(cLabel, "tool") || strings.Contains(cLabel, "function") {
|
|
mapping.ParamType = "tools"
|
|
discovery.FunctionsJSONIndex = idx
|
|
} else if strings.Contains(pName, "preserved") || strings.Contains(cLabel, "preserved") {
|
|
mapping.ParamType = "preserved_thinking"
|
|
discovery.PreservedThinkingIndex = idx
|
|
} else if strings.Contains(pName, "think_level") || strings.Contains(pName, "thinking") || strings.Contains(pLabel, "think") || strings.Contains(cLabel, "think") {
|
|
mapping.ParamType = "think_level"
|
|
discovery.ThinkLevelIndex = idx
|
|
} else if strings.Contains(pName, "system") || strings.Contains(pLabel, "system") || strings.Contains(cLabel, "system") || strings.Contains(cLabel, "instruction") {
|
|
mapping.ParamType = "system_prompt"
|
|
discovery.SystemIndex = idx
|
|
if comp.Props != nil {
|
|
if val, ok := comp.Props["value"].(string); ok && strings.TrimSpace(val) != "" {
|
|
discovery.DefaultSystemPrompt = strings.TrimSpace(val)
|
|
}
|
|
}
|
|
} else if strings.Contains(pName, "history") || strings.Contains(pLabel, "history") || strings.Contains(cLabel, "history") || strings.Contains(cLabel, "chat") || cType == "chatbot" {
|
|
mapping.ParamType = "history"
|
|
discovery.HistoryIndex = idx
|
|
} else if cType == "multimodaltextbox" || strings.Contains(pComp, "multimodal") {
|
|
mapping.ParamType = "message"
|
|
discovery.MessageIndex = idx
|
|
discovery.MessageIsMultimodal = true
|
|
} else if strings.Contains(pName, "message") || strings.Contains(pLabel, "message") || strings.Contains(cLabel, "message") || strings.Contains(cLabel, "prompt") || strings.Contains(cLabel, "query") || (discovery.MessageIndex == -1 && idx == 0 && (cType == "textbox" || pComp == "textbox")) {
|
|
mapping.ParamType = "message"
|
|
discovery.MessageIndex = idx
|
|
discovery.MessageIsMultimodal = false
|
|
} else if strings.Contains(pName, "temp") || strings.Contains(cLabel, "temp") {
|
|
mapping.ParamType = "temperature"
|
|
discovery.TempIndex = idx
|
|
} else if strings.Contains(pName, "token") || strings.Contains(cLabel, "token") || strings.Contains(cLabel, "max") {
|
|
mapping.ParamType = "max_tokens"
|
|
discovery.MaxTokensIndex = idx
|
|
} else if strings.Contains(pName, "top_p") || strings.Contains(cLabel, "top_p") || strings.Contains(cLabel, "top-p") || strings.Contains(cLabel, "top p") {
|
|
mapping.ParamType = "top_p"
|
|
discovery.TopPIndex = idx
|
|
} else if strings.Contains(pName, "stream") || strings.Contains(cLabel, "stream") {
|
|
mapping.ParamType = "stream"
|
|
discovery.StreamIndex = idx
|
|
} else if strings.Contains(pName, "search") || strings.Contains(pLabel, "search") || strings.Contains(cLabel, "search") || strings.Contains(pName, "browse") || strings.Contains(pLabel, "browse") || strings.Contains(cLabel, "browse") || strings.Contains(pName, "web") || strings.Contains(pLabel, "web") || strings.Contains(cLabel, "web") {
|
|
mapping.ParamType = "web_search"
|
|
discovery.WebSearchIndex = idx
|
|
} else if cType == "state" {
|
|
mapping.ParamType = "state"
|
|
}
|
|
}
|
|
discovery.ParamMappings = append(discovery.ParamMappings, mapping)
|
|
}
|
|
|
|
if discovery.MessageIndex == -1 {
|
|
if discovery.HistoryIndex != 0 && discovery.SystemIndex != 0 && discovery.FunctionsJSONIndex != 0 {
|
|
discovery.MessageIndex = 0
|
|
if len(bestMatchingDep.Inputs) > 0 {
|
|
if comp, exists := compMap[bestMatchingDep.Inputs[0]]; exists {
|
|
if strings.ToLower(comp.Type) == "multimodaltextbox" {
|
|
discovery.MessageIsMultimodal = true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
discovery.RawDefaultInputs = make([]interface{}, len(discovery.DefaultInputs))
|
|
copy(discovery.RawDefaultInputs, discovery.DefaultInputs)
|
|
|
|
// If the endpoint has canonical parameters exposed via /gradio_api/info,
|
|
// and trailing inputs in bestMatchingDep.Inputs are unexposed server-side state components,
|
|
// do not pad them with null so Gradio preserves its internal server state.
|
|
if bestEndpointInfo != nil && len(bestEndpointInfo.Parameters) > 0 && len(bestMatchingDep.Inputs) > len(bestEndpointInfo.Parameters) {
|
|
allTrailingAreState := true
|
|
for i := len(bestEndpointInfo.Parameters); i < len(bestMatchingDep.Inputs); i++ {
|
|
cID := bestMatchingDep.Inputs[i]
|
|
if comp, exists := compMap[cID]; exists {
|
|
cType := strings.ToLower(comp.Type)
|
|
if cType != "state" && cType != "browserstate" {
|
|
allTrailingAreState = false
|
|
break
|
|
}
|
|
}
|
|
}
|
|
if allTrailingAreState {
|
|
discovery.TotalInputs = len(bestEndpointInfo.Parameters)
|
|
discovery.DefaultInputs = discovery.DefaultInputs[:discovery.TotalInputs]
|
|
if len(discovery.ParamMappings) > discovery.TotalInputs {
|
|
discovery.ParamMappings = discovery.ParamMappings[:discovery.TotalInputs]
|
|
}
|
|
if discovery.MessageIndex >= discovery.TotalInputs {
|
|
discovery.MessageIndex = -1
|
|
}
|
|
if discovery.HistoryIndex >= discovery.TotalInputs {
|
|
discovery.HistoryIndex = -1
|
|
}
|
|
if discovery.SystemIndex >= discovery.TotalInputs {
|
|
discovery.SystemIndex = -1
|
|
}
|
|
if discovery.FunctionsJSONIndex >= discovery.TotalInputs {
|
|
discovery.FunctionsJSONIndex = -1
|
|
}
|
|
if discovery.TempIndex >= discovery.TotalInputs {
|
|
discovery.TempIndex = -1
|
|
}
|
|
if discovery.MaxTokensIndex >= discovery.TotalInputs {
|
|
discovery.MaxTokensIndex = -1
|
|
}
|
|
if discovery.TopPIndex >= discovery.TotalInputs {
|
|
discovery.TopPIndex = -1
|
|
}
|
|
if discovery.StreamIndex >= discovery.TotalInputs {
|
|
discovery.StreamIndex = -1
|
|
}
|
|
if discovery.WebSearchIndex >= discovery.TotalInputs {
|
|
discovery.WebSearchIndex = -1
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Refine history format or discover tools from bestEndpointInfo
|
|
if bestEndpointInfo != nil {
|
|
for _, p := range bestEndpointInfo.Parameters {
|
|
pName := strings.ToLower(p.ParameterName)
|
|
pLabel := strings.ToLower(p.Label)
|
|
pComp := strings.ToLower(p.Component)
|
|
if strings.Contains(pName, "history") || strings.Contains(pLabel, "history") || strings.Contains(pName, "chat") || strings.Contains(pName, "messages") || pComp == "chatbot" {
|
|
bType, _ := json.Marshal(p.Type)
|
|
bPyType, _ := json.Marshal(p.PythonType)
|
|
pPyType := strings.ToLower(string(bPyType))
|
|
bTypeStr := strings.ToLower(string(bType))
|
|
if strings.Contains(pPyType, "list[tuple[") || strings.Contains(pPyType, "list[list[") || strings.Contains(bTypeStr, "tuple") {
|
|
discovery.HistoryFormat = "pairs"
|
|
} else if strings.Contains(pPyType, "textmessage") || strings.Contains(pPyType, "dict(text: str") || strings.Contains(bTypeStr, "textmessage") || strings.Contains(bTypeStr, "chatbotdatamessages") {
|
|
discovery.HistoryFormat = "gradio_messages"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Fallback: If config.dependencies did not provide matchingDep, map directly from bestEndpointInfo.Parameters
|
|
if bestMatchingDep == nil && bestEndpointInfo != nil {
|
|
discovery.TotalInputs = len(bestEndpointInfo.Parameters)
|
|
discovery.DefaultInputs = make([]interface{}, len(bestEndpointInfo.Parameters))
|
|
discovery.ParamMappings = nil
|
|
discovery.MessageIndex = -1
|
|
|
|
for idx, p := range bestEndpointInfo.Parameters {
|
|
pName := strings.ToLower(p.ParameterName)
|
|
pLabel := strings.ToLower(p.Label)
|
|
pComp := strings.ToLower(p.Component)
|
|
|
|
if p.ParameterDefault != nil {
|
|
discovery.DefaultInputs[idx] = p.ParameterDefault
|
|
}
|
|
|
|
mapping := SpaceParamMapping{
|
|
InputIndex: idx,
|
|
ComponentType: p.Component,
|
|
Label: p.Label,
|
|
ParamName: p.ParameterName,
|
|
ParamType: "other",
|
|
DefaultValue: p.ParameterDefault,
|
|
}
|
|
|
|
if strings.Contains(pComp, "multimodal") {
|
|
if discovery.MessageIndex == -1 || strings.Contains(pLabel, "message") || strings.Contains(pName, "message") {
|
|
mapping.ParamType = "message"
|
|
discovery.MessageIndex = idx
|
|
discovery.MessageIsMultimodal = true
|
|
}
|
|
} else if strings.Contains(pName, "system") || strings.Contains(pLabel, "system") || strings.Contains(pLabel, "instruction") {
|
|
discovery.SystemIndex = idx
|
|
mapping.ParamType = "system_prompt"
|
|
if p.ParameterDefault != nil && discovery.DefaultSystemPrompt == "" {
|
|
if defStr, ok := p.ParameterDefault.(string); ok && strings.TrimSpace(defStr) != "" {
|
|
discovery.DefaultSystemPrompt = strings.TrimSpace(defStr)
|
|
}
|
|
}
|
|
} else if strings.Contains(pName, "history") || strings.Contains(pLabel, "history") || strings.Contains(pName, "chat") || strings.Contains(pName, "messages") || strings.Contains(pName, "conversation") || pComp == "chatbot" {
|
|
discovery.HistoryIndex = idx
|
|
mapping.ParamType = "history"
|
|
bType, _ := json.Marshal(p.Type)
|
|
bPyType, _ := json.Marshal(p.PythonType)
|
|
pPyType := strings.ToLower(string(bPyType))
|
|
bTypeStr := strings.ToLower(string(bType))
|
|
if strings.Contains(pPyType, "list[tuple[") || strings.Contains(pPyType, "list[list[") || strings.Contains(bTypeStr, "tuple") {
|
|
discovery.HistoryFormat = "pairs"
|
|
} else if strings.Contains(pPyType, "textmessage") || strings.Contains(pPyType, "dict(text: str") || strings.Contains(bTypeStr, "textmessage") || strings.Contains(bTypeStr, "chatbotdatamessages") {
|
|
discovery.HistoryFormat = "gradio_messages"
|
|
}
|
|
} else if strings.Contains(pLabel, "message") || strings.Contains(pName, "message") || (strings.Contains(pLabel, "prompt") && !strings.Contains(pLabel, "system")) || (strings.Contains(pName, "prompt") && !strings.Contains(pName, "system")) || strings.Contains(pLabel, "query") || strings.Contains(pName, "query") || strings.Contains(pLabel, "question") || strings.Contains(pName, "question") || (discovery.MessageIndex == -1 && (pComp == "textbox" || (idx == 0 && pComp != "chatbot"))) {
|
|
discovery.MessageIndex = idx
|
|
mapping.ParamType = "message"
|
|
discovery.MessageIsMultimodal = false
|
|
} else if strings.Contains(pName, "think_level") || strings.Contains(pName, "thinking_level") || strings.Contains(pLabel, "think") {
|
|
discovery.ThinkLevelIndex = idx
|
|
mapping.ParamType = "think_level"
|
|
} else if strings.Contains(pName, "functions") || strings.Contains(pName, "tools") || strings.Contains(pLabel, "tools") || strings.Contains(pLabel, "functions") {
|
|
discovery.FunctionsJSONIndex = idx
|
|
mapping.ParamType = "tools"
|
|
} else if strings.Contains(pName, "preserved") {
|
|
discovery.PreservedThinkingIndex = idx
|
|
} else if strings.Contains(pName, "temp") || strings.Contains(pLabel, "temp") {
|
|
discovery.TempIndex = idx
|
|
mapping.ParamType = "temperature"
|
|
} else if strings.Contains(pName, "token") || strings.Contains(pLabel, "token") {
|
|
discovery.MaxTokensIndex = idx
|
|
mapping.ParamType = "max_tokens"
|
|
} else if strings.Contains(pName, "top_p") || strings.Contains(pLabel, "top_p") || strings.Contains(pLabel, "top p") {
|
|
discovery.TopPIndex = idx
|
|
mapping.ParamType = "top_p"
|
|
} else if strings.Contains(pName, "stream") || strings.Contains(pLabel, "stream") {
|
|
discovery.StreamIndex = idx
|
|
mapping.ParamType = "stream"
|
|
} else if strings.Contains(pName, "search") || strings.Contains(pLabel, "search") || strings.Contains(pName, "browse") || strings.Contains(pLabel, "browse") || strings.Contains(pName, "web") || strings.Contains(pLabel, "web") {
|
|
discovery.WebSearchIndex = idx
|
|
mapping.ParamType = "web_search"
|
|
}
|
|
|
|
if typeMap, ok := p.Type.(map[string]interface{}); ok {
|
|
if enumArr, ok := typeMap["enum"].([]interface{}); ok {
|
|
for _, e := range enumArr {
|
|
if s, ok := e.(string); ok {
|
|
mapping.Choices = append(mapping.Choices, s)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
discovery.ParamMappings = append(discovery.ParamMappings, mapping)
|
|
}
|
|
|
|
if discovery.MessageIndex == -1 {
|
|
if discovery.HistoryIndex != 0 && discovery.SystemIndex != 0 && discovery.FunctionsJSONIndex != 0 {
|
|
discovery.MessageIndex = 0
|
|
}
|
|
}
|
|
}
|
|
|
|
// 7. Resolve default history format if not set by type inspection
|
|
if discovery.HistoryIndex != -1 {
|
|
if discovery.HistoryFormat == "" || discovery.HistoryFormat == "messages" {
|
|
if strings.HasPrefix(discovery.GradioVersion, "5.") || strings.HasPrefix(discovery.GradioVersion, "6.") {
|
|
discovery.HistoryFormat = "gradio_messages"
|
|
} else {
|
|
discovery.HistoryFormat = "pairs"
|
|
}
|
|
}
|
|
} else {
|
|
discovery.HistoryFormat = "none"
|
|
}
|
|
|
|
// 8. Hunyuan3 / Tencent Hy3 overrides
|
|
if discovery.FunctionsJSONIndex != -1 || discovery.ThinkLevelIndex != -1 || strings.Contains(cleanURL, "hy3") || strings.Contains(cleanURL, "hunyuan") {
|
|
discovery.IsHunyuan3 = true
|
|
discovery.HistoryFormat = "messages"
|
|
discovery.Models = append(discovery.Models, "hy3", "hunyuan3", "tencent/Hy3")
|
|
if discovery.PrimaryModel == "gradio-chat" || discovery.PrimaryModel == "" {
|
|
discovery.PrimaryModel = "hy3"
|
|
}
|
|
if discovery.Protocol == "call_v2" {
|
|
discovery.Protocol = "call"
|
|
}
|
|
}
|
|
|
|
// 9. Tool Calling Support Picture Resolution
|
|
if discovery.FunctionsJSONIndex >= 0 {
|
|
discovery.ToolCallMode = "native_slot"
|
|
} else if discovery.SystemIndex >= 0 {
|
|
discovery.ToolCallMode = "prompt_augmented_system"
|
|
} else if discovery.HistoryIndex >= 0 {
|
|
discovery.ToolCallMode = "prompt_augmented_first_turn"
|
|
} else {
|
|
discovery.ToolCallMode = "prompt_augmented_single_prompt"
|
|
}
|
|
|
|
// Ensure total inputs is at least 1
|
|
if discovery.TotalInputs < 1 {
|
|
discovery.TotalInputs = 1
|
|
}
|
|
for len(discovery.DefaultInputs) < discovery.TotalInputs {
|
|
discovery.DefaultInputs = append(discovery.DefaultInputs, nil)
|
|
}
|
|
|
|
return discovery, nil
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Universal Gradio gateway engine
|
|
// ---------------------------------------------------------------------------
|
|
|
|
type GradioJoinResponse struct {
|
|
EventID string `json:"event_id"`
|
|
}
|
|
|
|
type GradioGateway struct {
|
|
mu sync.RWMutex
|
|
defaultURL string
|
|
client *http.Client
|
|
discoveries map[string]*SpaceDiscovery
|
|
proxyURL string
|
|
}
|
|
|
|
func NewGradioGateway(defaultSpaceURL, proxyURL string, timeout time.Duration) *GradioGateway {
|
|
cleanDefault := strings.TrimRight(defaultSpaceURL, "/")
|
|
if !strings.HasPrefix(cleanDefault, "http://") && !strings.HasPrefix(cleanDefault, "https://") {
|
|
cleanDefault = "https://" + cleanDefault
|
|
}
|
|
|
|
transport := &http.Transport{
|
|
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
|
|
if proxyURL != "" {
|
|
return DialSOCKS5(ctx, proxyURL, addr)
|
|
}
|
|
var d net.Dialer
|
|
return d.DialContext(ctx, network, addr)
|
|
},
|
|
MaxIdleConns: 100,
|
|
IdleConnTimeout: 90 * time.Second,
|
|
TLSHandshakeTimeout: 15 * time.Second,
|
|
}
|
|
|
|
gw := &GradioGateway{
|
|
defaultURL: cleanDefault,
|
|
client: &http.Client{Transport: transport, Timeout: timeout},
|
|
discoveries: make(map[string]*SpaceDiscovery),
|
|
proxyURL: proxyURL,
|
|
}
|
|
|
|
// Pre-discover the default space
|
|
disc, err := InspectSpace(gw.client, cleanDefault, DefaultUserAgent)
|
|
if err == nil && disc != nil {
|
|
gw.discoveries[cleanDefault] = disc
|
|
log.Printf("\n%s\n", disc.Summary())
|
|
} else if err != nil {
|
|
log.Printf("Warning: initial space discovery for %s encountered error: %v (will retry on demand)", cleanDefault, err)
|
|
}
|
|
|
|
return gw
|
|
}
|
|
|
|
func (g *GradioGateway) GetDiscovery(spaceURL, userAgent string) *SpaceDiscovery {
|
|
target := spaceURL
|
|
if target == "" {
|
|
target = g.defaultURL
|
|
}
|
|
cleanTarget := strings.TrimRight(target, "/")
|
|
|
|
g.mu.RLock()
|
|
disc, exists := g.discoveries[cleanTarget]
|
|
g.mu.RUnlock()
|
|
|
|
if exists && disc != nil && time.Since(disc.LastDiscovered) < 30*time.Minute {
|
|
return disc
|
|
}
|
|
|
|
g.mu.Lock()
|
|
defer g.mu.Unlock()
|
|
|
|
// Double-check under lock
|
|
disc, exists = g.discoveries[cleanTarget]
|
|
if exists && disc != nil && time.Since(disc.LastDiscovered) < 30*time.Minute {
|
|
return disc
|
|
}
|
|
|
|
newDisc, err := InspectSpace(g.client, cleanTarget, userAgent)
|
|
if err == nil && newDisc != nil {
|
|
g.discoveries[cleanTarget] = newDisc
|
|
log.Printf("\n%s\n", newDisc.Summary())
|
|
return newDisc
|
|
}
|
|
|
|
if disc != nil {
|
|
return disc
|
|
}
|
|
|
|
// Fallback discovery
|
|
fallback := NewDefaultSpaceDiscovery(cleanTarget)
|
|
fallback.TotalInputs = 2
|
|
fallback.PrimaryModel = "gradio-chat"
|
|
fallback.Models = []string{"gradio-chat"}
|
|
g.discoveries[cleanTarget] = fallback
|
|
return fallback
|
|
}
|
|
|
|
// BuildGradioPayload packages OpenAI messages and parameters into the target Gradio input array.
|
|
func (g *GradioGateway) BuildGradioPayload(disc *SpaceDiscovery, req ChatCompletionRequest) ([]interface{}, error) {
|
|
var transformed []ChatMessage
|
|
var toolInstruction string
|
|
if disc.IsHunyuan3 && disc.FunctionsJSONIndex != -1 {
|
|
for _, msg := range req.Messages {
|
|
transformed = append(transformed, ChatMessage{
|
|
Role: msg.Role,
|
|
Content: msg.GetContentString(),
|
|
ReasoningContent: msg.ReasoningContent,
|
|
ToolCalls: msg.ToolCalls,
|
|
ToolCallID: msg.ToolCallID,
|
|
Name: msg.Name,
|
|
})
|
|
}
|
|
} else {
|
|
// When no native tool calling support is detected, augment system prompt
|
|
transformed, toolInstruction, _ = TransformMessages(req)
|
|
}
|
|
|
|
hasClientSystem := false
|
|
for _, m := range req.Messages {
|
|
if m.Role == "system" {
|
|
hasClientSystem = true
|
|
break
|
|
}
|
|
}
|
|
|
|
var systemPromptStr string
|
|
var historyArray []map[string]interface{}
|
|
var lastUserMessage string
|
|
|
|
var nonSystem []ChatMessage
|
|
for _, m := range transformed {
|
|
cStr := m.GetContentString()
|
|
if m.Role == "system" {
|
|
if systemPromptStr == "" {
|
|
systemPromptStr = cStr
|
|
} else {
|
|
systemPromptStr += "\n\n" + cStr
|
|
}
|
|
} else {
|
|
nonSystem = append(nonSystem, m)
|
|
}
|
|
}
|
|
|
|
// If the space has a DefaultSystemPrompt and client provided no system message,
|
|
// retain and augment the default system prompt:
|
|
if !hasClientSystem && disc.DefaultSystemPrompt != "" {
|
|
if systemPromptStr != "" {
|
|
systemPromptStr = disc.DefaultSystemPrompt + "\n\n" + systemPromptStr
|
|
} else {
|
|
systemPromptStr = disc.DefaultSystemPrompt
|
|
}
|
|
}
|
|
|
|
// If the space has NO native system prompt input (disc.SystemIndex == -1),
|
|
// but we have system instructions (from system message or tool instructions):
|
|
if disc.SystemIndex == -1 && systemPromptStr != "" && len(nonSystem) > 0 {
|
|
// If the space supports conversation history, prepend system instructions to the first turn
|
|
if disc.HistoryIndex != -1 {
|
|
if toolInstruction != "" && len(nonSystem) == 1 {
|
|
nonSystem[0].Content = fmt.Sprintf("%s\n\nQuery: %s", systemPromptStr, nonSystem[0].GetContentString())
|
|
} else {
|
|
nonSystem[0].Content = systemPromptStr + "\n\n" + nonSystem[0].GetContentString()
|
|
}
|
|
}
|
|
}
|
|
|
|
if len(nonSystem) > 0 {
|
|
for i := 0; i < len(nonSystem)-1; i++ {
|
|
m := nonSystem[i]
|
|
cStr := m.GetContentString()
|
|
item := map[string]interface{}{"role": m.Role}
|
|
switch m.Role {
|
|
case "assistant":
|
|
if cStr != "" {
|
|
item["content"] = cStr
|
|
} else {
|
|
item["content"] = nil
|
|
}
|
|
if m.ReasoningContent != "" {
|
|
item["reasoning_content"] = m.ReasoningContent
|
|
}
|
|
if len(m.ToolCalls) > 0 {
|
|
item["tool_calls"] = m.ToolCalls
|
|
}
|
|
case "tool", "function":
|
|
item["role"] = "tool"
|
|
item["content"] = cStr
|
|
toolID := m.ToolCallID
|
|
if toolID == "" {
|
|
toolID = m.Name
|
|
}
|
|
if toolID != "" {
|
|
item["tool_call_id"] = toolID
|
|
}
|
|
if m.Name != "" {
|
|
item["name"] = m.Name
|
|
}
|
|
default:
|
|
item["content"] = cStr
|
|
}
|
|
historyArray = append(historyArray, item)
|
|
}
|
|
|
|
lastMsg := nonSystem[len(nonSystem)-1]
|
|
lastContent := lastMsg.GetContentString()
|
|
if lastMsg.Role == "tool" || lastMsg.Role == "function" {
|
|
toolName := lastMsg.Name
|
|
if toolName == "" {
|
|
toolName = lastMsg.ToolCallID
|
|
}
|
|
if disc.IsHunyuan3 {
|
|
if toolName != "" {
|
|
lastUserMessage = fmt.Sprintf("Tool result for %s: %s", toolName, lastContent)
|
|
} else {
|
|
lastUserMessage = lastContent
|
|
}
|
|
} else {
|
|
if toolName != "" {
|
|
lastUserMessage = fmt.Sprintf("Tool result for %s: %s\nPlease answer the user's request based on the tool result.", toolName, lastContent)
|
|
} else {
|
|
lastUserMessage = fmt.Sprintf("Tool result: %s\nPlease answer the user's request based on the tool result.", lastContent)
|
|
}
|
|
}
|
|
} else {
|
|
lastUserMessage = lastContent
|
|
}
|
|
} else if systemPromptStr != "" {
|
|
lastUserMessage = systemPromptStr
|
|
}
|
|
|
|
var promptMessageText string
|
|
isToolReturn := strings.HasPrefix(lastUserMessage, "<tool_response>") || strings.HasPrefix(lastUserMessage, "Tool result")
|
|
if disc.HistoryIndex != -1 {
|
|
if disc.SystemIndex == -1 {
|
|
if len(nonSystem) > 1 && toolInstruction != "" {
|
|
if !isToolReturn {
|
|
promptMessageText = fmt.Sprintf("[System Directive: Tool calling mode active.]\n\nQuery: %s", lastUserMessage)
|
|
} else {
|
|
promptMessageText = fmt.Sprintf("[System Directive: Tool calling mode active.]\n\n%s", lastUserMessage)
|
|
}
|
|
} else {
|
|
promptMessageText = lastUserMessage
|
|
}
|
|
} else {
|
|
if toolInstruction != "" && !isToolReturn {
|
|
promptMessageText = fmt.Sprintf("Query: %s", lastUserMessage)
|
|
} else {
|
|
promptMessageText = lastUserMessage
|
|
}
|
|
}
|
|
} else {
|
|
// Single message space: compose multi-turn history into the prompt
|
|
if len(nonSystem) <= 1 {
|
|
if systemPromptStr != "" && len(nonSystem) == 1 {
|
|
if toolInstruction != "" && !isToolReturn {
|
|
promptMessageText = fmt.Sprintf("%s\n\nQuery: %s", systemPromptStr, lastUserMessage)
|
|
} else {
|
|
promptMessageText = systemPromptStr + "\n\n" + lastUserMessage
|
|
}
|
|
} else if systemPromptStr != "" {
|
|
promptMessageText = systemPromptStr
|
|
} else {
|
|
promptMessageText = lastUserMessage
|
|
}
|
|
} else {
|
|
var sb strings.Builder
|
|
if systemPromptStr != "" {
|
|
sb.WriteString("# Instructions\n" + systemPromptStr + "\n\n")
|
|
}
|
|
sb.WriteString("# Conversation History\n")
|
|
for i := 0; i < len(nonSystem)-1; i++ {
|
|
m := nonSystem[i]
|
|
roleLabel := "User"
|
|
if m.Role == "assistant" {
|
|
roleLabel = "Assistant"
|
|
}
|
|
sb.WriteString(fmt.Sprintf("%s: %s\n\n", roleLabel, m.GetContentString()))
|
|
}
|
|
lastRoleLabel := "User"
|
|
if len(nonSystem) > 0 && nonSystem[len(nonSystem)-1].Role == "assistant" {
|
|
lastRoleLabel = "Assistant"
|
|
}
|
|
if toolInstruction != "" && lastRoleLabel == "User" && !isToolReturn {
|
|
sb.WriteString(fmt.Sprintf("# Current Request\nQuery: %s", lastUserMessage))
|
|
} else {
|
|
sb.WriteString(fmt.Sprintf("# Current Request\n%s: %s", lastRoleLabel, lastUserMessage))
|
|
}
|
|
promptMessageText = sb.String()
|
|
}
|
|
}
|
|
|
|
// Allocate input array matching TotalInputs
|
|
totalInputs := disc.TotalInputs
|
|
if totalInputs < 1 {
|
|
totalInputs = 1
|
|
}
|
|
data := make([]interface{}, totalInputs)
|
|
|
|
// Initialize with space default inputs if available
|
|
if len(disc.DefaultInputs) == totalInputs {
|
|
for i := 0; i < totalInputs; i++ {
|
|
data[i] = disc.DefaultInputs[i]
|
|
}
|
|
}
|
|
|
|
// Extract multimodal files if message input is multimodal
|
|
var messageFiles []interface{}
|
|
if disc.MessageIsMultimodal && len(nonSystem) > 0 {
|
|
lastMsg := nonSystem[len(nonSystem)-1]
|
|
if parts, ok := lastMsg.Content.([]interface{}); ok {
|
|
for _, p := range parts {
|
|
if itemMap, ok := p.(map[string]interface{}); ok {
|
|
if itemMap["type"] == "image_url" {
|
|
imgURL := ""
|
|
if iuMap, ok := itemMap["image_url"].(map[string]interface{}); ok {
|
|
if u, ok := iuMap["url"].(string); ok {
|
|
imgURL = u
|
|
}
|
|
} else if iuStr, ok := itemMap["image_url"].(string); ok {
|
|
imgURL = iuStr
|
|
}
|
|
if imgURL != "" {
|
|
messageFiles = append(messageFiles, map[string]interface{}{
|
|
"path": imgURL,
|
|
"url": imgURL,
|
|
"meta": map[string]interface{}{"_type": "gradio.FileData"},
|
|
})
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if messageFiles == nil {
|
|
messageFiles = []interface{}{}
|
|
}
|
|
|
|
// Populate mapped fields
|
|
msgIdx := disc.MessageIndex
|
|
if msgIdx >= 0 && msgIdx < len(data) {
|
|
if disc.MessageIsMultimodal {
|
|
data[msgIdx] = map[string]interface{}{
|
|
"text": promptMessageText,
|
|
"files": messageFiles,
|
|
}
|
|
} else {
|
|
data[msgIdx] = promptMessageText
|
|
}
|
|
}
|
|
|
|
if disc.HistoryIndex >= 0 && disc.HistoryIndex < len(data) {
|
|
if disc.HistoryFormat == "pairs" {
|
|
var pairs [][]string
|
|
for i := 0; i < len(historyArray); i += 2 {
|
|
u := ""
|
|
a := ""
|
|
if i < len(historyArray) {
|
|
u, _ = historyArray[i]["content"].(string)
|
|
}
|
|
if i+1 < len(historyArray) {
|
|
a, _ = historyArray[i+1]["content"].(string)
|
|
}
|
|
pairs = append(pairs, []string{u, a})
|
|
}
|
|
if disc.MessageIndex == -1 && promptMessageText != "" {
|
|
pairs = append(pairs, []string{promptMessageText, ""})
|
|
}
|
|
data[disc.HistoryIndex] = pairs
|
|
} else if disc.HistoryFormat == "gradio_messages" {
|
|
var gMsgs []map[string]interface{}
|
|
for _, item := range historyArray {
|
|
role, _ := item["role"].(string)
|
|
cStr := ""
|
|
if s, ok := item["content"].(string); ok {
|
|
cStr = s
|
|
}
|
|
gMsg := map[string]interface{}{
|
|
"role": role,
|
|
"content": []map[string]string{
|
|
{"text": cStr, "type": "text"},
|
|
},
|
|
}
|
|
gMsgs = append(gMsgs, gMsg)
|
|
}
|
|
if disc.MessageIndex == -1 && promptMessageText != "" {
|
|
gMsgs = append(gMsgs, map[string]interface{}{
|
|
"role": "user",
|
|
"content": []map[string]string{
|
|
{"text": promptMessageText, "type": "text"},
|
|
},
|
|
})
|
|
}
|
|
if gMsgs == nil {
|
|
gMsgs = []map[string]interface{}{}
|
|
}
|
|
data[disc.HistoryIndex] = gMsgs
|
|
} else {
|
|
if disc.MessageIndex == -1 && promptMessageText != "" {
|
|
historyArray = append(historyArray, map[string]interface{}{
|
|
"role": "user",
|
|
"content": promptMessageText,
|
|
})
|
|
}
|
|
data[disc.HistoryIndex] = historyArray
|
|
}
|
|
}
|
|
|
|
// If neither message nor history slot was mapped, populate slot 0 with the prompt
|
|
if disc.MessageIndex == -1 && disc.HistoryIndex == -1 && len(data) > 0 {
|
|
data[0] = promptMessageText
|
|
}
|
|
|
|
if disc.SystemIndex >= 0 && disc.SystemIndex < len(data) {
|
|
data[disc.SystemIndex] = systemPromptStr
|
|
}
|
|
|
|
if disc.ThinkLevelIndex >= 0 && disc.ThinkLevelIndex != disc.MessageIndex && disc.ThinkLevelIndex != disc.HistoryIndex && disc.ThinkLevelIndex < len(data) {
|
|
thinkLevel := "high"
|
|
if req.ReasoningEffort != "" {
|
|
effort := strings.ToLower(req.ReasoningEffort)
|
|
switch effort {
|
|
case "none", "off", "no_think", "0":
|
|
thinkLevel = "no_think"
|
|
case "low", "1":
|
|
thinkLevel = "low"
|
|
case "medium", "high", "2", "3":
|
|
thinkLevel = "high"
|
|
default:
|
|
thinkLevel = effort
|
|
}
|
|
}
|
|
data[disc.ThinkLevelIndex] = thinkLevel
|
|
}
|
|
|
|
if disc.PreservedThinkingIndex >= 0 && disc.PreservedThinkingIndex != disc.MessageIndex && disc.PreservedThinkingIndex != disc.HistoryIndex && disc.PreservedThinkingIndex != disc.SystemIndex && disc.PreservedThinkingIndex < len(data) {
|
|
data[disc.PreservedThinkingIndex] = nil
|
|
}
|
|
|
|
if disc.TempIndex >= 0 && disc.TempIndex != disc.MessageIndex && disc.TempIndex != disc.HistoryIndex && disc.TempIndex < len(data) {
|
|
if req.Temperature != nil {
|
|
data[disc.TempIndex] = *req.Temperature
|
|
} else if disc.IsHunyuan3 {
|
|
data[disc.TempIndex] = nil
|
|
} else if data[disc.TempIndex] == nil {
|
|
data[disc.TempIndex] = 0.7
|
|
}
|
|
}
|
|
|
|
if disc.MaxTokensIndex >= 0 && disc.MaxTokensIndex != disc.MessageIndex && disc.MaxTokensIndex != disc.HistoryIndex && disc.MaxTokensIndex < len(data) {
|
|
if req.MaxTokens > 0 || req.MaxCompletionTokens > 0 {
|
|
data[disc.MaxTokensIndex] = ResolveMaxTokens(req)
|
|
} else if data[disc.MaxTokensIndex] == nil {
|
|
data[disc.MaxTokensIndex] = ResolveMaxTokens(req)
|
|
}
|
|
}
|
|
|
|
if disc.TopPIndex >= 0 && disc.TopPIndex != disc.MessageIndex && disc.TopPIndex != disc.HistoryIndex && disc.TopPIndex < len(data) {
|
|
if req.TopP != nil {
|
|
data[disc.TopPIndex] = *req.TopP
|
|
} else if disc.IsHunyuan3 {
|
|
data[disc.TopPIndex] = 0
|
|
} else if data[disc.TopPIndex] == nil {
|
|
data[disc.TopPIndex] = 1.0
|
|
}
|
|
}
|
|
|
|
if disc.StreamIndex > 0 && disc.StreamIndex != disc.MessageIndex && disc.StreamIndex < len(data) {
|
|
data[disc.StreamIndex] = false
|
|
}
|
|
|
|
if disc.FunctionsJSONIndex >= 0 && disc.FunctionsJSONIndex != disc.MessageIndex && disc.FunctionsJSONIndex != disc.HistoryIndex && disc.FunctionsJSONIndex != disc.SystemIndex && disc.FunctionsJSONIndex < len(data) {
|
|
functionsJSONStr := ""
|
|
if len(req.Tools) > 0 {
|
|
b, err := json.Marshal(req.Tools)
|
|
if err == nil {
|
|
functionsJSONStr = string(b)
|
|
}
|
|
}
|
|
data[disc.FunctionsJSONIndex] = functionsJSONStr
|
|
}
|
|
|
|
if disc.WebSearchIndex >= 0 && disc.WebSearchIndex != disc.MessageIndex && disc.WebSearchIndex != disc.HistoryIndex && disc.WebSearchIndex != disc.SystemIndex && disc.WebSearchIndex < len(data) && len(req.Tools) > 0 {
|
|
var mapping *SpaceParamMapping
|
|
for i := range disc.ParamMappings {
|
|
if disc.ParamMappings[i].InputIndex == disc.WebSearchIndex {
|
|
mapping = &disc.ParamMappings[i]
|
|
break
|
|
}
|
|
}
|
|
disabledSet := false
|
|
if mapping != nil && len(mapping.Choices) > 0 {
|
|
for _, choice := range mapping.Choices {
|
|
cLower := strings.ToLower(choice)
|
|
if cLower == "direct" || cLower == "off" || cLower == "disabled" || cLower == "none" || cLower == "false" || cLower == "no" || strings.Contains(cLower, "direct") || strings.Contains(cLower, "no search") || strings.Contains(cLower, "disable") {
|
|
data[disc.WebSearchIndex] = choice
|
|
disabledSet = true
|
|
break
|
|
}
|
|
}
|
|
}
|
|
if !disabledSet {
|
|
switch data[disc.WebSearchIndex].(type) {
|
|
case bool:
|
|
data[disc.WebSearchIndex] = false
|
|
case string:
|
|
strVal := strings.ToLower(data[disc.WebSearchIndex].(string))
|
|
if strings.Contains(strVal, "search") {
|
|
data[disc.WebSearchIndex] = "Direct"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return data, nil
|
|
}
|
|
|
|
// GradioOutputFrame holds parsed elements from a Gradio SSE output chunk
|
|
type GradioOutputFrame struct {
|
|
Content string
|
|
Reasoning string
|
|
ToolCalls []ToolCall
|
|
IsDelta bool
|
|
OK bool
|
|
}
|
|
|
|
func toInt(v interface{}) (int, bool) {
|
|
switch n := v.(type) {
|
|
case int:
|
|
return n, true
|
|
case int64:
|
|
return int(n), true
|
|
case float64:
|
|
return int(n), true
|
|
case json.Number:
|
|
i, err := n.Int64()
|
|
return int(i), err == nil
|
|
default:
|
|
return 0, false
|
|
}
|
|
}
|
|
|
|
func isReasoningPath(path interface{}) bool {
|
|
switch p := path.(type) {
|
|
case []interface{}:
|
|
for i, elem := range p {
|
|
if s, ok := elem.(string); ok {
|
|
sLow := strings.ToLower(s)
|
|
if strings.Contains(sLow, "reason") || strings.Contains(sLow, "thought") || strings.Contains(sLow, "think") {
|
|
return true
|
|
}
|
|
}
|
|
if num, ok := toInt(elem); ok {
|
|
if len(p) == 1 && num == 1 {
|
|
return true
|
|
}
|
|
if len(p) == 2 && i == 1 && num == 1 {
|
|
if p0, ok0 := toInt(p[0]); ok0 && p0 == 0 {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
case string:
|
|
sLow := strings.ToLower(p)
|
|
if strings.Contains(sLow, "reason") || strings.Contains(sLow, "thought") || strings.Contains(sLow, "think") {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func isContentPath(path interface{}) bool {
|
|
switch p := path.(type) {
|
|
case []interface{}:
|
|
if len(p) == 0 {
|
|
return true
|
|
}
|
|
for _, elem := range p {
|
|
if s, ok := elem.(string); ok {
|
|
sLow := strings.ToLower(s)
|
|
if strings.Contains(sLow, "content") || strings.Contains(sLow, "text") || strings.Contains(sLow, "value") {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
if num, ok := toInt(p[0]); ok && num == 0 {
|
|
if len(p) == 1 {
|
|
return true
|
|
}
|
|
if len(p) >= 2 {
|
|
if p1, ok1 := toInt(p[1]); ok1 && p1 == 0 {
|
|
return true
|
|
}
|
|
if s1, ok1 := p[1].(string); ok1 && (strings.Contains(s1, "content") || strings.Contains(s1, "text")) {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
case string:
|
|
sLow := strings.ToLower(p)
|
|
if strings.Contains(sLow, "content") || strings.Contains(sLow, "text") || strings.Contains(sLow, "value") {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// extractGradioDiffDelta extracts delta string from Gradio 6 streaming diff operations
|
|
func extractGradioDiffDelta(v []interface{}) (contentDelta string, reasoningDelta string, ok bool) {
|
|
var diffItems [][]interface{}
|
|
var findOps func(items []interface{})
|
|
findOps = func(items []interface{}) {
|
|
if len(items) >= 3 {
|
|
if op, ok := items[0].(string); ok && (op == "append" || op == "add") {
|
|
diffItems = append(diffItems, items)
|
|
return
|
|
}
|
|
}
|
|
for _, it := range items {
|
|
if sub, ok := it.([]interface{}); ok {
|
|
findOps(sub)
|
|
}
|
|
}
|
|
}
|
|
findOps(v)
|
|
|
|
if len(diffItems) == 0 {
|
|
return "", "", false
|
|
}
|
|
|
|
hasMatch := false
|
|
for _, item := range diffItems {
|
|
delta, isStr := item[2].(string)
|
|
if !isStr || delta == "" {
|
|
continue
|
|
}
|
|
path := item[1]
|
|
if isReasoningPath(path) {
|
|
if reasoningDelta == "" {
|
|
reasoningDelta = delta
|
|
hasMatch = true
|
|
}
|
|
} else if isContentPath(path) {
|
|
if contentDelta == "" {
|
|
contentDelta = delta
|
|
hasMatch = true
|
|
}
|
|
} else {
|
|
if contentDelta == "" {
|
|
contentDelta = delta
|
|
hasMatch = true
|
|
}
|
|
}
|
|
}
|
|
return contentDelta, reasoningDelta, hasMatch
|
|
}
|
|
|
|
// ParseGradioStreamOutput extracts structured content, reasoning, and tool calls from Gradio output
|
|
func ParseGradioStreamOutput(rawJSON string) (frame GradioOutputFrame) {
|
|
defer func() {
|
|
if frame.OK && len(frame.ToolCalls) == 0 && frame.Content != "" {
|
|
tcs, clean, has := DetectToolCalls(frame.Content)
|
|
if has && len(tcs) > 0 {
|
|
frame.ToolCalls = tcs
|
|
frame.Content = clean
|
|
}
|
|
}
|
|
}()
|
|
|
|
var val interface{}
|
|
if err := json.Unmarshal([]byte(rawJSON), &val); err != nil {
|
|
return frame
|
|
}
|
|
|
|
switch v := val.(type) {
|
|
case string:
|
|
frame.Content = v
|
|
frame.OK = true
|
|
return frame
|
|
|
|
case []interface{}:
|
|
if len(v) == 0 {
|
|
return frame
|
|
}
|
|
|
|
if cDelta, rDelta, ok := extractGradioDiffDelta(v); ok {
|
|
frame.Content = cDelta
|
|
frame.Reasoning = rDelta
|
|
frame.IsDelta = true
|
|
frame.OK = true
|
|
return frame
|
|
}
|
|
|
|
// 1. Check if v[0] is an inner slice with len >= 2 (e.g. Hy3: [[content, reasoning, tool_calls, history]])
|
|
if inner, ok := v[0].([]interface{}); ok && len(inner) >= 2 {
|
|
s0, ok0 := inner[0].(string)
|
|
s1, ok1 := inner[1].(string)
|
|
if ok0 || ok1 {
|
|
frame.Content = s0
|
|
frame.Reasoning = s1
|
|
if len(inner) >= 3 && inner[2] != nil {
|
|
b, err := json.Marshal(inner[2])
|
|
if err == nil {
|
|
var tcs []ToolCall
|
|
if json.Unmarshal(b, &tcs) == nil && len(tcs) > 0 {
|
|
frame.ToolCalls = tcs
|
|
}
|
|
}
|
|
}
|
|
frame.OK = true
|
|
return frame
|
|
}
|
|
}
|
|
|
|
// 2. Check if any element of v is a Chatbot message list or Chatbot pair list
|
|
for _, elem := range v {
|
|
if msgList, ok := elem.([]interface{}); ok && len(msgList) > 0 {
|
|
allMaps := true
|
|
var maps []map[string]interface{}
|
|
for _, item := range msgList {
|
|
if m, ok := item.(map[string]interface{}); ok {
|
|
if _, hasRole := m["role"]; hasRole {
|
|
maps = append(maps, m)
|
|
continue
|
|
}
|
|
}
|
|
allMaps = false
|
|
break
|
|
}
|
|
if allMaps && len(maps) > 0 {
|
|
var targetMsg map[string]interface{}
|
|
for i := len(maps) - 1; i >= 0; i-- {
|
|
if r, _ := maps[i]["role"].(string); r == "assistant" {
|
|
targetMsg = maps[i]
|
|
break
|
|
}
|
|
}
|
|
if targetMsg == nil {
|
|
targetMsg = maps[len(maps)-1]
|
|
}
|
|
|
|
cText := extractContentString(targetMsg["content"])
|
|
frame.Content = cText
|
|
|
|
if r, ok := targetMsg["reasoning_content"].(string); ok {
|
|
frame.Reasoning = r
|
|
} else if meta, ok := targetMsg["metadata"].(map[string]interface{}); ok {
|
|
if logStr, ok := meta["log"].(string); ok && logStr != "" {
|
|
frame.Reasoning = logStr
|
|
}
|
|
}
|
|
|
|
if tcsRaw, ok := targetMsg["tool_calls"].([]interface{}); ok && len(tcsRaw) > 0 {
|
|
b, err := json.Marshal(tcsRaw)
|
|
if err == nil {
|
|
var tcs []ToolCall
|
|
if json.Unmarshal(b, &tcs) == nil {
|
|
frame.ToolCalls = tcs
|
|
}
|
|
}
|
|
}
|
|
|
|
if len(frame.ToolCalls) == 0 {
|
|
tcs, clean, has := DetectToolCalls(cText)
|
|
if has && len(tcs) > 0 {
|
|
frame.ToolCalls = tcs
|
|
frame.Content = clean
|
|
}
|
|
}
|
|
|
|
frame.OK = true
|
|
return frame
|
|
}
|
|
|
|
// Check if elements are pairs [user, assistant]
|
|
allPairs := true
|
|
var pairs [][]interface{}
|
|
for _, item := range msgList {
|
|
if p, ok := item.([]interface{}); ok && len(p) == 2 {
|
|
pairs = append(pairs, p)
|
|
continue
|
|
}
|
|
allPairs = false
|
|
break
|
|
}
|
|
if allPairs && len(pairs) > 0 {
|
|
lastPair := pairs[len(pairs)-1]
|
|
cText := extractContentString(lastPair[1])
|
|
frame.Content = cText
|
|
tcs, clean, has := DetectToolCalls(cText)
|
|
if has && len(tcs) > 0 {
|
|
frame.ToolCalls = tcs
|
|
frame.Content = clean
|
|
}
|
|
frame.OK = true
|
|
return frame
|
|
}
|
|
}
|
|
}
|
|
|
|
// 3. Check if v itself is a list of message maps: [{"role": "assistant", ...}]
|
|
allMaps := true
|
|
var maps []map[string]interface{}
|
|
for _, item := range v {
|
|
if m, ok := item.(map[string]interface{}); ok {
|
|
if _, hasRole := m["role"]; hasRole {
|
|
maps = append(maps, m)
|
|
continue
|
|
}
|
|
}
|
|
allMaps = false
|
|
break
|
|
}
|
|
if allMaps && len(maps) > 0 {
|
|
var targetMsg map[string]interface{}
|
|
for i := len(maps) - 1; i >= 0; i-- {
|
|
if r, _ := maps[i]["role"].(string); r == "assistant" {
|
|
targetMsg = maps[i]
|
|
break
|
|
}
|
|
}
|
|
if targetMsg == nil {
|
|
targetMsg = maps[len(maps)-1]
|
|
}
|
|
cText := extractContentString(targetMsg["content"])
|
|
frame.Content = cText
|
|
if r, ok := targetMsg["reasoning_content"].(string); ok {
|
|
frame.Reasoning = r
|
|
}
|
|
if tcsRaw, ok := targetMsg["tool_calls"].([]interface{}); ok && len(tcsRaw) > 0 {
|
|
b, err := json.Marshal(tcsRaw)
|
|
if err == nil {
|
|
var tcs []ToolCall
|
|
if json.Unmarshal(b, &tcs) == nil {
|
|
frame.ToolCalls = tcs
|
|
}
|
|
}
|
|
}
|
|
if len(frame.ToolCalls) == 0 {
|
|
tcs, clean, has := DetectToolCalls(cText)
|
|
if has && len(tcs) > 0 {
|
|
frame.ToolCalls = tcs
|
|
frame.Content = clean
|
|
}
|
|
}
|
|
frame.OK = true
|
|
return frame
|
|
}
|
|
|
|
|
|
// 5. Check if v[0] is a non-empty string or single output
|
|
if s, ok := v[0].(string); ok && (s != "" || len(v) == 1) {
|
|
frame.Content = s
|
|
frame.OK = true
|
|
return frame
|
|
}
|
|
|
|
case map[string]interface{}:
|
|
if outMap, ok := v["output"].(map[string]interface{}); ok {
|
|
if dataArr, ok := outMap["data"].([]interface{}); ok {
|
|
b, err := json.Marshal(dataArr)
|
|
if err == nil {
|
|
subFrame := ParseGradioStreamOutput(string(b))
|
|
if subFrame.OK {
|
|
if r, ok := v["reasoning_content"].(string); ok && subFrame.Reasoning == "" {
|
|
subFrame.Reasoning = r
|
|
}
|
|
return subFrame
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if dataArr, ok := v["data"].([]interface{}); ok {
|
|
b, err := json.Marshal(dataArr)
|
|
if err == nil {
|
|
subFrame := ParseGradioStreamOutput(string(b))
|
|
if subFrame.OK {
|
|
if r, ok := v["reasoning_content"].(string); ok && subFrame.Reasoning == "" {
|
|
subFrame.Reasoning = r
|
|
}
|
|
return subFrame
|
|
}
|
|
}
|
|
}
|
|
|
|
for _, key := range []string{"text", "content", "response", "data", "value"} {
|
|
if s, ok := v[key].(string); ok {
|
|
frame.Content = s
|
|
frame.OK = true
|
|
break
|
|
}
|
|
}
|
|
if r, ok := v["reasoning_content"].(string); ok {
|
|
frame.Reasoning = r
|
|
} else if r, ok := v["reasoning"].(string); ok {
|
|
frame.Reasoning = r
|
|
}
|
|
if tcsRaw, ok := v["tool_calls"].([]interface{}); ok && len(tcsRaw) > 0 {
|
|
b, err := json.Marshal(tcsRaw)
|
|
if err == nil {
|
|
var tcs []ToolCall
|
|
if err := json.Unmarshal(b, &tcs); err == nil {
|
|
frame.ToolCalls = tcs
|
|
}
|
|
}
|
|
}
|
|
if frame.OK {
|
|
return frame
|
|
}
|
|
}
|
|
|
|
return frame
|
|
}
|
|
|
|
// ExtractTextFromGradioOutput extracts the assistant text string from Gradio output chunks (compatibility wrapper)
|
|
func ExtractTextFromGradioOutput(rawJSON string) (string, bool) {
|
|
frame := ParseGradioStreamOutput(rawJSON)
|
|
if frame.OK {
|
|
return frame.Content, true
|
|
}
|
|
return "", false
|
|
}
|
|
|
|
// executePredictCompletion handles completions using Gradio 3 direct /run/predict or /api/predict protocol.
|
|
func (g *GradioGateway) executePredictCompletion(w http.ResponseWriter, r *http.Request, disc *SpaceDiscovery, gradioData []interface{}, req ChatCompletionRequest, completionID string, createdTime int64, modelName, effUA string) error {
|
|
fnIndex := disc.FnIndex
|
|
if fnIndex < 0 {
|
|
fnIndex = 0
|
|
}
|
|
|
|
predictData := make([]interface{}, len(gradioData))
|
|
copy(predictData, gradioData)
|
|
if disc.RawTotalInputs > len(predictData) {
|
|
for i := len(predictData); i < disc.RawTotalInputs; i++ {
|
|
var defVal interface{}
|
|
if i < len(disc.RawDefaultInputs) {
|
|
defVal = disc.RawDefaultInputs[i]
|
|
} else if i < len(disc.DefaultInputs) {
|
|
defVal = disc.DefaultInputs[i]
|
|
}
|
|
predictData = append(predictData, defVal)
|
|
}
|
|
}
|
|
|
|
payloadMap := map[string]interface{}{
|
|
"data": predictData,
|
|
"fn_index": fnIndex,
|
|
"session_hash": GenerateUUID(),
|
|
}
|
|
|
|
jsonPayload, err := json.Marshal(payloadMap)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to marshal predict payload: %w", err)
|
|
}
|
|
|
|
var candidateURLs []string
|
|
if disc.APIPrefix != "" {
|
|
candidateURLs = append(candidateURLs, fmt.Sprintf("%s%s/run/predict", disc.SpaceURL, disc.APIPrefix))
|
|
}
|
|
candidateURLs = append(candidateURLs,
|
|
fmt.Sprintf("%s/run/predict", disc.SpaceURL),
|
|
fmt.Sprintf("%s/api/predict", disc.SpaceURL),
|
|
)
|
|
|
|
var resp *http.Response
|
|
var lastErr error
|
|
|
|
for _, targetURL := range candidateURLs {
|
|
curURL := targetURL
|
|
makeReq := func() (*http.Request, error) {
|
|
req, err := http.NewRequest("POST", curURL, bytes.NewBuffer(jsonPayload))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
req.Header.Set("Content-Type", "application/json")
|
|
req.Header.Set("User-Agent", effUA)
|
|
return req, nil
|
|
}
|
|
resp, lastErr = DoWithFibonacciRetry(g.client, makeReq, 3)
|
|
if lastErr == nil && resp != nil && resp.StatusCode == http.StatusOK {
|
|
break
|
|
}
|
|
if resp != nil {
|
|
resp.Body.Close()
|
|
resp = nil
|
|
}
|
|
}
|
|
|
|
if resp == nil {
|
|
return fmt.Errorf("upstream Gradio predict error: %w", lastErr)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
bodyBytes, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to read Gradio predict response: %w", err)
|
|
}
|
|
|
|
frame := ParseGradioStreamOutput(string(bodyBytes))
|
|
if !frame.OK {
|
|
return fmt.Errorf("upstream Gradio space returned empty or unparseable response")
|
|
}
|
|
|
|
finalContent, reasoning, toolCalls, finishReason := finalizeOutput(frame)
|
|
|
|
if !req.Stream {
|
|
WriteCompletionResponse(w, completionID, createdTime, modelName, FinalOutput{
|
|
Content: finalContent,
|
|
ReasoningContent: reasoning,
|
|
ToolCalls: toolCalls,
|
|
FinishReason: finishReason,
|
|
})
|
|
return nil
|
|
}
|
|
|
|
flusher, _ := w.(http.Flusher)
|
|
streamer := NewStreamer(w, flusher, completionID, createdTime, modelName)
|
|
if reasoning != "" {
|
|
streamer.Reasoning(reasoning)
|
|
}
|
|
if len(toolCalls) > 0 {
|
|
for i, tc := range toolCalls {
|
|
iCopy := i
|
|
tc.Index = &iCopy
|
|
streamer.ToolCallDelta(tc)
|
|
}
|
|
}
|
|
if finalContent != nil {
|
|
if s, ok := finalContent.(string); ok && s != "" {
|
|
streamer.Content(s)
|
|
}
|
|
}
|
|
streamer.Finish(finishReason)
|
|
streamer.Done()
|
|
return nil
|
|
}
|
|
|
|
// executeQueueCompletion handles completions using Gradio queue SSE protocol (/queue/join + /queue/data).
|
|
func (g *GradioGateway) executeQueueCompletion(w http.ResponseWriter, r *http.Request, disc *SpaceDiscovery, gradioData []interface{}, req ChatCompletionRequest, completionID string, createdTime int64, modelName, effUA string) error {
|
|
fnIndex := disc.FnIndex
|
|
if fnIndex < 0 {
|
|
fnIndex = 0
|
|
}
|
|
|
|
queueData := make([]interface{}, len(gradioData))
|
|
copy(queueData, gradioData)
|
|
if disc.RawTotalInputs > len(queueData) {
|
|
for i := len(queueData); i < disc.RawTotalInputs; i++ {
|
|
var defVal interface{}
|
|
if i < len(disc.RawDefaultInputs) {
|
|
defVal = disc.RawDefaultInputs[i]
|
|
} else if i < len(disc.DefaultInputs) {
|
|
defVal = disc.DefaultInputs[i]
|
|
}
|
|
queueData = append(queueData, defVal)
|
|
}
|
|
}
|
|
|
|
sessionHash := GenerateUUID()
|
|
payloadMap := map[string]interface{}{
|
|
"data": queueData,
|
|
"fn_index": fnIndex,
|
|
"session_hash": sessionHash,
|
|
}
|
|
|
|
jsonPayload, err := json.Marshal(payloadMap)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to marshal queue payload: %w", err)
|
|
}
|
|
|
|
var joinURLs []string
|
|
if disc.APIPrefix != "" {
|
|
joinURLs = append(joinURLs, fmt.Sprintf("%s%s/queue/join", disc.SpaceURL, disc.APIPrefix))
|
|
}
|
|
joinURLs = append(joinURLs, fmt.Sprintf("%s/queue/join", disc.SpaceURL))
|
|
|
|
var resp *http.Response
|
|
var lastErr error
|
|
|
|
for _, targetURL := range joinURLs {
|
|
curURL := targetURL
|
|
makeReq := func() (*http.Request, error) {
|
|
req, err := http.NewRequest("POST", curURL, bytes.NewBuffer(jsonPayload))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
req.Header.Set("Content-Type", "application/json")
|
|
req.Header.Set("User-Agent", effUA)
|
|
return req, nil
|
|
}
|
|
resp, lastErr = DoWithFibonacciRetry(g.client, makeReq, 3)
|
|
if lastErr == nil && resp != nil && resp.StatusCode == http.StatusOK {
|
|
break
|
|
}
|
|
if resp != nil {
|
|
resp.Body.Close()
|
|
resp = nil
|
|
}
|
|
}
|
|
|
|
if resp == nil {
|
|
log.Printf("Gradio /queue/join unavailable (%v), falling back to /run/predict protocol...", lastErr)
|
|
return g.executePredictCompletion(w, r, disc, gradioData, req, completionID, createdTime, modelName, effUA)
|
|
}
|
|
resp.Body.Close()
|
|
|
|
// 2. Connect to Gradio SSE queue data stream
|
|
var dataURLs []string
|
|
if disc.APIPrefix != "" {
|
|
dataURLs = append(dataURLs, fmt.Sprintf("%s%s/queue/data?session_hash=%s", disc.SpaceURL, disc.APIPrefix, sessionHash))
|
|
}
|
|
dataURLs = append(dataURLs, fmt.Sprintf("%s/queue/data?session_hash=%s", disc.SpaceURL, sessionHash))
|
|
|
|
var streamResp *http.Response
|
|
for _, targetURL := range dataURLs {
|
|
curURL := targetURL
|
|
makeStreamReq := func() (*http.Request, error) {
|
|
req, err := http.NewRequest("GET", curURL, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
req.Header.Set("Accept", "text/event-stream")
|
|
req.Header.Set("User-Agent", effUA)
|
|
return req, nil
|
|
}
|
|
streamResp, lastErr = DoWithFibonacciRetry(g.client, makeStreamReq, 3)
|
|
if lastErr == nil && streamResp != nil && streamResp.StatusCode == http.StatusOK {
|
|
break
|
|
}
|
|
if streamResp != nil {
|
|
streamResp.Body.Close()
|
|
streamResp = nil
|
|
}
|
|
}
|
|
|
|
if streamResp == nil {
|
|
log.Printf("Gradio /queue/data unavailable (%v), falling back to /run/predict protocol...", lastErr)
|
|
return g.executePredictCompletion(w, r, disc, gradioData, req, completionID, createdTime, modelName, effUA)
|
|
}
|
|
defer streamResp.Body.Close()
|
|
|
|
type queueMsg struct {
|
|
Msg string `json:"msg"`
|
|
EventID string `json:"event_id,omitempty"`
|
|
Success *bool `json:"success,omitempty"`
|
|
Output map[string]interface{} `json:"output,omitempty"`
|
|
}
|
|
|
|
// 3. Handle Non-Streaming vs Streaming
|
|
if !req.Stream {
|
|
reader := bufio.NewReader(streamResp.Body)
|
|
var latestFrame GradioOutputFrame
|
|
|
|
for {
|
|
line, err := reader.ReadString('\n')
|
|
if err != nil {
|
|
break
|
|
}
|
|
line = strings.TrimRight(line, "\r\n")
|
|
|
|
if strings.HasPrefix(line, "data: ") {
|
|
dataStr := strings.TrimPrefix(line, "data: ")
|
|
var qMsg queueMsg
|
|
if err := json.Unmarshal([]byte(dataStr), &qMsg); err != nil {
|
|
continue
|
|
}
|
|
|
|
if qMsg.Msg == "close_stream" {
|
|
break
|
|
}
|
|
|
|
if qMsg.Msg == "process_generating" || qMsg.Msg == "process_completed" {
|
|
if qMsg.Success != nil && !*qMsg.Success {
|
|
errBytes, _ := json.Marshal(qMsg.Output)
|
|
errMsg := extractGradioErrorMessage(string(errBytes))
|
|
log.Printf("Upstream Gradio error: %s", errMsg)
|
|
return fmt.Errorf("upstream Gradio error: %s", errMsg)
|
|
}
|
|
|
|
if qMsg.Output != nil && qMsg.Output["data"] != nil {
|
|
dataBytes, _ := json.Marshal(qMsg.Output["data"])
|
|
frame := ParseGradioStreamOutput(string(dataBytes))
|
|
if frame.OK {
|
|
if frame.IsDelta {
|
|
latestFrame.Content += frame.Content
|
|
latestFrame.Reasoning += frame.Reasoning
|
|
} else {
|
|
if len(frame.ToolCalls) > 0 {
|
|
latestFrame.Content = frame.Content
|
|
} else if frame.Content != "" || latestFrame.Content == "" {
|
|
latestFrame.Content = frame.Content
|
|
}
|
|
if frame.Reasoning != "" || latestFrame.Reasoning == "" {
|
|
latestFrame.Reasoning = frame.Reasoning
|
|
}
|
|
}
|
|
if len(frame.ToolCalls) > 0 {
|
|
latestFrame.ToolCalls = frame.ToolCalls
|
|
}
|
|
latestFrame.OK = true
|
|
}
|
|
}
|
|
if qMsg.Msg == "process_completed" {
|
|
break
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if !latestFrame.OK {
|
|
return fmt.Errorf("upstream Gradio space returned empty or unparseable response")
|
|
}
|
|
|
|
finalContent, reasoning, toolCalls, finishReason := finalizeOutput(latestFrame)
|
|
|
|
WriteCompletionResponse(w, completionID, createdTime, modelName, FinalOutput{
|
|
Content: finalContent,
|
|
ReasoningContent: reasoning,
|
|
ToolCalls: toolCalls,
|
|
FinishReason: finishReason,
|
|
})
|
|
disc.Protocol = "queue"
|
|
return nil
|
|
}
|
|
|
|
// 4. Streaming Mode
|
|
flusher, _ := w.(http.Flusher)
|
|
streamer := NewStreamer(w, flusher, completionID, createdTime, modelName)
|
|
|
|
thinkFilter := NewStreamThinkingFilter()
|
|
toolFilter := NewStreamToolCallFilter()
|
|
|
|
reader := bufio.NewReader(streamResp.Body)
|
|
var prevContent string
|
|
var prevReasoning string
|
|
prevToolArgs := make(map[int]string)
|
|
nativeReasoningSeen := disc.IsHunyuan3
|
|
nativeToolCallsSeen := false
|
|
var streamErr error
|
|
|
|
for {
|
|
line, err := reader.ReadString('\n')
|
|
if err != nil {
|
|
break
|
|
}
|
|
line = strings.TrimRight(line, "\r\n")
|
|
|
|
if strings.HasPrefix(line, "data: ") {
|
|
dataStr := strings.TrimPrefix(line, "data: ")
|
|
var qMsg queueMsg
|
|
if err := json.Unmarshal([]byte(dataStr), &qMsg); err != nil {
|
|
continue
|
|
}
|
|
|
|
if qMsg.Msg == "close_stream" {
|
|
break
|
|
}
|
|
|
|
if qMsg.Msg == "process_generating" || qMsg.Msg == "process_completed" {
|
|
if qMsg.Success != nil && !*qMsg.Success {
|
|
errBytes, _ := json.Marshal(qMsg.Output)
|
|
errMsg := extractGradioErrorMessage(string(errBytes))
|
|
log.Printf("Upstream Gradio error: %s", errMsg)
|
|
if !streamer.started {
|
|
return fmt.Errorf("upstream Gradio error: %s", errMsg)
|
|
}
|
|
streamer.Error(errMsg)
|
|
streamErr = fmt.Errorf("upstream Gradio error: %s", errMsg)
|
|
break
|
|
}
|
|
|
|
if qMsg.Output != nil && qMsg.Output["data"] != nil {
|
|
dataBytes, _ := json.Marshal(qMsg.Output["data"])
|
|
frame := ParseGradioStreamOutput(string(dataBytes))
|
|
if frame.OK {
|
|
// 1. Native reasoning handling
|
|
if frame.Reasoning != "" || nativeReasoningSeen {
|
|
nativeReasoningSeen = true
|
|
var deltaReasoning string
|
|
if frame.IsDelta {
|
|
deltaReasoning = frame.Reasoning
|
|
prevReasoning += deltaReasoning
|
|
} else {
|
|
if strings.HasPrefix(frame.Reasoning, prevReasoning) {
|
|
deltaReasoning = frame.Reasoning[len(prevReasoning):]
|
|
} else if prevReasoning == "" {
|
|
deltaReasoning = frame.Reasoning
|
|
} else {
|
|
deltaReasoning = frame.Reasoning
|
|
}
|
|
prevReasoning = frame.Reasoning
|
|
}
|
|
if deltaReasoning != "" {
|
|
streamer.Reasoning(deltaReasoning)
|
|
}
|
|
}
|
|
|
|
// 2. Native tool call handling
|
|
if len(frame.ToolCalls) > 0 || nativeToolCallsSeen {
|
|
nativeToolCallsSeen = true
|
|
for i, tc := range frame.ToolCalls {
|
|
prevArgs := prevToolArgs[i]
|
|
fullArgs := tc.Function.Arguments
|
|
if len(fullArgs) > len(prevArgs) && strings.HasPrefix(fullArgs, prevArgs) {
|
|
deltaArgs := fullArgs[len(prevArgs):]
|
|
iCopy := i
|
|
streamer.ToolCallDelta(ToolCall{
|
|
Index: &iCopy,
|
|
ID: tc.ID,
|
|
Type: tc.Type,
|
|
Function: ToolCallFunction{
|
|
Name: tc.Function.Name,
|
|
Arguments: deltaArgs,
|
|
},
|
|
})
|
|
prevToolArgs[i] = fullArgs
|
|
} else if prevArgs == "" {
|
|
iCopy := i
|
|
streamer.ToolCallDelta(ToolCall{
|
|
Index: &iCopy,
|
|
ID: tc.ID,
|
|
Type: tc.Type,
|
|
Function: ToolCallFunction{
|
|
Name: tc.Function.Name,
|
|
Arguments: fullArgs,
|
|
},
|
|
})
|
|
prevToolArgs[i] = fullArgs
|
|
}
|
|
}
|
|
}
|
|
|
|
// 3. Content handling
|
|
currentText := frame.Content
|
|
var delta string
|
|
if frame.IsDelta {
|
|
delta = frame.Content
|
|
prevContent += delta
|
|
} else {
|
|
if strings.HasPrefix(currentText, prevContent) {
|
|
delta = currentText[len(prevContent):]
|
|
} else if prevContent == "" {
|
|
delta = currentText
|
|
} else {
|
|
delta = currentText
|
|
}
|
|
prevContent = currentText
|
|
}
|
|
|
|
if delta != "" {
|
|
if nativeReasoningSeen || nativeToolCallsSeen {
|
|
if nativeReasoningSeen && nativeToolCallsSeen {
|
|
streamer.Content(delta)
|
|
} else if nativeReasoningSeen {
|
|
toolFilter.Feed(delta, func(cleanChunk string) {
|
|
if cleanChunk != "" {
|
|
streamer.Content(cleanChunk)
|
|
}
|
|
}, func(tc ToolCall) {
|
|
streamer.ToolCallDelta(tc)
|
|
})
|
|
} else {
|
|
thinkFilter.Feed(delta, func(contentChunk string) {
|
|
if contentChunk != "" {
|
|
streamer.Content(contentChunk)
|
|
}
|
|
}, func(reasoningChunk string) {
|
|
if reasoningChunk != "" {
|
|
streamer.Reasoning(reasoningChunk)
|
|
}
|
|
})
|
|
}
|
|
} else {
|
|
thinkFilter.Feed(delta, func(contentChunk string) {
|
|
toolFilter.Feed(contentChunk, func(cleanChunk string) {
|
|
if cleanChunk != "" {
|
|
streamer.Content(cleanChunk)
|
|
}
|
|
}, func(tc ToolCall) {
|
|
streamer.ToolCallDelta(tc)
|
|
})
|
|
}, func(reasoningChunk string) {
|
|
if reasoningChunk != "" {
|
|
streamer.Reasoning(reasoningChunk)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if qMsg.Msg == "process_completed" {
|
|
break
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if streamErr != nil {
|
|
return streamErr
|
|
}
|
|
|
|
// Flush remaining tokens in filters if used
|
|
if !nativeReasoningSeen {
|
|
thinkFilter.Flush(func(contentChunk string) {
|
|
if !nativeToolCallsSeen {
|
|
toolFilter.Feed(contentChunk, func(cleanChunk string) {
|
|
if cleanChunk != "" {
|
|
streamer.Content(cleanChunk)
|
|
}
|
|
}, func(tc ToolCall) {
|
|
streamer.ToolCallDelta(tc)
|
|
})
|
|
} else if contentChunk != "" {
|
|
streamer.Content(contentChunk)
|
|
}
|
|
}, func(reasoningChunk string) {
|
|
if reasoningChunk != "" {
|
|
streamer.Reasoning(reasoningChunk)
|
|
}
|
|
})
|
|
}
|
|
|
|
if !nativeToolCallsSeen {
|
|
toolFilter.Flush(func(cleanChunk string) {
|
|
if cleanChunk != "" {
|
|
streamer.Content(cleanChunk)
|
|
}
|
|
}, func(tc ToolCall) {
|
|
streamer.ToolCallDelta(tc)
|
|
})
|
|
}
|
|
|
|
finishReason := "stop"
|
|
if nativeToolCallsSeen || toolFilter.emittedCall {
|
|
finishReason = "tool_calls"
|
|
}
|
|
|
|
streamer.Finish(finishReason)
|
|
streamer.Done()
|
|
disc.Protocol = "queue"
|
|
return nil
|
|
}
|
|
|
|
// ExecuteChatCompletion handles both streaming and non-streaming requests.
|
|
func (g *GradioGateway) ExecuteChatCompletion(w http.ResponseWriter, r *http.Request, req ChatCompletionRequest) error {
|
|
effUA := EffectiveUserAgent(r)
|
|
|
|
// Target space selection: check request headers or fallback to default space
|
|
spaceURL := g.defaultURL
|
|
if hdr := r.Header.Get("X-Gradio-Space"); hdr != "" {
|
|
spaceURL = hdr
|
|
} else if hdr := r.Header.Get("X-Space-URL"); hdr != "" {
|
|
spaceURL = hdr
|
|
}
|
|
|
|
disc := g.GetDiscovery(spaceURL, effUA)
|
|
|
|
modelName := req.Model
|
|
if modelName == "" {
|
|
if ConfiguredModelName != "" {
|
|
modelName = ConfiguredModelName
|
|
} else {
|
|
modelName = disc.PrimaryModel
|
|
}
|
|
}
|
|
|
|
gradioData, err := g.BuildGradioPayload(disc, req)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to build Gradio payload: %w", err)
|
|
}
|
|
|
|
completionID := "chatcmpl-" + GenerateUUID()
|
|
createdTime := time.Now().Unix()
|
|
|
|
// If the resolved protocol is predict (e.g. Gradio 3), execute predict completion directly
|
|
if disc.Protocol == "predict" {
|
|
return g.executePredictCompletion(w, r, disc, gradioData, req, completionID, createdTime, modelName, effUA)
|
|
}
|
|
if disc.Protocol == "queue" {
|
|
return g.executeQueueCompletion(w, r, disc, gradioData, req, completionID, createdTime, modelName, effUA)
|
|
}
|
|
|
|
isV2 := disc.Protocol == "call_v2"
|
|
buildPayload := func(v2 bool) ([]byte, error) {
|
|
if v2 {
|
|
v2Payload := make(map[string]interface{})
|
|
for idx, val := range gradioData {
|
|
if idx < len(disc.ParamMappings) && disc.ParamMappings[idx].ParamType == "state" {
|
|
continue
|
|
}
|
|
pName := ""
|
|
if idx < len(disc.ParamMappings) {
|
|
if disc.ParamMappings[idx].ParamName != "" {
|
|
pName = disc.ParamMappings[idx].ParamName
|
|
} else if disc.ParamMappings[idx].Label != "" {
|
|
pName = disc.ParamMappings[idx].Label
|
|
}
|
|
}
|
|
if pName == "" {
|
|
pName = fmt.Sprintf("param_%d", idx)
|
|
}
|
|
v2Payload[pName] = val
|
|
}
|
|
return json.Marshal(v2Payload)
|
|
}
|
|
payloadMap := map[string]interface{}{
|
|
"data": gradioData,
|
|
}
|
|
return json.Marshal(payloadMap)
|
|
}
|
|
|
|
callPath := "call"
|
|
if isV2 {
|
|
callPath = "call/v2"
|
|
}
|
|
jsonPayload, err := buildPayload(isV2)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to encode request: %w", err)
|
|
}
|
|
|
|
// 1. Submit to /call/{endpoint} or /call/v2/{endpoint}
|
|
callURL := fmt.Sprintf("%s%s/%s/%s", disc.SpaceURL, disc.APIPrefix, callPath, disc.CleanEndpoint)
|
|
makeCallReq := func() (*http.Request, error) {
|
|
r, err := http.NewRequest("POST", callURL, bytes.NewBuffer(jsonPayload))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
r.Header.Set("Content-Type", "application/json")
|
|
r.Header.Set("User-Agent", effUA)
|
|
return r, nil
|
|
}
|
|
|
|
resp, err := DoWithFibonacciRetry(g.client, makeCallReq, 5)
|
|
if err != nil && isV2 {
|
|
log.Printf("Gradio /call/v2 endpoint failed (%v), falling back to /call...", err)
|
|
isV2 = false
|
|
callPath = "call"
|
|
jsonPayload, _ = buildPayload(false)
|
|
callURL = fmt.Sprintf("%s%s/%s/%s", disc.SpaceURL, disc.APIPrefix, callPath, disc.CleanEndpoint)
|
|
resp, err = DoWithFibonacciRetry(g.client, makeCallReq, 3)
|
|
}
|
|
if err != nil {
|
|
if fg, ok := extractFailedGeneration(err.Error()); ok {
|
|
if tcs, _, has := DetectToolCalls(fg); has && len(tcs) > 0 {
|
|
if !req.Stream {
|
|
WriteCompletionResponse(w, completionID, createdTime, modelName, FinalOutput{
|
|
ToolCalls: tcs,
|
|
FinishReason: "tool_calls",
|
|
})
|
|
return nil
|
|
}
|
|
flusher, _ := w.(http.Flusher)
|
|
streamer := NewStreamer(w, flusher, completionID, createdTime, modelName)
|
|
for i, tc := range tcs {
|
|
iCopy := i
|
|
tc.Index = &iCopy
|
|
streamer.ToolCallDelta(tc)
|
|
}
|
|
streamer.Finish("tool_calls")
|
|
streamer.Done()
|
|
return nil
|
|
}
|
|
}
|
|
// If call failed, try without APIPrefix
|
|
altCallURL := fmt.Sprintf("%s/%s/%s", disc.SpaceURL, callPath, disc.CleanEndpoint)
|
|
makeAltReq := func() (*http.Request, error) {
|
|
r, err := http.NewRequest("POST", altCallURL, bytes.NewBuffer(jsonPayload))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
r.Header.Set("Content-Type", "application/json")
|
|
r.Header.Set("User-Agent", effUA)
|
|
return r, nil
|
|
}
|
|
resp, err = DoWithFibonacciRetry(g.client, makeAltReq, 3)
|
|
if err != nil {
|
|
if fg, ok := extractFailedGeneration(err.Error()); ok {
|
|
if tcs, _, has := DetectToolCalls(fg); has && len(tcs) > 0 {
|
|
if !req.Stream {
|
|
WriteCompletionResponse(w, completionID, createdTime, modelName, FinalOutput{
|
|
ToolCalls: tcs,
|
|
FinishReason: "tool_calls",
|
|
})
|
|
return nil
|
|
}
|
|
flusher, _ := w.(http.Flusher)
|
|
streamer := NewStreamer(w, flusher, completionID, createdTime, modelName)
|
|
for i, tc := range tcs {
|
|
iCopy := i
|
|
tc.Index = &iCopy
|
|
streamer.ToolCallDelta(tc)
|
|
}
|
|
streamer.Finish("tool_calls")
|
|
streamer.Done()
|
|
return nil
|
|
}
|
|
}
|
|
if strings.Contains(err.Error(), "404") || strings.Contains(err.Error(), "405") || strings.Contains(err.Error(), "500") {
|
|
log.Printf("Gradio /call endpoint unavailable (%v), falling back to /queue/join protocol...", err)
|
|
return g.executeQueueCompletion(w, r, disc, gradioData, req, completionID, createdTime, modelName, effUA)
|
|
}
|
|
return fmt.Errorf("upstream Gradio call error: %w", err)
|
|
}
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
var joinRes GradioJoinResponse
|
|
if err := json.NewDecoder(resp.Body).Decode(&joinRes); err != nil || joinRes.EventID == "" {
|
|
log.Printf("Gradio /call returned non-SSE response, falling back to /queue/join protocol...")
|
|
return g.executeQueueCompletion(w, r, disc, gradioData, req, completionID, createdTime, modelName, effUA)
|
|
}
|
|
|
|
// 2. Connect to Gradio SSE EventStream
|
|
streamURL := fmt.Sprintf("%s%s/call/%s/%s", disc.SpaceURL, disc.APIPrefix, disc.CleanEndpoint, joinRes.EventID)
|
|
makeStreamReq := func() (*http.Request, error) {
|
|
r, err := http.NewRequest("GET", streamURL, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
r.Header.Set("Accept", "text/event-stream")
|
|
r.Header.Set("User-Agent", effUA)
|
|
return r, nil
|
|
}
|
|
|
|
streamResp, err := DoWithFibonacciRetry(g.client, makeStreamReq, 5)
|
|
if err != nil {
|
|
if fg, ok := extractFailedGeneration(err.Error()); ok {
|
|
if tcs, _, has := DetectToolCalls(fg); has && len(tcs) > 0 {
|
|
if !req.Stream {
|
|
WriteCompletionResponse(w, completionID, createdTime, modelName, FinalOutput{
|
|
ToolCalls: tcs,
|
|
FinishReason: "tool_calls",
|
|
})
|
|
return nil
|
|
}
|
|
flusher, _ := w.(http.Flusher)
|
|
streamer := NewStreamer(w, flusher, completionID, createdTime, modelName)
|
|
for i, tc := range tcs {
|
|
iCopy := i
|
|
tc.Index = &iCopy
|
|
streamer.ToolCallDelta(tc)
|
|
}
|
|
streamer.Finish("tool_calls")
|
|
streamer.Done()
|
|
return nil
|
|
}
|
|
}
|
|
log.Printf("Gradio /call stream connection failed (%v), falling back to /queue/join protocol...", err)
|
|
return g.executeQueueCompletion(w, r, disc, gradioData, req, completionID, createdTime, modelName, effUA)
|
|
}
|
|
defer streamResp.Body.Close()
|
|
|
|
// 3. Handle Non-Streaming vs Streaming
|
|
if !req.Stream {
|
|
reader := bufio.NewReader(streamResp.Body)
|
|
var latestFrame GradioOutputFrame
|
|
currentEvent := ""
|
|
|
|
for {
|
|
line, err := reader.ReadString('\n')
|
|
if err != nil {
|
|
break
|
|
}
|
|
line = strings.TrimRight(line, "\r\n")
|
|
|
|
if strings.HasPrefix(line, "event: ") {
|
|
currentEvent = strings.TrimPrefix(line, "event: ")
|
|
continue
|
|
}
|
|
|
|
if strings.HasPrefix(line, "data: ") {
|
|
dataStr := strings.TrimPrefix(line, "data: ")
|
|
if currentEvent == "error" {
|
|
if fg, ok := extractFailedGeneration(dataStr); ok {
|
|
if tcs, _, has := DetectToolCalls(fg); has && len(tcs) > 0 {
|
|
latestFrame = GradioOutputFrame{
|
|
ToolCalls: tcs,
|
|
OK: true,
|
|
}
|
|
break
|
|
}
|
|
}
|
|
errMsg := extractGradioErrorMessage(dataStr)
|
|
if isProtocolOrInputError(errMsg) {
|
|
log.Printf("Upstream Gradio /call stream error (%s), falling back to /queue/join protocol...", errMsg)
|
|
return g.executeQueueCompletion(w, r, disc, gradioData, req, completionID, createdTime, modelName, effUA)
|
|
}
|
|
log.Printf("Upstream Gradio error: %s", errMsg)
|
|
return fmt.Errorf("upstream Gradio error: %s", errMsg)
|
|
}
|
|
if frame := ParseGradioStreamOutput(dataStr); frame.OK {
|
|
if frame.IsDelta {
|
|
latestFrame.Content += frame.Content
|
|
latestFrame.Reasoning += frame.Reasoning
|
|
} else {
|
|
if len(frame.ToolCalls) > 0 {
|
|
latestFrame.Content = frame.Content
|
|
} else if frame.Content != "" || latestFrame.Content == "" {
|
|
latestFrame.Content = frame.Content
|
|
}
|
|
if frame.Reasoning != "" || latestFrame.Reasoning == "" {
|
|
latestFrame.Reasoning = frame.Reasoning
|
|
}
|
|
}
|
|
if len(frame.ToolCalls) > 0 {
|
|
latestFrame.ToolCalls = frame.ToolCalls
|
|
}
|
|
latestFrame.OK = true
|
|
}
|
|
if currentEvent == "complete" {
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
if !latestFrame.OK {
|
|
log.Printf("Gradio /call returned empty or unparseable response, falling back to /queue/join protocol...")
|
|
return g.executeQueueCompletion(w, r, disc, gradioData, req, completionID, createdTime, modelName, effUA)
|
|
}
|
|
|
|
finalContent, reasoning, toolCalls, finishReason := finalizeOutput(latestFrame)
|
|
|
|
WriteCompletionResponse(w, completionID, createdTime, modelName, FinalOutput{
|
|
Content: finalContent,
|
|
ReasoningContent: reasoning,
|
|
ToolCalls: toolCalls,
|
|
FinishReason: finishReason,
|
|
})
|
|
return nil
|
|
}
|
|
|
|
// 4. Streaming Mode
|
|
flusher, _ := w.(http.Flusher)
|
|
streamer := NewStreamer(w, flusher, completionID, createdTime, modelName)
|
|
|
|
thinkFilter := NewStreamThinkingFilter()
|
|
toolFilter := NewStreamToolCallFilter()
|
|
|
|
reader := bufio.NewReader(streamResp.Body)
|
|
var prevContent string
|
|
var prevReasoning string
|
|
prevToolArgs := make(map[int]string)
|
|
nativeReasoningSeen := disc.IsHunyuan3
|
|
nativeToolCallsSeen := false
|
|
currentEvent := ""
|
|
var streamErr error
|
|
|
|
for {
|
|
line, err := reader.ReadString('\n')
|
|
if err != nil {
|
|
break
|
|
}
|
|
line = strings.TrimRight(line, "\r\n")
|
|
|
|
if strings.HasPrefix(line, "event: ") {
|
|
currentEvent = strings.TrimPrefix(line, "event: ")
|
|
continue
|
|
}
|
|
|
|
if strings.HasPrefix(line, "data: ") {
|
|
dataStr := strings.TrimPrefix(line, "data: ")
|
|
if currentEvent == "error" {
|
|
if fg, ok := extractFailedGeneration(dataStr); ok {
|
|
if tcs, _, has := DetectToolCalls(fg); has && len(tcs) > 0 {
|
|
for i, tc := range tcs {
|
|
iCopy := i
|
|
tc.Index = &iCopy
|
|
streamer.ToolCallDelta(tc)
|
|
}
|
|
nativeToolCallsSeen = true
|
|
break
|
|
}
|
|
}
|
|
errMsg := extractGradioErrorMessage(dataStr)
|
|
if !streamer.started {
|
|
if isProtocolOrInputError(errMsg) {
|
|
log.Printf("Upstream Gradio /call stream error (%s), falling back to /queue/join protocol...", errMsg)
|
|
return g.executeQueueCompletion(w, r, disc, gradioData, req, completionID, createdTime, modelName, effUA)
|
|
}
|
|
log.Printf("Upstream Gradio error: %s", errMsg)
|
|
return fmt.Errorf("upstream Gradio error: %s", errMsg)
|
|
}
|
|
log.Printf("Upstream Gradio error: %s", errMsg)
|
|
streamer.Error(errMsg)
|
|
streamErr = fmt.Errorf("upstream Gradio error: %s", errMsg)
|
|
break
|
|
}
|
|
|
|
frame := ParseGradioStreamOutput(dataStr)
|
|
if frame.OK {
|
|
// 1. Native reasoning handling
|
|
if frame.Reasoning != "" || nativeReasoningSeen {
|
|
nativeReasoningSeen = true
|
|
var deltaReasoning string
|
|
if frame.IsDelta {
|
|
deltaReasoning = frame.Reasoning
|
|
prevReasoning += deltaReasoning
|
|
} else {
|
|
if strings.HasPrefix(frame.Reasoning, prevReasoning) {
|
|
deltaReasoning = frame.Reasoning[len(prevReasoning):]
|
|
} else if prevReasoning == "" {
|
|
deltaReasoning = frame.Reasoning
|
|
} else {
|
|
deltaReasoning = frame.Reasoning
|
|
}
|
|
prevReasoning = frame.Reasoning
|
|
}
|
|
if deltaReasoning != "" {
|
|
streamer.Reasoning(deltaReasoning)
|
|
}
|
|
}
|
|
|
|
// 2. Native tool calls handling
|
|
if len(frame.ToolCalls) > 0 {
|
|
nativeToolCallsSeen = true
|
|
for idx, tc := range frame.ToolCalls {
|
|
prevArgs, started := prevToolArgs[idx]
|
|
currArgs := tc.Function.Arguments
|
|
idxCopy := idx
|
|
if !started {
|
|
tcDelta := ToolCall{
|
|
Index: &idxCopy,
|
|
ID: tc.ID,
|
|
Type: tc.Type,
|
|
Function: ToolCallFunction{
|
|
Name: tc.Function.Name,
|
|
Arguments: currArgs,
|
|
},
|
|
}
|
|
streamer.ToolCallDelta(tcDelta)
|
|
prevToolArgs[idx] = currArgs
|
|
} else if len(currArgs) > len(prevArgs) {
|
|
var argDelta string
|
|
if strings.HasPrefix(currArgs, prevArgs) {
|
|
argDelta = currArgs[len(prevArgs):]
|
|
} else {
|
|
argDelta = currArgs[len(prevArgs):]
|
|
}
|
|
if argDelta != "" {
|
|
tcDelta := ToolCall{
|
|
Index: &idxCopy,
|
|
Function: ToolCallFunction{
|
|
Arguments: argDelta,
|
|
},
|
|
}
|
|
streamer.ToolCallDelta(tcDelta)
|
|
}
|
|
prevToolArgs[idx] = currArgs
|
|
}
|
|
}
|
|
}
|
|
|
|
// 3. Content handling
|
|
currentText := frame.Content
|
|
var delta string
|
|
if frame.IsDelta {
|
|
delta = currentText
|
|
prevContent += delta
|
|
} else {
|
|
if strings.HasPrefix(currentText, prevContent) {
|
|
delta = currentText[len(prevContent):]
|
|
} else if prevContent == "" {
|
|
delta = currentText
|
|
} else {
|
|
delta = currentText
|
|
}
|
|
prevContent = currentText
|
|
}
|
|
|
|
if delta != "" {
|
|
if nativeReasoningSeen || nativeToolCallsSeen {
|
|
if nativeReasoningSeen && nativeToolCallsSeen {
|
|
streamer.Content(delta)
|
|
} else if nativeReasoningSeen {
|
|
toolFilter.Feed(delta, func(cleanChunk string) {
|
|
if cleanChunk != "" {
|
|
streamer.Content(cleanChunk)
|
|
}
|
|
}, func(tc ToolCall) {
|
|
streamer.ToolCallDelta(tc)
|
|
})
|
|
} else {
|
|
thinkFilter.Feed(delta, func(contentChunk string) {
|
|
if contentChunk != "" {
|
|
streamer.Content(contentChunk)
|
|
}
|
|
}, func(reasoningChunk string) {
|
|
if reasoningChunk != "" {
|
|
streamer.Reasoning(reasoningChunk)
|
|
}
|
|
})
|
|
}
|
|
} else {
|
|
thinkFilter.Feed(delta, func(contentChunk string) {
|
|
toolFilter.Feed(contentChunk, func(cleanChunk string) {
|
|
if cleanChunk != "" {
|
|
streamer.Content(cleanChunk)
|
|
}
|
|
}, func(tc ToolCall) {
|
|
streamer.ToolCallDelta(tc)
|
|
})
|
|
}, func(reasoningChunk string) {
|
|
if reasoningChunk != "" {
|
|
streamer.Reasoning(reasoningChunk)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
if currentEvent == "complete" {
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
if streamErr != nil {
|
|
return nil
|
|
}
|
|
|
|
// Flush remaining tokens in filters if used
|
|
if !nativeReasoningSeen {
|
|
thinkFilter.Flush(func(contentChunk string) {
|
|
if !nativeToolCallsSeen {
|
|
toolFilter.Feed(contentChunk, func(cleanChunk string) {
|
|
if cleanChunk != "" {
|
|
streamer.Content(cleanChunk)
|
|
}
|
|
}, func(tc ToolCall) {
|
|
streamer.ToolCallDelta(tc)
|
|
})
|
|
} else if contentChunk != "" {
|
|
streamer.Content(contentChunk)
|
|
}
|
|
}, func(reasoningChunk string) {
|
|
if reasoningChunk != "" {
|
|
streamer.Reasoning(reasoningChunk)
|
|
}
|
|
})
|
|
}
|
|
|
|
if !nativeToolCallsSeen {
|
|
toolFilter.Flush(func(cleanChunk string) {
|
|
if cleanChunk != "" {
|
|
streamer.Content(cleanChunk)
|
|
}
|
|
}, func(tc ToolCall) {
|
|
streamer.ToolCallDelta(tc)
|
|
})
|
|
}
|
|
|
|
if !streamer.started {
|
|
log.Printf("Gradio /call closed stream without sending content, falling back to /queue/join protocol...")
|
|
return g.executeQueueCompletion(w, r, disc, gradioData, req, completionID, createdTime, modelName, effUA)
|
|
}
|
|
|
|
if nativeToolCallsSeen || toolFilter.emittedCall {
|
|
streamer.Finish("tool_calls")
|
|
} else {
|
|
streamer.Finish("stop")
|
|
}
|
|
streamer.Done()
|
|
|
|
return nil
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// HTTP routes & main server
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func main() {
|
|
spaceFlag := flag.String("space", DefaultSpaceURL, "Target Gradio Space URL")
|
|
flag.StringVar(spaceFlag, "url", DefaultSpaceURL, "Alias for -space")
|
|
flag.StringVar(spaceFlag, "endpoint", DefaultSpaceURL, "Alias for -space")
|
|
modelFlag := flag.String("model", "", "Exposed model name override (default: auto-detected)")
|
|
portFlag := flag.Int("port", 8080, "Gateway HTTP server port")
|
|
hostFlag := flag.String("host", "0.0.0.0", "Gateway HTTP server host")
|
|
socksFlag := flag.String("socks", "", "Optional SOCKS5 proxy URL (e.g. socks5://127.0.0.1:1080)")
|
|
flag.StringVar(socksFlag, "proxy", "", "Alias for -socks")
|
|
flag.StringVar(socksFlag, "socks5", "", "Alias for -socks")
|
|
uaFlag := flag.String("user-agent", "", "Custom User-Agent header")
|
|
flag.StringVar(uaFlag, "ua", "", "Alias for -user-agent")
|
|
timeoutFlag := flag.Int("timeout", 300, "Upstream timeout in seconds")
|
|
flag.Parse()
|
|
|
|
// Environment variable fallbacks
|
|
if envSpace := os.Getenv("GRADIO_SPACE_URL"); envSpace != "" && *spaceFlag == DefaultSpaceURL {
|
|
*spaceFlag = envSpace
|
|
}
|
|
if *socksFlag == "" {
|
|
for _, envName := range []string{"ALL_PROXY", "all_proxy", "SOCKS5_PROXY", "socks5_proxy", "SOCKS_PROXY", "socks_proxy"} {
|
|
if p := os.Getenv(envName); p != "" {
|
|
*socksFlag = p
|
|
break
|
|
}
|
|
}
|
|
}
|
|
if *uaFlag != "" {
|
|
ConfiguredUserAgent = *uaFlag
|
|
}
|
|
if *modelFlag != "" {
|
|
ConfiguredModelName = *modelFlag
|
|
}
|
|
|
|
gateway := NewGradioGateway(*spaceFlag, *socksFlag, time.Duration(*timeoutFlag)*time.Second)
|
|
|
|
mux := http.NewServeMux()
|
|
|
|
// Health and Info
|
|
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
|
EnableCORS(w)
|
|
if r.Method == "OPTIONS" {
|
|
w.WriteHeader(http.StatusOK)
|
|
return
|
|
}
|
|
if r.URL.Path != "/" {
|
|
http.NotFound(w, r)
|
|
return
|
|
}
|
|
disc := gateway.GetDiscovery("", EffectiveUserAgent(r))
|
|
w.Header().Set("Content-Type", "application/json")
|
|
json.NewEncoder(w).Encode(map[string]interface{}{
|
|
"status": "running",
|
|
"service": "gr2gw",
|
|
"space_url": disc.SpaceURL,
|
|
"title": disc.Title,
|
|
"gradio_version": disc.GradioVersion,
|
|
"flavor": disc.Flavor,
|
|
"protocol": disc.Protocol,
|
|
"endpoint": disc.Endpoint,
|
|
"primary_model": disc.PrimaryModel,
|
|
"models": disc.Models,
|
|
"total_inputs": disc.TotalInputs,
|
|
"history_format": disc.HistoryFormat,
|
|
"tool_call_mode": disc.ToolCallMode,
|
|
"param_mappings": disc.ParamMappings,
|
|
})
|
|
})
|
|
|
|
// Models list
|
|
handleModels := func(w http.ResponseWriter, r *http.Request) {
|
|
EnableCORS(w)
|
|
if r.Method == "OPTIONS" {
|
|
w.WriteHeader(http.StatusOK)
|
|
return
|
|
}
|
|
spaceURL := gateway.defaultURL
|
|
if hdr := r.Header.Get("X-Gradio-Space"); hdr != "" {
|
|
spaceURL = hdr
|
|
} else if hdr := r.Header.Get("X-Space-URL"); hdr != "" {
|
|
spaceURL = hdr
|
|
}
|
|
disc := gateway.GetDiscovery(spaceURL, EffectiveUserAgent(r))
|
|
resp := ModelsResponse{
|
|
Object: "list",
|
|
Data: disc.GetModelList(),
|
|
}
|
|
w.Header().Set("Content-Type", "application/json")
|
|
json.NewEncoder(w).Encode(resp)
|
|
}
|
|
|
|
mux.HandleFunc("/models", handleModels)
|
|
mux.HandleFunc("/v1/models", handleModels)
|
|
|
|
// Chat completions
|
|
handleCompletions := func(w http.ResponseWriter, r *http.Request) {
|
|
EnableCORS(w)
|
|
if r.Method == "OPTIONS" {
|
|
w.WriteHeader(http.StatusOK)
|
|
return
|
|
}
|
|
if r.Method != "POST" {
|
|
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
|
return
|
|
}
|
|
|
|
var req ChatCompletionRequest
|
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
|
http.Error(w, fmt.Sprintf("Invalid JSON request: %v", err), http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
if err := gateway.ExecuteChatCompletion(w, r, req); err != nil {
|
|
log.Printf("Chat completion error: %v", err)
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(http.StatusBadGateway)
|
|
json.NewEncoder(w).Encode(map[string]interface{}{
|
|
"error": map[string]interface{}{
|
|
"message": err.Error(),
|
|
"type": "upstream_error",
|
|
"code": http.StatusBadGateway,
|
|
},
|
|
})
|
|
return
|
|
}
|
|
}
|
|
|
|
mux.HandleFunc("/chat/completions", handleCompletions)
|
|
mux.HandleFunc("/v1/chat/completions", handleCompletions)
|
|
|
|
addr := fmt.Sprintf("%s:%d", *hostFlag, *portFlag)
|
|
log.Printf("gr2gw listening on %s (target space: %s)", addr, *spaceFlag)
|
|
if *socksFlag != "" {
|
|
log.Printf("Using SOCKS5 proxy: %s", *socksFlag)
|
|
}
|
|
|
|
server := &http.Server{
|
|
Addr: addr,
|
|
Handler: mux,
|
|
ReadTimeout: time.Duration(*timeoutFlag+30) * time.Second,
|
|
WriteTimeout: time.Duration(*timeoutFlag+30) * time.Second,
|
|
}
|
|
|
|
if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
|
log.Fatalf("Server failed: %v", err)
|
|
}
|
|
}
|