// gr2gw: Universal Gradio to OpenAI LLM proxy gateway in Go
// Created by Luxferre in 2026, released into the public domain
package main
import (
"bufio"
"bytes"
"context"
"crypto/rand"
"encoding/json"
"flag"
"fmt"
"io"
"log"
"net"
"net/http"
"net/url"
"os"
"regexp"
"sort"
"strconv"
"strings"
"sync"
"time"
)
var (
DefaultSpaceURL = "https://tencent-hy3.hf.space"
DefaultUserAgent = "Mozilla/5.0 (X11; Linux x86_64; rv:153.0) Gecko/20100101 Firefox/153.0"
ConfiguredUserAgent string
ConfiguredModelName string
)
// ---------------------------------------------------------------------------
// OpenAI API data structures
// ---------------------------------------------------------------------------
type ModelItem struct {
ID string `json:"id"`
Object string `json:"object"`
Created int64 `json:"created"`
OwnedBy string `json:"owned_by"`
}
type ModelsResponse struct {
Object string `json:"object"`
Data []ModelItem `json:"data"`
}
type ToolCallFunction struct {
Name string `json:"name,omitempty"`
Arguments string `json:"arguments"`
}
type ToolCall struct {
Index *int `json:"index,omitempty"`
ID string `json:"id,omitempty"`
Type string `json:"type,omitempty"`
Function ToolCallFunction `json:"function"`
}
type Tool struct {
Type string `json:"type"`
Function interface{} `json:"function"`
}
type ChatMessage struct {
Role string `json:"role"`
Content interface{} `json:"content"`
ReasoningContent string `json:"reasoning_content,omitempty"`
ToolCalls []ToolCall `json:"tool_calls,omitempty"`
ToolCallID string `json:"tool_call_id,omitempty"`
Name string `json:"name,omitempty"`
}
func extractContentString(val interface{}) string {
if val == nil {
return ""
}
if str, ok := val.(string); ok {
return str
}
if parts, ok := val.([]interface{}); ok {
var sb strings.Builder
for _, p := range parts {
if str, ok := p.(string); ok {
sb.WriteString(str)
} else if itemMap, ok := p.(map[string]interface{}); ok {
if textVal, ok := itemMap["text"].(string); ok {
sb.WriteString(textVal)
} else if valStr, ok := itemMap["value"].(string); ok {
sb.WriteString(valStr)
}
}
}
return sb.String()
}
if itemMap, ok := val.(map[string]interface{}); ok {
if textVal, ok := itemMap["text"].(string); ok {
return textVal
} else if valStr, ok := itemMap["value"].(string); ok {
return valStr
}
}
b, err := json.Marshal(val)
if err == nil {
return string(b)
}
return fmt.Sprintf("%v", val)
}
func (m *ChatMessage) GetContentString() string {
return extractContentString(m.Content)
}
type ChatCompletionRequest struct {
Model string `json:"model"`
Messages []ChatMessage `json:"messages"`
Tools []Tool `json:"tools,omitempty"`
ToolChoice interface{} `json:"tool_choice,omitempty"`
Stream bool `json:"stream"`
MaxTokens int `json:"max_tokens"`
MaxCompletionTokens int `json:"max_completion_tokens"`
Temperature *float64 `json:"temperature,omitempty"`
TopP *float64 `json:"top_p,omitempty"`
ReasoningEffort string `json:"reasoning_effort,omitempty"`
}
type ChatCompletionResponseChoice struct {
Index int `json:"index"`
Message ChatMessage `json:"message"`
FinishReason string `json:"finish_reason"`
}
type Usage struct {
PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"`
TotalTokens int `json:"total_tokens"`
}
type ChatCompletionResponse struct {
ID string `json:"id"`
Object string `json:"object"`
Created int64 `json:"created"`
Model string `json:"model"`
Choices []ChatCompletionResponseChoice `json:"choices"`
Usage Usage `json:"usage"`
}
type StreamDelta struct {
Role string `json:"role,omitempty"`
Content string `json:"content,omitempty"`
ReasoningContent string `json:"reasoning_content,omitempty"`
ToolCalls []ToolCall `json:"tool_calls,omitempty"`
}
type StreamChoice struct {
Index int `json:"index"`
Delta StreamDelta `json:"delta"`
FinishReason *string `json:"finish_reason,omitempty"`
}
type StreamResponse struct {
ID string `json:"id"`
Object string `json:"object"`
Created int64 `json:"created"`
Model string `json:"model"`
Choices []StreamChoice `json:"choices"`
}
// ---------------------------------------------------------------------------
// SOCKS5 proxy client (RFC 1928 / RFC 1929)
// ---------------------------------------------------------------------------
type SOCKS5Config struct {
Address string
Username string
Password string
}
func ParseSOCKS5URL(proxyURL string) (*SOCKS5Config, error) {
cleanURL := strings.TrimSpace(proxyURL)
if cleanURL == "" {
return nil, nil
}
if strings.HasPrefix(cleanURL, "socks5://") {
cleanURL = strings.TrimPrefix(cleanURL, "socks5://")
} else if strings.HasPrefix(cleanURL, "socks5h://") {
cleanURL = strings.TrimPrefix(cleanURL, "socks5h://")
}
cfg := &SOCKS5Config{}
if atIdx := strings.LastIndex(cleanURL, "@"); atIdx != -1 {
userPass := cleanURL[:atIdx]
cfg.Address = cleanURL[atIdx+1:]
if colonIdx := strings.Index(userPass, ":"); colonIdx != -1 {
cfg.Username = userPass[:colonIdx]
cfg.Password = userPass[colonIdx+1:]
} else {
cfg.Username = userPass
}
} else {
cfg.Address = cleanURL
}
if !strings.Contains(cfg.Address, ":") {
cfg.Address = cfg.Address + ":1080"
}
return cfg, nil
}
func DialSOCKS5(ctx context.Context, proxyURL, targetAddr string) (net.Conn, error) {
cfg, err := ParseSOCKS5URL(proxyURL)
if err != nil {
return nil, fmt.Errorf("invalid socks5 proxy configuration: %w", err)
}
if cfg == nil {
var d net.Dialer
return d.DialContext(ctx, "tcp", targetAddr)
}
var d net.Dialer
conn, err := d.DialContext(ctx, "tcp", cfg.Address)
if err != nil {
return nil, fmt.Errorf("failed to connect to socks5 proxy at %s: %w", cfg.Address, err)
}
deadline, ok := ctx.Deadline()
if !ok {
deadline = time.Now().Add(30 * time.Second)
}
conn.SetDeadline(deadline)
defer conn.SetDeadline(time.Time{})
// 1. Negotiation Greeting (RFC 1928)
var greeting []byte
if cfg.Username != "" {
greeting = []byte{0x05, 0x02, 0x00, 0x02}
} else {
greeting = []byte{0x05, 0x01, 0x00}
}
if _, err := conn.Write(greeting); err != nil {
conn.Close()
return nil, fmt.Errorf("failed to write socks5 greeting: %w", err)
}
resp := make([]byte, 2)
if _, err := io.ReadFull(conn, resp); err != nil {
conn.Close()
return nil, fmt.Errorf("failed to read socks5 greeting response: %w", err)
}
if resp[0] != 0x05 {
conn.Close()
return nil, fmt.Errorf("unsupported socks version: 0x%02x", resp[0])
}
// 2. Authentication if required (RFC 1929)
if resp[1] == 0x02 {
if cfg.Username == "" {
conn.Close()
return nil, fmt.Errorf("socks5 proxy requires authentication, but no credentials provided")
}
uLen := byte(len(cfg.Username))
pLen := byte(len(cfg.Password))
authReq := []byte{0x01, uLen}
authReq = append(authReq, []byte(cfg.Username)...)
authReq = append(authReq, pLen)
authReq = append(authReq, []byte(cfg.Password)...)
if _, err := conn.Write(authReq); err != nil {
conn.Close()
return nil, fmt.Errorf("failed to send socks5 authentication: %w", err)
}
authResp := make([]byte, 2)
if _, err := io.ReadFull(conn, authResp); err != nil {
conn.Close()
return nil, fmt.Errorf("failed to read socks5 auth response: %w", err)
}
if authResp[1] != 0x00 {
conn.Close()
return nil, fmt.Errorf("socks5 authentication failed with status 0x%02x", authResp[1])
}
} else if resp[1] != 0x00 {
conn.Close()
return nil, fmt.Errorf("socks5 proxy rejected authentication methods: 0x%02x", resp[1])
}
// 3. Connection Request (CONNECT command)
host, portStr, err := net.SplitHostPort(targetAddr)
if err != nil {
conn.Close()
return nil, fmt.Errorf("invalid target address %s: %w", targetAddr, err)
}
port, err := strconv.Atoi(portStr)
if err != nil || port < 1 || port > 65535 {
conn.Close()
return nil, fmt.Errorf("invalid port in target address %s", targetAddr)
}
reqBuf := []byte{0x05, 0x01, 0x00}
ip := net.ParseIP(host)
if ip4 := ip.To4(); ip4 != nil {
reqBuf = append(reqBuf, 0x01)
reqBuf = append(reqBuf, ip4...)
} else if ip6 := ip.To16(); ip6 != nil {
reqBuf = append(reqBuf, 0x04)
reqBuf = append(reqBuf, ip6...)
} else {
reqBuf = append(reqBuf, 0x03, byte(len(host)))
reqBuf = append(reqBuf, []byte(host)...)
}
reqBuf = append(reqBuf, byte(port>>8), byte(port&0xFF))
if _, err := conn.Write(reqBuf); err != nil {
conn.Close()
return nil, fmt.Errorf("failed to send socks5 connect request: %w", err)
}
// 4. Connection Response
respHdr := make([]byte, 4)
if _, err := io.ReadFull(conn, respHdr); err != nil {
conn.Close()
return nil, fmt.Errorf("failed to read socks5 connect response: %w", err)
}
if respHdr[1] != 0x00 {
conn.Close()
return nil, fmt.Errorf("socks5 connect failed with reply code 0x%02x", respHdr[1])
}
switch respHdr[3] {
case 0x01:
bnd := make([]byte, 6)
io.ReadFull(conn, bnd)
case 0x03:
lenBuf := make([]byte, 1)
io.ReadFull(conn, lenBuf)
bnd := make([]byte, int(lenBuf[0])+2)
io.ReadFull(conn, bnd)
case 0x04:
bnd := make([]byte, 18)
io.ReadFull(conn, bnd)
}
return conn, nil
}
// ---------------------------------------------------------------------------
// Helper utilities
// ---------------------------------------------------------------------------
func GenerateUUID() string {
var b [16]byte
_, err := rand.Read(b[:])
if err != nil {
return fmt.Sprintf("%d", time.Now().UnixNano())
}
b[6] = (b[6] & 0x0f) | 0x40
b[8] = (b[8] & 0x3f) | 0x80
return fmt.Sprintf("%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:])
}
func GenerateSessionHash() string {
const chars = "abcdefghijklmnopqrstuvwxyz0123456789"
var b [12]byte
rand.Read(b[:])
var sb strings.Builder
for _, v := range b {
sb.WriteByte(chars[int(v)%len(chars)])
}
return sb.String()
}
func FibonacciDelay(attempt int) time.Duration {
if attempt <= 0 {
return 1 * time.Second
}
a, b := 1, 1
for i := 1; i < attempt; i++ {
a, b = b, a+b
}
return time.Duration(a) * time.Second
}
func DoWithFibonacciRetry(client *http.Client, makeReq func() (*http.Request, error), maxRetries int) (*http.Response, error) {
var lastErr error
for attempt := 1; attempt <= maxRetries; attempt++ {
req, err := makeReq()
if err != nil {
return nil, err
}
resp, err := client.Do(req)
if err == nil && resp.StatusCode == http.StatusOK {
return resp, nil
}
if resp != nil {
respBody, _ := io.ReadAll(resp.Body)
resp.Body.Close()
lastErr = fmt.Errorf("HTTP status %d: %s", resp.StatusCode, string(respBody))
if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusMethodNotAllowed {
break
}
if _, ok := extractFailedGeneration(string(respBody)); ok {
break
}
} else {
lastErr = err
}
if attempt < maxRetries {
delay := FibonacciDelay(attempt)
time.Sleep(delay)
}
}
return nil, fmt.Errorf("request failed after %d retries: %v", maxRetries, lastErr)
}
func EnableCORS(w http.ResponseWriter) {
w.Header().Set("Access-Control-Allow-Origin", "*")
w.Header().Set("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Authorization, api-key, X-User-Agent, X-Gradio-Space, X-Space-URL")
}
func ResolveMaxTokens(req ChatCompletionRequest) int {
mt := req.MaxTokens
if mt == 0 && req.MaxCompletionTokens > 0 {
mt = req.MaxCompletionTokens
}
if mt <= 0 {
mt = 131072
}
return mt
}
func EffectiveUserAgent(r *http.Request) string {
if r != nil {
if c := r.Header.Get("X-User-Agent"); c != "" {
return c
}
}
if ConfiguredUserAgent != "" {
return ConfiguredUserAgent
}
return DefaultUserAgent
}
// ---------------------------------------------------------------------------
// Tool and message processing
// ---------------------------------------------------------------------------
func BuildToolInstruction(tools []Tool, toolChoice interface{}) string {
if len(tools) == 0 {
return ""
}
if tcStr, ok := toolChoice.(string); ok && tcStr == "none" {
return ""
}
toolsBytes, _ := json.MarshalIndent(tools, "", " ")
sampleFnName := ""
for _, t := range tools {
if fnMap, ok := t.Function.(map[string]interface{}); ok {
if n, ok := fnMap["name"].(string); ok && n != "" {
sampleFnName = n
break
}
}
}
if sampleFnName == "" {
sampleFnName = "function_name"
}
directive := "If a tool is relevant, emit the tool call XML. If no tools are relevant, answer the query directly."
if tcStr, ok := toolChoice.(string); ok && tcStr == "required" {
directive = "You MUST call one of the available tools for this query and emit the tool call XML."
} else if tcMap, ok := toolChoice.(map[string]interface{}); ok {
if fnMap, ok := tcMap["function"].(map[string]interface{}); ok {
if fnName, ok := fnMap["name"].(string); ok && fnName != "" {
directive = fmt.Sprintf("You MUST call the %s tool for this query and emit the tool call XML.", fnName)
}
}
}
return fmt.Sprintf(`You are an API router and assistant. Convert the user query into the appropriate tool call XML using the available tools.
Available Tools:
%s
Syntax:
{"name": "%s", "arguments": {...}}
%s`, string(toolsBytes), sampleFnName, directive)
}
func TransformMessages(req ChatCompletionRequest) (processed []ChatMessage, toolInstruction string, hasSystem bool) {
// 1. Build lookup map from tool_call_id to function name across all assistant messages
toolIDToName := make(map[string]string)
for _, msg := range req.Messages {
for _, tc := range msg.ToolCalls {
if tc.ID != "" && tc.Function.Name != "" {
toolIDToName[tc.ID] = tc.Function.Name
}
}
}
toolInstruction = BuildToolInstruction(req.Tools, req.ToolChoice)
// 2. Process and coalesce messages preserving turn parity
var staged []ChatMessage
for i := 0; i < len(req.Messages); i++ {
msg := req.Messages[i]
contentStr := msg.GetContentString()
switch msg.Role {
case "system":
hasSystem = true
staged = append(staged, ChatMessage{Role: "system", Content: contentStr})
case "assistant":
var sb strings.Builder
if contentStr != "" {
sb.WriteString(contentStr)
}
for _, tc := range msg.ToolCalls {
if sb.Len() > 0 {
sb.WriteString("\n")
}
args := tc.Function.Arguments
if strings.TrimSpace(args) == "" {
args = "{}"
}
sb.WriteString(fmt.Sprintf("\n{\"name\": %q, \"arguments\": %s}\n", tc.Function.Name, args))
}
staged = append(staged, ChatMessage{
Role: "assistant",
Content: sb.String(),
ReasoningContent: msg.ReasoningContent,
ToolCalls: msg.ToolCalls,
})
case "tool", "function":
// Gather consecutive tool returns into a coalesced turn
var toolResponses []string
j := i
for j < len(req.Messages) && (req.Messages[j].Role == "tool" || req.Messages[j].Role == "function") {
tMsg := req.Messages[j]
tContent := tMsg.GetContentString()
tName := tMsg.Name
if tName == "" && tMsg.ToolCallID != "" {
if mapped, ok := toolIDToName[tMsg.ToolCallID]; ok {
tName = mapped
} else {
tName = tMsg.ToolCallID
}
}
var contentJSON []byte
if json.Valid([]byte(tContent)) {
contentJSON = []byte(tContent)
} else {
contentJSON, _ = json.Marshal(tContent)
}
toolResponses = append(toolResponses, fmt.Sprintf("\n{\"name\": %q, \"content\": %s}\n", tName, string(contentJSON)))
j++
}
i = j - 1 // advance loop
promptSuffix := "Please answer the user's request based on the tool result."
if len(toolResponses) > 1 {
promptSuffix = "Please answer the user's request based on the tool results."
}
coalesced := strings.Join(toolResponses, "\n") + "\n\n" + promptSuffix
staged = append(staged, ChatMessage{
Role: "user",
Content: coalesced,
})
default: // "user" or other roles
staged = append(staged, ChatMessage{Role: msg.Role, Content: contentStr})
}
}
// 3. Inject tool instructions into system prompt
if toolInstruction != "" {
if hasSystem {
systemInjected := false
for i, m := range staged {
if m.Role == "system" {
staged[i].Content = m.GetContentString() + "\n\n" + strings.TrimSpace(toolInstruction)
systemInjected = true
break
}
}
if !systemInjected {
staged = append([]ChatMessage{
{Role: "system", Content: strings.TrimSpace(toolInstruction)},
}, staged...)
}
} else {
staged = append([]ChatMessage{
{Role: "system", Content: strings.TrimSpace(toolInstruction)},
}, staged...)
hasSystem = true
}
}
return staged, toolInstruction, hasSystem
}
func cleanJSONBlock(input string) string {
s := strings.TrimSpace(input)
if strings.HasPrefix(s, "```") {
lines := strings.Split(s, "\n")
if len(lines) >= 2 {
if strings.HasPrefix(lines[len(lines)-1], "```") {
lines = lines[1 : len(lines)-1]
} else {
lines = lines[1:]
}
s = strings.TrimSpace(strings.Join(lines, "\n"))
}
}
return s
}
func sanitizeJSONValue(v interface{}) interface{} {
switch val := v.(type) {
case string:
return strings.TrimSpace(val)
case map[string]interface{}:
res := make(map[string]interface{})
for k, item := range val {
res[k] = sanitizeJSONValue(item)
}
return res
case []interface{}:
res := make([]interface{}, len(val))
for i, item := range val {
res[i] = sanitizeJSONValue(item)
}
return res
default:
return v
}
}
type ToolTagPair struct {
Start string
End string
}
var ToolTagPairs = []ToolTagPair{
{Start: "", End: ""},
{Start: "", End: ""},
{Start: "", End: ""},
{Start: "[TOOL_CALLS]", End: "[/TOOL_CALLS]"},
}
func getToolStartPrefixes() []string {
seen := make(map[string]bool)
var prefixes []string
for _, pair := range ToolTagPairs {
for i := 1; i <= len(pair.Start); i++ {
pref := pair.Start[:i]
if !seen[pref] {
seen[pref] = true
prefixes = append(prefixes, pref)
}
}
}
return prefixes
}
var toolStartPrefixes = getToolStartPrefixes()
func repairToolCallJSON(input string) (ToolCall, bool) {
s := strings.TrimSpace(input)
reName := regexp.MustCompile(`"(?:name|function|action|call)"\s*:\s*"([^"]+)"`)
matches := reName.FindStringSubmatch(s)
if len(matches) < 2 {
return ToolCall{}, false
}
fnName := matches[1]
reArgsObj := regexp.MustCompile(`"(?:arguments|parameters|args|input)"\s*:\s*(\{[\s\S]*\})`)
argMatches := reArgsObj.FindStringSubmatch(s)
argsStr := "{}"
if len(argMatches) >= 2 {
candidate := argMatches[1]
var dummy map[string]interface{}
if json.Unmarshal([]byte(candidate), &dummy) == nil {
argsStr = candidate
}
} else {
reArgsStr := regexp.MustCompile(`"(?:arguments|parameters|args|input)"\s*:\s*"((?:\\.|[^"\\])*)"`)
strMatches := reArgsStr.FindStringSubmatch(s)
if len(strMatches) >= 2 {
var unescaped string
if json.Unmarshal([]byte(`"`+strMatches[1]+`"`), &unescaped) == nil {
argsStr = unescaped
}
}
}
return ToolCall{
ID: "call_" + GenerateUUID()[:8],
Type: "function",
Function: ToolCallFunction{
Name: fnName,
Arguments: argsStr,
},
}, true
}
func parseSingleToolCall(jsonStr string) (ToolCall, bool) {
cleaned := cleanJSONBlock(jsonStr)
var raw map[string]interface{}
if err := json.Unmarshal([]byte(cleaned), &raw); err == nil {
sanitizedRaw, ok := sanitizeJSONValue(raw).(map[string]interface{})
if !ok {
sanitizedRaw = raw
}
for _, wrapperKey := range []string{"function", "function_call", "tool_call"} {
if fnObj, ok := sanitizedRaw[wrapperKey].(map[string]interface{}); ok {
if nameVal, ok := fnObj["name"].(string); ok && nameVal != "" {
argsStr := "{}"
var argsVal interface{}
if a, hasA := fnObj["arguments"]; hasA {
argsVal = a
} else if p, hasP := fnObj["parameters"]; hasP {
argsVal = p
} else if args, hasArgs := fnObj["args"]; hasArgs {
argsVal = args
}
if argsVal != nil {
if s, isStr := argsVal.(string); isStr {
var innerObj interface{}
if json.Unmarshal([]byte(s), &innerObj) == nil {
b, _ := json.Marshal(sanitizeJSONValue(innerObj))
argsStr = string(b)
} else {
argsStr = strings.TrimSpace(s)
}
} else {
b, _ := json.Marshal(sanitizeJSONValue(argsVal))
argsStr = string(b)
}
}
return ToolCall{
ID: "call_" + GenerateUUID()[:8],
Type: "function",
Function: ToolCallFunction{
Name: nameVal,
Arguments: argsStr,
},
}, true
}
}
}
nameVal := ""
for _, key := range []string{"name", "function", "action", "call"} {
if n, ok := sanitizedRaw[key].(string); ok && n != "" {
nameVal = n
break
}
}
if nameVal != "" {
argsStr := "{}"
var argsVal interface{}
for _, key := range []string{"arguments", "parameters", "args", "input"} {
if a, ok := sanitizedRaw[key]; ok {
argsVal = a
break
}
}
if argsVal != nil {
if s, isStr := argsVal.(string); isStr {
var innerObj interface{}
if json.Unmarshal([]byte(s), &innerObj) == nil {
b, _ := json.Marshal(sanitizeJSONValue(innerObj))
argsStr = string(b)
} else {
argsStr = strings.TrimSpace(s)
}
} else {
b, _ := json.Marshal(sanitizeJSONValue(argsVal))
argsStr = string(b)
}
} else {
argsMap := make(map[string]interface{})
for k, v := range sanitizedRaw {
if k != "name" && k != "function" && k != "type" && k != "action" && k != "call" {
argsMap[k] = v
}
}
if len(argsMap) > 0 {
b, _ := json.Marshal(sanitizeJSONValue(argsMap))
argsStr = string(b)
}
}
return ToolCall{
ID: "call_" + GenerateUUID()[:8],
Type: "function",
Function: ToolCallFunction{
Name: nameVal,
Arguments: argsStr,
},
}, true
}
}
return repairToolCallJSON(cleaned)
}
func parseMultipleToolCalls(raw string) ([]ToolCall, bool) {
cleaned := cleanJSONBlock(raw)
if cleaned == "" {
return nil, false
}
// 1. Direct JSON array: [{"name":...}, ...]
var rawList []interface{}
if err := json.Unmarshal([]byte(cleaned), &rawList); err == nil {
var calls []ToolCall
for _, item := range rawList {
b, err := json.Marshal(item)
if err == nil {
if tc, ok := parseSingleToolCall(string(b)); ok {
calls = append(calls, tc)
}
}
}
if len(calls) > 0 {
return calls, true
}
}
// 2. Wrapper object with tool_calls / calls array
var rawMap map[string]interface{}
if err := json.Unmarshal([]byte(cleaned), &rawMap); err == nil {
for _, listKey := range []string{"tool_calls", "calls", "functions"} {
if subArr, ok := rawMap[listKey].([]interface{}); ok && len(subArr) > 0 {
var calls []ToolCall
for _, item := range subArr {
b, err := json.Marshal(item)
if err == nil {
if tc, ok := parseSingleToolCall(string(b)); ok {
calls = append(calls, tc)
}
}
}
if len(calls) > 0 {
return calls, true
}
}
}
}
// 3. Single tool call
if tc, ok := parseSingleToolCall(cleaned); ok {
return []ToolCall{tc}, true
}
return nil, false
}
func parseXMLToolCall(block string) ([]ToolCall, bool) {
inner := strings.TrimSpace(block)
for _, pair := range ToolTagPairs {
inner = strings.ReplaceAll(inner, pair.Start, "")
inner = strings.ReplaceAll(inner, pair.End, "")
}
inner = cleanJSONBlock(inner)
if calls, ok := parseMultipleToolCalls(inner); ok && len(calls) > 0 {
return calls, true
}
var fnName string
if strings.Contains(inner, "") && strings.Contains(inner, "") {
nStart := strings.Index(inner, "") + len("")
nEnd := strings.Index(inner, "")
if nStart < nEnd {
fnName = strings.TrimSpace(inner[nStart:nEnd])
}
}
var argsStr string
if strings.Contains(inner, "") && strings.Contains(inner, "") {
aStart := strings.Index(inner, "") + len("")
aEnd := strings.Index(inner, "")
if aStart < aEnd {
argsStr = strings.TrimSpace(inner[aStart:aEnd])
}
}
if fnName != "" {
if argsStr == "" {
argsStr = "{}"
}
return []ToolCall{{
ID: "call_" + GenerateUUID()[:8],
Type: "function",
Function: ToolCallFunction{
Name: fnName,
Arguments: argsStr,
},
}}, true
}
return nil, false
}
func ExtractToolCallBlocks(content string) (blocks []string, remaining string) {
remaining = content
for _, pair := range ToolTagPairs {
for strings.Contains(remaining, pair.Start) {
sIdx := strings.Index(remaining, pair.Start)
rest := remaining[sIdx+len(pair.Start):]
relNextSIdx := strings.Index(rest, pair.Start)
var nextSIdx int
if relNextSIdx != -1 {
nextSIdx = sIdx + len(pair.Start) + relNextSIdx
} else {
nextSIdx = -1
}
relEIdx := strings.Index(rest, pair.End)
var eIdx int
if relEIdx != -1 {
eIdx = sIdx + len(pair.Start) + relEIdx
} else {
eIdx = -1
}
var blockText string
if eIdx != -1 && (nextSIdx == -1 || eIdx < nextSIdx) {
blockEndPos := eIdx + len(pair.End)
blockText = remaining[sIdx:blockEndPos]
remaining = strings.TrimSpace(remaining[:sIdx] + remaining[blockEndPos:])
} else if nextSIdx != -1 {
blockEndPos := nextSIdx
blockText = remaining[sIdx:blockEndPos]
remaining = strings.TrimSpace(remaining[:sIdx] + remaining[blockEndPos:])
} else {
blockText = remaining[sIdx:]
remaining = strings.TrimSpace(remaining[:sIdx])
}
blocks = append(blocks, blockText)
}
}
return blocks, remaining
}
func DetectToolCalls(content string) ([]ToolCall, string, bool) {
blocks, remaining := ExtractToolCallBlocks(content)
var calls []ToolCall
for _, block := range blocks {
if tcs, ok := parseXMLToolCall(block); ok {
calls = append(calls, tcs...)
}
}
if len(calls) > 0 {
return calls, remaining, true
}
if tcs, ok := parseMultipleToolCalls(strings.TrimSpace(content)); ok && len(tcs) > 0 {
return tcs, "", true
}
return nil, content, false
}
func ExtractThinking(content string) (string, string) {
if strings.Contains(content, "") && strings.Contains(content, "") {
start := strings.Index(content, "")
end := strings.Index(content, "")
if start < end {
reasoning := content[start+len("") : end]
rem := content[:start] + content[end+len(""):]
rem = strings.TrimPrefix(rem, "\n\n")
rem = strings.TrimPrefix(rem, "\n")
return rem, reasoning
}
}
return content, ""
}
// ---------------------------------------------------------------------------
// Response framing & streamer
// ---------------------------------------------------------------------------
type FinalOutput struct {
Content interface{}
ReasoningContent string
ToolCalls []ToolCall
FinishReason string
}
func WriteCompletionResponse(w http.ResponseWriter, completionID string, created int64, model string, out FinalOutput) {
finish := out.FinishReason
if finish == "" {
finish = "stop"
}
resp := ChatCompletionResponse{
ID: completionID,
Object: "chat.completion",
Created: created,
Model: model,
Choices: []ChatCompletionResponseChoice{
{
Index: 0,
Message: ChatMessage{
Role: "assistant",
Content: out.Content,
ReasoningContent: out.ReasoningContent,
ToolCalls: out.ToolCalls,
},
FinishReason: finish,
},
},
Usage: Usage{
PromptTokens: 0,
CompletionTokens: 0,
TotalTokens: 0,
},
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(resp)
}
func extractFailedGeneration(raw string) (string, bool) {
var obj map[string]interface{}
if err := json.Unmarshal([]byte(raw), &obj); err == nil {
if fg, ok := obj["failed_generation"].(string); ok && fg != "" {
return strings.TrimSpace(fg), true
}
if errVal, ok := obj["error"]; ok {
if errMap, ok := errVal.(map[string]interface{}); ok {
if fg, ok := errMap["failed_generation"].(string); ok && fg != "" {
return strings.TrimSpace(fg), true
}
} else if errStr, ok := errVal.(string); ok {
if fg, ok := extractFailedGeneration(errStr); ok {
return fg, true
}
}
}
}
keyIdx := strings.Index(raw, `"failed_generation"`)
if keyIdx == -1 {
keyIdx = strings.Index(raw, `'failed_generation'`)
}
if keyIdx == -1 {
keyIdx = strings.Index(raw, `failed_generation`)
}
if keyIdx == -1 {
return "", false
}
colonIdx := strings.Index(raw[keyIdx:], ":")
if colonIdx == -1 {
return "", false
}
valStart := keyIdx + colonIdx + 1
for valStart < len(raw) && (raw[valStart] == ' ' || raw[valStart] == '\t' || raw[valStart] == '\r' || raw[valStart] == '\n') {
valStart++
}
if valStart >= len(raw) {
return "", false
}
firstChar := raw[valStart]
var candidate string
if firstChar == '\'' || firstChar == '"' {
quoteChar := firstChar
var b strings.Builder
escaped := false
for i := valStart + 1; i < len(raw); i++ {
ch := raw[i]
if escaped {
switch ch {
case 'n':
b.WriteByte('\n')
case 'r':
b.WriteByte('\r')
case 't':
b.WriteByte('\t')
case '\\':
b.WriteByte('\\')
case '\'':
b.WriteByte('\'')
case '"':
b.WriteByte('"')
default:
b.WriteByte('\\')
b.WriteByte(ch)
}
escaped = false
} else if ch == '\\' {
escaped = true
} else if ch == quoteChar {
break
} else {
b.WriteByte(ch)
}
}
candidate = strings.TrimSpace(b.String())
} else if firstChar == '{' || firstChar == '[' {
openChar := firstChar
closeChar := byte('}')
if openChar == '[' {
closeChar = ']'
}
depth := 0
inStr := false
var strQuote byte
escaped := false
endIdx := -1
for i := valStart; i < len(raw); i++ {
ch := raw[i]
if inStr {
if escaped {
escaped = false
} else if ch == '\\' {
escaped = true
} else if ch == strQuote {
inStr = false
}
} else {
if ch == '"' || ch == '\'' {
inStr = true
strQuote = ch
} else if ch == openChar {
depth++
} else if ch == closeChar {
depth--
if depth == 0 {
endIdx = i + 1
break
}
}
}
}
if endIdx != -1 {
candidate = strings.TrimSpace(raw[valStart:endIdx])
}
}
if candidate != "" {
if strings.Contains(candidate, `\"`) {
candidate = strings.ReplaceAll(candidate, `\"`, `"`)
}
if strings.Contains(candidate, `\n`) {
candidate = strings.ReplaceAll(candidate, `\n`, "\n")
}
return candidate, true
}
return "", false
}
func extractGradioErrorMessage(dataStr string) string {
var strVal string
if err := json.Unmarshal([]byte(dataStr), &strVal); err == nil && strVal != "" {
return strVal
}
var errObj map[string]interface{}
if err := json.Unmarshal([]byte(dataStr), &errObj); err == nil {
if e, ok := errObj["error"].(string); ok && e != "" {
return e
}
if m, ok := errObj["message"].(string); ok && m != "" {
return m
}
if eNull, ok := errObj["error"]; ok && eNull == nil {
if t, ok := errObj["title"].(string); ok && t != "" {
return t
}
return "internal space error (check Gradio inputs/types)"
}
}
clean := strings.TrimSpace(dataStr)
if clean == "null" || clean == "" {
return "upstream Gradio space returned null error (space may have show_error=False or failed input validation)"
}
return clean
}
func isProtocolOrInputError(errMsg string) bool {
lower := strings.ToLower(errMsg)
if strings.Contains(lower, "unauthorized") ||
strings.Contains(lower, "forbidden") ||
strings.Contains(lower, "quota exceeded") ||
strings.Contains(lower, "payment required") ||
strings.Contains(lower, "rate limit") {
return false
}
return true
}
type Streamer struct {
w http.ResponseWriter
flusher http.Flusher
id string
created int64
model string
started bool
}
func NewStreamer(w http.ResponseWriter, flusher http.Flusher, id string, created int64, model string) *Streamer {
return &Streamer{w: w, flusher: flusher, id: id, created: created, model: model}
}
func (s *Streamer) EnsureStarted() {
if s.started {
return
}
s.w.Header().Set("Content-Type", "text/event-stream")
s.w.Header().Set("Cache-Control", "no-cache")
s.w.Header().Set("Connection", "keep-alive")
s.started = true
s.Role()
}
func (s *Streamer) Role() {
sendStreamDelta(s.w, s.flusher, s.id, s.created, s.model, StreamDelta{Role: "assistant"})
}
func (s *Streamer) Reasoning(text string) {
s.EnsureStarted()
sendStreamDelta(s.w, s.flusher, s.id, s.created, s.model, StreamDelta{ReasoningContent: text})
}
func (s *Streamer) Content(text string) {
s.EnsureStarted()
sendStreamDelta(s.w, s.flusher, s.id, s.created, s.model, StreamDelta{Content: text})
}
func (s *Streamer) ToolCallDelta(tc ToolCall) {
s.EnsureStarted()
sendStreamDelta(s.w, s.flusher, s.id, s.created, s.model, StreamDelta{ToolCalls: []ToolCall{tc}})
}
func (s *Streamer) Finish(reason string) {
if !s.started {
return
}
sendStreamChunk(s.w, s.flusher, s.id, s.created, s.model, StreamDelta{}, &reason)
}
func (s *Streamer) Done() {
if !s.started {
return
}
fmt.Fprintf(s.w, "data: [DONE]\n\n")
if s.flusher != nil {
s.flusher.Flush()
}
}
func (s *Streamer) Error(errMsg string) {
s.EnsureStarted()
errChunk := map[string]interface{}{
"error": map[string]interface{}{
"message": errMsg,
"type": "upstream_error",
"code": 502,
},
}
b, _ := json.Marshal(errChunk)
fmt.Fprintf(s.w, "data: %s\n\n", b)
if s.flusher != nil {
s.flusher.Flush()
}
}
func sendStreamDelta(w http.ResponseWriter, flusher http.Flusher, completionID string, createdTime int64, modelName string, delta StreamDelta) {
sendStreamChunk(w, flusher, completionID, createdTime, modelName, delta, nil)
}
func sendStreamChunk(w http.ResponseWriter, flusher http.Flusher, completionID string, createdTime int64, modelName string, delta StreamDelta, finishReason *string) {
chunk := StreamResponse{
ID: completionID,
Object: "chat.completion.chunk",
Created: createdTime,
Model: modelName,
Choices: []StreamChoice{
{
Index: 0,
Delta: delta,
FinishReason: finishReason,
},
},
}
b, _ := json.Marshal(chunk)
fmt.Fprintf(w, "data: %s\n\n", b)
if flusher != nil {
flusher.Flush()
}
}
// ---------------------------------------------------------------------------
// Stateful thinking tag filter for streaming
// ---------------------------------------------------------------------------
type StreamThinkingFilter struct {
inThinking bool
buf string
}
func NewStreamThinkingFilter() *StreamThinkingFilter {
return &StreamThinkingFilter{}
}
func hasPrefixOf(target string, prefixes []string) int {
maxMatch := 0
for _, p := range prefixes {
if strings.HasSuffix(target, p) && len(p) > maxMatch {
maxMatch = len(p)
}
}
return maxMatch
}
func hasSuffixPrefixOf(target string, tag string) int {
maxMatch := 0
for i := 1; i < len(tag); i++ {
p := tag[:i]
if strings.HasSuffix(target, p) && len(p) > maxMatch {
maxMatch = len(p)
}
}
return maxMatch
}
func (f *StreamThinkingFilter) Feed(chunk string, onContent func(string), onReasoning func(string)) {
f.buf += chunk
thinkStartTag := ""
thinkEndTag := ""
thinkStartPrefixes := []string{"<", " 0 {
if !f.inThinking {
if idx := strings.Index(f.buf, thinkStartTag); idx != -1 {
before := f.buf[:idx]
if before != "" {
onContent(before)
}
f.inThinking = true
f.buf = f.buf[idx+len(thinkStartTag):]
} else if matchLen := hasPrefixOf(f.buf, thinkStartPrefixes); matchLen > 0 {
safe := f.buf[:len(f.buf)-matchLen]
if safe != "" {
onContent(safe)
}
f.buf = f.buf[len(f.buf)-matchLen:]
break
} else {
onContent(f.buf)
f.buf = ""
break
}
} else {
if idx := strings.Index(f.buf, thinkEndTag); idx != -1 {
before := f.buf[:idx]
if before != "" {
onReasoning(before)
}
f.inThinking = false
f.buf = f.buf[idx+len(thinkEndTag):]
f.buf = strings.TrimPrefix(f.buf, "\n\n")
f.buf = strings.TrimPrefix(f.buf, "\n")
} else if matchLen := hasPrefixOf(f.buf, thinkEndPrefixes); matchLen > 0 {
safe := f.buf[:len(f.buf)-matchLen]
if safe != "" {
onReasoning(safe)
}
f.buf = f.buf[len(f.buf)-matchLen:]
break
} else {
onReasoning(f.buf)
f.buf = ""
break
}
}
}
}
func (f *StreamThinkingFilter) Flush(onContent func(string), onReasoning func(string)) {
if len(f.buf) > 0 {
if f.inThinking {
onReasoning(f.buf)
} else {
onContent(f.buf)
}
f.buf = ""
}
}
// ---------------------------------------------------------------------------
// Stateful tool call tag filter for streaming
// ---------------------------------------------------------------------------
type StreamToolCallFilter struct {
inToolCall bool
buf string
toolCallBuf string
toolIndex int
emittedCall bool
activePair ToolTagPair
activeEndTag string
}
func NewStreamToolCallFilter() *StreamToolCallFilter {
return &StreamToolCallFilter{}
}
func (f *StreamToolCallFilter) Feed(chunk string, onContent func(string), onToolCall func(ToolCall)) {
f.buf += chunk
for len(f.buf) > 0 {
if !f.inToolCall {
earliestIdx := -1
var matchedPair ToolTagPair
for _, pair := range ToolTagPairs {
if idx := strings.Index(f.buf, pair.Start); idx != -1 {
if earliestIdx == -1 || idx < earliestIdx {
earliestIdx = idx
matchedPair = pair
}
}
}
if earliestIdx != -1 {
before := f.buf[:earliestIdx]
if before != "" {
onContent(before)
}
f.inToolCall = true
f.activePair = matchedPair
f.activeEndTag = matchedPair.End
f.buf = f.buf[earliestIdx+len(matchedPair.Start):]
} else if matchLen := hasPrefixOf(f.buf, toolStartPrefixes); matchLen > 0 {
safe := f.buf[:len(f.buf)-matchLen]
if safe != "" {
onContent(safe)
}
f.buf = f.buf[len(f.buf)-matchLen:]
break
} else {
onContent(f.buf)
f.buf = ""
break
}
} else {
if idx := strings.Index(f.buf, f.activeEndTag); idx != -1 {
f.toolCallBuf += f.buf[:idx]
f.buf = f.buf[idx+len(f.activeEndTag):]
f.inToolCall = false
if tcs, ok := parseMultipleToolCalls(f.toolCallBuf); ok && len(tcs) > 0 {
for _, tc := range tcs {
idxCopy := f.toolIndex
tc.Index = &idxCopy
f.toolIndex++
f.emittedCall = true
onToolCall(tc)
}
} else if tcs2, ok2 := parseXMLToolCall(f.activePair.Start + f.toolCallBuf + f.activePair.End); ok2 && len(tcs2) > 0 {
for _, tc := range tcs2 {
idxCopy := f.toolIndex
tc.Index = &idxCopy
f.toolIndex++
f.emittedCall = true
onToolCall(tc)
}
} else {
onContent(f.activePair.Start + f.toolCallBuf + f.activePair.End)
}
f.toolCallBuf = ""
} else if matchLen := hasSuffixPrefixOf(f.buf, f.activeEndTag); matchLen > 0 {
safe := f.buf[:len(f.buf)-matchLen]
f.toolCallBuf += safe
f.buf = f.buf[len(f.buf)-matchLen:]
break
} else {
f.toolCallBuf += f.buf
f.buf = ""
break
}
}
}
}
func (f *StreamToolCallFilter) Flush(onContent func(string), onToolCall func(ToolCall)) {
if f.inToolCall && len(f.toolCallBuf) > 0 {
if tcs, ok := parseMultipleToolCalls(f.toolCallBuf); ok && len(tcs) > 0 {
for _, tc := range tcs {
idxCopy := f.toolIndex
tc.Index = &idxCopy
f.toolIndex++
f.emittedCall = true
onToolCall(tc)
}
} else if tcs2, ok2 := parseXMLToolCall(f.activePair.Start + f.toolCallBuf + f.activePair.End); ok2 && len(tcs2) > 0 {
for _, tc := range tcs2 {
idxCopy := f.toolIndex
tc.Index = &idxCopy
f.toolIndex++
f.emittedCall = true
onToolCall(tc)
}
} else {
onContent(f.activePair.Start + f.toolCallBuf)
}
f.toolCallBuf = ""
}
if len(f.buf) > 0 {
onContent(f.buf)
f.buf = ""
}
}
// ---------------------------------------------------------------------------
// Universal Gradio space inspector & metadata discovery
// ---------------------------------------------------------------------------
type GradioParamInfo struct {
Label string `json:"label"`
ParameterName string `json:"parameter_name"`
ParameterDefault interface{} `json:"parameter_default,omitempty"`
Component string `json:"component"`
Type interface{} `json:"type,omitempty"`
PythonType interface{} `json:"python_type,omitempty"`
}
type GradioEndpointInfo struct {
Parameters []GradioParamInfo `json:"parameters"`
Returns []GradioParamInfo `json:"returns"`
APIVisibility string `json:"api_visibility"`
Description string `json:"description"`
CodeSnippets map[string]interface{} `json:"code_snippets,omitempty"`
}
type GradioAPIInfoResponse struct {
NamedEndpoints map[string]GradioEndpointInfo `json:"named_endpoints"`
UnnamedEndpoints map[string]GradioEndpointInfo `json:"unnamed_endpoints"`
}
type GradioComponent struct {
ID int `json:"id"`
Type string `json:"type"`
Props map[string]interface{} `json:"props"`
SkipAPI bool `json:"skip_api"`
}
type GradioDependencyTypes struct {
Generator bool `json:"generator"`
Cancel bool `json:"cancel"`
}
type GradioDependency struct {
ID int `json:"id"`
APIName interface{} `json:"api_name"`
Inputs []int `json:"inputs"`
Outputs []int `json:"outputs"`
Queue interface{} `json:"queue"`
Types GradioDependencyTypes `json:"types"`
APIVisibility string `json:"api_visibility"`
}
type GradioConfigResponse struct {
Version string `json:"version"`
APIPrefix string `json:"api_prefix"`
Mode string `json:"mode"`
Title string `json:"title"`
Components []GradioComponent `json:"components"`
Dependencies []GradioDependency `json:"dependencies"`
}
type HFSpaceCardData struct {
Title string `json:"title"`
ShortDescription string `json:"short_description"`
}
type HFSpaceInfoResponse struct {
ID string `json:"id"`
Models []string `json:"models"`
CardData HFSpaceCardData `json:"cardData"`
}
type SpaceParamMapping struct {
InputIndex int `json:"input_index"`
ComponentID int `json:"component_id"`
ComponentType string `json:"component_type,omitempty"`
Label string `json:"label,omitempty"`
ParamName string `json:"param_name,omitempty"`
ParamType string `json:"param_type"` // "message", "history", "system_prompt", "temperature", "max_tokens", "top_p", "think_level", "tools", "stream", "state", "other"
DefaultValue interface{} `json:"default_value,omitempty"`
}
type SpaceDiscovery struct {
SpaceURL string `json:"space_url"`
Title string `json:"title"`
GradioVersion string `json:"gradio_version"`
Flavor string `json:"flavor"`
Protocol string `json:"protocol"` // "call", "queue", "predict"
APIPrefix string `json:"api_prefix"` // e.g. "/gradio_api" or ""
Endpoint string `json:"endpoint"` // e.g. "/chat_fn" or "/chat" or "/run/predict"
CleanEndpoint string `json:"clean_endpoint"` // e.g. "chat_fn" or "chat"
FnIndex int `json:"fn_index"` // -1 if not applicable
Models []string `json:"models"`
PrimaryModel string `json:"primary_model"`
TotalInputs int `json:"total_inputs"`
RawTotalInputs int `json:"raw_total_inputs,omitempty"`
ParamMappings []SpaceParamMapping `json:"param_mappings"`
DefaultInputs []interface{} `json:"default_inputs"`
MessageIsMultimodal bool `json:"message_is_multimodal"`
HistoryIndex int `json:"history_index"` // -1 if none
MessageIndex int `json:"message_index"` // index for user message text
SystemIndex int `json:"system_index"` // -1 if none
DefaultSystemPrompt string `json:"default_system_prompt,omitempty"` // default space system prompt if present
TempIndex int `json:"temp_index"` // -1 if none
MaxTokensIndex int `json:"max_tokens_index"` // -1 if none
TopPIndex int `json:"top_p_index"` // -1 if none
StreamIndex int `json:"stream_index"` // -1 if none
ThinkLevelIndex int `json:"think_level_index"` // -1 if none
FunctionsJSONIndex int `json:"functions_json_index"` // -1 if none
PreservedThinkingIndex int `json:"preserved_thinking_index"` // -1 if none
IsHunyuan3 bool `json:"is_hunyuan3"`
HistoryFormat string `json:"history_format"` // "messages", "pairs", "gradio_messages", "none"
ToolCallMode string `json:"tool_call_mode"` // "native_slot", "prompt_augmented_system", "prompt_augmented_first_turn", "prompt_augmented_single_prompt"
LastDiscovered time.Time `json:"last_discovered"`
}
func (d *SpaceDiscovery) Summary() string {
var sb strings.Builder
sb.WriteString("================================================================================\n")
sb.WriteString("Gradio Space Resolution Picture\n")
sb.WriteString("--------------------------------------------------------------------------------\n")
sb.WriteString(fmt.Sprintf("Space URL: %s\n", d.SpaceURL))
if d.Title != "" {
sb.WriteString(fmt.Sprintf("Title: %s\n", d.Title))
}
sb.WriteString(fmt.Sprintf("Gradio Version: %s\n", d.GradioVersion))
sb.WriteString(fmt.Sprintf("UI Flavor: %s\n", d.Flavor))
sb.WriteString(fmt.Sprintf("Protocol: %s\n", d.Protocol))
sb.WriteString(fmt.Sprintf("API Prefix: %s\n", d.APIPrefix))
sb.WriteString(fmt.Sprintf("Resolved Endpoint: %s\n", d.Endpoint))
if d.FnIndex >= 0 {
sb.WriteString(fmt.Sprintf("Function Index: %d\n", d.FnIndex))
}
sb.WriteString(fmt.Sprintf("Primary Model: %s\n", d.PrimaryModel))
if len(d.Models) > 0 {
sb.WriteString(fmt.Sprintf("Exposed Models: %s\n", strings.Join(d.Models, ", ")))
}
sb.WriteString(fmt.Sprintf("History Format: %s\n", d.HistoryFormat))
sb.WriteString(fmt.Sprintf("Tool Call Support: %s\n", d.ToolCallMode))
sb.WriteString(fmt.Sprintf("Total Input Slots: %d\n", d.TotalInputs))
if len(d.ParamMappings) > 0 {
sb.WriteString("Input Slot Mappings:\n")
for _, m := range d.ParamMappings {
desc := m.ComponentType
if desc == "" {
desc = "Unknown"
}
if m.Label != "" {
desc += fmt.Sprintf(" (label=%q)", m.Label)
}
sb.WriteString(fmt.Sprintf(" [%d] Component ID %-4d %-30s -> %s\n", m.InputIndex, m.ComponentID, desc, m.ParamType))
}
}
sb.WriteString("================================================================================")
return sb.String()
}
func (d *SpaceDiscovery) GetModelList() []ModelItem {
now := time.Now().Unix()
var items []ModelItem
seen := make(map[string]bool)
if ConfiguredModelName != "" {
seen[ConfiguredModelName] = true
items = append(items, ModelItem{
ID: ConfiguredModelName,
Object: "model",
Created: now,
OwnedBy: "gradio",
})
}
for _, m := range d.Models {
if m != "" && !seen[m] {
seen[m] = true
items = append(items, ModelItem{
ID: m,
Object: "model",
Created: now,
OwnedBy: "gradio",
})
}
}
if d.PrimaryModel != "" && !seen[d.PrimaryModel] {
seen[d.PrimaryModel] = true
items = append(items, ModelItem{
ID: d.PrimaryModel,
Object: "model",
Created: now,
OwnedBy: "gradio",
})
}
if len(items) == 0 {
items = append(items, ModelItem{
ID: "gradio-chat",
Object: "model",
Created: now,
OwnedBy: "gradio",
})
}
return items
}
func (d *SpaceDiscovery) MatchesModel(requested string) bool {
if requested == "" {
return true
}
cleanReq := strings.TrimPrefix(requested, "models/")
cleanReq = strings.TrimPrefix(cleanReq, "openai/")
cleanReq = strings.ToLower(cleanReq)
if ConfiguredModelName != "" && strings.ToLower(ConfiguredModelName) == cleanReq {
return true
}
for _, m := range d.Models {
mClean := strings.TrimPrefix(m, "models/")
mClean = strings.TrimPrefix(mClean, "openai/")
if strings.ToLower(mClean) == cleanReq || strings.ToLower(m) == cleanReq {
return true
}
}
if strings.ToLower(d.PrimaryModel) == cleanReq {
return true
}
return cleanReq == "default" || cleanReq == "gradio" || cleanReq == "gradio-chat"
}
func NewDefaultSpaceDiscovery(spaceURL string) *SpaceDiscovery {
cleanURL := strings.TrimRight(spaceURL, "/")
if cleanURL != "" && !strings.HasPrefix(cleanURL, "http://") && !strings.HasPrefix(cleanURL, "https://") {
cleanURL = "https://" + cleanURL
}
return &SpaceDiscovery{
SpaceURL: cleanURL,
GradioVersion: "unknown",
Flavor: "generic",
Protocol: "call",
APIPrefix: "/gradio_api",
Endpoint: "/chat_fn",
CleanEndpoint: "chat_fn",
FnIndex: -1,
TotalInputs: 1,
HistoryIndex: -1,
MessageIndex: 0,
SystemIndex: -1,
DefaultSystemPrompt: "",
TempIndex: -1,
MaxTokensIndex: -1,
TopPIndex: -1,
StreamIndex: -1,
ThinkLevelIndex: -1,
FunctionsJSONIndex: -1,
PreservedThinkingIndex: -1,
HistoryFormat: "messages",
ToolCallMode: "prompt_augmented_single_prompt",
LastDiscovered: time.Now(),
}
}
// DetectGradioFlavor classifies the architecture of the Gradio space.
func DetectGradioFlavor(configResp GradioConfigResponse, compMap map[int]GradioComponent) string {
mode := strings.ToLower(strings.TrimSpace(configResp.Mode))
hasChatbot := false
hasMultimodal := false
for _, c := range compMap {
cType := strings.ToLower(c.Type)
if cType == "chatbot" {
hasChatbot = true
} else if cType == "multimodaltextbox" {
hasMultimodal = true
}
}
switch mode {
case "chat_interface":
if hasMultimodal {
return "ChatInterface (Multimodal)"
}
return "ChatInterface"
case "blocks":
if hasChatbot {
if hasMultimodal {
return "Blocks (Multimodal Chat)"
}
return "Blocks (Chat)"
}
return "Blocks"
case "interface":
return "Interface"
default:
if mode != "" {
return mode
}
if hasChatbot {
return "ChatInterface"
}
return "Generic"
}
}
// ScoreCandidateEndpoint computes a heuristic score for an endpoint based on its API name,
// parameter signatures, and dependency graph topology.
func ScoreCandidateEndpoint(apiName string, parameters []GradioParamInfo, dep *GradioDependency, compMap map[int]GradioComponent) int {
score := 0
cleanName := strings.TrimPrefix(apiName, "/")
lowerName := strings.ToLower(cleanName)
hasInputs := len(parameters) > 0 || (dep != nil && len(dep.Inputs) > 0)
if !hasInputs {
return -1000
}
if strings.Contains(lowerName, "clear") || strings.Contains(lowerName, "reset") ||
strings.Contains(lowerName, "init") || strings.Contains(lowerName, "undo") ||
strings.Contains(lowerName, "delete") || strings.Contains(lowerName, "remove") ||
strings.Contains(lowerName, "pop") || strings.Contains(lowerName, "clean") {
score -= 600
}
if strings.Contains(lowerName, "vote") || strings.Contains(lowerName, "like") ||
strings.Contains(lowerName, "dislike") || strings.Contains(lowerName, "flag") ||
strings.Contains(lowerName, "feedback") || strings.Contains(lowerName, "report") {
score -= 500
}
if strings.Contains(lowerName, "download") || strings.Contains(lowerName, "export") ||
strings.Contains(lowerName, "save") || strings.Contains(lowerName, "upload") ||
strings.Contains(lowerName, "auth") || strings.Contains(lowerName, "login") ||
strings.Contains(lowerName, "theme") || strings.Contains(lowerName, "token") {
score -= 400
}
if strings.Contains(lowerName, "whisper") || strings.Contains(lowerName, "transcribe") ||
strings.Contains(lowerName, "tts") || strings.Contains(lowerName, "speech") ||
strings.Contains(lowerName, "diffusion") || strings.Contains(lowerName, "draw") ||
strings.Contains(lowerName, "sdxl") || strings.Contains(lowerName, "upscale") {
score -= 300
}
if strings.Contains(lowerName, "lambda") {
score -= 200
}
// Penalize user-submission helper endpoints (common in Gradio Blocks multi-step chat interfaces
// where a "user" function simply appends input to history and clears the textbox)
isUserHandler := lowerName == "user" || strings.HasPrefix(lowerName, "user_") ||
strings.Contains(lowerName, "add_text") || strings.Contains(lowerName, "add_msg") ||
strings.Contains(lowerName, "add_message") || strings.Contains(lowerName, "append_to_history")
if isUserHandler {
score -= 600
} else if strings.HasPrefix(lowerName, "user") {
// Check if name is user followed by digits/underscores (e.g. user2, user3, user_1)
suffix := strings.TrimPrefix(lowerName, "user")
isDigitsOrUnder := true
for _, r := range suffix {
if (r < '0' || r > '9') && r != '_' {
isDigitsOrUnder = false
break
}
}
if isDigitsOrUnder && len(suffix) > 0 {
score -= 600
}
}
if lowerName == "chat" || lowerName == "chat_fn" || lowerName == "bot" || lowerName == "bot_fn" {
score += 150
} else if strings.Contains(lowerName, "chat") || strings.Contains(lowerName, "conversation") ||
strings.Contains(lowerName, "dialogue") || strings.Contains(lowerName, "chatbot") ||
strings.Contains(lowerName, "bot") {
score += 120
}
if lowerName == "predict" || lowerName == "generate" {
score += 80
}
if strings.Contains(lowerName, "respond") || strings.Contains(lowerName, "reply") || strings.Contains(lowerName, "answer") {
score += 90
}
if strings.Contains(lowerName, "generate") || strings.Contains(lowerName, "completion") || strings.Contains(lowerName, "infer") || strings.Contains(lowerName, "predict") {
score += 70
}
if strings.Contains(lowerName, "ask") || strings.Contains(lowerName, "query") || strings.Contains(lowerName, "prompt") || strings.Contains(lowerName, "talk") || strings.Contains(lowerName, "run") {
score += 50
}
if strings.Contains(lowerName, "submit") {
score += 40
}
for _, p := range parameters {
pLower := strings.ToLower(p.ParameterName)
pLabel := strings.ToLower(p.Label)
pComp := strings.ToLower(p.Component)
if pComp == "multimodaltextbox" {
score += 80
} else if pComp == "textbox" {
if strings.Contains(pLower, "message") || strings.Contains(pLabel, "message") ||
strings.Contains(pLower, "prompt") || strings.Contains(pLabel, "prompt") ||
strings.Contains(pLower, "query") || strings.Contains(pLabel, "query") ||
strings.Contains(pLower, "input") || strings.Contains(pLabel, "input") {
score += 60
} else {
score += 30
}
} else if pComp == "chatbot" {
score += 70
} else if pComp == "state" {
score += 20
} else if pComp == "slider" || pComp == "number" {
if strings.Contains(pLower, "temp") || strings.Contains(pLabel, "temp") ||
strings.Contains(pLower, "token") || strings.Contains(pLabel, "token") ||
strings.Contains(pLower, "top_p") || strings.Contains(pLabel, "top_p") {
score += 20
}
}
if strings.Contains(pLower, "tool") || strings.Contains(pLabel, "tool") ||
strings.Contains(pLower, "function") || strings.Contains(pLabel, "function") {
score += 50
}
}
if dep != nil {
if dep.Types.Generator {
score += 150
}
// Check if any textbox component in dep.Inputs is also in dep.Outputs (textbox-clearing UI handler)
for _, inID := range dep.Inputs {
if comp, exists := compMap[inID]; exists {
if strings.ToLower(comp.Type) == "textbox" {
for _, outID := range dep.Outputs {
if outID == inID {
score -= 600
break
}
}
}
}
}
for _, inID := range dep.Inputs {
if comp, exists := compMap[inID]; exists {
cType := strings.ToLower(comp.Type)
cLabel := ""
if comp.Props != nil {
if l, ok := comp.Props["label"].(string); ok {
cLabel = strings.ToLower(l)
}
}
if cType == "multimodaltextbox" {
score += 80
} else if cType == "textbox" {
if strings.Contains(cLabel, "message") || strings.Contains(cLabel, "prompt") ||
strings.Contains(cLabel, "query") || strings.Contains(cLabel, "input") {
score += 60
} else if strings.Contains(cLabel, "system") || strings.Contains(cLabel, "instruction") {
score += 30
} else {
score += 20
}
} else if cType == "chatbot" {
score += 70
} else if cType == "state" {
score += 20
} else if cType == "slider" || cType == "number" {
if strings.Contains(cLabel, "temp") || strings.Contains(cLabel, "token") || strings.Contains(cLabel, "top") {
score += 20
}
}
if strings.Contains(cLabel, "tool") || strings.Contains(cLabel, "function") {
score += 50
}
}
}
for _, outID := range dep.Outputs {
if comp, exists := compMap[outID]; exists {
cType := strings.ToLower(comp.Type)
if cType == "chatbot" {
score += 100
} else if cType == "textbox" || cType == "markdown" {
isSharedInput := false
for _, inID := range dep.Inputs {
if inID == outID {
isSharedInput = true
break
}
}
if !isSharedInput {
score += 50
}
}
}
}
}
return score
}
// InspectSpace queries Gradio's /gradio_api/info, /config, and HuggingFace Space APIs
// to build an adaptive schema mapping for any Gradio space.
func InspectSpace(client *http.Client, rawURL, userAgent string) (*SpaceDiscovery, error) {
cleanURL := strings.TrimRight(rawURL, "/")
if !strings.HasPrefix(cleanURL, "http://") && !strings.HasPrefix(cleanURL, "https://") {
cleanURL = "https://" + cleanURL
}
discovery := NewDefaultSpaceDiscovery(cleanURL)
// 1. Try fetching /gradio_api/info or /info
var infoResp GradioAPIInfoResponse
infoFetched := false
for _, path := range []string{"/gradio_api/info", "/info"} {
infoURL := cleanURL + path
req, err := http.NewRequest("GET", infoURL, nil)
if err == nil {
req.Header.Set("User-Agent", userAgent)
resp, err := client.Do(req)
if err == nil && resp.StatusCode == http.StatusOK {
if json.NewDecoder(resp.Body).Decode(&infoResp) == nil {
infoFetched = true
if path == "/gradio_api/info" {
discovery.APIPrefix = "/gradio_api"
} else {
discovery.APIPrefix = ""
}
}
resp.Body.Close()
if infoFetched {
break
}
} else if resp != nil {
resp.Body.Close()
}
}
}
// 2. Try fetching /config
var configResp GradioConfigResponse
configFetched := false
for _, path := range []string{"/config", "/gradio_api/config"} {
cfgURL := cleanURL + path
req, err := http.NewRequest("GET", cfgURL, nil)
if err == nil {
req.Header.Set("User-Agent", userAgent)
resp, err := client.Do(req)
if err == nil && resp.StatusCode == http.StatusOK {
if json.NewDecoder(resp.Body).Decode(&configResp) == nil {
configFetched = true
if configResp.APIPrefix != "" {
discovery.APIPrefix = configResp.APIPrefix
}
if configResp.Title != "" {
discovery.Title = configResp.Title
}
}
resp.Body.Close()
if configFetched {
break
}
} else if resp != nil {
resp.Body.Close()
}
}
}
// 3. Inspect Hugging Face Space Metadata if hosted on HF
parsedURL, _ := url.Parse(cleanURL)
if parsedURL != nil && (strings.HasSuffix(parsedURL.Host, ".hf.space") || strings.Contains(parsedURL.Host, "huggingface.co")) {
subdomain := strings.TrimSuffix(parsedURL.Host, ".hf.space")
var owner, name string
dashIdx := strings.Index(subdomain, "-")
if dashIdx != -1 {
owner = subdomain[:dashIdx]
name = subdomain[dashIdx+1:]
}
if owner != "" && name != "" {
hfAPIURL := fmt.Sprintf("https://huggingface.co/api/spaces/%s/%s", owner, name)
req, err := http.NewRequest("GET", hfAPIURL, nil)
if err == nil {
req.Header.Set("User-Agent", userAgent)
resp, err := client.Do(req)
if err == nil && resp.StatusCode == http.StatusOK {
var hfResp HFSpaceInfoResponse
if json.NewDecoder(resp.Body).Decode(&hfResp) == nil {
for _, m := range hfResp.Models {
discovery.Models = append(discovery.Models, m)
cleanM := strings.TrimPrefix(m, "openai/")
cleanM = strings.TrimPrefix(cleanM, "models/")
if cleanM != m {
discovery.Models = append(discovery.Models, cleanM)
}
}
if hfResp.CardData.Title != "" && discovery.Title == "" {
discovery.Title = hfResp.CardData.Title
}
if len(discovery.Models) > 0 {
discovery.PrimaryModel = discovery.Models[0]
}
}
resp.Body.Close()
} else if resp != nil {
resp.Body.Close()
}
}
}
}
// Fallback model names if not discovered
if len(discovery.Models) == 0 {
if parsedURL != nil && strings.HasSuffix(parsedURL.Host, ".hf.space") {
sub := strings.TrimSuffix(parsedURL.Host, ".hf.space")
parts := strings.Split(sub, "-")
if len(parts) > 1 {
cleanModel := strings.Join(parts[1:], "-")
discovery.Models = append(discovery.Models, cleanModel)
discovery.PrimaryModel = cleanModel
}
}
}
if discovery.PrimaryModel == "" {
if len(discovery.Models) > 0 {
discovery.PrimaryModel = discovery.Models[0]
} else {
discovery.PrimaryModel = "gradio-chat"
discovery.Models = append(discovery.Models, "gradio-chat")
}
}
// 4. Detect Gradio version and UI flavor
compMap := make(map[int]GradioComponent)
if configFetched {
for _, comp := range configResp.Components {
compMap[comp.ID] = comp
}
if configResp.Version != "" {
discovery.GradioVersion = configResp.Version
} else if infoFetched {
discovery.GradioVersion = "4+ (inferred from /gradio_api/info)"
}
discovery.Flavor = DetectGradioFlavor(configResp, compMap)
} else if infoFetched {
discovery.GradioVersion = "4+ (inferred from /gradio_api/info)"
discovery.Flavor = "Generic"
}
// 5. Score and select the best chat completion endpoint
bestEndpoint := ""
bestScore := -1000
var bestEndpointInfo *GradioEndpointInfo
var bestMatchingDep *GradioDependency
// Try named endpoints from /gradio_api/info first
if infoFetched && len(infoResp.NamedEndpoints) > 0 {
epNames := make([]string, 0, len(infoResp.NamedEndpoints))
for epName := range infoResp.NamedEndpoints {
epNames = append(epNames, epName)
}
sort.Strings(epNames)
for _, epName := range epNames {
epInfo := infoResp.NamedEndpoints[epName]
cleanTarget := strings.TrimPrefix(epName, "/")
var matchingDep *GradioDependency
if configFetched {
for _, dep := range configResp.Dependencies {
depAPIName := ""
if s, ok := dep.APIName.(string); ok {
depAPIName = strings.TrimPrefix(s, "/")
}
if depAPIName == cleanTarget {
depCopy := dep
matchingDep = &depCopy
break
}
}
}
score := ScoreCandidateEndpoint(epName, epInfo.Parameters, matchingDep, compMap)
if score > bestScore {
bestScore = score
bestEndpoint = epName
epCopy := epInfo
bestEndpointInfo = &epCopy
bestMatchingDep = matchingDep
}
}
}
// Fallback: if no named endpoint from /info, inspect dependencies in config
if (bestEndpoint == "" || bestScore <= 0) && configFetched && len(configResp.Dependencies) > 0 {
depScore := -1000
for _, dep := range configResp.Dependencies {
apiName := ""
if s, ok := dep.APIName.(string); ok {
apiName = s
}
score := ScoreCandidateEndpoint(apiName, nil, &dep, compMap)
if score > depScore {
depScore = score
depCopy := dep
bestMatchingDep = &depCopy
if apiName != "" {
bestEndpoint = "/" + strings.TrimPrefix(apiName, "/")
} else {
bestEndpoint = fmt.Sprintf("/%d", dep.ID)
}
}
}
}
if bestMatchingDep != nil {
discovery.FnIndex = bestMatchingDep.ID
}
// Select protocol
isCallV2 := false
if strings.HasPrefix(discovery.GradioVersion, "6.") {
isCallV2 = true
}
if bestEndpointInfo != nil && bestEndpointInfo.CodeSnippets != nil {
if bashSnippet, ok := bestEndpointInfo.CodeSnippets["bash"].(string); ok {
if strings.Contains(bashSnippet, "/call/v2/") {
isCallV2 = true
}
}
}
if strings.HasPrefix(discovery.GradioVersion, "3.") {
discovery.Protocol = "predict"
discovery.APIPrefix = ""
discovery.Endpoint = "/run/predict"
discovery.CleanEndpoint = "run/predict"
} else if isCallV2 {
discovery.Protocol = "call_v2"
if bestEndpoint != "" {
discovery.Endpoint = bestEndpoint
discovery.CleanEndpoint = strings.TrimPrefix(bestEndpoint, "/")
}
} else {
discovery.Protocol = "call"
if bestEndpoint != "" {
discovery.Endpoint = bestEndpoint
discovery.CleanEndpoint = strings.TrimPrefix(bestEndpoint, "/")
}
}
// 6. Correlate with config.dependencies to determine exact input count & state padding
if bestMatchingDep != nil {
discovery.TotalInputs = len(bestMatchingDep.Inputs)
discovery.RawTotalInputs = len(bestMatchingDep.Inputs)
discovery.DefaultInputs = make([]interface{}, len(bestMatchingDep.Inputs))
discovery.ParamMappings = nil
discovery.MessageIndex = -1
for idx, compID := range bestMatchingDep.Inputs {
mapping := SpaceParamMapping{
InputIndex: idx,
ComponentID: compID,
ParamType: "other",
}
var pName, pLabel, pComp string
if bestEndpointInfo != nil && idx < len(bestEndpointInfo.Parameters) {
p := bestEndpointInfo.Parameters[idx]
pName = strings.ToLower(p.ParameterName)
pLabel = strings.ToLower(p.Label)
pComp = strings.ToLower(p.Component)
if p.ParameterDefault != nil && discovery.DefaultInputs[idx] == nil {
discovery.DefaultInputs[idx] = p.ParameterDefault
mapping.DefaultValue = p.ParameterDefault
}
mapping.Label = p.ParameterName
mapping.ParamName = p.ParameterName
if p.Component != "" {
mapping.ComponentType = p.Component
}
}
if comp, exists := compMap[compID]; exists {
if mapping.ComponentType == "" {
mapping.ComponentType = comp.Type
}
cType := strings.ToLower(comp.Type)
cLabel := ""
if comp.Props != nil {
if l, ok := comp.Props["label"].(string); ok {
cLabel = strings.ToLower(l)
if mapping.Label == "" {
mapping.Label = l
}
}
if val, ok := comp.Props["value"]; ok {
discovery.DefaultInputs[idx] = val
mapping.DefaultValue = val
}
}
if strings.Contains(pName, "function") || strings.Contains(pName, "tool") || strings.Contains(pLabel, "tool") || strings.Contains(pLabel, "function") || strings.Contains(cLabel, "tool") || strings.Contains(cLabel, "function") {
mapping.ParamType = "tools"
discovery.FunctionsJSONIndex = idx
} else if strings.Contains(pName, "think_level") || strings.Contains(pName, "thinking") || strings.Contains(pLabel, "think") || strings.Contains(cLabel, "think") {
mapping.ParamType = "think_level"
discovery.ThinkLevelIndex = idx
} else if strings.Contains(pName, "preserved") || strings.Contains(cLabel, "preserved") {
discovery.PreservedThinkingIndex = idx
} else if strings.Contains(pName, "system") || strings.Contains(pLabel, "system") || strings.Contains(cLabel, "system") || strings.Contains(cLabel, "instruction") {
mapping.ParamType = "system_prompt"
discovery.SystemIndex = idx
if comp.Props != nil {
if val, ok := comp.Props["value"].(string); ok && strings.TrimSpace(val) != "" {
discovery.DefaultSystemPrompt = strings.TrimSpace(val)
}
}
} else if strings.Contains(pName, "history") || strings.Contains(pLabel, "history") || strings.Contains(cLabel, "history") || strings.Contains(cLabel, "chat") || cType == "chatbot" {
mapping.ParamType = "history"
discovery.HistoryIndex = idx
} else if cType == "multimodaltextbox" || strings.Contains(pComp, "multimodal") {
mapping.ParamType = "message"
discovery.MessageIndex = idx
discovery.MessageIsMultimodal = true
} else if strings.Contains(pName, "message") || strings.Contains(pLabel, "message") || strings.Contains(cLabel, "message") || strings.Contains(cLabel, "prompt") || strings.Contains(cLabel, "query") || (discovery.MessageIndex == -1 && idx == 0 && (cType == "textbox" || pComp == "textbox")) {
mapping.ParamType = "message"
discovery.MessageIndex = idx
discovery.MessageIsMultimodal = false
} else if strings.Contains(pName, "temp") || strings.Contains(cLabel, "temp") {
mapping.ParamType = "temperature"
discovery.TempIndex = idx
} else if strings.Contains(pName, "token") || strings.Contains(cLabel, "token") || strings.Contains(cLabel, "max") {
mapping.ParamType = "max_tokens"
discovery.MaxTokensIndex = idx
} else if strings.Contains(pName, "top_p") || strings.Contains(cLabel, "top_p") || strings.Contains(cLabel, "top-p") || strings.Contains(cLabel, "top p") {
mapping.ParamType = "top_p"
discovery.TopPIndex = idx
} else if strings.Contains(pName, "stream") || strings.Contains(cLabel, "stream") {
mapping.ParamType = "stream"
discovery.StreamIndex = idx
} else if cType == "state" {
mapping.ParamType = "state"
}
}
discovery.ParamMappings = append(discovery.ParamMappings, mapping)
}
if discovery.MessageIndex == -1 {
if discovery.HistoryIndex != 0 && discovery.SystemIndex != 0 && discovery.FunctionsJSONIndex != 0 {
discovery.MessageIndex = 0
if len(bestMatchingDep.Inputs) > 0 {
if comp, exists := compMap[bestMatchingDep.Inputs[0]]; exists {
if strings.ToLower(comp.Type) == "multimodaltextbox" {
discovery.MessageIsMultimodal = true
}
}
}
}
}
// If the endpoint has canonical parameters exposed via /gradio_api/info,
// and trailing inputs in bestMatchingDep.Inputs are unexposed server-side state components,
// do not pad them with null so Gradio preserves its internal server state.
if bestEndpointInfo != nil && len(bestEndpointInfo.Parameters) > 0 && len(bestMatchingDep.Inputs) > len(bestEndpointInfo.Parameters) {
allTrailingAreState := true
for i := len(bestEndpointInfo.Parameters); i < len(bestMatchingDep.Inputs); i++ {
cID := bestMatchingDep.Inputs[i]
if comp, exists := compMap[cID]; exists {
cType := strings.ToLower(comp.Type)
if cType != "state" && cType != "browserstate" {
allTrailingAreState = false
break
}
}
}
if allTrailingAreState {
discovery.TotalInputs = len(bestEndpointInfo.Parameters)
discovery.DefaultInputs = discovery.DefaultInputs[:discovery.TotalInputs]
if len(discovery.ParamMappings) > discovery.TotalInputs {
discovery.ParamMappings = discovery.ParamMappings[:discovery.TotalInputs]
}
if discovery.MessageIndex >= discovery.TotalInputs {
discovery.MessageIndex = -1
}
if discovery.HistoryIndex >= discovery.TotalInputs {
discovery.HistoryIndex = -1
}
if discovery.SystemIndex >= discovery.TotalInputs {
discovery.SystemIndex = -1
}
if discovery.FunctionsJSONIndex >= discovery.TotalInputs {
discovery.FunctionsJSONIndex = -1
}
if discovery.TempIndex >= discovery.TotalInputs {
discovery.TempIndex = -1
}
if discovery.MaxTokensIndex >= discovery.TotalInputs {
discovery.MaxTokensIndex = -1
}
if discovery.TopPIndex >= discovery.TotalInputs {
discovery.TopPIndex = -1
}
if discovery.StreamIndex >= discovery.TotalInputs {
discovery.StreamIndex = -1
}
}
}
}
// Refine history format or discover tools from bestEndpointInfo
if bestEndpointInfo != nil {
for _, p := range bestEndpointInfo.Parameters {
pName := strings.ToLower(p.ParameterName)
pLabel := strings.ToLower(p.Label)
pComp := strings.ToLower(p.Component)
if strings.Contains(pName, "history") || strings.Contains(pLabel, "history") || strings.Contains(pName, "chat") || strings.Contains(pName, "messages") || pComp == "chatbot" {
bType, _ := json.Marshal(p.Type)
bPyType, _ := json.Marshal(p.PythonType)
pPyType := strings.ToLower(string(bPyType))
bTypeStr := strings.ToLower(string(bType))
if strings.Contains(pPyType, "list[tuple[") || strings.Contains(pPyType, "list[list[") || strings.Contains(bTypeStr, "tuple") {
discovery.HistoryFormat = "pairs"
} else if strings.Contains(pPyType, "textmessage") || strings.Contains(pPyType, "dict(text: str") || strings.Contains(bTypeStr, "textmessage") || strings.Contains(bTypeStr, "chatbotdatamessages") {
discovery.HistoryFormat = "gradio_messages"
}
}
}
}
// Fallback: If config.dependencies did not provide matchingDep, map directly from bestEndpointInfo.Parameters
if bestMatchingDep == nil && bestEndpointInfo != nil {
discovery.TotalInputs = len(bestEndpointInfo.Parameters)
discovery.DefaultInputs = make([]interface{}, len(bestEndpointInfo.Parameters))
discovery.ParamMappings = nil
discovery.MessageIndex = -1
for idx, p := range bestEndpointInfo.Parameters {
pName := strings.ToLower(p.ParameterName)
pLabel := strings.ToLower(p.Label)
pComp := strings.ToLower(p.Component)
if p.ParameterDefault != nil {
discovery.DefaultInputs[idx] = p.ParameterDefault
}
mapping := SpaceParamMapping{
InputIndex: idx,
ComponentType: p.Component,
Label: p.Label,
ParamName: p.ParameterName,
ParamType: "other",
DefaultValue: p.ParameterDefault,
}
if strings.Contains(pComp, "multimodal") {
if discovery.MessageIndex == -1 || strings.Contains(pLabel, "message") || strings.Contains(pName, "message") {
mapping.ParamType = "message"
discovery.MessageIndex = idx
discovery.MessageIsMultimodal = true
}
} else if strings.Contains(pName, "system") || strings.Contains(pLabel, "system") || strings.Contains(pLabel, "instruction") {
discovery.SystemIndex = idx
mapping.ParamType = "system_prompt"
if p.ParameterDefault != nil && discovery.DefaultSystemPrompt == "" {
if defStr, ok := p.ParameterDefault.(string); ok && strings.TrimSpace(defStr) != "" {
discovery.DefaultSystemPrompt = strings.TrimSpace(defStr)
}
}
} else if strings.Contains(pName, "history") || strings.Contains(pLabel, "history") || strings.Contains(pName, "chat") || strings.Contains(pName, "messages") || strings.Contains(pName, "conversation") || pComp == "chatbot" {
discovery.HistoryIndex = idx
mapping.ParamType = "history"
bType, _ := json.Marshal(p.Type)
bPyType, _ := json.Marshal(p.PythonType)
pPyType := strings.ToLower(string(bPyType))
bTypeStr := strings.ToLower(string(bType))
if strings.Contains(pPyType, "list[tuple[") || strings.Contains(pPyType, "list[list[") || strings.Contains(bTypeStr, "tuple") {
discovery.HistoryFormat = "pairs"
} else if strings.Contains(pPyType, "textmessage") || strings.Contains(pPyType, "dict(text: str") || strings.Contains(bTypeStr, "textmessage") || strings.Contains(bTypeStr, "chatbotdatamessages") {
discovery.HistoryFormat = "gradio_messages"
}
} else if strings.Contains(pLabel, "message") || strings.Contains(pName, "message") || (strings.Contains(pLabel, "prompt") && !strings.Contains(pLabel, "system")) || (strings.Contains(pName, "prompt") && !strings.Contains(pName, "system")) || strings.Contains(pLabel, "query") || strings.Contains(pName, "query") || strings.Contains(pLabel, "question") || strings.Contains(pName, "question") || (discovery.MessageIndex == -1 && (pComp == "textbox" || (idx == 0 && pComp != "chatbot"))) {
discovery.MessageIndex = idx
mapping.ParamType = "message"
discovery.MessageIsMultimodal = false
} else if strings.Contains(pName, "think_level") || strings.Contains(pName, "thinking_level") || strings.Contains(pLabel, "think") {
discovery.ThinkLevelIndex = idx
mapping.ParamType = "think_level"
} else if strings.Contains(pName, "functions") || strings.Contains(pName, "tools") || strings.Contains(pLabel, "tools") || strings.Contains(pLabel, "functions") {
discovery.FunctionsJSONIndex = idx
mapping.ParamType = "tools"
} else if strings.Contains(pName, "preserved") {
discovery.PreservedThinkingIndex = idx
} else if strings.Contains(pName, "temp") || strings.Contains(pLabel, "temp") {
discovery.TempIndex = idx
mapping.ParamType = "temperature"
} else if strings.Contains(pName, "token") || strings.Contains(pLabel, "token") {
discovery.MaxTokensIndex = idx
mapping.ParamType = "max_tokens"
} else if strings.Contains(pName, "top_p") || strings.Contains(pLabel, "top_p") || strings.Contains(pLabel, "top p") {
discovery.TopPIndex = idx
mapping.ParamType = "top_p"
} else if strings.Contains(pName, "stream") || strings.Contains(pLabel, "stream") {
discovery.StreamIndex = idx
mapping.ParamType = "stream"
}
discovery.ParamMappings = append(discovery.ParamMappings, mapping)
}
if discovery.MessageIndex == -1 {
if discovery.HistoryIndex != 0 && discovery.SystemIndex != 0 && discovery.FunctionsJSONIndex != 0 {
discovery.MessageIndex = 0
}
}
}
// 7. Resolve default history format if not set by type inspection
if discovery.HistoryIndex != -1 {
if discovery.HistoryFormat == "" || discovery.HistoryFormat == "messages" {
if strings.HasPrefix(discovery.GradioVersion, "5.") || strings.HasPrefix(discovery.GradioVersion, "6.") {
discovery.HistoryFormat = "gradio_messages"
} else {
discovery.HistoryFormat = "pairs"
}
}
} else {
discovery.HistoryFormat = "none"
}
// 8. Hunyuan3 / Tencent Hy3 overrides
if discovery.FunctionsJSONIndex != -1 || discovery.ThinkLevelIndex != -1 || strings.Contains(cleanURL, "hy3") || strings.Contains(cleanURL, "hunyuan") {
discovery.IsHunyuan3 = true
discovery.HistoryFormat = "messages"
discovery.Models = append(discovery.Models, "hy3", "hunyuan3", "tencent/Hy3")
if discovery.PrimaryModel == "gradio-chat" || discovery.PrimaryModel == "" {
discovery.PrimaryModel = "hy3"
}
}
// 9. Tool Calling Support Picture Resolution
if discovery.FunctionsJSONIndex >= 0 {
discovery.ToolCallMode = "native_slot"
} else if discovery.SystemIndex >= 0 {
discovery.ToolCallMode = "prompt_augmented_system"
} else if discovery.HistoryIndex >= 0 {
discovery.ToolCallMode = "prompt_augmented_first_turn"
} else {
discovery.ToolCallMode = "prompt_augmented_single_prompt"
}
// Ensure total inputs is at least 1
if discovery.TotalInputs < 1 {
discovery.TotalInputs = 1
}
for len(discovery.DefaultInputs) < discovery.TotalInputs {
discovery.DefaultInputs = append(discovery.DefaultInputs, nil)
}
return discovery, nil
}
// ---------------------------------------------------------------------------
// Universal Gradio gateway engine
// ---------------------------------------------------------------------------
type GradioJoinResponse struct {
EventID string `json:"event_id"`
}
type GradioGateway struct {
mu sync.RWMutex
defaultURL string
client *http.Client
discoveries map[string]*SpaceDiscovery
proxyURL string
}
func NewGradioGateway(defaultSpaceURL, proxyURL string, timeout time.Duration) *GradioGateway {
cleanDefault := strings.TrimRight(defaultSpaceURL, "/")
if !strings.HasPrefix(cleanDefault, "http://") && !strings.HasPrefix(cleanDefault, "https://") {
cleanDefault = "https://" + cleanDefault
}
transport := &http.Transport{
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
if proxyURL != "" {
return DialSOCKS5(ctx, proxyURL, addr)
}
var d net.Dialer
return d.DialContext(ctx, network, addr)
},
MaxIdleConns: 100,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 15 * time.Second,
}
gw := &GradioGateway{
defaultURL: cleanDefault,
client: &http.Client{Transport: transport, Timeout: timeout},
discoveries: make(map[string]*SpaceDiscovery),
proxyURL: proxyURL,
}
// Pre-discover the default space
disc, err := InspectSpace(gw.client, cleanDefault, DefaultUserAgent)
if err == nil && disc != nil {
gw.discoveries[cleanDefault] = disc
log.Printf("\n%s\n", disc.Summary())
} else if err != nil {
log.Printf("Warning: initial space discovery for %s encountered error: %v (will retry on demand)", cleanDefault, err)
}
return gw
}
func (g *GradioGateway) GetDiscovery(spaceURL, userAgent string) *SpaceDiscovery {
target := spaceURL
if target == "" {
target = g.defaultURL
}
cleanTarget := strings.TrimRight(target, "/")
g.mu.RLock()
disc, exists := g.discoveries[cleanTarget]
g.mu.RUnlock()
if exists && disc != nil && time.Since(disc.LastDiscovered) < 30*time.Minute {
return disc
}
g.mu.Lock()
defer g.mu.Unlock()
// Double-check under lock
disc, exists = g.discoveries[cleanTarget]
if exists && disc != nil && time.Since(disc.LastDiscovered) < 30*time.Minute {
return disc
}
newDisc, err := InspectSpace(g.client, cleanTarget, userAgent)
if err == nil && newDisc != nil {
g.discoveries[cleanTarget] = newDisc
log.Printf("\n%s\n", newDisc.Summary())
return newDisc
}
if disc != nil {
return disc
}
// Fallback discovery
fallback := NewDefaultSpaceDiscovery(cleanTarget)
fallback.TotalInputs = 2
fallback.PrimaryModel = "gradio-chat"
fallback.Models = []string{"gradio-chat"}
g.discoveries[cleanTarget] = fallback
return fallback
}
// BuildGradioPayload packages OpenAI messages and parameters into the target Gradio input array.
func (g *GradioGateway) BuildGradioPayload(disc *SpaceDiscovery, req ChatCompletionRequest) ([]interface{}, error) {
var transformed []ChatMessage
var toolInstruction string
if disc.IsHunyuan3 && disc.FunctionsJSONIndex != -1 {
for _, msg := range req.Messages {
transformed = append(transformed, ChatMessage{
Role: msg.Role,
Content: msg.GetContentString(),
ReasoningContent: msg.ReasoningContent,
ToolCalls: msg.ToolCalls,
ToolCallID: msg.ToolCallID,
Name: msg.Name,
})
}
} else {
// When no native tool calling support is detected, augment system prompt
transformed, toolInstruction, _ = TransformMessages(req)
}
hasClientSystem := false
for _, m := range req.Messages {
if m.Role == "system" {
hasClientSystem = true
break
}
}
var systemPromptStr string
var historyArray []map[string]interface{}
var lastUserMessage string
var nonSystem []ChatMessage
for _, m := range transformed {
cStr := m.GetContentString()
if m.Role == "system" {
if systemPromptStr == "" {
systemPromptStr = cStr
} else {
systemPromptStr += "\n\n" + cStr
}
} else {
nonSystem = append(nonSystem, m)
}
}
// If the space has a DefaultSystemPrompt and client provided no system message,
// retain and augment the default system prompt:
if !hasClientSystem && disc.DefaultSystemPrompt != "" {
if systemPromptStr != "" {
systemPromptStr = disc.DefaultSystemPrompt + "\n\n" + systemPromptStr
} else {
systemPromptStr = disc.DefaultSystemPrompt
}
}
// If the space has NO native system prompt input (disc.SystemIndex == -1),
// but we have system instructions (from system message or tool instructions):
if disc.SystemIndex == -1 && systemPromptStr != "" && len(nonSystem) > 0 {
// If the space supports conversation history, prepend system instructions to the first turn
if disc.HistoryIndex != -1 {
if toolInstruction != "" && len(nonSystem) == 1 {
nonSystem[0].Content = fmt.Sprintf("%s\n\nQuery: %s", systemPromptStr, nonSystem[0].GetContentString())
} else {
nonSystem[0].Content = systemPromptStr + "\n\n" + nonSystem[0].GetContentString()
}
}
}
if len(nonSystem) > 0 {
for i := 0; i < len(nonSystem)-1; i++ {
m := nonSystem[i]
cStr := m.GetContentString()
item := map[string]interface{}{"role": m.Role}
switch m.Role {
case "assistant":
if cStr != "" {
item["content"] = cStr
} else {
item["content"] = nil
}
if m.ReasoningContent != "" {
item["reasoning_content"] = m.ReasoningContent
}
if len(m.ToolCalls) > 0 {
item["tool_calls"] = m.ToolCalls
}
case "tool", "function":
item["role"] = "tool"
item["content"] = cStr
toolID := m.ToolCallID
if toolID == "" {
toolID = m.Name
}
if toolID != "" {
item["tool_call_id"] = toolID
}
if m.Name != "" {
item["name"] = m.Name
}
default:
item["content"] = cStr
}
historyArray = append(historyArray, item)
}
lastMsg := nonSystem[len(nonSystem)-1]
lastContent := lastMsg.GetContentString()
if lastMsg.Role == "tool" || lastMsg.Role == "function" {
toolName := lastMsg.Name
if toolName == "" {
toolName = lastMsg.ToolCallID
}
if disc.IsHunyuan3 {
if toolName != "" {
lastUserMessage = fmt.Sprintf("Tool result for %s: %s", toolName, lastContent)
} else {
lastUserMessage = lastContent
}
} else {
if toolName != "" {
lastUserMessage = fmt.Sprintf("Tool result for %s: %s\nPlease answer the user's request based on the tool result.", toolName, lastContent)
} else {
lastUserMessage = fmt.Sprintf("Tool result: %s\nPlease answer the user's request based on the tool result.", lastContent)
}
}
} else {
lastUserMessage = lastContent
}
} else if systemPromptStr != "" {
lastUserMessage = systemPromptStr
}
var promptMessageText string
if disc.HistoryIndex != -1 {
if disc.SystemIndex == -1 {
if len(nonSystem) > 1 && toolInstruction != "" {
if !strings.HasPrefix(lastUserMessage, "Tool result") {
promptMessageText = fmt.Sprintf("[System Directive: Tool calling mode active.]\n\nQuery: %s", lastUserMessage)
} else {
promptMessageText = fmt.Sprintf("[System Directive: Tool calling mode active.]\n\n%s", lastUserMessage)
}
} else {
promptMessageText = lastUserMessage
}
} else {
if toolInstruction != "" && !strings.HasPrefix(lastUserMessage, "Tool result") {
promptMessageText = fmt.Sprintf("Query: %s", lastUserMessage)
} else {
promptMessageText = lastUserMessage
}
}
} else {
// Single message space: compose multi-turn history into the prompt
if len(nonSystem) <= 1 {
if systemPromptStr != "" && len(nonSystem) == 1 {
if toolInstruction != "" && !strings.HasPrefix(lastUserMessage, "Tool result") {
promptMessageText = fmt.Sprintf("%s\n\nQuery: %s", systemPromptStr, lastUserMessage)
} else {
promptMessageText = systemPromptStr + "\n\n" + lastUserMessage
}
} else if systemPromptStr != "" {
promptMessageText = systemPromptStr
} else {
promptMessageText = lastUserMessage
}
} else {
var sb strings.Builder
if systemPromptStr != "" {
sb.WriteString("# Instructions\n" + systemPromptStr + "\n\n")
}
sb.WriteString("# Conversation History\n")
for i := 0; i < len(nonSystem)-1; i++ {
m := nonSystem[i]
roleLabel := "User"
if m.Role == "assistant" {
roleLabel = "Assistant"
}
sb.WriteString(fmt.Sprintf("%s: %s\n\n", roleLabel, m.GetContentString()))
}
lastRoleLabel := "User"
if len(nonSystem) > 0 && nonSystem[len(nonSystem)-1].Role == "assistant" {
lastRoleLabel = "Assistant"
}
if toolInstruction != "" && lastRoleLabel == "User" && !strings.HasPrefix(lastUserMessage, "Tool result") {
sb.WriteString(fmt.Sprintf("# Current Request\nQuery: %s", lastUserMessage))
} else {
sb.WriteString(fmt.Sprintf("# Current Request\n%s: %s", lastRoleLabel, lastUserMessage))
}
promptMessageText = sb.String()
}
}
// Allocate input array matching TotalInputs
totalInputs := disc.TotalInputs
if totalInputs < 1 {
totalInputs = 1
}
data := make([]interface{}, totalInputs)
// Initialize with space default inputs if available
if len(disc.DefaultInputs) == totalInputs {
for i := 0; i < totalInputs; i++ {
data[i] = disc.DefaultInputs[i]
}
}
// Extract multimodal files if message input is multimodal
var messageFiles []interface{}
if disc.MessageIsMultimodal && len(nonSystem) > 0 {
lastMsg := nonSystem[len(nonSystem)-1]
if parts, ok := lastMsg.Content.([]interface{}); ok {
for _, p := range parts {
if itemMap, ok := p.(map[string]interface{}); ok {
if itemMap["type"] == "image_url" {
imgURL := ""
if iuMap, ok := itemMap["image_url"].(map[string]interface{}); ok {
if u, ok := iuMap["url"].(string); ok {
imgURL = u
}
} else if iuStr, ok := itemMap["image_url"].(string); ok {
imgURL = iuStr
}
if imgURL != "" {
messageFiles = append(messageFiles, map[string]interface{}{
"path": imgURL,
"url": imgURL,
"meta": map[string]interface{}{"_type": "gradio.FileData"},
})
}
}
}
}
}
}
if messageFiles == nil {
messageFiles = []interface{}{}
}
// Populate mapped fields
msgIdx := disc.MessageIndex
if msgIdx >= 0 && msgIdx < len(data) {
if disc.MessageIsMultimodal {
data[msgIdx] = map[string]interface{}{
"text": promptMessageText,
"files": messageFiles,
}
} else {
data[msgIdx] = promptMessageText
}
}
if disc.HistoryIndex >= 0 && disc.HistoryIndex < len(data) {
if disc.HistoryFormat == "pairs" {
var pairs [][]string
for i := 0; i < len(historyArray); i += 2 {
u := ""
a := ""
if i < len(historyArray) {
u, _ = historyArray[i]["content"].(string)
}
if i+1 < len(historyArray) {
a, _ = historyArray[i+1]["content"].(string)
}
pairs = append(pairs, []string{u, a})
}
if disc.MessageIndex == -1 && promptMessageText != "" {
pairs = append(pairs, []string{promptMessageText, ""})
}
data[disc.HistoryIndex] = pairs
} else if disc.HistoryFormat == "gradio_messages" {
var gMsgs []map[string]interface{}
for _, item := range historyArray {
role, _ := item["role"].(string)
cStr := ""
if s, ok := item["content"].(string); ok {
cStr = s
}
gMsg := map[string]interface{}{
"role": role,
"content": []map[string]string{
{"text": cStr, "type": "text"},
},
}
gMsgs = append(gMsgs, gMsg)
}
if disc.MessageIndex == -1 && promptMessageText != "" {
gMsgs = append(gMsgs, map[string]interface{}{
"role": "user",
"content": []map[string]string{
{"text": promptMessageText, "type": "text"},
},
})
}
if gMsgs == nil {
gMsgs = []map[string]interface{}{}
}
data[disc.HistoryIndex] = gMsgs
} else {
if disc.MessageIndex == -1 && promptMessageText != "" {
historyArray = append(historyArray, map[string]interface{}{
"role": "user",
"content": promptMessageText,
})
}
data[disc.HistoryIndex] = historyArray
}
}
// If neither message nor history slot was mapped, populate slot 0 with the prompt
if disc.MessageIndex == -1 && disc.HistoryIndex == -1 && len(data) > 0 {
data[0] = promptMessageText
}
if disc.SystemIndex >= 0 && disc.SystemIndex < len(data) {
data[disc.SystemIndex] = systemPromptStr
}
if disc.ThinkLevelIndex >= 0 && disc.ThinkLevelIndex < len(data) {
thinkLevel := "high"
if req.ReasoningEffort != "" {
effort := strings.ToLower(req.ReasoningEffort)
switch effort {
case "none", "off", "no_think", "0":
thinkLevel = "no_think"
case "low", "1":
thinkLevel = "low"
case "medium", "high", "2", "3":
thinkLevel = "high"
default:
thinkLevel = effort
}
}
data[disc.ThinkLevelIndex] = thinkLevel
}
if disc.TempIndex >= 0 && disc.TempIndex < len(data) {
if req.Temperature != nil {
data[disc.TempIndex] = *req.Temperature
} else if disc.IsHunyuan3 {
data[disc.TempIndex] = nil
} else if data[disc.TempIndex] == nil {
data[disc.TempIndex] = 0.7
}
}
if disc.MaxTokensIndex >= 0 && disc.MaxTokensIndex < len(data) {
if req.MaxTokens > 0 || req.MaxCompletionTokens > 0 {
data[disc.MaxTokensIndex] = ResolveMaxTokens(req)
} else if data[disc.MaxTokensIndex] == nil {
data[disc.MaxTokensIndex] = ResolveMaxTokens(req)
}
}
if disc.TopPIndex >= 0 && disc.TopPIndex < len(data) {
if req.TopP != nil {
data[disc.TopPIndex] = *req.TopP
} else if disc.IsHunyuan3 {
data[disc.TopPIndex] = 0
} else if data[disc.TopPIndex] == nil {
data[disc.TopPIndex] = 1.0
}
}
if disc.StreamIndex > 0 && disc.StreamIndex != disc.MessageIndex && disc.StreamIndex < len(data) {
data[disc.StreamIndex] = false
}
if disc.FunctionsJSONIndex >= 0 && disc.FunctionsJSONIndex < len(data) {
functionsJSONStr := ""
if len(req.Tools) > 0 {
b, err := json.Marshal(req.Tools)
if err == nil {
functionsJSONStr = string(b)
}
}
data[disc.FunctionsJSONIndex] = functionsJSONStr
}
return data, nil
}
// GradioOutputFrame holds parsed elements from a Gradio SSE output chunk
type GradioOutputFrame struct {
Content string
Reasoning string
ToolCalls []ToolCall
IsDelta bool
OK bool
}
// extractGradioDiffDelta extracts delta string from Gradio 6 streaming diff operations
func extractGradioDiffDelta(v []interface{}) (string, bool) {
if len(v) == 0 {
return "", false
}
if len(v) >= 3 {
if op, ok := v[0].(string); ok && (op == "append" || op == "add") {
if delta, ok := v[2].(string); ok {
return delta, true
}
}
}
for _, item := range v {
if innerList, ok := item.([]interface{}); ok && len(innerList) > 0 {
for _, opItem := range innerList {
if opArr, ok := opItem.([]interface{}); ok && len(opArr) >= 3 {
if op, ok := opArr[0].(string); ok && (op == "append" || op == "add") {
if delta, ok := opArr[2].(string); ok {
return delta, true
}
}
}
}
if len(innerList) >= 3 {
if op, ok := innerList[0].(string); ok && (op == "append" || op == "add") {
if delta, ok := innerList[2].(string); ok {
return delta, true
}
}
}
}
}
return "", false
}
// ParseGradioStreamOutput extracts structured content, reasoning, and tool calls from Gradio output
func ParseGradioStreamOutput(rawJSON string) (frame GradioOutputFrame) {
defer func() {
if frame.OK && len(frame.ToolCalls) == 0 && frame.Content != "" {
tcs, clean, has := DetectToolCalls(frame.Content)
if has && len(tcs) > 0 {
frame.ToolCalls = tcs
frame.Content = clean
}
}
}()
var val interface{}
if err := json.Unmarshal([]byte(rawJSON), &val); err != nil {
return frame
}
switch v := val.(type) {
case string:
frame.Content = v
frame.OK = true
return frame
case []interface{}:
if len(v) == 0 {
return frame
}
if delta, ok := extractGradioDiffDelta(v); ok {
frame.Content = delta
frame.IsDelta = true
frame.OK = true
return frame
}
// 1. Check if v[0] is an inner slice with len >= 3 (e.g. Hy3: [[content, reasoning, tool_calls, history]])
if inner, ok := v[0].([]interface{}); ok && len(inner) >= 3 {
s0, _ := inner[0].(string)
s1, _ := inner[1].(string)
frame.Content = s0
frame.Reasoning = s1
if inner[2] != nil {
b, err := json.Marshal(inner[2])
if err == nil {
var tcs []ToolCall
if json.Unmarshal(b, &tcs) == nil && len(tcs) > 0 {
frame.ToolCalls = tcs
}
}
}
frame.OK = true
return frame
}
// 2. Check if any element of v is a Chatbot message list or Chatbot pair list
for _, elem := range v {
if msgList, ok := elem.([]interface{}); ok && len(msgList) > 0 {
allMaps := true
var maps []map[string]interface{}
for _, item := range msgList {
if m, ok := item.(map[string]interface{}); ok {
if _, hasRole := m["role"]; hasRole {
maps = append(maps, m)
continue
}
}
allMaps = false
break
}
if allMaps && len(maps) > 0 {
var targetMsg map[string]interface{}
for i := len(maps) - 1; i >= 0; i-- {
if r, _ := maps[i]["role"].(string); r == "assistant" {
targetMsg = maps[i]
break
}
}
if targetMsg == nil {
targetMsg = maps[len(maps)-1]
}
cText := extractContentString(targetMsg["content"])
frame.Content = cText
if r, ok := targetMsg["reasoning_content"].(string); ok {
frame.Reasoning = r
} else if meta, ok := targetMsg["metadata"].(map[string]interface{}); ok {
if logStr, ok := meta["log"].(string); ok && logStr != "" {
frame.Reasoning = logStr
}
}
if tcsRaw, ok := targetMsg["tool_calls"].([]interface{}); ok && len(tcsRaw) > 0 {
b, err := json.Marshal(tcsRaw)
if err == nil {
var tcs []ToolCall
if json.Unmarshal(b, &tcs) == nil {
frame.ToolCalls = tcs
}
}
}
if len(frame.ToolCalls) == 0 {
tcs, clean, has := DetectToolCalls(cText)
if has && len(tcs) > 0 {
frame.ToolCalls = tcs
frame.Content = clean
}
}
frame.OK = true
return frame
}
// Check if elements are pairs [user, assistant]
allPairs := true
var pairs [][]interface{}
for _, item := range msgList {
if p, ok := item.([]interface{}); ok && len(p) == 2 {
pairs = append(pairs, p)
continue
}
allPairs = false
break
}
if allPairs && len(pairs) > 0 {
lastPair := pairs[len(pairs)-1]
cText := extractContentString(lastPair[1])
frame.Content = cText
tcs, clean, has := DetectToolCalls(cText)
if has && len(tcs) > 0 {
frame.ToolCalls = tcs
frame.Content = clean
}
frame.OK = true
return frame
}
}
}
// 3. Check if v itself is a list of message maps: [{"role": "assistant", ...}]
allMaps := true
var maps []map[string]interface{}
for _, item := range v {
if m, ok := item.(map[string]interface{}); ok {
if _, hasRole := m["role"]; hasRole {
maps = append(maps, m)
continue
}
}
allMaps = false
break
}
if allMaps && len(maps) > 0 {
var targetMsg map[string]interface{}
for i := len(maps) - 1; i >= 0; i-- {
if r, _ := maps[i]["role"].(string); r == "assistant" {
targetMsg = maps[i]
break
}
}
if targetMsg == nil {
targetMsg = maps[len(maps)-1]
}
cText := extractContentString(targetMsg["content"])
frame.Content = cText
if r, ok := targetMsg["reasoning_content"].(string); ok {
frame.Reasoning = r
}
if tcsRaw, ok := targetMsg["tool_calls"].([]interface{}); ok && len(tcsRaw) > 0 {
b, err := json.Marshal(tcsRaw)
if err == nil {
var tcs []ToolCall
if json.Unmarshal(b, &tcs) == nil {
frame.ToolCalls = tcs
}
}
}
if len(frame.ToolCalls) == 0 {
tcs, clean, has := DetectToolCalls(cText)
if has && len(tcs) > 0 {
frame.ToolCalls = tcs
frame.Content = clean
}
}
frame.OK = true
return frame
}
// 4. Check if v[0] is an inner pair [content, reasoning]
if inner, ok := v[0].([]interface{}); ok && len(inner) == 2 {
s0, ok0 := inner[0].(string)
s1, ok1 := inner[1].(string)
if ok0 && ok1 {
frame.Content = s0
frame.Reasoning = s1
frame.OK = true
return frame
}
if ok1 {
frame.Content = s1
frame.OK = true
return frame
}
}
// 5. Check if v[0] is a non-empty string or single output
if s, ok := v[0].(string); ok && (s != "" || len(v) == 1) {
frame.Content = s
frame.OK = true
return frame
}
case map[string]interface{}:
if outMap, ok := v["output"].(map[string]interface{}); ok {
if dataArr, ok := outMap["data"].([]interface{}); ok {
b, err := json.Marshal(dataArr)
if err == nil {
subFrame := ParseGradioStreamOutput(string(b))
if subFrame.OK {
if r, ok := v["reasoning_content"].(string); ok && subFrame.Reasoning == "" {
subFrame.Reasoning = r
}
return subFrame
}
}
}
}
if dataArr, ok := v["data"].([]interface{}); ok {
b, err := json.Marshal(dataArr)
if err == nil {
subFrame := ParseGradioStreamOutput(string(b))
if subFrame.OK {
if r, ok := v["reasoning_content"].(string); ok && subFrame.Reasoning == "" {
subFrame.Reasoning = r
}
return subFrame
}
}
}
for _, key := range []string{"text", "content", "response", "data", "value"} {
if s, ok := v[key].(string); ok {
frame.Content = s
frame.OK = true
break
}
}
if r, ok := v["reasoning_content"].(string); ok {
frame.Reasoning = r
} else if r, ok := v["reasoning"].(string); ok {
frame.Reasoning = r
}
if tcsRaw, ok := v["tool_calls"].([]interface{}); ok && len(tcsRaw) > 0 {
b, err := json.Marshal(tcsRaw)
if err == nil {
var tcs []ToolCall
if err := json.Unmarshal(b, &tcs); err == nil {
frame.ToolCalls = tcs
}
}
}
if frame.OK {
return frame
}
}
return frame
}
// ExtractTextFromGradioOutput extracts the assistant text string from Gradio output chunks (compatibility wrapper)
func ExtractTextFromGradioOutput(rawJSON string) (string, bool) {
frame := ParseGradioStreamOutput(rawJSON)
if frame.OK {
return frame.Content, true
}
return "", false
}
// executePredictCompletion handles completions using Gradio 3 direct /run/predict or /api/predict protocol.
func (g *GradioGateway) executePredictCompletion(w http.ResponseWriter, r *http.Request, disc *SpaceDiscovery, gradioData []interface{}, req ChatCompletionRequest, completionID string, createdTime int64, modelName, effUA string) error {
fnIndex := disc.FnIndex
if fnIndex < 0 {
fnIndex = 0
}
predictData := make([]interface{}, len(gradioData))
copy(predictData, gradioData)
if disc.RawTotalInputs > len(predictData) {
for i := len(predictData); i < disc.RawTotalInputs; i++ {
var defVal interface{}
if i < len(disc.DefaultInputs) {
defVal = disc.DefaultInputs[i]
}
predictData = append(predictData, defVal)
}
}
payloadMap := map[string]interface{}{
"data": predictData,
"fn_index": fnIndex,
"session_hash": GenerateUUID(),
}
jsonPayload, err := json.Marshal(payloadMap)
if err != nil {
return fmt.Errorf("failed to marshal predict payload: %w", err)
}
var candidateURLs []string
if disc.APIPrefix != "" {
candidateURLs = append(candidateURLs, fmt.Sprintf("%s%s/run/predict", disc.SpaceURL, disc.APIPrefix))
}
candidateURLs = append(candidateURLs,
fmt.Sprintf("%s/run/predict", disc.SpaceURL),
fmt.Sprintf("%s/api/predict", disc.SpaceURL),
)
var resp *http.Response
var lastErr error
for _, targetURL := range candidateURLs {
curURL := targetURL
makeReq := func() (*http.Request, error) {
req, err := http.NewRequest("POST", curURL, bytes.NewBuffer(jsonPayload))
if err != nil {
return nil, err
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("User-Agent", effUA)
return req, nil
}
resp, lastErr = DoWithFibonacciRetry(g.client, makeReq, 3)
if lastErr == nil && resp != nil && resp.StatusCode == http.StatusOK {
break
}
if resp != nil {
resp.Body.Close()
resp = nil
}
}
if resp == nil {
return fmt.Errorf("upstream Gradio predict error: %w", lastErr)
}
defer resp.Body.Close()
bodyBytes, err := io.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("failed to read Gradio predict response: %w", err)
}
frame := ParseGradioStreamOutput(string(bodyBytes))
if !frame.OK {
return fmt.Errorf("upstream Gradio space returned empty or unparseable response")
}
cleanText := frame.Content
reasoning := frame.Reasoning
toolCalls := frame.ToolCalls
hasTools := len(toolCalls) > 0
if reasoning == "" {
cleanText, reasoning = ExtractThinking(cleanText)
}
if !hasTools {
toolCalls, cleanText, hasTools = DetectToolCalls(cleanText)
}
finishReason := "stop"
var finalContent interface{} = cleanText
if hasTools && len(toolCalls) > 0 {
finishReason = "tool_calls"
if strings.TrimSpace(cleanText) == "" {
finalContent = nil
}
}
if !req.Stream {
WriteCompletionResponse(w, completionID, createdTime, modelName, FinalOutput{
Content: finalContent,
ReasoningContent: reasoning,
ToolCalls: toolCalls,
FinishReason: finishReason,
})
return nil
}
flusher, _ := w.(http.Flusher)
streamer := NewStreamer(w, flusher, completionID, createdTime, modelName)
if reasoning != "" {
streamer.Reasoning(reasoning)
}
if hasTools && len(toolCalls) > 0 {
for i, tc := range toolCalls {
iCopy := i
tc.Index = &iCopy
streamer.ToolCallDelta(tc)
}
} else if cleanText != "" {
streamer.Content(cleanText)
}
streamer.Finish(finishReason)
streamer.Done()
return nil
}
// executeQueueCompletion handles completions using Gradio queue SSE protocol (/queue/join + /queue/data).
func (g *GradioGateway) executeQueueCompletion(w http.ResponseWriter, r *http.Request, disc *SpaceDiscovery, gradioData []interface{}, req ChatCompletionRequest, completionID string, createdTime int64, modelName, effUA string) error {
fnIndex := disc.FnIndex
if fnIndex < 0 {
fnIndex = 0
}
queueData := make([]interface{}, len(gradioData))
copy(queueData, gradioData)
if disc.RawTotalInputs > len(queueData) {
for i := len(queueData); i < disc.RawTotalInputs; i++ {
var defVal interface{}
if i < len(disc.DefaultInputs) {
defVal = disc.DefaultInputs[i]
}
queueData = append(queueData, defVal)
}
}
sessionHash := GenerateUUID()
payloadMap := map[string]interface{}{
"data": queueData,
"fn_index": fnIndex,
"session_hash": sessionHash,
}
jsonPayload, err := json.Marshal(payloadMap)
if err != nil {
return fmt.Errorf("failed to marshal queue payload: %w", err)
}
var joinURLs []string
if disc.APIPrefix != "" {
joinURLs = append(joinURLs, fmt.Sprintf("%s%s/queue/join", disc.SpaceURL, disc.APIPrefix))
}
joinURLs = append(joinURLs, fmt.Sprintf("%s/queue/join", disc.SpaceURL))
var resp *http.Response
var lastErr error
for _, targetURL := range joinURLs {
curURL := targetURL
makeReq := func() (*http.Request, error) {
req, err := http.NewRequest("POST", curURL, bytes.NewBuffer(jsonPayload))
if err != nil {
return nil, err
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("User-Agent", effUA)
return req, nil
}
resp, lastErr = DoWithFibonacciRetry(g.client, makeReq, 3)
if lastErr == nil && resp != nil && resp.StatusCode == http.StatusOK {
break
}
if resp != nil {
resp.Body.Close()
resp = nil
}
}
if resp == nil {
log.Printf("Gradio /queue/join unavailable (%v), falling back to /run/predict protocol...", lastErr)
return g.executePredictCompletion(w, r, disc, gradioData, req, completionID, createdTime, modelName, effUA)
}
resp.Body.Close()
// 2. Connect to Gradio SSE queue data stream
var dataURLs []string
if disc.APIPrefix != "" {
dataURLs = append(dataURLs, fmt.Sprintf("%s%s/queue/data?session_hash=%s", disc.SpaceURL, disc.APIPrefix, sessionHash))
}
dataURLs = append(dataURLs, fmt.Sprintf("%s/queue/data?session_hash=%s", disc.SpaceURL, sessionHash))
var streamResp *http.Response
for _, targetURL := range dataURLs {
curURL := targetURL
makeStreamReq := func() (*http.Request, error) {
req, err := http.NewRequest("GET", curURL, nil)
if err != nil {
return nil, err
}
req.Header.Set("Accept", "text/event-stream")
req.Header.Set("User-Agent", effUA)
return req, nil
}
streamResp, lastErr = DoWithFibonacciRetry(g.client, makeStreamReq, 3)
if lastErr == nil && streamResp != nil && streamResp.StatusCode == http.StatusOK {
break
}
if streamResp != nil {
streamResp.Body.Close()
streamResp = nil
}
}
if streamResp == nil {
log.Printf("Gradio /queue/data unavailable (%v), falling back to /run/predict protocol...", lastErr)
return g.executePredictCompletion(w, r, disc, gradioData, req, completionID, createdTime, modelName, effUA)
}
defer streamResp.Body.Close()
type queueMsg struct {
Msg string `json:"msg"`
EventID string `json:"event_id,omitempty"`
Success *bool `json:"success,omitempty"`
Output map[string]interface{} `json:"output,omitempty"`
}
// 3. Handle Non-Streaming vs Streaming
if !req.Stream {
reader := bufio.NewReader(streamResp.Body)
var latestFrame GradioOutputFrame
for {
line, err := reader.ReadString('\n')
if err != nil {
break
}
line = strings.TrimRight(line, "\r\n")
if strings.HasPrefix(line, "data: ") {
dataStr := strings.TrimPrefix(line, "data: ")
var qMsg queueMsg
if err := json.Unmarshal([]byte(dataStr), &qMsg); err != nil {
continue
}
if qMsg.Msg == "close_stream" {
break
}
if qMsg.Msg == "process_generating" || qMsg.Msg == "process_completed" {
if qMsg.Success != nil && !*qMsg.Success {
errBytes, _ := json.Marshal(qMsg.Output)
errMsg := extractGradioErrorMessage(string(errBytes))
log.Printf("Upstream Gradio error: %s", errMsg)
return fmt.Errorf("upstream Gradio error: %s", errMsg)
}
if qMsg.Output != nil && qMsg.Output["data"] != nil {
dataBytes, _ := json.Marshal(qMsg.Output["data"])
frame := ParseGradioStreamOutput(string(dataBytes))
if frame.OK {
if frame.IsDelta {
latestFrame.Content += frame.Content
} else {
if frame.Content != "" || latestFrame.Content == "" {
latestFrame.Content = frame.Content
}
}
if frame.Reasoning != "" || latestFrame.Reasoning == "" {
latestFrame.Reasoning = frame.Reasoning
}
if len(frame.ToolCalls) > 0 {
latestFrame.ToolCalls = frame.ToolCalls
}
latestFrame.OK = true
}
}
if qMsg.Msg == "process_completed" {
break
}
}
}
}
if !latestFrame.OK {
return fmt.Errorf("upstream Gradio space returned empty or unparseable response")
}
cleanText := latestFrame.Content
reasoning := latestFrame.Reasoning
toolCalls := latestFrame.ToolCalls
hasTools := len(toolCalls) > 0
if reasoning == "" {
cleanText, reasoning = ExtractThinking(cleanText)
}
if !hasTools {
toolCalls, cleanText, hasTools = DetectToolCalls(cleanText)
}
finishReason := "stop"
var finalContent interface{} = cleanText
if hasTools && len(toolCalls) > 0 {
finishReason = "tool_calls"
if strings.TrimSpace(cleanText) == "" {
finalContent = nil
}
}
WriteCompletionResponse(w, completionID, createdTime, modelName, FinalOutput{
Content: finalContent,
ReasoningContent: reasoning,
ToolCalls: toolCalls,
FinishReason: finishReason,
})
disc.Protocol = "queue"
return nil
}
// 4. Streaming Mode
flusher, _ := w.(http.Flusher)
streamer := NewStreamer(w, flusher, completionID, createdTime, modelName)
thinkFilter := NewStreamThinkingFilter()
toolFilter := NewStreamToolCallFilter()
reader := bufio.NewReader(streamResp.Body)
var prevContent string
var prevReasoning string
prevToolArgs := make(map[int]string)
nativeReasoningSeen := false
nativeToolCallsSeen := false
var streamErr error
for {
line, err := reader.ReadString('\n')
if err != nil {
break
}
line = strings.TrimRight(line, "\r\n")
if strings.HasPrefix(line, "data: ") {
dataStr := strings.TrimPrefix(line, "data: ")
var qMsg queueMsg
if err := json.Unmarshal([]byte(dataStr), &qMsg); err != nil {
continue
}
if qMsg.Msg == "close_stream" {
break
}
if qMsg.Msg == "process_generating" || qMsg.Msg == "process_completed" {
if qMsg.Success != nil && !*qMsg.Success {
errBytes, _ := json.Marshal(qMsg.Output)
errMsg := extractGradioErrorMessage(string(errBytes))
log.Printf("Upstream Gradio error: %s", errMsg)
if !streamer.started {
return fmt.Errorf("upstream Gradio error: %s", errMsg)
}
streamer.Error(errMsg)
streamErr = fmt.Errorf("upstream Gradio error: %s", errMsg)
break
}
if qMsg.Output != nil && qMsg.Output["data"] != nil {
dataBytes, _ := json.Marshal(qMsg.Output["data"])
frame := ParseGradioStreamOutput(string(dataBytes))
if frame.OK {
// 1. Native reasoning handling
if frame.Reasoning != "" || nativeReasoningSeen {
nativeReasoningSeen = true
if len(frame.Reasoning) > len(prevReasoning) && strings.HasPrefix(frame.Reasoning, prevReasoning) {
delta := frame.Reasoning[len(prevReasoning):]
streamer.Reasoning(delta)
prevReasoning = frame.Reasoning
} else if prevReasoning == "" && frame.Reasoning != "" {
streamer.Reasoning(frame.Reasoning)
prevReasoning = frame.Reasoning
}
}
// 2. Native tool call handling
if len(frame.ToolCalls) > 0 || nativeToolCallsSeen {
nativeToolCallsSeen = true
for i, tc := range frame.ToolCalls {
prevArgs := prevToolArgs[i]
fullArgs := tc.Function.Arguments
if len(fullArgs) > len(prevArgs) && strings.HasPrefix(fullArgs, prevArgs) {
deltaArgs := fullArgs[len(prevArgs):]
iCopy := i
streamer.ToolCallDelta(ToolCall{
Index: &iCopy,
ID: tc.ID,
Type: tc.Type,
Function: ToolCallFunction{
Name: tc.Function.Name,
Arguments: deltaArgs,
},
})
prevToolArgs[i] = fullArgs
} else if prevArgs == "" {
iCopy := i
streamer.ToolCallDelta(ToolCall{
Index: &iCopy,
ID: tc.ID,
Type: tc.Type,
Function: ToolCallFunction{
Name: tc.Function.Name,
Arguments: fullArgs,
},
})
prevToolArgs[i] = fullArgs
}
}
}
// 3. Content handling
currentText := frame.Content
var delta string
if frame.IsDelta {
delta = frame.Content
prevContent += delta
} else {
if strings.HasPrefix(currentText, prevContent) {
delta = currentText[len(prevContent):]
} else if prevContent == "" {
delta = currentText
} else {
delta = currentText
}
prevContent = currentText
}
if delta != "" {
if nativeReasoningSeen || nativeToolCallsSeen {
if nativeReasoningSeen && nativeToolCallsSeen {
streamer.Content(delta)
} else if nativeReasoningSeen {
toolFilter.Feed(delta, func(cleanChunk string) {
if cleanChunk != "" {
streamer.Content(cleanChunk)
}
}, func(tc ToolCall) {
streamer.ToolCallDelta(tc)
})
} else {
thinkFilter.Feed(delta, func(contentChunk string) {
if contentChunk != "" {
streamer.Content(contentChunk)
}
}, func(reasoningChunk string) {
if reasoningChunk != "" {
streamer.Reasoning(reasoningChunk)
}
})
}
} else {
thinkFilter.Feed(delta, func(contentChunk string) {
toolFilter.Feed(contentChunk, func(cleanChunk string) {
if cleanChunk != "" {
streamer.Content(cleanChunk)
}
}, func(tc ToolCall) {
streamer.ToolCallDelta(tc)
})
}, func(reasoningChunk string) {
if reasoningChunk != "" {
streamer.Reasoning(reasoningChunk)
}
})
}
}
}
}
if qMsg.Msg == "process_completed" {
break
}
}
}
}
if streamErr != nil {
return streamErr
}
// Flush remaining tokens in filters if used
if !nativeReasoningSeen {
thinkFilter.Flush(func(contentChunk string) {
if !nativeToolCallsSeen {
toolFilter.Feed(contentChunk, func(cleanChunk string) {
if cleanChunk != "" {
streamer.Content(cleanChunk)
}
}, func(tc ToolCall) {
streamer.ToolCallDelta(tc)
})
} else if contentChunk != "" {
streamer.Content(contentChunk)
}
}, func(reasoningChunk string) {
if reasoningChunk != "" {
streamer.Reasoning(reasoningChunk)
}
})
}
if !nativeToolCallsSeen {
toolFilter.Flush(func(cleanChunk string) {
if cleanChunk != "" {
streamer.Content(cleanChunk)
}
}, func(tc ToolCall) {
streamer.ToolCallDelta(tc)
})
}
finishReason := "stop"
if nativeToolCallsSeen || toolFilter.emittedCall {
finishReason = "tool_calls"
}
streamer.Finish(finishReason)
streamer.Done()
disc.Protocol = "queue"
return nil
}
// ExecuteChatCompletion handles both streaming and non-streaming requests.
func (g *GradioGateway) ExecuteChatCompletion(w http.ResponseWriter, r *http.Request, req ChatCompletionRequest) error {
effUA := EffectiveUserAgent(r)
// Target space selection: check request headers or fallback to default space
spaceURL := g.defaultURL
if hdr := r.Header.Get("X-Gradio-Space"); hdr != "" {
spaceURL = hdr
} else if hdr := r.Header.Get("X-Space-URL"); hdr != "" {
spaceURL = hdr
}
disc := g.GetDiscovery(spaceURL, effUA)
modelName := req.Model
if modelName == "" {
if ConfiguredModelName != "" {
modelName = ConfiguredModelName
} else {
modelName = disc.PrimaryModel
}
}
gradioData, err := g.BuildGradioPayload(disc, req)
if err != nil {
return fmt.Errorf("failed to build Gradio payload: %w", err)
}
completionID := "chatcmpl-" + GenerateUUID()
createdTime := time.Now().Unix()
// If the resolved protocol is predict (e.g. Gradio 3), execute predict completion directly
if disc.Protocol == "predict" {
return g.executePredictCompletion(w, r, disc, gradioData, req, completionID, createdTime, modelName, effUA)
}
if disc.Protocol == "queue" {
return g.executeQueueCompletion(w, r, disc, gradioData, req, completionID, createdTime, modelName, effUA)
}
isV2 := disc.Protocol == "call_v2"
buildPayload := func(v2 bool) ([]byte, error) {
if v2 {
v2Payload := make(map[string]interface{})
for idx, val := range gradioData {
if idx < len(disc.ParamMappings) && disc.ParamMappings[idx].ParamType == "state" {
continue
}
pName := ""
if idx < len(disc.ParamMappings) {
if disc.ParamMappings[idx].ParamName != "" {
pName = disc.ParamMappings[idx].ParamName
} else if disc.ParamMappings[idx].Label != "" {
pName = disc.ParamMappings[idx].Label
}
}
if pName == "" {
pName = fmt.Sprintf("param_%d", idx)
}
v2Payload[pName] = val
}
return json.Marshal(v2Payload)
}
payloadMap := map[string]interface{}{
"data": gradioData,
"session_hash": GenerateUUID(),
}
return json.Marshal(payloadMap)
}
callPath := "call"
if isV2 {
callPath = "call/v2"
}
jsonPayload, err := buildPayload(isV2)
if err != nil {
return fmt.Errorf("failed to encode request: %w", err)
}
// 1. Submit to /call/{endpoint} or /call/v2/{endpoint}
callURL := fmt.Sprintf("%s%s/%s/%s", disc.SpaceURL, disc.APIPrefix, callPath, disc.CleanEndpoint)
makeCallReq := func() (*http.Request, error) {
r, err := http.NewRequest("POST", callURL, bytes.NewBuffer(jsonPayload))
if err != nil {
return nil, err
}
r.Header.Set("Content-Type", "application/json")
r.Header.Set("User-Agent", effUA)
return r, nil
}
resp, err := DoWithFibonacciRetry(g.client, makeCallReq, 5)
if err != nil && isV2 {
log.Printf("Gradio /call/v2 endpoint failed (%v), falling back to /call...", err)
isV2 = false
callPath = "call"
jsonPayload, _ = buildPayload(false)
callURL = fmt.Sprintf("%s%s/%s/%s", disc.SpaceURL, disc.APIPrefix, callPath, disc.CleanEndpoint)
resp, err = DoWithFibonacciRetry(g.client, makeCallReq, 3)
}
if err != nil {
if fg, ok := extractFailedGeneration(err.Error()); ok {
if tcs, _, has := DetectToolCalls(fg); has && len(tcs) > 0 {
if !req.Stream {
WriteCompletionResponse(w, completionID, createdTime, modelName, FinalOutput{
ToolCalls: tcs,
FinishReason: "tool_calls",
})
return nil
}
flusher, _ := w.(http.Flusher)
streamer := NewStreamer(w, flusher, completionID, createdTime, modelName)
for i, tc := range tcs {
iCopy := i
tc.Index = &iCopy
streamer.ToolCallDelta(tc)
}
streamer.Finish("tool_calls")
streamer.Done()
return nil
}
}
// If call failed, try without APIPrefix
altCallURL := fmt.Sprintf("%s/%s/%s", disc.SpaceURL, callPath, disc.CleanEndpoint)
makeAltReq := func() (*http.Request, error) {
r, err := http.NewRequest("POST", altCallURL, bytes.NewBuffer(jsonPayload))
if err != nil {
return nil, err
}
r.Header.Set("Content-Type", "application/json")
r.Header.Set("User-Agent", effUA)
return r, nil
}
resp, err = DoWithFibonacciRetry(g.client, makeAltReq, 3)
if err != nil {
if fg, ok := extractFailedGeneration(err.Error()); ok {
if tcs, _, has := DetectToolCalls(fg); has && len(tcs) > 0 {
if !req.Stream {
WriteCompletionResponse(w, completionID, createdTime, modelName, FinalOutput{
ToolCalls: tcs,
FinishReason: "tool_calls",
})
return nil
}
flusher, _ := w.(http.Flusher)
streamer := NewStreamer(w, flusher, completionID, createdTime, modelName)
for i, tc := range tcs {
iCopy := i
tc.Index = &iCopy
streamer.ToolCallDelta(tc)
}
streamer.Finish("tool_calls")
streamer.Done()
return nil
}
}
if strings.Contains(err.Error(), "404") || strings.Contains(err.Error(), "405") || strings.Contains(err.Error(), "500") {
log.Printf("Gradio /call endpoint unavailable (%v), falling back to /queue/join protocol...", err)
return g.executeQueueCompletion(w, r, disc, gradioData, req, completionID, createdTime, modelName, effUA)
}
return fmt.Errorf("upstream Gradio call error: %w", err)
}
}
defer resp.Body.Close()
var joinRes GradioJoinResponse
if err := json.NewDecoder(resp.Body).Decode(&joinRes); err != nil || joinRes.EventID == "" {
log.Printf("Gradio /call returned non-SSE response, falling back to /queue/join protocol...")
return g.executeQueueCompletion(w, r, disc, gradioData, req, completionID, createdTime, modelName, effUA)
}
// 2. Connect to Gradio SSE EventStream
streamURL := fmt.Sprintf("%s%s/call/%s/%s", disc.SpaceURL, disc.APIPrefix, disc.CleanEndpoint, joinRes.EventID)
makeStreamReq := func() (*http.Request, error) {
r, err := http.NewRequest("GET", streamURL, nil)
if err != nil {
return nil, err
}
r.Header.Set("Accept", "text/event-stream")
r.Header.Set("User-Agent", effUA)
return r, nil
}
streamResp, err := DoWithFibonacciRetry(g.client, makeStreamReq, 5)
if err != nil {
if fg, ok := extractFailedGeneration(err.Error()); ok {
if tcs, _, has := DetectToolCalls(fg); has && len(tcs) > 0 {
if !req.Stream {
WriteCompletionResponse(w, completionID, createdTime, modelName, FinalOutput{
ToolCalls: tcs,
FinishReason: "tool_calls",
})
return nil
}
flusher, _ := w.(http.Flusher)
streamer := NewStreamer(w, flusher, completionID, createdTime, modelName)
for i, tc := range tcs {
iCopy := i
tc.Index = &iCopy
streamer.ToolCallDelta(tc)
}
streamer.Finish("tool_calls")
streamer.Done()
return nil
}
}
log.Printf("Gradio /call stream connection failed (%v), falling back to /queue/join protocol...", err)
return g.executeQueueCompletion(w, r, disc, gradioData, req, completionID, createdTime, modelName, effUA)
}
defer streamResp.Body.Close()
// 3. Handle Non-Streaming vs Streaming
if !req.Stream {
reader := bufio.NewReader(streamResp.Body)
var latestFrame GradioOutputFrame
currentEvent := ""
for {
line, err := reader.ReadString('\n')
if err != nil {
break
}
line = strings.TrimRight(line, "\r\n")
if strings.HasPrefix(line, "event: ") {
currentEvent = strings.TrimPrefix(line, "event: ")
continue
}
if strings.HasPrefix(line, "data: ") {
dataStr := strings.TrimPrefix(line, "data: ")
if currentEvent == "error" {
if fg, ok := extractFailedGeneration(dataStr); ok {
if tcs, _, has := DetectToolCalls(fg); has && len(tcs) > 0 {
latestFrame = GradioOutputFrame{
ToolCalls: tcs,
OK: true,
}
break
}
}
errMsg := extractGradioErrorMessage(dataStr)
if isProtocolOrInputError(errMsg) {
log.Printf("Upstream Gradio /call stream error (%s), falling back to /queue/join protocol...", errMsg)
return g.executeQueueCompletion(w, r, disc, gradioData, req, completionID, createdTime, modelName, effUA)
}
log.Printf("Upstream Gradio error: %s", errMsg)
return fmt.Errorf("upstream Gradio error: %s", errMsg)
}
if frame := ParseGradioStreamOutput(dataStr); frame.OK {
if frame.Content != "" || latestFrame.Content == "" {
latestFrame.Content = frame.Content
}
if frame.Reasoning != "" || latestFrame.Reasoning == "" {
latestFrame.Reasoning = frame.Reasoning
}
if len(frame.ToolCalls) > 0 {
latestFrame.ToolCalls = frame.ToolCalls
}
latestFrame.OK = true
}
if currentEvent == "complete" {
break
}
}
}
if !latestFrame.OK {
log.Printf("Gradio /call returned empty or unparseable response, falling back to /queue/join protocol...")
return g.executeQueueCompletion(w, r, disc, gradioData, req, completionID, createdTime, modelName, effUA)
}
cleanText := latestFrame.Content
reasoning := latestFrame.Reasoning
toolCalls := latestFrame.ToolCalls
hasTools := len(toolCalls) > 0
if reasoning == "" {
cleanText, reasoning = ExtractThinking(cleanText)
}
if !hasTools {
toolCalls, cleanText, hasTools = DetectToolCalls(cleanText)
}
finishReason := "stop"
var finalContent interface{} = cleanText
if hasTools && len(toolCalls) > 0 {
finishReason = "tool_calls"
if strings.TrimSpace(cleanText) == "" {
finalContent = nil
}
}
WriteCompletionResponse(w, completionID, createdTime, modelName, FinalOutput{
Content: finalContent,
ReasoningContent: reasoning,
ToolCalls: toolCalls,
FinishReason: finishReason,
})
return nil
}
// 4. Streaming Mode
flusher, _ := w.(http.Flusher)
streamer := NewStreamer(w, flusher, completionID, createdTime, modelName)
thinkFilter := NewStreamThinkingFilter()
toolFilter := NewStreamToolCallFilter()
reader := bufio.NewReader(streamResp.Body)
var prevContent string
var prevReasoning string
prevToolArgs := make(map[int]string)
nativeReasoningSeen := false
nativeToolCallsSeen := false
currentEvent := ""
var streamErr error
for {
line, err := reader.ReadString('\n')
if err != nil {
break
}
line = strings.TrimRight(line, "\r\n")
if strings.HasPrefix(line, "event: ") {
currentEvent = strings.TrimPrefix(line, "event: ")
continue
}
if strings.HasPrefix(line, "data: ") {
dataStr := strings.TrimPrefix(line, "data: ")
if currentEvent == "error" {
if fg, ok := extractFailedGeneration(dataStr); ok {
if tcs, _, has := DetectToolCalls(fg); has && len(tcs) > 0 {
for i, tc := range tcs {
iCopy := i
tc.Index = &iCopy
streamer.ToolCallDelta(tc)
}
nativeToolCallsSeen = true
break
}
}
errMsg := extractGradioErrorMessage(dataStr)
if !streamer.started {
if isProtocolOrInputError(errMsg) {
log.Printf("Upstream Gradio /call stream error (%s), falling back to /queue/join protocol...", errMsg)
return g.executeQueueCompletion(w, r, disc, gradioData, req, completionID, createdTime, modelName, effUA)
}
log.Printf("Upstream Gradio error: %s", errMsg)
return fmt.Errorf("upstream Gradio error: %s", errMsg)
}
log.Printf("Upstream Gradio error: %s", errMsg)
streamer.Error(errMsg)
streamErr = fmt.Errorf("upstream Gradio error: %s", errMsg)
break
}
frame := ParseGradioStreamOutput(dataStr)
if frame.OK {
// 1. Native reasoning handling
if frame.Reasoning != "" || nativeReasoningSeen {
nativeReasoningSeen = true
var deltaReasoning string
if strings.HasPrefix(frame.Reasoning, prevReasoning) {
deltaReasoning = frame.Reasoning[len(prevReasoning):]
} else if prevReasoning == "" {
deltaReasoning = frame.Reasoning
} else {
deltaReasoning = frame.Reasoning
}
prevReasoning = frame.Reasoning
if deltaReasoning != "" {
streamer.Reasoning(deltaReasoning)
}
}
// 2. Native tool calls handling
if len(frame.ToolCalls) > 0 {
nativeToolCallsSeen = true
for idx, tc := range frame.ToolCalls {
prevArgs, started := prevToolArgs[idx]
currArgs := tc.Function.Arguments
idxCopy := idx
if !started {
tcDelta := ToolCall{
Index: &idxCopy,
ID: tc.ID,
Type: tc.Type,
Function: ToolCallFunction{
Name: tc.Function.Name,
Arguments: currArgs,
},
}
streamer.ToolCallDelta(tcDelta)
prevToolArgs[idx] = currArgs
} else if len(currArgs) > len(prevArgs) {
var argDelta string
if strings.HasPrefix(currArgs, prevArgs) {
argDelta = currArgs[len(prevArgs):]
} else {
argDelta = currArgs[len(prevArgs):]
}
if argDelta != "" {
tcDelta := ToolCall{
Index: &idxCopy,
Function: ToolCallFunction{
Arguments: argDelta,
},
}
streamer.ToolCallDelta(tcDelta)
}
prevToolArgs[idx] = currArgs
}
}
}
// 3. Content handling
currentText := frame.Content
var delta string
if strings.HasPrefix(currentText, prevContent) {
delta = currentText[len(prevContent):]
} else if prevContent == "" {
delta = currentText
} else {
delta = currentText
}
prevContent = currentText
if delta != "" {
if nativeReasoningSeen || nativeToolCallsSeen {
if nativeReasoningSeen && nativeToolCallsSeen {
streamer.Content(delta)
} else if nativeReasoningSeen {
toolFilter.Feed(delta, func(cleanChunk string) {
if cleanChunk != "" {
streamer.Content(cleanChunk)
}
}, func(tc ToolCall) {
streamer.ToolCallDelta(tc)
})
} else {
thinkFilter.Feed(delta, func(contentChunk string) {
if contentChunk != "" {
streamer.Content(contentChunk)
}
}, func(reasoningChunk string) {
if reasoningChunk != "" {
streamer.Reasoning(reasoningChunk)
}
})
}
} else {
thinkFilter.Feed(delta, func(contentChunk string) {
toolFilter.Feed(contentChunk, func(cleanChunk string) {
if cleanChunk != "" {
streamer.Content(cleanChunk)
}
}, func(tc ToolCall) {
streamer.ToolCallDelta(tc)
})
}, func(reasoningChunk string) {
if reasoningChunk != "" {
streamer.Reasoning(reasoningChunk)
}
})
}
}
}
if currentEvent == "complete" {
break
}
}
}
if streamErr != nil {
return nil
}
// Flush remaining tokens in filters if used
if !nativeReasoningSeen {
thinkFilter.Flush(func(contentChunk string) {
if !nativeToolCallsSeen {
toolFilter.Feed(contentChunk, func(cleanChunk string) {
if cleanChunk != "" {
streamer.Content(cleanChunk)
}
}, func(tc ToolCall) {
streamer.ToolCallDelta(tc)
})
} else if contentChunk != "" {
streamer.Content(contentChunk)
}
}, func(reasoningChunk string) {
if reasoningChunk != "" {
streamer.Reasoning(reasoningChunk)
}
})
}
if !nativeToolCallsSeen {
toolFilter.Flush(func(cleanChunk string) {
if cleanChunk != "" {
streamer.Content(cleanChunk)
}
}, func(tc ToolCall) {
streamer.ToolCallDelta(tc)
})
}
if !streamer.started {
log.Printf("Gradio /call closed stream without sending content, falling back to /queue/join protocol...")
return g.executeQueueCompletion(w, r, disc, gradioData, req, completionID, createdTime, modelName, effUA)
}
if nativeToolCallsSeen || toolFilter.emittedCall {
streamer.Finish("tool_calls")
} else {
streamer.Finish("stop")
}
streamer.Done()
return nil
}
// ---------------------------------------------------------------------------
// HTTP routes & main server
// ---------------------------------------------------------------------------
func main() {
spaceFlag := flag.String("space", DefaultSpaceURL, "Target Gradio Space URL")
flag.StringVar(spaceFlag, "url", DefaultSpaceURL, "Alias for -space")
flag.StringVar(spaceFlag, "endpoint", DefaultSpaceURL, "Alias for -space")
modelFlag := flag.String("model", "", "Exposed model name override (default: auto-detected)")
portFlag := flag.Int("port", 8080, "Gateway HTTP server port")
hostFlag := flag.String("host", "0.0.0.0", "Gateway HTTP server host")
socksFlag := flag.String("socks", "", "Optional SOCKS5 proxy URL (e.g. socks5://127.0.0.1:1080)")
flag.StringVar(socksFlag, "proxy", "", "Alias for -socks")
flag.StringVar(socksFlag, "socks5", "", "Alias for -socks")
uaFlag := flag.String("user-agent", "", "Custom User-Agent header")
flag.StringVar(uaFlag, "ua", "", "Alias for -user-agent")
timeoutFlag := flag.Int("timeout", 300, "Upstream timeout in seconds")
flag.Parse()
// Environment variable fallbacks
if envSpace := os.Getenv("GRADIO_SPACE_URL"); envSpace != "" && *spaceFlag == DefaultSpaceURL {
*spaceFlag = envSpace
}
if *socksFlag == "" {
for _, envName := range []string{"ALL_PROXY", "all_proxy", "SOCKS5_PROXY", "socks5_proxy", "SOCKS_PROXY", "socks_proxy"} {
if p := os.Getenv(envName); p != "" {
*socksFlag = p
break
}
}
}
if *uaFlag != "" {
ConfiguredUserAgent = *uaFlag
}
if *modelFlag != "" {
ConfiguredModelName = *modelFlag
}
gateway := NewGradioGateway(*spaceFlag, *socksFlag, time.Duration(*timeoutFlag)*time.Second)
mux := http.NewServeMux()
// Health and Info
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
EnableCORS(w)
if r.Method == "OPTIONS" {
w.WriteHeader(http.StatusOK)
return
}
if r.URL.Path != "/" {
http.NotFound(w, r)
return
}
disc := gateway.GetDiscovery("", EffectiveUserAgent(r))
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(map[string]interface{}{
"status": "running",
"service": "gr2gw",
"space_url": disc.SpaceURL,
"title": disc.Title,
"gradio_version": disc.GradioVersion,
"flavor": disc.Flavor,
"protocol": disc.Protocol,
"endpoint": disc.Endpoint,
"primary_model": disc.PrimaryModel,
"models": disc.Models,
"total_inputs": disc.TotalInputs,
"history_format": disc.HistoryFormat,
"tool_call_mode": disc.ToolCallMode,
"param_mappings": disc.ParamMappings,
})
})
// Models list
handleModels := func(w http.ResponseWriter, r *http.Request) {
EnableCORS(w)
if r.Method == "OPTIONS" {
w.WriteHeader(http.StatusOK)
return
}
spaceURL := gateway.defaultURL
if hdr := r.Header.Get("X-Gradio-Space"); hdr != "" {
spaceURL = hdr
} else if hdr := r.Header.Get("X-Space-URL"); hdr != "" {
spaceURL = hdr
}
disc := gateway.GetDiscovery(spaceURL, EffectiveUserAgent(r))
resp := ModelsResponse{
Object: "list",
Data: disc.GetModelList(),
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(resp)
}
mux.HandleFunc("/models", handleModels)
mux.HandleFunc("/v1/models", handleModels)
// Chat completions
handleCompletions := func(w http.ResponseWriter, r *http.Request) {
EnableCORS(w)
if r.Method == "OPTIONS" {
w.WriteHeader(http.StatusOK)
return
}
if r.Method != "POST" {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
var req ChatCompletionRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
http.Error(w, fmt.Sprintf("Invalid JSON request: %v", err), http.StatusBadRequest)
return
}
if err := gateway.ExecuteChatCompletion(w, r, req); err != nil {
log.Printf("Chat completion error: %v", err)
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusBadGateway)
json.NewEncoder(w).Encode(map[string]interface{}{
"error": map[string]interface{}{
"message": err.Error(),
"type": "upstream_error",
"code": http.StatusBadGateway,
},
})
return
}
}
mux.HandleFunc("/chat/completions", handleCompletions)
mux.HandleFunc("/v1/chat/completions", handleCompletions)
addr := fmt.Sprintf("%s:%d", *hostFlag, *portFlag)
log.Printf("gr2gw listening on %s (target space: %s)", addr, *spaceFlag)
if *socksFlag != "" {
log.Printf("Using SOCKS5 proxy: %s", *socksFlag)
}
server := &http.Server{
Addr: addr,
Handler: mux,
ReadTimeout: time.Duration(*timeoutFlag+30) * time.Second,
WriteTimeout: time.Duration(*timeoutFlag+30) * time.Second,
}
if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
log.Fatalf("Server failed: %v", err)
}
}