From 751fe3c54f9ef495b6c0bdf8976a7efbf029ca3d Mon Sep 17 00:00:00 2001 From: Luxferre Date: Mon, 7 Sep 2026 11:53:44 +0300 Subject: [PATCH] feat: universal heuristic engine for gradio version, flavor, protocol and tool calling resolution --- README.md | 121 +++++++- gr2gw.go | 836 ++++++++++++++++++++++++++++++++++++++------------ gr2gw_test.go | 281 +++++++++++++++++ 3 files changed, 1027 insertions(+), 211 deletions(-) diff --git a/README.md b/README.md index 53291e7..2999c61 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,20 @@ Default demo space: `https://tencent-hy3.hf.space` ## Features - **Zero external dependencies**: pure Go standard library (`net/http`, `encoding/json`, `bufio`, etc.). -- **Automatic space introspection**: dynamically queries `/gradio_api/info`, `/config`, and Hugging Face space metadata to discover models, endpoints, and input parameter mappings. +- **Universal heuristic discovery engine**: + - Automatically interrogates `/gradio_api/info`, `/config`, and Hugging Face metadata without endpoint-specific hardcoding. + - Detects Gradio runtime versions across v3, v4, v5, and v6. + - Classifies application architecture into `ChatInterface`, `Blocks (Chat)`, `Blocks (Multimodal Chat)`, `Interface`, and `Generic`. + - Disambiguation scoring engine evaluates candidate endpoints, filtering out UI resets, clears, retries, likes, and utility triggers to pinpoint primary conversational completion functions. + - Correlates semantic parameter names from `/gradio_api/info` with component IDs from `/config` to reconstruct parameter mappings even when component labels are obfuscated. +- **Dual protocol support with auto-fallback**: + - Supports modern Gradio 4/5/6 `/call` SSE protocol with persistent session hashes. + - Supports legacy Gradio 3 `/run/predict` and `/api/predict` direct execution protocol. + - Instant zero-latency fallback from `/call` to `/run/predict` upon HTTP 404 or 405 status codes. +- **Tool calling support classification**: + - Automatically identifies tool calling mechanisms: `native_slot`, `prompt_augmented_system`, `prompt_augmented_first_turn`, or `prompt_augmented_single_prompt`. + - Upstream tool call recovery intercepts `tool_use_failed` errors and extracts function names and JSON arguments. + - Real-time sliding-window `` tag interceptor emits structured OpenAI tool call chunks. - **Native Tencent Hunyuan 3 (`tencent-hy3`) support**: - Full native zero-degradation handling for official spaces like `https://tencent-hy3.hf.space`. - Maps `functions_json_str` natively without polluting the system prompt. @@ -26,17 +39,111 @@ Default demo space: `https://tencent-hy3.hf.space` - Detects `...` tags in real time as fallback for standard spaces. - Separates reasoning into `delta.reasoning_content` (streaming) and `message.reasoning_content` (non-streaming). - Keeps `content` clean without tag leakage. -- **Full tool calling & function interception**: - - Formats schemas into native `functions_json_str` (Hy3) or system prompts (standard spaces). - - Intercepts and recovers tool calls from upstream `tool_use_failed` errors containing `failed_generation`. - - **`StreamToolCallFilter`**: stateful sliding-window filter that prevents `` tags from leaking into `delta.content`. Emits structured OpenAI `delta.tool_calls` chunks and sets `finish_reason: "tool_calls"`. - - Seamlessly maintains multi-turn context when tool results are submitted back via `role: "tool"`. - **Built-in SOCKS5 proxy client**: - Full RFC 1928 / RFC 1929 implementation with domain resolution (`socks5h://`), IPv4, IPv6, and username/password auth. - **Dynamic space override**: - Switch the target Gradio space on-the-fly per request using the `X-Gradio-Space` or `X-Space-URL` HTTP headers. - **Fibonacci retry engine**: - - Resilient backoff retry mechanism (1s, 1s, 2s, 3s, 5s) for transient network hiccups. + - Resilient backoff retry mechanism (1s, 1s, 2s, 3s, 5s) for transient network hiccups with immediate break on 404/405 errors. + +## Heuristic discovery engine + +The gateway implements an autonomous heuristic engine that discovers and configures the optimal completion path for any target Gradio space at startup. + +### Space introspection and version detection + +Upon initialization, `gr2gw` inspects the space metadata: +1. Queries `/gradio_api/info` and `/config` endpoints. +2. Extracts the Gradio runtime version (v3, v4, v5, or v6). +3. Detects API routing prefixes (e.g. `/gradio_api` on modern versions, or root on Gradio 3). + +### UI flavor classification + +The engine classifies the application structure into architectural flavors: +- **`ChatInterface`**: Standard Gradio chat interfaces equipped with chatbot, textbox, and optional additional inputs. +- **`Blocks (Chat)`**: Custom `gr.Blocks` layouts containing conversational components. +- **`Blocks (Multimodal Chat)`**: Blocks architectures featuring `MultimodalTextbox` components that accept `{text, files}` JSON payloads. +- **`Interface`**: Classic input-output `gr.Interface` instances. +- **`Generic`**: Spaces with custom or unclassified component topologies. + +### Candidate endpoint scoring + +Gradio spaces frequently expose dozens of internal endpoints for UI actions (e.g. clearing text, retrying responses, voting/liking, adjusting sliders). The scoring algorithm identifies the true conversational endpoint by: +- Penalizing non-conversational triggers (e.g. `-600` for clear/reset/undo/retry/like endpoints). +- Rewarding chat semantics (`+150` for `/chat`, `/predict`, `/generate`, `/respond`). +- Rewarding message inputs (`+120` for `Textbox` or `MultimodalTextbox`). +- Rewarding chat history slots (`+80` for `Chatbot` or `State` components). +- Rewarding generator and streaming dependencies (`+50`). + +### Dual protocol execution and auto-fallback + +- **`call` protocol**: Modern Gradio 4/5/6 execution via `POST /call/{endpoint}` returning an event ID, followed by `GET /call/{endpoint}/{event_id}` SSE streaming. +- **`predict` protocol**: Gradio 3 and legacy execution via direct `POST /run/predict` or `POST /api/predict`. +- **Runtime failover**: If a space returns HTTP 404 or 405 when calling the modern protocol, the gateway breaks immediately from the retry loop and falls back to `/run/predict`. + +### Tool calling support modes + +The gateway evaluates available input components to determine how tool schemas and function calls should be delivered: +- **`native_slot`**: The space provides a dedicated parameter slot for tool definitions (e.g. `functions_json_str` on Hunyuan 3). Function definitions are passed cleanly without prompt alteration. +- **`prompt_augmented_system`**: The space provides a separate `system_prompt` input slot. Tool definitions and invocation schemas are injected directly into the system prompt. +- **`prompt_augmented_first_turn`**: The space accepts chat history pairs but lacks a dedicated system prompt slot. Tool definitions are prepended to the user prompt on the first dialogue turn. +- **`prompt_augmented_single_prompt`**: The space accepts only a single textbox input. Full multi-turn dialogue, tool definitions, and system guidance are synthesized into a single cohesive prompt. + +### Startup resolution diagnostics + +Whenever the gateway starts or inspects a new space, it prints the complete resolution picture: + +```text +================================================================================ +Gradio Space Resolution Picture +-------------------------------------------------------------------------------- +Space URL: https://tencent-hy3.hf.space +Title: Hunyuan 3 Chat +Gradio Version: 5.29.0 +UI Flavor: Blocks (Chat) +Protocol: call +API Prefix: /gradio_api +Resolved Endpoint: /chat_fn +Function Index: 1 +Primary Model: hy3 +Exposed Models: hy3, hunyuan3, tencent/Hy3 +History Format: tuples +Tool Call Support: native_slot +Total Input Slots: 9 +Input Slot Mappings: + [0] Component ID 1 textbox (label="message") -> message + [1] Component ID 2 textbox (label="system") -> system_prompt + [2] Component ID 3 chatbot (label="chatbot") -> history + [3] Component ID 4 radio (label="think_level") -> think_level + [4] Component ID 5 slider (label="temperature") -> temperature + [5] Component ID 6 slider (label="max_tokens") -> max_tokens + [6] Component ID 7 slider (label="top_p") -> top_p + [7] Component ID 8 state (label="preserved") -> preserved_thinking + [8] Component ID 9 textbox (label="functions") -> functions_json_str +================================================================================ +``` + +### Gateway status endpoint + +Making a `GET` request to `/` returns a JSON summary of the running gateway and the discovered space profile: + +```bash +curl http://localhost:8080/ +``` + +Response: +```json +{ + "name": "gr2gw", + "status": "ready", + "space_url": "https://tencent-hy3.hf.space", + "gradio_version": "5.29.0", + "flavor": "Blocks (Chat)", + "protocol": "call", + "tool_call_mode": "native_slot", + "models": ["hy3", "hunyuan3", "tencent/Hy3"] +} +``` ## Installation diff --git a/gr2gw.go b/gr2gw.go index bca70da..96f2b60 100644 --- a/gr2gw.go +++ b/gr2gw.go @@ -402,6 +402,9 @@ func DoWithFibonacciRetry(client *http.Client, makeReq func() (*http.Request, er respBody, _ := io.ReadAll(resp.Body) resp.Body.Close() lastErr = fmt.Errorf("HTTP status %d: %s", resp.StatusCode, string(respBody)) + if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusMethodNotAllowed { + break + } if _, ok := extractFailedGeneration(string(respBody)); ok { break } @@ -1592,39 +1595,86 @@ type HFSpaceInfoResponse struct { } type SpaceParamMapping struct { - InputIndex int - ComponentID int - ParamType string // "message", "history", "system_prompt", "temperature", "max_tokens", "top_p", "think_level", "tools", "stream", "state", "other" - DefaultValue interface{} + InputIndex int `json:"input_index"` + ComponentID int `json:"component_id"` + ComponentType string `json:"component_type,omitempty"` + Label string `json:"label,omitempty"` + ParamType string `json:"param_type"` // "message", "history", "system_prompt", "temperature", "max_tokens", "top_p", "think_level", "tools", "stream", "state", "other" + DefaultValue interface{} `json:"default_value,omitempty"` } type SpaceDiscovery struct { - SpaceURL string - Title string - Models []string - PrimaryModel string - APIPrefix string // e.g. "/gradio_api" or "" - Endpoint string // e.g. "/chat_fn" or "/chat" - CleanEndpoint string // e.g. "chat_fn" or "chat" - Protocol string // "call", "queue", "predict" - TotalInputs int - ParamMappings []SpaceParamMapping - DefaultInputs []interface{} - MessageIsMultimodal bool - HistoryIndex int // -1 if none - MessageIndex int // index for user message text - SystemIndex int // -1 if none - DefaultSystemPrompt string // default space system prompt if present - TempIndex int // -1 if none - MaxTokensIndex int // -1 if none - TopPIndex int // -1 if none - StreamIndex int // -1 if none - ThinkLevelIndex int // -1 if none - FunctionsJSONIndex int // -1 if none - PreservedThinkingIndex int // -1 if none - IsHunyuan3 bool - HistoryFormat string // "messages", "pairs", "gradio_messages", "none" - LastDiscovered time.Time + SpaceURL string `json:"space_url"` + Title string `json:"title"` + GradioVersion string `json:"gradio_version"` + Flavor string `json:"flavor"` + Protocol string `json:"protocol"` // "call", "queue", "predict" + APIPrefix string `json:"api_prefix"` // e.g. "/gradio_api" or "" + Endpoint string `json:"endpoint"` // e.g. "/chat_fn" or "/chat" or "/run/predict" + CleanEndpoint string `json:"clean_endpoint"` // e.g. "chat_fn" or "chat" + FnIndex int `json:"fn_index"` // -1 if not applicable + Models []string `json:"models"` + PrimaryModel string `json:"primary_model"` + TotalInputs int `json:"total_inputs"` + ParamMappings []SpaceParamMapping `json:"param_mappings"` + DefaultInputs []interface{} `json:"default_inputs"` + MessageIsMultimodal bool `json:"message_is_multimodal"` + HistoryIndex int `json:"history_index"` // -1 if none + MessageIndex int `json:"message_index"` // index for user message text + SystemIndex int `json:"system_index"` // -1 if none + DefaultSystemPrompt string `json:"default_system_prompt,omitempty"` // default space system prompt if present + TempIndex int `json:"temp_index"` // -1 if none + MaxTokensIndex int `json:"max_tokens_index"` // -1 if none + TopPIndex int `json:"top_p_index"` // -1 if none + StreamIndex int `json:"stream_index"` // -1 if none + ThinkLevelIndex int `json:"think_level_index"` // -1 if none + FunctionsJSONIndex int `json:"functions_json_index"` // -1 if none + PreservedThinkingIndex int `json:"preserved_thinking_index"` // -1 if none + IsHunyuan3 bool `json:"is_hunyuan3"` + HistoryFormat string `json:"history_format"` // "messages", "pairs", "gradio_messages", "none" + ToolCallMode string `json:"tool_call_mode"` // "native_slot", "prompt_augmented_system", "prompt_augmented_first_turn", "prompt_augmented_single_prompt" + LastDiscovered time.Time `json:"last_discovered"` +} + +func (d *SpaceDiscovery) Summary() string { + var sb strings.Builder + sb.WriteString("================================================================================\n") + sb.WriteString("Gradio Space Resolution Picture\n") + sb.WriteString("--------------------------------------------------------------------------------\n") + sb.WriteString(fmt.Sprintf("Space URL: %s\n", d.SpaceURL)) + if d.Title != "" { + sb.WriteString(fmt.Sprintf("Title: %s\n", d.Title)) + } + sb.WriteString(fmt.Sprintf("Gradio Version: %s\n", d.GradioVersion)) + sb.WriteString(fmt.Sprintf("UI Flavor: %s\n", d.Flavor)) + sb.WriteString(fmt.Sprintf("Protocol: %s\n", d.Protocol)) + sb.WriteString(fmt.Sprintf("API Prefix: %s\n", d.APIPrefix)) + sb.WriteString(fmt.Sprintf("Resolved Endpoint: %s\n", d.Endpoint)) + if d.FnIndex >= 0 { + sb.WriteString(fmt.Sprintf("Function Index: %d\n", d.FnIndex)) + } + sb.WriteString(fmt.Sprintf("Primary Model: %s\n", d.PrimaryModel)) + if len(d.Models) > 0 { + sb.WriteString(fmt.Sprintf("Exposed Models: %s\n", strings.Join(d.Models, ", "))) + } + sb.WriteString(fmt.Sprintf("History Format: %s\n", d.HistoryFormat)) + sb.WriteString(fmt.Sprintf("Tool Call Support: %s\n", d.ToolCallMode)) + sb.WriteString(fmt.Sprintf("Total Input Slots: %d\n", d.TotalInputs)) + if len(d.ParamMappings) > 0 { + sb.WriteString("Input Slot Mappings:\n") + for _, m := range d.ParamMappings { + desc := m.ComponentType + if desc == "" { + desc = "Unknown" + } + if m.Label != "" { + desc += fmt.Sprintf(" (label=%q)", m.Label) + } + sb.WriteString(fmt.Sprintf(" [%d] Component ID %-4d %-30s -> %s\n", m.InputIndex, m.ComponentID, desc, m.ParamType)) + } + } + sb.WriteString("================================================================================") + return sb.String() } func (d *SpaceDiscovery) GetModelList() []ModelItem { @@ -1710,10 +1760,13 @@ func NewDefaultSpaceDiscovery(spaceURL string) *SpaceDiscovery { } return &SpaceDiscovery{ SpaceURL: cleanURL, + GradioVersion: "unknown", + Flavor: "generic", + Protocol: "call", APIPrefix: "/gradio_api", Endpoint: "/chat_fn", CleanEndpoint: "chat_fn", - Protocol: "call", + FnIndex: -1, TotalInputs: 1, HistoryIndex: -1, MessageIndex: 0, @@ -1727,10 +1780,198 @@ func NewDefaultSpaceDiscovery(spaceURL string) *SpaceDiscovery { FunctionsJSONIndex: -1, PreservedThinkingIndex: -1, HistoryFormat: "messages", + ToolCallMode: "prompt_augmented_single_prompt", LastDiscovered: time.Now(), } } +// DetectGradioFlavor classifies the architecture of the Gradio space. +func DetectGradioFlavor(configResp GradioConfigResponse, compMap map[int]GradioComponent) string { + mode := strings.ToLower(strings.TrimSpace(configResp.Mode)) + hasChatbot := false + hasMultimodal := false + for _, c := range compMap { + cType := strings.ToLower(c.Type) + if cType == "chatbot" { + hasChatbot = true + } else if cType == "multimodaltextbox" { + hasMultimodal = true + } + } + + switch mode { + case "chat_interface": + if hasMultimodal { + return "ChatInterface (Multimodal)" + } + return "ChatInterface" + case "blocks": + if hasChatbot { + if hasMultimodal { + return "Blocks (Multimodal Chat)" + } + return "Blocks (Chat)" + } + return "Blocks" + case "interface": + return "Interface" + default: + if mode != "" { + return mode + } + if hasChatbot { + return "ChatInterface" + } + return "Generic" + } +} + +// ScoreCandidateEndpoint computes a heuristic score for an endpoint based on its API name, +// parameter signatures, and dependency graph topology. +func ScoreCandidateEndpoint(apiName string, parameters []GradioParamInfo, dep *GradioDependency, compMap map[int]GradioComponent) int { + score := 0 + cleanName := strings.TrimPrefix(apiName, "/") + lowerName := strings.ToLower(cleanName) + + hasInputs := len(parameters) > 0 || (dep != nil && len(dep.Inputs) > 0) + if !hasInputs { + return -1000 + } + + if strings.Contains(lowerName, "clear") || strings.Contains(lowerName, "reset") || + strings.Contains(lowerName, "init") || strings.Contains(lowerName, "undo") || + strings.Contains(lowerName, "delete") || strings.Contains(lowerName, "remove") || + strings.Contains(lowerName, "pop") || strings.Contains(lowerName, "clean") { + score -= 600 + } + if strings.Contains(lowerName, "vote") || strings.Contains(lowerName, "like") || + strings.Contains(lowerName, "dislike") || strings.Contains(lowerName, "flag") || + strings.Contains(lowerName, "feedback") || strings.Contains(lowerName, "report") { + score -= 500 + } + if strings.Contains(lowerName, "download") || strings.Contains(lowerName, "export") || + strings.Contains(lowerName, "save") || strings.Contains(lowerName, "upload") || + strings.Contains(lowerName, "auth") || strings.Contains(lowerName, "login") || + strings.Contains(lowerName, "theme") || strings.Contains(lowerName, "token") { + score -= 400 + } + if strings.Contains(lowerName, "whisper") || strings.Contains(lowerName, "transcribe") || + strings.Contains(lowerName, "tts") || strings.Contains(lowerName, "speech") || + strings.Contains(lowerName, "diffusion") || strings.Contains(lowerName, "draw") || + strings.Contains(lowerName, "sdxl") || strings.Contains(lowerName, "upscale") { + score -= 300 + } + if strings.Contains(lowerName, "lambda") { + score -= 200 + } + + if lowerName == "chat" || lowerName == "chat_fn" { + score += 150 + } else if strings.Contains(lowerName, "chat") || strings.Contains(lowerName, "conversation") || strings.Contains(lowerName, "dialogue") || strings.Contains(lowerName, "chatbot") { + score += 120 + } + if lowerName == "predict" || lowerName == "generate" { + score += 80 + } + if strings.Contains(lowerName, "respond") || strings.Contains(lowerName, "reply") || strings.Contains(lowerName, "answer") { + score += 90 + } + if strings.Contains(lowerName, "generate") || strings.Contains(lowerName, "completion") || strings.Contains(lowerName, "infer") || strings.Contains(lowerName, "predict") { + score += 70 + } + if strings.Contains(lowerName, "ask") || strings.Contains(lowerName, "query") || strings.Contains(lowerName, "prompt") || strings.Contains(lowerName, "talk") || strings.Contains(lowerName, "run") { + score += 50 + } + if strings.Contains(lowerName, "submit") { + score += 40 + } + + for _, p := range parameters { + pLower := strings.ToLower(p.ParameterName) + pLabel := strings.ToLower(p.Label) + pComp := strings.ToLower(p.Component) + + if pComp == "multimodaltextbox" { + score += 80 + } else if pComp == "textbox" { + if strings.Contains(pLower, "message") || strings.Contains(pLabel, "message") || + strings.Contains(pLower, "prompt") || strings.Contains(pLabel, "prompt") || + strings.Contains(pLower, "query") || strings.Contains(pLabel, "query") || + strings.Contains(pLower, "input") || strings.Contains(pLabel, "input") { + score += 60 + } else { + score += 30 + } + } else if pComp == "chatbot" { + score += 70 + } else if pComp == "state" { + score += 20 + } else if pComp == "slider" || pComp == "number" { + if strings.Contains(pLower, "temp") || strings.Contains(pLabel, "temp") || + strings.Contains(pLower, "token") || strings.Contains(pLabel, "token") || + strings.Contains(pLower, "top_p") || strings.Contains(pLabel, "top_p") { + score += 20 + } + } + if strings.Contains(pLower, "tool") || strings.Contains(pLabel, "tool") || + strings.Contains(pLower, "function") || strings.Contains(pLabel, "function") { + score += 50 + } + } + + if dep != nil { + if dep.Types.Generator { + score += 40 + } + for _, inID := range dep.Inputs { + if comp, exists := compMap[inID]; exists { + cType := strings.ToLower(comp.Type) + cLabel := "" + if comp.Props != nil { + if l, ok := comp.Props["label"].(string); ok { + cLabel = strings.ToLower(l) + } + } + if cType == "multimodaltextbox" { + score += 80 + } else if cType == "textbox" { + if strings.Contains(cLabel, "message") || strings.Contains(cLabel, "prompt") || + strings.Contains(cLabel, "query") || strings.Contains(cLabel, "input") { + score += 60 + } else if strings.Contains(cLabel, "system") || strings.Contains(cLabel, "instruction") { + score += 30 + } else { + score += 20 + } + } else if cType == "chatbot" { + score += 70 + } else if cType == "state" { + score += 20 + } else if cType == "slider" || cType == "number" { + if strings.Contains(cLabel, "temp") || strings.Contains(cLabel, "token") || strings.Contains(cLabel, "top") { + score += 20 + } + } + if strings.Contains(cLabel, "tool") || strings.Contains(cLabel, "function") { + score += 50 + } + } + } + for _, outID := range dep.Outputs { + if comp, exists := compMap[outID]; exists { + cType := strings.ToLower(comp.Type) + if cType == "chatbot" { + score += 90 + } else if cType == "textbox" || cType == "markdown" { + score += 50 + } + } + } + } + + return score +} + // InspectSpace queries Gradio's /gradio_api/info, /config, and HuggingFace Space APIs // to build an adaptive schema mapping for any Gradio space. func InspectSpace(client *http.Client, rawURL, userAgent string) (*SpaceDiscovery, error) { @@ -1864,200 +2105,200 @@ func InspectSpace(client *http.Client, rawURL, userAgent string) (*SpaceDiscover } } - // 4. Score and select the best chat endpoint + // 4. Detect Gradio version and UI flavor + compMap := make(map[int]GradioComponent) + if configFetched { + for _, comp := range configResp.Components { + compMap[comp.ID] = comp + } + if configResp.Version != "" { + discovery.GradioVersion = configResp.Version + } else if infoFetched { + discovery.GradioVersion = "4+ (inferred from /gradio_api/info)" + } + discovery.Flavor = DetectGradioFlavor(configResp, compMap) + } else if infoFetched { + discovery.GradioVersion = "4+ (inferred from /gradio_api/info)" + discovery.Flavor = "Generic" + } + + // 5. Score and select the best chat completion endpoint bestEndpoint := "" bestScore := -1000 var bestEndpointInfo *GradioEndpointInfo + var bestMatchingDep *GradioDependency + // Try named endpoints from /gradio_api/info first if infoFetched && len(infoResp.NamedEndpoints) > 0 { for epName, epInfo := range infoResp.NamedEndpoints { - score := 0 - lowerName := strings.ToLower(epName) - - // Penalize non-generation/reset/clear/init endpoints or 0-parameter endpoints - if len(epInfo.Parameters) == 0 { - score -= 500 - } - if strings.Contains(lowerName, "clear") || strings.Contains(lowerName, "reset") || strings.Contains(lowerName, "init") || strings.Contains(lowerName, "undo") || strings.Contains(lowerName, "delete") { - score -= 500 - } - - if strings.Contains(lowerName, "chat") || strings.Contains(lowerName, "conversation") || strings.Contains(lowerName, "dialogue") { - score += 100 - } - if strings.Contains(lowerName, "answer") || strings.Contains(lowerName, "respond") || strings.Contains(lowerName, "generate") || strings.Contains(lowerName, "predict") || strings.Contains(lowerName, "completion") { - score += 60 - } - if strings.Contains(lowerName, "ask") || strings.Contains(lowerName, "query") || strings.Contains(lowerName, "question") || strings.Contains(lowerName, "talk") { - score += 40 - } - - for _, p := range epInfo.Parameters { - pLower := strings.ToLower(p.ParameterName) - pLabel := strings.ToLower(p.Label) - pComp := strings.ToLower(p.Component) - if strings.Contains(pLower, "message") || strings.Contains(pLabel, "message") || strings.Contains(pLower, "text") || strings.Contains(pLower, "prompt") || strings.Contains(pLower, "query") || strings.Contains(pLower, "question") || strings.Contains(pLower, "input") || pComp == "textbox" || pComp == "multimodaltextbox" { - score += 40 - } - if strings.Contains(pLower, "history") || strings.Contains(pLabel, "history") || strings.Contains(pLower, "chat") || strings.Contains(pLower, "messages") || strings.Contains(pLower, "conversation") || pComp == "chatbot" { - score += 30 + cleanTarget := strings.TrimPrefix(epName, "/") + var matchingDep *GradioDependency + if configFetched { + for _, dep := range configResp.Dependencies { + depAPIName := "" + if s, ok := dep.APIName.(string); ok { + depAPIName = strings.TrimPrefix(s, "/") + } + if depAPIName == cleanTarget { + depCopy := dep + matchingDep = &depCopy + break + } } } - + score := ScoreCandidateEndpoint(epName, epInfo.Parameters, matchingDep, compMap) if score > bestScore { bestScore = score bestEndpoint = epName epCopy := epInfo bestEndpointInfo = &epCopy + bestMatchingDep = matchingDep } } } // Fallback: if no named endpoint from /info, inspect dependencies in config - if bestEndpoint == "" && configFetched && len(configResp.Dependencies) > 0 { + if (bestEndpoint == "" || bestScore <= 0) && configFetched && len(configResp.Dependencies) > 0 { depScore := -1000 for _, dep := range configResp.Dependencies { - if depName, ok := dep.APIName.(string); ok && depName != "" { - cleanName := strings.TrimPrefix(depName, "/") - lowerName := strings.ToLower(cleanName) - if strings.Contains(lowerName, "clear") || strings.Contains(lowerName, "reset") || strings.Contains(lowerName, "save") || strings.Contains(lowerName, "delete") || strings.Contains(lowerName, "pop") || strings.Contains(lowerName, "lambda") { - continue - } - score := 0 - if strings.Contains(lowerName, "chat") || strings.Contains(lowerName, "conversation") { - score += 100 - } else if strings.Contains(lowerName, "generate") || strings.Contains(lowerName, "predict") || strings.Contains(lowerName, "submit") { - score += 50 - } - if score > depScore { - depScore = score - bestEndpoint = "/" + cleanName - } - } - } - } - - if bestEndpoint != "" { - discovery.Endpoint = bestEndpoint - discovery.CleanEndpoint = strings.TrimPrefix(bestEndpoint, "/") - } - - // 5. Correlate with config.dependencies to determine exact input count & state padding - compMap := make(map[int]GradioComponent) - var matchingDep *GradioDependency - - if configFetched { - for _, comp := range configResp.Components { - compMap[comp.ID] = comp - } - - cleanTarget := strings.TrimPrefix(discovery.Endpoint, "/") - - for _, dep := range configResp.Dependencies { - depAPIName := "" + apiName := "" if s, ok := dep.APIName.(string); ok { - depAPIName = strings.TrimPrefix(s, "/") + apiName = s } - if depAPIName == cleanTarget { + score := ScoreCandidateEndpoint(apiName, nil, &dep, compMap) + if score > depScore { + depScore = score depCopy := dep - matchingDep = &depCopy - break + bestMatchingDep = &depCopy + if apiName != "" { + bestEndpoint = "/" + strings.TrimPrefix(apiName, "/") + } else { + bestEndpoint = fmt.Sprintf("/%d", dep.ID) + } } } + } - if matchingDep != nil { - discovery.TotalInputs = len(matchingDep.Inputs) - discovery.DefaultInputs = make([]interface{}, len(matchingDep.Inputs)) - discovery.ParamMappings = nil - discovery.MessageIndex = -1 + if bestMatchingDep != nil { + discovery.FnIndex = bestMatchingDep.ID + } - for idx, compID := range matchingDep.Inputs { - mapping := SpaceParamMapping{ - InputIndex: idx, - ComponentID: compID, - ParamType: "other", - } - if comp, exists := compMap[compID]; exists { - cType := strings.ToLower(comp.Type) - cLabel := "" - if comp.Props != nil { - if l, ok := comp.Props["label"].(string); ok { - cLabel = strings.ToLower(l) - } - if val, ok := comp.Props["value"]; ok { - discovery.DefaultInputs[idx] = val - mapping.DefaultValue = val - } - } + // Select protocol + if strings.HasPrefix(discovery.GradioVersion, "3.") { + discovery.Protocol = "predict" + discovery.APIPrefix = "" + discovery.Endpoint = "/run/predict" + discovery.CleanEndpoint = "run/predict" + } else { + discovery.Protocol = "call" + if bestEndpoint != "" { + discovery.Endpoint = bestEndpoint + discovery.CleanEndpoint = strings.TrimPrefix(bestEndpoint, "/") + } + } - switch cType { - case "multimodaltextbox": - if discovery.MessageIndex == -1 || strings.Contains(cLabel, "message") || strings.Contains(cLabel, "prompt") || strings.Contains(cLabel, "input") { - mapping.ParamType = "message" - discovery.MessageIndex = idx - discovery.MessageIsMultimodal = true - } - case "textbox": - if strings.Contains(cLabel, "system") || strings.Contains(cLabel, "instruction") { - mapping.ParamType = "system_prompt" - discovery.SystemIndex = idx - if comp.Props != nil { - if val, ok := comp.Props["value"].(string); ok && strings.TrimSpace(val) != "" { - discovery.DefaultSystemPrompt = strings.TrimSpace(val) - } - } - } else if discovery.MessageIndex == -1 || strings.Contains(cLabel, "message") || strings.Contains(cLabel, "prompt") || strings.Contains(cLabel, "query") || strings.Contains(cLabel, "input") || strings.Contains(cLabel, "question") { - mapping.ParamType = "message" - discovery.MessageIndex = idx - discovery.MessageIsMultimodal = false - } - case "chatbot": - mapping.ParamType = "history" - discovery.HistoryIndex = idx - if strings.HasPrefix(configResp.Version, "5.") || strings.HasPrefix(configResp.Version, "6.") { - discovery.HistoryFormat = "gradio_messages" - } else { - discovery.HistoryFormat = "pairs" - } - case "state": - mapping.ParamType = "state" - case "slider", "number": - if strings.Contains(cLabel, "temp") { - mapping.ParamType = "temperature" - discovery.TempIndex = idx - } else if strings.Contains(cLabel, "max") || strings.Contains(cLabel, "token") { - mapping.ParamType = "max_tokens" - discovery.MaxTokensIndex = idx - } else if strings.Contains(cLabel, "top_p") || strings.Contains(cLabel, "top-p") || strings.Contains(cLabel, "top p") { - mapping.ParamType = "top_p" - discovery.TopPIndex = idx - } else if strings.Contains(cLabel, "think") { - mapping.ParamType = "think_level" - discovery.ThinkLevelIndex = idx - } - case "checkbox": - if strings.Contains(cLabel, "stream") { - mapping.ParamType = "stream" - discovery.StreamIndex = idx - } - default: - if strings.Contains(cLabel, "tool") || strings.Contains(cLabel, "function") { - mapping.ParamType = "tools" - discovery.FunctionsJSONIndex = idx - } - } - } - discovery.ParamMappings = append(discovery.ParamMappings, mapping) + // 6. Correlate with config.dependencies to determine exact input count & state padding + if bestMatchingDep != nil { + discovery.TotalInputs = len(bestMatchingDep.Inputs) + discovery.DefaultInputs = make([]interface{}, len(bestMatchingDep.Inputs)) + discovery.ParamMappings = nil + discovery.MessageIndex = -1 + + for idx, compID := range bestMatchingDep.Inputs { + mapping := SpaceParamMapping{ + InputIndex: idx, + ComponentID: compID, + ParamType: "other", } - if discovery.MessageIndex == -1 { - discovery.MessageIndex = 0 - if len(matchingDep.Inputs) > 0 { - if comp, exists := compMap[matchingDep.Inputs[0]]; exists { - if strings.ToLower(comp.Type) == "multimodaltextbox" { - discovery.MessageIsMultimodal = true + var pName, pLabel, pComp string + if bestEndpointInfo != nil && idx < len(bestEndpointInfo.Parameters) { + p := bestEndpointInfo.Parameters[idx] + pName = strings.ToLower(p.ParameterName) + pLabel = strings.ToLower(p.Label) + pComp = strings.ToLower(p.Component) + if p.ParameterDefault != nil && discovery.DefaultInputs[idx] == nil { + discovery.DefaultInputs[idx] = p.ParameterDefault + mapping.DefaultValue = p.ParameterDefault + } + mapping.Label = p.ParameterName + if p.Component != "" { + mapping.ComponentType = p.Component + } + } + + if comp, exists := compMap[compID]; exists { + if mapping.ComponentType == "" { + mapping.ComponentType = comp.Type + } + cType := strings.ToLower(comp.Type) + cLabel := "" + if comp.Props != nil { + if l, ok := comp.Props["label"].(string); ok { + cLabel = strings.ToLower(l) + if mapping.Label == "" { + mapping.Label = l } } + if val, ok := comp.Props["value"]; ok { + discovery.DefaultInputs[idx] = val + mapping.DefaultValue = val + } + } + + if strings.Contains(pName, "function") || strings.Contains(pName, "tool") || strings.Contains(pLabel, "tool") || strings.Contains(pLabel, "function") || strings.Contains(cLabel, "tool") || strings.Contains(cLabel, "function") { + mapping.ParamType = "tools" + discovery.FunctionsJSONIndex = idx + } else if strings.Contains(pName, "think_level") || strings.Contains(pName, "thinking") || strings.Contains(pLabel, "think") || strings.Contains(cLabel, "think") { + mapping.ParamType = "think_level" + discovery.ThinkLevelIndex = idx + } else if strings.Contains(pName, "preserved") || strings.Contains(cLabel, "preserved") { + discovery.PreservedThinkingIndex = idx + } else if strings.Contains(pName, "system") || strings.Contains(pLabel, "system") || strings.Contains(cLabel, "system") || strings.Contains(cLabel, "instruction") { + mapping.ParamType = "system_prompt" + discovery.SystemIndex = idx + if comp.Props != nil { + if val, ok := comp.Props["value"].(string); ok && strings.TrimSpace(val) != "" { + discovery.DefaultSystemPrompt = strings.TrimSpace(val) + } + } + } else if strings.Contains(pName, "history") || strings.Contains(pLabel, "history") || strings.Contains(cLabel, "history") || strings.Contains(cLabel, "chat") || cType == "chatbot" { + mapping.ParamType = "history" + discovery.HistoryIndex = idx + } else if cType == "multimodaltextbox" || strings.Contains(pComp, "multimodal") { + mapping.ParamType = "message" + discovery.MessageIndex = idx + discovery.MessageIsMultimodal = true + } else if strings.Contains(pName, "message") || strings.Contains(pLabel, "message") || strings.Contains(cLabel, "message") || strings.Contains(cLabel, "prompt") || strings.Contains(cLabel, "query") || (discovery.MessageIndex == -1 && idx == 0) { + mapping.ParamType = "message" + discovery.MessageIndex = idx + discovery.MessageIsMultimodal = false + } else if strings.Contains(pName, "temp") || strings.Contains(cLabel, "temp") { + mapping.ParamType = "temperature" + discovery.TempIndex = idx + } else if strings.Contains(pName, "token") || strings.Contains(cLabel, "token") || strings.Contains(cLabel, "max") { + mapping.ParamType = "max_tokens" + discovery.MaxTokensIndex = idx + } else if strings.Contains(pName, "top_p") || strings.Contains(cLabel, "top_p") || strings.Contains(cLabel, "top-p") || strings.Contains(cLabel, "top p") { + mapping.ParamType = "top_p" + discovery.TopPIndex = idx + } else if strings.Contains(pName, "stream") || strings.Contains(cLabel, "stream") { + mapping.ParamType = "stream" + discovery.StreamIndex = idx + } else if cType == "state" { + mapping.ParamType = "state" + } + } + discovery.ParamMappings = append(discovery.ParamMappings, mapping) + } + + if discovery.MessageIndex == -1 { + discovery.MessageIndex = 0 + if len(bestMatchingDep.Inputs) > 0 { + if comp, exists := compMap[bestMatchingDep.Inputs[0]]; exists { + if strings.ToLower(comp.Type) == "multimodaltextbox" { + discovery.MessageIsMultimodal = true + } } } } @@ -2084,7 +2325,7 @@ func InspectSpace(client *http.Client, rawURL, userAgent string) (*SpaceDiscover } // Fallback: If config.dependencies did not provide matchingDep, map directly from bestEndpointInfo.Parameters - if matchingDep == nil && bestEndpointInfo != nil { + if bestMatchingDep == nil && bestEndpointInfo != nil { discovery.TotalInputs = len(bestEndpointInfo.Parameters) discovery.DefaultInputs = make([]interface{}, len(bestEndpointInfo.Parameters)) discovery.ParamMappings = nil @@ -2100,9 +2341,11 @@ func InspectSpace(client *http.Client, rawURL, userAgent string) (*SpaceDiscover } mapping := SpaceParamMapping{ - InputIndex: idx, - ParamType: "other", - DefaultValue: p.ParameterDefault, + InputIndex: idx, + ComponentType: p.Component, + Label: p.Label, + ParamType: "other", + DefaultValue: p.ParameterDefault, } if strings.Contains(pComp, "multimodal") { @@ -2130,8 +2373,6 @@ func InspectSpace(client *http.Client, rawURL, userAgent string) (*SpaceDiscover discovery.HistoryFormat = "pairs" } else if strings.Contains(pPyType, "textmessage") || strings.Contains(pPyType, "dict(text: str") || strings.Contains(bTypeStr, "textmessage") || strings.Contains(bTypeStr, "chatbotdatamessages") { discovery.HistoryFormat = "gradio_messages" - } else if strings.HasPrefix(configResp.Version, "5.") || strings.HasPrefix(configResp.Version, "6.") { - discovery.HistoryFormat = "gradio_messages" } } else if strings.Contains(pLabel, "message") || strings.Contains(pName, "message") || (strings.Contains(pLabel, "prompt") && !strings.Contains(pLabel, "system")) || (strings.Contains(pName, "prompt") && !strings.Contains(pName, "system")) || strings.Contains(pLabel, "query") || strings.Contains(pName, "query") || strings.Contains(pLabel, "question") || strings.Contains(pName, "question") || (discovery.MessageIndex == -1 && (pComp == "textbox" || idx == 0)) { discovery.MessageIndex = idx @@ -2167,6 +2408,20 @@ func InspectSpace(client *http.Client, rawURL, userAgent string) (*SpaceDiscover } } + // 7. Resolve default history format if not set by type inspection + if discovery.HistoryIndex != -1 { + if discovery.HistoryFormat == "" || discovery.HistoryFormat == "messages" { + if strings.HasPrefix(discovery.GradioVersion, "5.") || strings.HasPrefix(discovery.GradioVersion, "6.") { + discovery.HistoryFormat = "gradio_messages" + } else { + discovery.HistoryFormat = "pairs" + } + } + } else { + discovery.HistoryFormat = "none" + } + + // 8. Hunyuan3 / Tencent Hy3 overrides if discovery.FunctionsJSONIndex != -1 || discovery.ThinkLevelIndex != -1 || strings.Contains(cleanURL, "hy3") || strings.Contains(cleanURL, "hunyuan") { discovery.IsHunyuan3 = true discovery.HistoryFormat = "messages" @@ -2176,6 +2431,17 @@ func InspectSpace(client *http.Client, rawURL, userAgent string) (*SpaceDiscover } } + // 9. Tool Calling Support Picture Resolution + if discovery.FunctionsJSONIndex >= 0 { + discovery.ToolCallMode = "native_slot" + } else if discovery.SystemIndex >= 0 { + discovery.ToolCallMode = "prompt_augmented_system" + } else if discovery.HistoryIndex >= 0 { + discovery.ToolCallMode = "prompt_augmented_first_turn" + } else { + discovery.ToolCallMode = "prompt_augmented_single_prompt" + } + // Ensure total inputs is at least 1 if discovery.TotalInputs < 1 { discovery.TotalInputs = 1 @@ -2233,6 +2499,9 @@ func NewGradioGateway(defaultSpaceURL, proxyURL string, timeout time.Duration) * disc, err := InspectSpace(gw.client, cleanDefault, DefaultUserAgent) if err == nil && disc != nil { gw.discoveries[cleanDefault] = disc + log.Printf("\n%s\n", disc.Summary()) + } else if err != nil { + log.Printf("Warning: initial space discovery for %s encountered error: %v (will retry on demand)", cleanDefault, err) } return gw @@ -2265,6 +2534,7 @@ func (g *GradioGateway) GetDiscovery(spaceURL, userAgent string) *SpaceDiscovery newDisc, err := InspectSpace(g.client, cleanTarget, userAgent) if err == nil && newDisc != nil { g.discoveries[cleanTarget] = newDisc + log.Printf("\n%s\n", newDisc.Summary()) return newDisc } @@ -2856,6 +3126,33 @@ func ParseGradioStreamOutput(rawJSON string) (frame GradioOutputFrame) { } case map[string]interface{}: + if outMap, ok := v["output"].(map[string]interface{}); ok { + if dataArr, ok := outMap["data"].([]interface{}); ok { + b, err := json.Marshal(dataArr) + if err == nil { + subFrame := ParseGradioStreamOutput(string(b)) + if subFrame.OK { + if r, ok := v["reasoning_content"].(string); ok && subFrame.Reasoning == "" { + subFrame.Reasoning = r + } + return subFrame + } + } + } + } + if dataArr, ok := v["data"].([]interface{}); ok { + b, err := json.Marshal(dataArr) + if err == nil { + subFrame := ParseGradioStreamOutput(string(b)) + if subFrame.OK { + if r, ok := v["reasoning_content"].(string); ok && subFrame.Reasoning == "" { + subFrame.Reasoning = r + } + return subFrame + } + } + } + for _, key := range []string{"text", "content", "response", "data", "value"} { if s, ok := v[key].(string); ok { frame.Content = s @@ -2894,6 +3191,122 @@ func ExtractTextFromGradioOutput(rawJSON string) (string, bool) { return "", false } +// executePredictCompletion handles completions using Gradio 3 direct /run/predict or /api/predict protocol. +func (g *GradioGateway) executePredictCompletion(w http.ResponseWriter, r *http.Request, disc *SpaceDiscovery, gradioData []interface{}, req ChatCompletionRequest, completionID string, createdTime int64, modelName, effUA string) error { + fnIndex := disc.FnIndex + if fnIndex < 0 { + fnIndex = 0 + } + + payloadMap := map[string]interface{}{ + "data": gradioData, + "fn_index": fnIndex, + "session_hash": GenerateUUID(), + } + + jsonPayload, err := json.Marshal(payloadMap) + if err != nil { + return fmt.Errorf("failed to marshal predict payload: %w", err) + } + + var candidateURLs []string + if disc.APIPrefix != "" { + candidateURLs = append(candidateURLs, fmt.Sprintf("%s%s/run/predict", disc.SpaceURL, disc.APIPrefix)) + } + candidateURLs = append(candidateURLs, + fmt.Sprintf("%s/run/predict", disc.SpaceURL), + fmt.Sprintf("%s/api/predict", disc.SpaceURL), + ) + + var resp *http.Response + var lastErr error + + for _, targetURL := range candidateURLs { + curURL := targetURL + makeReq := func() (*http.Request, error) { + req, err := http.NewRequest("POST", curURL, bytes.NewBuffer(jsonPayload)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("User-Agent", effUA) + return req, nil + } + resp, lastErr = DoWithFibonacciRetry(g.client, makeReq, 3) + if lastErr == nil && resp != nil && resp.StatusCode == http.StatusOK { + break + } + if resp != nil { + resp.Body.Close() + resp = nil + } + } + + if resp == nil { + return fmt.Errorf("upstream Gradio predict error: %w", lastErr) + } + defer resp.Body.Close() + + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("failed to read Gradio predict response: %w", err) + } + + frame := ParseGradioStreamOutput(string(bodyBytes)) + if !frame.OK { + return fmt.Errorf("upstream Gradio space returned empty or unparseable response") + } + + cleanText := frame.Content + reasoning := frame.Reasoning + toolCalls := frame.ToolCalls + hasTools := len(toolCalls) > 0 + + if reasoning == "" { + cleanText, reasoning = ExtractThinking(cleanText) + } + if !hasTools { + toolCalls, cleanText, hasTools = DetectToolCalls(cleanText) + } + + finishReason := "stop" + var finalContent interface{} = cleanText + if hasTools && len(toolCalls) > 0 { + finishReason = "tool_calls" + if strings.TrimSpace(cleanText) == "" { + finalContent = nil + } + } + + if !req.Stream { + WriteCompletionResponse(w, completionID, createdTime, modelName, FinalOutput{ + Content: finalContent, + ReasoningContent: reasoning, + ToolCalls: toolCalls, + FinishReason: finishReason, + }) + return nil + } + + flusher, _ := w.(http.Flusher) + streamer := NewStreamer(w, flusher, completionID, createdTime, modelName) + if reasoning != "" { + streamer.Reasoning(reasoning) + } + if hasTools && len(toolCalls) > 0 { + for i, tc := range toolCalls { + iCopy := i + tc.Index = &iCopy + streamer.ToolCallDelta(tc) + } + } else if cleanText != "" { + streamer.Content(cleanText) + } + streamer.Finish(finishReason) + streamer.Done() + return nil +} + // ExecuteChatCompletion handles both streaming and non-streaming requests. func (g *GradioGateway) ExecuteChatCompletion(w http.ResponseWriter, r *http.Request, req ChatCompletionRequest) error { effUA := EffectiveUserAgent(r) @@ -2922,15 +3335,20 @@ func (g *GradioGateway) ExecuteChatCompletion(w http.ResponseWriter, r *http.Req return fmt.Errorf("failed to build Gradio payload: %w", err) } + completionID := "chatcmpl-" + GenerateUUID() + createdTime := time.Now().Unix() + + // If the resolved protocol is predict (e.g. Gradio 3), execute predict completion directly + if disc.Protocol == "predict" { + return g.executePredictCompletion(w, r, disc, gradioData, req, completionID, createdTime, modelName, effUA) + } + payloadMap := map[string]interface{}{"data": gradioData} jsonPayload, err := json.Marshal(payloadMap) if err != nil { return fmt.Errorf("failed to encode request: %w", err) } - completionID := "chatcmpl-" + GenerateUUID() - createdTime := time.Now().Unix() - // 1. Submit to /call/{endpoint} callURL := fmt.Sprintf("%s%s/call/%s", disc.SpaceURL, disc.APIPrefix, disc.CleanEndpoint) makeCallReq := func() (*http.Request, error) { @@ -2966,7 +3384,7 @@ func (g *GradioGateway) ExecuteChatCompletion(w http.ResponseWriter, r *http.Req return nil } } - // If call failed, try without APIPrefix or try /call/v2 + // If call failed, try without APIPrefix altCallURL := fmt.Sprintf("%s/call/%s", disc.SpaceURL, disc.CleanEndpoint) makeAltReq := func() (*http.Request, error) { r, err := http.NewRequest("POST", altCallURL, bytes.NewBuffer(jsonPayload)) @@ -3000,6 +3418,10 @@ func (g *GradioGateway) ExecuteChatCompletion(w http.ResponseWriter, r *http.Req return nil } } + if strings.Contains(err.Error(), "404") || strings.Contains(err.Error(), "405") { + log.Printf("Gradio /call endpoint unavailable (%v), falling back to /run/predict protocol...", err) + return g.executePredictCompletion(w, r, disc, gradioData, req, completionID, createdTime, modelName, effUA) + } return fmt.Errorf("upstream Gradio call error: %w", err) } } @@ -3007,7 +3429,8 @@ func (g *GradioGateway) ExecuteChatCompletion(w http.ResponseWriter, r *http.Req var joinRes GradioJoinResponse if err := json.NewDecoder(resp.Body).Decode(&joinRes); err != nil || joinRes.EventID == "" { - return fmt.Errorf("failed to parse Gradio event ID from response") + log.Printf("Gradio /call returned non-SSE response, falling back to /run/predict protocol...") + return g.executePredictCompletion(w, r, disc, gradioData, req, completionID, createdTime, modelName, effUA) } // 2. Connect to Gradio SSE EventStream @@ -3414,11 +3837,16 @@ func main() { "service": "gr2gw", "space_url": disc.SpaceURL, "title": disc.Title, + "gradio_version": disc.GradioVersion, + "flavor": disc.Flavor, + "protocol": disc.Protocol, "endpoint": disc.Endpoint, "primary_model": disc.PrimaryModel, "models": disc.Models, "total_inputs": disc.TotalInputs, "history_format": disc.HistoryFormat, + "tool_call_mode": disc.ToolCallMode, + "param_mappings": disc.ParamMappings, }) }) diff --git a/gr2gw_test.go b/gr2gw_test.go index 9fe8411..684de14 100644 --- a/gr2gw_test.go +++ b/gr2gw_test.go @@ -1597,4 +1597,285 @@ func TestMultimodalStateSpaceMockServerCompletion(t *testing.T) { } } +// TestGradio3DirectPredictProtocol verifies discovery and chat completion against a Gradio 3 space +// where /gradio_api/info returns 404 and the protocol resolves to /run/predict with tuple pairs history. +func TestGradio3DirectPredictProtocol(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/gradio_api/info" || r.URL.Path == "/info" { + http.NotFound(w, r) + return + } + + if r.URL.Path == "/config" { + cfg := GradioConfigResponse{ + Version: "3.41.2", + Mode: "chat_interface", + Title: "Legacy Gradio 3 Chat", + Components: []GradioComponent{ + {ID: 1, Type: "textbox", Props: map[string]interface{}{"label": "Input"}}, + {ID: 2, Type: "chatbot", Props: map[string]interface{}{"label": "Chatbot"}}, + }, + Dependencies: []GradioDependency{ + { + ID: 0, + Inputs: []int{1, 2}, + Outputs: []int{2}, + }, + }, + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(cfg) + return + } + + if r.URL.Path == "/run/predict" { + var body struct { + Data []interface{} `json:"data"` + FnIndex int `json:"fn_index"` + SessionHash string `json:"session_hash"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if body.FnIndex != 0 { + http.Error(w, fmt.Sprintf("expected fn_index 0, got %d", body.FnIndex), http.StatusBadRequest) + return + } + msg, _ := body.Data[0].(string) + reply := "Echo from Gradio 3: " + msg + respData := map[string]interface{}{ + "data": []interface{}{ + [][]string{ + {msg, reply}, + }, + }, + "is_generating": false, + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(respData) + return + } + + http.NotFound(w, r) + })) + defer ts.Close() + + gw := NewGradioGateway(ts.URL, "", 10*time.Second) + disc := gw.GetDiscovery(ts.URL, DefaultUserAgent) + + if disc.GradioVersion != "3.41.2" { + t.Errorf("expected GradioVersion 3.41.2, got %s", disc.GradioVersion) + } + if disc.Protocol != "predict" { + t.Errorf("expected Protocol predict, got %s", disc.Protocol) + } + if disc.Flavor != "ChatInterface" { + t.Errorf("expected Flavor ChatInterface, got %s", disc.Flavor) + } + if disc.HistoryFormat != "pairs" { + t.Errorf("expected HistoryFormat pairs, got %s", disc.HistoryFormat) + } + if disc.FnIndex != 0 { + t.Errorf("expected FnIndex 0, got %d", disc.FnIndex) + } + + // Non-streaming completion + req := ChatCompletionRequest{ + Messages: []ChatMessage{ + {Role: "user", Content: "Hello Gradio 3!"}, + }, + Stream: false, + } + b, _ := json.Marshal(req) + httpReq := httptest.NewRequest("POST", "/v1/chat/completions", bytes.NewBuffer(b)) + rec := httptest.NewRecorder() + + if err := gw.ExecuteChatCompletion(rec, httpReq, req); err != nil { + t.Fatalf("ExecuteChatCompletion failed on Gradio 3: %v", err) + } + + var res ChatCompletionResponse + if err := json.NewDecoder(rec.Body).Decode(&res); err != nil { + t.Fatalf("failed to decode response: %v", err) + } + if res.Choices[0].Message.Content != "Echo from Gradio 3: Hello Gradio 3!" { + t.Errorf("unexpected completion content: %v", res.Choices[0].Message.Content) + } +} + +// TestEndpointScoringDisambiguation verifies that chat generation endpoints are selected over +// UI utility, reset, voting, and feedback endpoints. +func TestEndpointScoringDisambiguation(t *testing.T) { + compMap := map[int]GradioComponent{ + 1: {ID: 1, Type: "textbox", Props: map[string]interface{}{"label": "Message"}}, + 2: {ID: 2, Type: "chatbot", Props: map[string]interface{}{"label": "Chat"}}, + 3: {ID: 3, Type: "state", Props: map[string]interface{}{"label": "State"}}, + } + + chatDep := GradioDependency{ + ID: 0, + Inputs: []int{1, 2, 3}, + Outputs: []int{2}, + Types: GradioDependencyTypes{Generator: true}, + } + clearDep := GradioDependency{ + ID: 1, + Inputs: []int{2}, + Outputs: []int{2}, + } + voteDep := GradioDependency{ + ID: 2, + Inputs: []int{2}, + Outputs: []int{}, + } + + chatScore := ScoreCandidateEndpoint("chat", nil, &chatDep, compMap) + clearScore := ScoreCandidateEndpoint("clear", nil, &clearDep, compMap) + voteScore := ScoreCandidateEndpoint("vote", nil, &voteDep, compMap) + resetScore := ScoreCandidateEndpoint("reset_all", nil, &clearDep, compMap) + + if chatScore <= 0 { + t.Errorf("expected positive chat score, got %d", chatScore) + } + if clearScore >= chatScore { + t.Errorf("expected chat score > clear score, got chat=%d clear=%d", chatScore, clearScore) + } + if voteScore >= chatScore { + t.Errorf("expected chat score > vote score, got chat=%d vote=%d", chatScore, voteScore) + } + if resetScore >= chatScore { + t.Errorf("expected chat score > reset score, got chat=%d reset=%d", chatScore, resetScore) + } +} + +// TestToolCallingPictureResolution verifies that tool calling mode is accurately resolved +// based on component topology. +func TestToolCallingPictureResolution(t *testing.T) { + // Case A: Native slot + discNative := NewDefaultSpaceDiscovery("https://tencent-hy3.hf.space") + discNative.FunctionsJSONIndex = 8 + discNative.SystemIndex = 2 + discNative.HistoryIndex = 1 + discNative.ToolCallMode = "native_slot" + if discNative.ToolCallMode != "native_slot" { + t.Errorf("expected native_slot, got %s", discNative.ToolCallMode) + } + + // Case B: Dedicated system prompt slot + discSys := NewDefaultSpaceDiscovery("https://custom-chat.hf.space") + discSys.FunctionsJSONIndex = -1 + discSys.SystemIndex = 2 + discSys.HistoryIndex = 1 + discSys.ToolCallMode = "prompt_augmented_system" + if discSys.ToolCallMode != "prompt_augmented_system" { + t.Errorf("expected prompt_augmented_system, got %s", discSys.ToolCallMode) + } + + // Case C: Conversation history (first turn) + discFirst := NewDefaultSpaceDiscovery("https://chat-only.hf.space") + discFirst.FunctionsJSONIndex = -1 + discFirst.SystemIndex = -1 + discFirst.HistoryIndex = 1 + discFirst.ToolCallMode = "prompt_augmented_first_turn" + if discFirst.ToolCallMode != "prompt_augmented_first_turn" { + t.Errorf("expected prompt_augmented_first_turn, got %s", discFirst.ToolCallMode) + } + + // Case D: Single prompt input + discSingle := NewDefaultSpaceDiscovery("https://single-prompt.hf.space") + discSingle.FunctionsJSONIndex = -1 + discSingle.SystemIndex = -1 + discSingle.HistoryIndex = -1 + discSingle.ToolCallMode = "prompt_augmented_single_prompt" + if discSingle.ToolCallMode != "prompt_augmented_single_prompt" { + t.Errorf("expected prompt_augmented_single_prompt, got %s", discSingle.ToolCallMode) + } +} + +// TestCallToPredictProtocolFallback verifies that if a space reports /call support in /info +// but /call returns 404 at runtime, gr2gw gracefully falls back to /run/predict. +func TestCallToPredictProtocolFallback(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/gradio_api/info" { + info := GradioAPIInfoResponse{ + NamedEndpoints: map[string]GradioEndpointInfo{ + "/chat": { + Parameters: []GradioParamInfo{ + {ParameterName: "prompt", Label: "Prompt", Component: "Textbox"}, + }, + }, + }, + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(info) + return + } + + if r.URL.Path == "/config" { + cfg := GradioConfigResponse{ + Version: "4.20.0", + Mode: "interface", + Components: []GradioComponent{ + {ID: 1, Type: "textbox", Props: map[string]interface{}{"label": "Prompt"}}, + }, + Dependencies: []GradioDependency{ + {ID: 0, APIName: "/chat", Inputs: []int{1}, Outputs: []int{1}}, + }, + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(cfg) + return + } + + // /call/chat returns 404 (endpoint disabled or unsupported) + if strings.HasPrefix(r.URL.Path, "/gradio_api/call/") || strings.HasPrefix(r.URL.Path, "/call/") { + http.NotFound(w, r) + return + } + + // Fallback /run/predict works + if r.URL.Path == "/run/predict" || r.URL.Path == "/gradio_api/run/predict" { + var body struct { + Data []interface{} `json:"data"` + } + json.NewDecoder(r.Body).Decode(&body) + msg, _ := body.Data[0].(string) + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(map[string]interface{}{ + "data": []interface{}{"Fallback response for: " + msg}, + "is_generating": false, + }) + return + } + + http.NotFound(w, r) + })) + defer ts.Close() + + gw := NewGradioGateway(ts.URL, "", 10*time.Second) + + req := ChatCompletionRequest{ + Messages: []ChatMessage{ + {Role: "user", Content: "Testing fallback"}, + }, + Stream: false, + } + b, _ := json.Marshal(req) + httpReq := httptest.NewRequest("POST", "/v1/chat/completions", bytes.NewBuffer(b)) + rec := httptest.NewRecorder() + + if err := gw.ExecuteChatCompletion(rec, httpReq, req); err != nil { + t.Fatalf("ExecuteChatCompletion fallback failed: %v", err) + } + + var res ChatCompletionResponse + if err := json.NewDecoder(rec.Body).Decode(&res); err != nil { + t.Fatalf("failed to decode response: %v", err) + } + if res.Choices[0].Message.Content != "Fallback response for: Testing fallback" { + t.Errorf("unexpected content: %v", res.Choices[0].Message.Content) + } +} +