Support multimodal textbox spaces with state inputs and default parameter mapping

README.md

@@ -17,6 +17,7 @@ Default demo space: `https://tencent-hy3.hf.space`
   - Automatically formats conversation history into structured inputs when the space supports them.
   - Transparently composes multi-turn dialogue (`System`, `User`, `Assistant`) into single prompt inputs when the space only accepts a single message textbox.
   - Automatically pads hidden/State inputs (e.g. Gradio State components) to prevent backend argument count mismatches.
+  - Automatically handles multimodal textbox inputs (`MultimodalData` with `{text, files}`) and space component defaults (radios, sliders, checkboxes).
 - **Real-time streaming & accumulation filter**:
   - Automatically computes token deltas from cumulative or incremental Gradio SSE output streams (including 2D Hy3 frames `[[content, reasoning, tool_calls, history]]`).
   - Emits standards-compliant `chat.completion.chunk` SSE events in real time.

gr2gw.go

@@ -1192,10 +1192,10 @@ func extractGradioErrorMessage(dataStr string) string {
     }
   }
   clean := strings.TrimSpace(dataStr)
-  if clean != "" && clean != "null" {
-    return clean
+  if clean == "null" || clean == "" {
+    return "upstream Gradio space returned null error (space may have show_error=False or failed input validation)"
   }
-  return "unknown upstream Gradio error"
+  return clean
 }
 
 type Streamer struct {
@@ -1594,7 +1594,7 @@ type HFSpaceInfoResponse struct {
 type SpaceParamMapping struct {
   InputIndex   int
   ComponentID  int
-  ParamType    string // "message", "history", "system_prompt", "temperature", "max_tokens", "top_p", "state", "other"
+  ParamType    string // "message", "history", "system_prompt", "temperature", "max_tokens", "top_p", "think_level", "tools", "stream", "state", "other"
   DefaultValue interface{}
 }
 
@@ -1609,6 +1609,8 @@ type SpaceDiscovery struct {
   Protocol               string // "call", "queue", "predict"
   TotalInputs            int
   ParamMappings          []SpaceParamMapping
+  DefaultInputs          []interface{}
+  MessageIsMultimodal    bool
   HistoryIndex           int    // -1 if none
   MessageIndex           int    // index for user message text
   SystemIndex            int    // -1 if none
@@ -1616,11 +1618,12 @@ type SpaceDiscovery struct {
   TempIndex              int    // -1 if none
   MaxTokensIndex         int    // -1 if none
   TopPIndex              int    // -1 if none
+  StreamIndex            int    // -1 if none
   ThinkLevelIndex        int    // -1 if none
   FunctionsJSONIndex     int    // -1 if none
   PreservedThinkingIndex int    // -1 if none
   IsHunyuan3             bool
-  HistoryFormat          string // "messages", "pairs", "none"
+  HistoryFormat          string // "messages", "pairs", "gradio_messages", "none"
   LastDiscovered         time.Time
 }
 
@@ -1663,7 +1666,7 @@ func (d *SpaceDiscovery) GetModelList() []ModelItem {
 
   if len(items) == 0 {
     items = append(items, ModelItem{
-      ID:      "default",
+      ID:      "gradio-chat",
       Object:  "model",
       Created: now,
       OwnedBy: "gradio",
@@ -1673,6 +1676,33 @@ func (d *SpaceDiscovery) GetModelList() []ModelItem {
   return items
 }
 
+func (d *SpaceDiscovery) MatchesModel(requested string) bool {
+  if requested == "" {
+    return true
+  }
+  cleanReq := strings.TrimPrefix(requested, "models/")
+  cleanReq = strings.TrimPrefix(cleanReq, "openai/")
+  cleanReq = strings.ToLower(cleanReq)
+
+  if ConfiguredModelName != "" && strings.ToLower(ConfiguredModelName) == cleanReq {
+    return true
+  }
+
+  for _, m := range d.Models {
+    mClean := strings.TrimPrefix(m, "models/")
+    mClean = strings.TrimPrefix(mClean, "openai/")
+    if strings.ToLower(mClean) == cleanReq || strings.ToLower(m) == cleanReq {
+      return true
+    }
+  }
+
+  if strings.ToLower(d.PrimaryModel) == cleanReq {
+    return true
+  }
+
+  return cleanReq == "default" || cleanReq == "gradio" || cleanReq == "gradio-chat"
+}
+
 func NewDefaultSpaceDiscovery(spaceURL string) *SpaceDiscovery {
   cleanURL := strings.TrimRight(spaceURL, "/")
   if cleanURL != "" && !strings.HasPrefix(cleanURL, "http://") && !strings.HasPrefix(cleanURL, "https://") {
@@ -1692,6 +1722,7 @@ func NewDefaultSpaceDiscovery(spaceURL string) *SpaceDiscovery {
     TempIndex:              -1,
     MaxTokensIndex:         -1,
     TopPIndex:              -1,
+    StreamIndex:            -1,
     ThinkLevelIndex:        -1,
     FunctionsJSONIndex:     -1,
     PreservedThinkingIndex: -1,
@@ -1863,11 +1894,12 @@ func InspectSpace(client *http.Client, rawURL, userAgent string) (*SpaceDiscover
 
       for _, p := range epInfo.Parameters {
         pLower := strings.ToLower(p.ParameterName)
+        pLabel := strings.ToLower(p.Label)
         pComp := strings.ToLower(p.Component)
-        if strings.Contains(pLower, "message") || strings.Contains(pLower, "text") || strings.Contains(pLower, "prompt") || strings.Contains(pLower, "query") || strings.Contains(pLower, "question") || strings.Contains(pLower, "input") || pComp == "textbox" {
+        if strings.Contains(pLower, "message") || strings.Contains(pLabel, "message") || strings.Contains(pLower, "text") || strings.Contains(pLower, "prompt") || strings.Contains(pLower, "query") || strings.Contains(pLower, "question") || strings.Contains(pLower, "input") || pComp == "textbox" || pComp == "multimodaltextbox" {
           score += 40
         }
-        if strings.Contains(pLower, "history") || strings.Contains(pLower, "chat") || strings.Contains(pLower, "messages") || strings.Contains(pLower, "conversation") || pComp == "chatbot" {
+        if strings.Contains(pLower, "history") || strings.Contains(pLabel, "history") || strings.Contains(pLower, "chat") || strings.Contains(pLower, "messages") || strings.Contains(pLower, "conversation") || pComp == "chatbot" {
           score += 30
         }
       }
@@ -1881,6 +1913,30 @@ func InspectSpace(client *http.Client, rawURL, userAgent string) (*SpaceDiscover
     }
   }
 
+  // Fallback: if no named endpoint from /info, inspect dependencies in config
+  if bestEndpoint == "" && configFetched && len(configResp.Dependencies) > 0 {
+    depScore := -1000
+    for _, dep := range configResp.Dependencies {
+      if depName, ok := dep.APIName.(string); ok && depName != "" {
+        cleanName := strings.TrimPrefix(depName, "/")
+        lowerName := strings.ToLower(cleanName)
+        if strings.Contains(lowerName, "clear") || strings.Contains(lowerName, "reset") || strings.Contains(lowerName, "save") || strings.Contains(lowerName, "delete") || strings.Contains(lowerName, "pop") || strings.Contains(lowerName, "lambda") {
+          continue
+        }
+        score := 0
+        if strings.Contains(lowerName, "chat") || strings.Contains(lowerName, "conversation") {
+          score += 100
+        } else if strings.Contains(lowerName, "generate") || strings.Contains(lowerName, "predict") || strings.Contains(lowerName, "submit") {
+          score += 50
+        }
+        if score > depScore {
+          depScore = score
+          bestEndpoint = "/" + cleanName
+        }
+      }
+    }
+  }
+
   if bestEndpoint != "" {
     discovery.Endpoint = bestEndpoint
     discovery.CleanEndpoint = strings.TrimPrefix(bestEndpoint, "/")
@@ -1888,12 +1944,13 @@ func InspectSpace(client *http.Client, rawURL, userAgent string) (*SpaceDiscover
 
   // 5. Correlate with config.dependencies to determine exact input count & state padding
   compMap := make(map[int]GradioComponent)
+  var matchingDep *GradioDependency
+
   if configFetched {
     for _, comp := range configResp.Components {
       compMap[comp.ID] = comp
     }
 
-    var matchingDep *GradioDependency
     cleanTarget := strings.TrimPrefix(discovery.Endpoint, "/")
 
     for _, dep := range configResp.Dependencies {
@@ -1910,6 +1967,10 @@ func InspectSpace(client *http.Client, rawURL, userAgent string) (*SpaceDiscover
 
     if matchingDep != nil {
       discovery.TotalInputs = len(matchingDep.Inputs)
+      discovery.DefaultInputs = make([]interface{}, len(matchingDep.Inputs))
+      discovery.ParamMappings = nil
+      discovery.MessageIndex = -1
+
       for idx, compID := range matchingDep.Inputs {
         mapping := SpaceParamMapping{
           InputIndex:  idx,
@@ -1918,11 +1979,26 @@ func InspectSpace(client *http.Client, rawURL, userAgent string) (*SpaceDiscover
         }
         if comp, exists := compMap[compID]; exists {
           cType := strings.ToLower(comp.Type)
+          cLabel := ""
+          if comp.Props != nil {
+            if l, ok := comp.Props["label"].(string); ok {
+              cLabel = strings.ToLower(l)
+            }
+            if val, ok := comp.Props["value"]; ok {
+              discovery.DefaultInputs[idx] = val
+              mapping.DefaultValue = val
+            }
+          }
+
           switch cType {
-          case "textbox", "multimodaltextbox":
-            if discovery.MessageIndex == 0 && idx == 0 {
+          case "multimodaltextbox":
+            if discovery.MessageIndex == -1 || strings.Contains(cLabel, "message") || strings.Contains(cLabel, "prompt") || strings.Contains(cLabel, "input") {
               mapping.ParamType = "message"
-            } else if discovery.SystemIndex == -1 {
+              discovery.MessageIndex = idx
+              discovery.MessageIsMultimodal = true
+            }
+          case "textbox":
+            if strings.Contains(cLabel, "system") || strings.Contains(cLabel, "instruction") {
               mapping.ParamType = "system_prompt"
               discovery.SystemIndex = idx
               if comp.Props != nil {
@@ -1930,56 +2006,122 @@ func InspectSpace(client *http.Client, rawURL, userAgent string) (*SpaceDiscover
                   discovery.DefaultSystemPrompt = strings.TrimSpace(val)
                 }
               }
+            } else if discovery.MessageIndex == -1 || strings.Contains(cLabel, "message") || strings.Contains(cLabel, "prompt") || strings.Contains(cLabel, "query") || strings.Contains(cLabel, "input") || strings.Contains(cLabel, "question") {
+              mapping.ParamType = "message"
+              discovery.MessageIndex = idx
+              discovery.MessageIsMultimodal = false
+            }
+          case "chatbot":
+            mapping.ParamType = "history"
+            discovery.HistoryIndex = idx
+            if strings.HasPrefix(configResp.Version, "5.") || strings.HasPrefix(configResp.Version, "6.") {
+              discovery.HistoryFormat = "gradio_messages"
+            } else {
+              discovery.HistoryFormat = "pairs"
             }
           case "state":
             mapping.ParamType = "state"
-            if idx == 1 && len(matchingDep.Inputs) == 2 {
-              // Standard Gradio ChatInterface: [textbox, state]
-              // Component 13 is state
-            }
           case "slider", "number":
-            label := ""
-            if comp.Props != nil {
-              if l, ok := comp.Props["label"].(string); ok {
-                label = strings.ToLower(l)
-              }
-            }
-            if strings.Contains(label, "temp") {
+            if strings.Contains(cLabel, "temp") {
               mapping.ParamType = "temperature"
               discovery.TempIndex = idx
-            } else if strings.Contains(label, "max") || strings.Contains(label, "token") {
+            } else if strings.Contains(cLabel, "max") || strings.Contains(cLabel, "token") {
               mapping.ParamType = "max_tokens"
               discovery.MaxTokensIndex = idx
-            } else if strings.Contains(label, "top_p") {
+            } else if strings.Contains(cLabel, "top_p") || strings.Contains(cLabel, "top-p") || strings.Contains(cLabel, "top p") {
               mapping.ParamType = "top_p"
               discovery.TopPIndex = idx
+            } else if strings.Contains(cLabel, "think") {
+              mapping.ParamType = "think_level"
+              discovery.ThinkLevelIndex = idx
+            }
+          case "checkbox":
+            if strings.Contains(cLabel, "stream") {
+              mapping.ParamType = "stream"
+              discovery.StreamIndex = idx
+            }
+          default:
+            if strings.Contains(cLabel, "tool") || strings.Contains(cLabel, "function") {
+              mapping.ParamType = "tools"
+              discovery.FunctionsJSONIndex = idx
             }
           }
         }
         discovery.ParamMappings = append(discovery.ParamMappings, mapping)
       }
-    } else if bestEndpointInfo != nil {
-      discovery.TotalInputs = len(bestEndpointInfo.Parameters)
+
+      if discovery.MessageIndex == -1 {
+        discovery.MessageIndex = 0
+        if len(matchingDep.Inputs) > 0 {
+          if comp, exists := compMap[matchingDep.Inputs[0]]; exists {
+            if strings.ToLower(comp.Type) == "multimodaltextbox" {
+              discovery.MessageIsMultimodal = true
+            }
+          }
+        }
+      }
     }
   }
 
-  // Check parameters in bestEndpointInfo for input indices and history support
+  // Refine history format or discover tools from bestEndpointInfo
   if bestEndpointInfo != nil {
-    if discovery.TotalInputs < len(bestEndpointInfo.Parameters) {
-      discovery.TotalInputs = len(bestEndpointInfo.Parameters)
+    for _, p := range bestEndpointInfo.Parameters {
+      pName := strings.ToLower(p.ParameterName)
+      pLabel := strings.ToLower(p.Label)
+      pComp := strings.ToLower(p.Component)
+      if strings.Contains(pName, "history") || strings.Contains(pLabel, "history") || strings.Contains(pName, "chat") || strings.Contains(pName, "messages") || pComp == "chatbot" {
+        bType, _ := json.Marshal(p.Type)
+        bPyType, _ := json.Marshal(p.PythonType)
+        pPyType := strings.ToLower(string(bPyType))
+        bTypeStr := strings.ToLower(string(bType))
+        if strings.Contains(pPyType, "list[tuple[") || strings.Contains(pPyType, "list[list[") || strings.Contains(bTypeStr, "tuple") {
+          discovery.HistoryFormat = "pairs"
+        } else if strings.Contains(pPyType, "textmessage") || strings.Contains(pPyType, "dict(text: str") || strings.Contains(bTypeStr, "textmessage") || strings.Contains(bTypeStr, "chatbotdatamessages") {
+          discovery.HistoryFormat = "gradio_messages"
         }
+      }
+    }
+  }
+
+  // Fallback: If config.dependencies did not provide matchingDep, map directly from bestEndpointInfo.Parameters
+  if matchingDep == nil && bestEndpointInfo != nil {
+    discovery.TotalInputs = len(bestEndpointInfo.Parameters)
+    discovery.DefaultInputs = make([]interface{}, len(bestEndpointInfo.Parameters))
+    discovery.ParamMappings = nil
+    discovery.MessageIndex = -1
+
     for idx, p := range bestEndpointInfo.Parameters {
       pName := strings.ToLower(p.ParameterName)
+      pLabel := strings.ToLower(p.Label)
       pComp := strings.ToLower(p.Component)
-      if strings.Contains(pName, "system") {
+
+      if p.ParameterDefault != nil {
+        discovery.DefaultInputs[idx] = p.ParameterDefault
+      }
+
+      mapping := SpaceParamMapping{
+        InputIndex:   idx,
+        ParamType:    "other",
+        DefaultValue: p.ParameterDefault,
+      }
+
+      if strings.Contains(pComp, "multimodal") {
+        if discovery.MessageIndex == -1 || strings.Contains(pLabel, "message") || strings.Contains(pName, "message") {
+          mapping.ParamType = "message"
+          discovery.MessageIndex = idx
+          discovery.MessageIsMultimodal = true
+        }
+      } else if strings.Contains(pName, "system") || strings.Contains(pLabel, "system") || strings.Contains(pLabel, "instruction") {
         discovery.SystemIndex = idx
+        mapping.ParamType = "system_prompt"
         if p.ParameterDefault != nil && discovery.DefaultSystemPrompt == "" {
           if defStr, ok := p.ParameterDefault.(string); ok && strings.TrimSpace(defStr) != "" {
             discovery.DefaultSystemPrompt = strings.TrimSpace(defStr)
           }
         }
-      } else if strings.Contains(pName, "history") || strings.Contains(pName, "chat") || strings.Contains(pName, "conversation") || strings.Contains(pName, "messages") || pComp == "chatbot" {
+      } else if strings.Contains(pName, "history") || strings.Contains(pLabel, "history") || strings.Contains(pName, "chat") || strings.Contains(pName, "messages") || strings.Contains(pName, "conversation") || pComp == "chatbot" {
         discovery.HistoryIndex = idx
+        mapping.ParamType = "history"
         bType, _ := json.Marshal(p.Type)
         bPyType, _ := json.Marshal(p.PythonType)
         pPyType := strings.ToLower(string(bPyType))
@@ -1991,21 +2133,37 @@ func InspectSpace(client *http.Client, rawURL, userAgent string) (*SpaceDiscover
         } else if strings.HasPrefix(configResp.Version, "5.") || strings.HasPrefix(configResp.Version, "6.") {
           discovery.HistoryFormat = "gradio_messages"
         }
-      } else if strings.Contains(pName, "message") || (strings.Contains(pName, "prompt") && !strings.Contains(pName, "system")) || strings.Contains(pName, "text") || strings.Contains(pName, "query") || strings.Contains(pName, "question") || strings.Contains(pName, "input") || pComp == "textbox" {
+      } else if strings.Contains(pLabel, "message") || strings.Contains(pName, "message") || (strings.Contains(pLabel, "prompt") && !strings.Contains(pLabel, "system")) || (strings.Contains(pName, "prompt") && !strings.Contains(pName, "system")) || strings.Contains(pLabel, "query") || strings.Contains(pName, "query") || strings.Contains(pLabel, "question") || strings.Contains(pName, "question") || (discovery.MessageIndex == -1 && (pComp == "textbox" || idx == 0)) {
         discovery.MessageIndex = idx
-      } else if strings.Contains(pName, "think_level") || strings.Contains(pName, "thinking_level") {
+        mapping.ParamType = "message"
+        discovery.MessageIsMultimodal = false
+      } else if strings.Contains(pName, "think_level") || strings.Contains(pName, "thinking_level") || strings.Contains(pLabel, "think") {
         discovery.ThinkLevelIndex = idx
-      } else if strings.Contains(pName, "functions") || strings.Contains(pName, "tools") {
+        mapping.ParamType = "think_level"
+      } else if strings.Contains(pName, "functions") || strings.Contains(pName, "tools") || strings.Contains(pLabel, "tools") || strings.Contains(pLabel, "functions") {
         discovery.FunctionsJSONIndex = idx
+        mapping.ParamType = "tools"
       } else if strings.Contains(pName, "preserved") {
         discovery.PreservedThinkingIndex = idx
-      } else if strings.Contains(pName, "temp") {
+      } else if strings.Contains(pName, "temp") || strings.Contains(pLabel, "temp") {
         discovery.TempIndex = idx
-      } else if strings.Contains(pName, "token") {
+        mapping.ParamType = "temperature"
+      } else if strings.Contains(pName, "token") || strings.Contains(pLabel, "token") {
         discovery.MaxTokensIndex = idx
-      } else if strings.Contains(pName, "top_p") {
+        mapping.ParamType = "max_tokens"
+      } else if strings.Contains(pName, "top_p") || strings.Contains(pLabel, "top_p") || strings.Contains(pLabel, "top p") {
         discovery.TopPIndex = idx
+        mapping.ParamType = "top_p"
+      } else if strings.Contains(pName, "stream") || strings.Contains(pLabel, "stream") {
+        discovery.StreamIndex = idx
+        mapping.ParamType = "stream"
       }
+
+      discovery.ParamMappings = append(discovery.ParamMappings, mapping)
+    }
+
+    if discovery.MessageIndex == -1 {
+      discovery.MessageIndex = 0
     }
   }
 
@@ -2022,6 +2180,9 @@ func InspectSpace(client *http.Client, rawURL, userAgent string) (*SpaceDiscover
   if discovery.TotalInputs < 1 {
     discovery.TotalInputs = 1
   }
+  for len(discovery.DefaultInputs) < discovery.TotalInputs {
+    discovery.DefaultInputs = append(discovery.DefaultInputs, nil)
+  }
 
   return discovery, nil
 }
@@ -2320,11 +2481,57 @@ func (g *GradioGateway) BuildGradioPayload(disc *SpaceDiscovery, req ChatComplet
   }
   data := make([]interface{}, totalInputs)
 
+  // Initialize with space default inputs if available
+  if len(disc.DefaultInputs) == totalInputs {
+    for i := 0; i < totalInputs; i++ {
+      data[i] = disc.DefaultInputs[i]
+    }
+  }
+
+  // Extract multimodal files if message input is multimodal
+  var messageFiles []interface{}
+  if disc.MessageIsMultimodal && len(nonSystem) > 0 {
+    lastMsg := nonSystem[len(nonSystem)-1]
+    if parts, ok := lastMsg.Content.([]interface{}); ok {
+      for _, p := range parts {
+        if itemMap, ok := p.(map[string]interface{}); ok {
+          if itemMap["type"] == "image_url" {
+            imgURL := ""
+            if iuMap, ok := itemMap["image_url"].(map[string]interface{}); ok {
+              if u, ok := iuMap["url"].(string); ok {
+                imgURL = u
+              }
+            } else if iuStr, ok := itemMap["image_url"].(string); ok {
+              imgURL = iuStr
+            }
+            if imgURL != "" {
+              messageFiles = append(messageFiles, map[string]interface{}{
+                "path": imgURL,
+                "url":  imgURL,
+                "meta": map[string]interface{}{"_type": "gradio.FileData"},
+              })
+            }
+          }
+        }
+      }
+    }
+  }
+  if messageFiles == nil {
+    messageFiles = []interface{}{}
+  }
+
   // Populate mapped fields
   msgIdx := disc.MessageIndex
   if msgIdx >= 0 && msgIdx < len(data) {
+    if disc.MessageIsMultimodal {
+      data[msgIdx] = map[string]interface{}{
+        "text":  promptMessageText,
+        "files": messageFiles,
+      }
+    } else {
       data[msgIdx] = promptMessageText
     }
+  }
 
   if disc.HistoryIndex >= 0 && disc.HistoryIndex < len(data) {
     if disc.HistoryFormat == "pairs" {
@@ -2393,13 +2600,17 @@ func (g *GradioGateway) BuildGradioPayload(disc *SpaceDiscovery, req ChatComplet
       data[disc.TempIndex] = *req.Temperature
     } else if disc.IsHunyuan3 {
       data[disc.TempIndex] = nil
-    } else {
+    } else if data[disc.TempIndex] == nil {
       data[disc.TempIndex] = 0.7
     }
   }
 
   if disc.MaxTokensIndex >= 0 && disc.MaxTokensIndex < len(data) {
+    if req.MaxTokens > 0 || req.MaxCompletionTokens > 0 {
       data[disc.MaxTokensIndex] = ResolveMaxTokens(req)
+    } else if data[disc.MaxTokensIndex] == nil {
+      data[disc.MaxTokensIndex] = ResolveMaxTokens(req)
+    }
   }
 
   if disc.TopPIndex >= 0 && disc.TopPIndex < len(data) {
@@ -2407,11 +2618,15 @@ func (g *GradioGateway) BuildGradioPayload(disc *SpaceDiscovery, req ChatComplet
       data[disc.TopPIndex] = *req.TopP
     } else if disc.IsHunyuan3 {
       data[disc.TopPIndex] = 0
-    } else {
+    } else if data[disc.TopPIndex] == nil {
       data[disc.TopPIndex] = 1.0
     }
   }
 
+  if disc.StreamIndex > 0 && disc.StreamIndex != disc.MessageIndex && disc.StreamIndex < len(data) {
+    data[disc.StreamIndex] = false
+  }
+
   if disc.FunctionsJSONIndex >= 0 && disc.FunctionsJSONIndex < len(data) {
     functionsJSONStr := ""
     if len(req.Tools) > 0 {
@@ -2435,8 +2650,17 @@ type GradioOutputFrame struct {
 }
 
 // ParseGradioStreamOutput extracts structured content, reasoning, and tool calls from Gradio output
-func ParseGradioStreamOutput(rawJSON string) GradioOutputFrame {
-  var frame GradioOutputFrame
+func ParseGradioStreamOutput(rawJSON string) (frame GradioOutputFrame) {
+  defer func() {
+    if frame.OK && len(frame.ToolCalls) == 0 && frame.Content != "" {
+      tcs, clean, has := DetectToolCalls(frame.Content)
+      if has && len(tcs) > 0 {
+        frame.ToolCalls = tcs
+        frame.Content = clean
+      }
+    }
+  }()
+
   var val interface{}
   if err := json.Unmarshal([]byte(rawJSON), &val); err != nil {
     return frame

gr2gw_test.go

@@ -1397,3 +1397,204 @@ func TestConfiguredModelName(t *testing.T) {
   }
 }
 
+func TestMultimodalStateSpaceMockServerCompletion(t *testing.T) {
+  ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+    if r.URL.Path == "/config" {
+      cfg := GradioConfigResponse{
+        Version: "5.29.0",
+        Dependencies: []GradioDependency{
+          {
+            ID:      6,
+            APIName: "chat",
+            Inputs:  []int{12, 16, 20, 21, 22, 23, 24, 25, 26},
+            Outputs: []int{14, 16},
+          },
+        },
+        Components: []GradioComponent{
+          {ID: 12, Type: "multimodaltextbox", Props: map[string]interface{}{"label": "Message"}},
+          {ID: 16, Type: "state"},
+          {ID: 20, Type: "radio", Props: map[string]interface{}{"label": "Model Type", "value": "Chat"}},
+          {ID: 21, Type: "checkbox", Props: map[string]interface{}{"label": "Use Internet", "value": false}},
+          {ID: 22, Type: "slider", Props: map[string]interface{}{"label": "Max Tokens", "value": 32768}},
+          {ID: 23, Type: "slider", Props: map[string]interface{}{"label": "Temperature", "value": 0.8}},
+          {ID: 24, Type: "slider", Props: map[string]interface{}{"label": "Top P", "value": 0.95}},
+          {ID: 25, Type: "checkbox", Props: map[string]interface{}{"label": "stream", "value": true}},
+          {ID: 26, Type: "textbox", Props: map[string]interface{}{"label": "user", "value": "null"}},
+        },
+      }
+      w.Header().Set("Content-Type", "application/json")
+      json.NewEncoder(w).Encode(cfg)
+      return
+    }
+
+    if r.URL.Path == "/gradio_api/info" {
+      info := GradioAPIInfoResponse{
+        NamedEndpoints: map[string]GradioEndpointInfo{
+          "/chat": {
+            Parameters: []GradioParamInfo{
+              {ParameterName: "param_0", Label: "Message", Component: "Multimodaltextbox"},
+              {ParameterName: "param_2", Label: "Model Type", Component: "Radio", ParameterDefault: "Chat"},
+              {ParameterName: "param_3", Label: "Use Internet", Component: "Checkbox", ParameterDefault: false},
+              {ParameterName: "param_4", Label: "Max Tokens", Component: "Slider", ParameterDefault: 32768},
+              {ParameterName: "param_5", Label: "Temperature", Component: "Slider", ParameterDefault: 0.8},
+              {ParameterName: "param_6", Label: "Top P", Component: "Slider", ParameterDefault: 0.95},
+              {ParameterName: "param_7", Label: "stream", Component: "Checkbox", ParameterDefault: true},
+              {ParameterName: "param_8", Label: "user", Component: "Textbox", ParameterDefault: "null"},
+            },
+          },
+        },
+      }
+      w.Header().Set("Content-Type", "application/json")
+      json.NewEncoder(w).Encode(info)
+      return
+    }
+
+    if r.URL.Path == "/gradio_api/call/chat" {
+      var body struct {
+        Data []interface{} `json:"data"`
+      }
+      if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
+        http.Error(w, err.Error(), http.StatusBadRequest)
+        return
+      }
+      if len(body.Data) != 9 {
+        http.Error(w, fmt.Sprintf("expected 9 inputs, got %d", len(body.Data)), http.StatusBadRequest)
+        return
+      }
+      // Check multimodal dict at index 0
+      mmDict, ok := body.Data[0].(map[string]interface{})
+      if !ok {
+        http.Error(w, "input 0 must be multimodal dict", http.StatusBadRequest)
+        return
+      }
+      txt, _ := mmDict["text"].(string)
+      evt := "evt_normal"
+      if strings.Contains(txt, "get_weather") {
+        evt = "evt_tool"
+      }
+      w.Header().Set("Content-Type", "application/json")
+      json.NewEncoder(w).Encode(GradioJoinResponse{EventID: evt})
+      return
+    }
+
+    if r.URL.Path == "/gradio_api/call/chat/evt_normal" {
+      w.Header().Set("Content-Type", "text/event-stream")
+      flusher, _ := w.(http.Flusher)
+      fmt.Fprintf(w, "event: complete\ndata: [\"Hello from multimodal space!\", null]\n\n")
+      flusher.Flush()
+      return
+    }
+
+    if r.URL.Path == "/gradio_api/call/chat/evt_tool" {
+      w.Header().Set("Content-Type", "text/event-stream")
+      flusher, _ := w.(http.Flusher)
+      fmt.Fprintf(w, "event: complete\ndata: [\"```json\\n[{\\\"name\\\": \\\"get_weather\\\", \\\"arguments\\\": {\\\"location\\\": \\\"Tokyo\\\"}}]\\n```\", null]\n\n")
+      flusher.Flush()
+      return
+    }
+
+    http.NotFound(w, r)
+  }))
+  defer ts.Close()
+
+  gw := NewGradioGateway(ts.URL, "", 10*time.Second)
+
+  // 1. Verify Space Inspection
+  disc, err := InspectSpace(gw.client, ts.URL, DefaultUserAgent)
+  if err != nil {
+    t.Fatalf("InspectSpace failed: %v", err)
+  }
+  if disc.TotalInputs != 9 {
+    t.Errorf("expected TotalInputs 9, got %d", disc.TotalInputs)
+  }
+  if disc.MessageIndex != 0 {
+    t.Errorf("expected MessageIndex 0, got %d", disc.MessageIndex)
+  }
+  if !disc.MessageIsMultimodal {
+    t.Errorf("expected MessageIsMultimodal true")
+  }
+  if disc.TempIndex != 5 {
+    t.Errorf("expected TempIndex 5, got %d", disc.TempIndex)
+  }
+  if disc.MaxTokensIndex != 4 {
+    t.Errorf("expected MaxTokensIndex 4, got %d", disc.MaxTokensIndex)
+  }
+  if disc.TopPIndex != 6 {
+    t.Errorf("expected TopPIndex 6, got %d", disc.TopPIndex)
+  }
+  if disc.StreamIndex != 7 {
+    t.Errorf("expected StreamIndex 7, got %d", disc.StreamIndex)
+  }
+
+  // 2. Normal Chat Completion
+  req1 := ChatCompletionRequest{
+    Model: "askcyph",
+    Messages: []ChatMessage{
+      {Role: "user", Content: "Hello world"},
+    },
+    Stream: false,
+  }
+  b1, _ := json.Marshal(req1)
+  httpReq1 := httptest.NewRequest("POST", "/v1/chat/completions", bytes.NewBuffer(b1))
+  rec1 := httptest.NewRecorder()
+
+  err = gw.ExecuteChatCompletion(rec1, httpReq1, req1)
+  if err != nil {
+    t.Fatalf("ExecuteChatCompletion failed: %v", err)
+  }
+  var resp1 ChatCompletionResponse
+  if err := json.NewDecoder(rec1.Body).Decode(&resp1); err != nil {
+    t.Fatalf("failed to decode response: %v", err)
+  }
+  if resp1.Choices[0].Message.Content != "Hello from multimodal space!" {
+    t.Errorf("unexpected content: %v", resp1.Choices[0].Message.Content)
+  }
+
+  // 3. Tool Calling Chat Completion
+  req2 := ChatCompletionRequest{
+    Model: "askcyph",
+    Messages: []ChatMessage{
+      {Role: "user", Content: "Weather in Tokyo?"},
+    },
+    Tools: []Tool{
+      {
+        Type: "function",
+        Function: map[string]interface{}{
+          "name":        "get_weather",
+          "description": "Get current weather",
+          "parameters": map[string]interface{}{
+            "type": "object",
+            "properties": map[string]interface{}{
+              "location": map[string]interface{}{"type": "string"},
+            },
+            "required": []string{"location"},
+          },
+        },
+      },
+    },
+    Stream: false,
+  }
+  b2, _ := json.Marshal(req2)
+  httpReq2 := httptest.NewRequest("POST", "/v1/chat/completions", bytes.NewBuffer(b2))
+  rec2 := httptest.NewRecorder()
+
+  err = gw.ExecuteChatCompletion(rec2, httpReq2, req2)
+  if err != nil {
+    t.Fatalf("ExecuteChatCompletion tool calling failed: %v", err)
+  }
+  var resp2 ChatCompletionResponse
+  if err := json.NewDecoder(rec2.Body).Decode(&resp2); err != nil {
+    t.Fatalf("failed to decode response: %v", err)
+  }
+  if resp2.Choices[0].FinishReason != "tool_calls" {
+    t.Fatalf("expected finish_reason 'tool_calls', got %q", resp2.Choices[0].FinishReason)
+  }
+  if len(resp2.Choices[0].Message.ToolCalls) != 1 {
+    t.Fatalf("expected 1 tool call, got %d", len(resp2.Choices[0].Message.ToolCalls))
+  }
+  if resp2.Choices[0].Message.ToolCalls[0].Function.Name != "get_weather" {
+    t.Errorf("expected get_weather, got %s", resp2.Choices[0].Message.ToolCalls[0].Function.Name)
+  }
+}
+
+