Support multimodal textbox spaces with state inputs and default parameter mapping

2026-09-07 11:38:37 +03:00
parent 9f2a5068f8
commit 491114b2bd
3 changed files with 479 additions and 53 deletions
@@ -1397,3 +1397,204 @@ func TestConfiguredModelName(t *testing.T) {
  }
 }

+func TestMultimodalStateSpaceMockServerCompletion(t *testing.T) {
+  ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+    if r.URL.Path == "/config" {
+      cfg := GradioConfigResponse{
+        Version: "5.29.0",
+        Dependencies: []GradioDependency{
+          {
+            ID:      6,
+            APIName: "chat",
+            Inputs:  []int{12, 16, 20, 21, 22, 23, 24, 25, 26},
+            Outputs: []int{14, 16},
+          },
+        },
+        Components: []GradioComponent{
+          {ID: 12, Type: "multimodaltextbox", Props: map[string]interface{}{"label": "Message"}},
+          {ID: 16, Type: "state"},
+          {ID: 20, Type: "radio", Props: map[string]interface{}{"label": "Model Type", "value": "Chat"}},
+          {ID: 21, Type: "checkbox", Props: map[string]interface{}{"label": "Use Internet", "value": false}},
+          {ID: 22, Type: "slider", Props: map[string]interface{}{"label": "Max Tokens", "value": 32768}},
+          {ID: 23, Type: "slider", Props: map[string]interface{}{"label": "Temperature", "value": 0.8}},
+          {ID: 24, Type: "slider", Props: map[string]interface{}{"label": "Top P", "value": 0.95}},
+          {ID: 25, Type: "checkbox", Props: map[string]interface{}{"label": "stream", "value": true}},
+          {ID: 26, Type: "textbox", Props: map[string]interface{}{"label": "user", "value": "null"}},
+        },
+      }
+      w.Header().Set("Content-Type", "application/json")
+      json.NewEncoder(w).Encode(cfg)
+      return
+    }
+
+    if r.URL.Path == "/gradio_api/info" {
+      info := GradioAPIInfoResponse{
+        NamedEndpoints: map[string]GradioEndpointInfo{
+          "/chat": {
+            Parameters: []GradioParamInfo{
+              {ParameterName: "param_0", Label: "Message", Component: "Multimodaltextbox"},
+              {ParameterName: "param_2", Label: "Model Type", Component: "Radio", ParameterDefault: "Chat"},
+              {ParameterName: "param_3", Label: "Use Internet", Component: "Checkbox", ParameterDefault: false},
+              {ParameterName: "param_4", Label: "Max Tokens", Component: "Slider", ParameterDefault: 32768},
+              {ParameterName: "param_5", Label: "Temperature", Component: "Slider", ParameterDefault: 0.8},
+              {ParameterName: "param_6", Label: "Top P", Component: "Slider", ParameterDefault: 0.95},
+              {ParameterName: "param_7", Label: "stream", Component: "Checkbox", ParameterDefault: true},
+              {ParameterName: "param_8", Label: "user", Component: "Textbox", ParameterDefault: "null"},
+            },
+          },
+        },
+      }
+      w.Header().Set("Content-Type", "application/json")
+      json.NewEncoder(w).Encode(info)
+      return
+    }
+
+    if r.URL.Path == "/gradio_api/call/chat" {
+      var body struct {
+        Data []interface{} `json:"data"`
+      }
+      if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
+        http.Error(w, err.Error(), http.StatusBadRequest)
+        return
+      }
+      if len(body.Data) != 9 {
+        http.Error(w, fmt.Sprintf("expected 9 inputs, got %d", len(body.Data)), http.StatusBadRequest)
+        return
+      }
+      // Check multimodal dict at index 0
+      mmDict, ok := body.Data[0].(map[string]interface{})
+      if !ok {
+        http.Error(w, "input 0 must be multimodal dict", http.StatusBadRequest)
+        return
+      }
+      txt, _ := mmDict["text"].(string)
+      evt := "evt_normal"
+      if strings.Contains(txt, "get_weather") {
+        evt = "evt_tool"
+      }
+      w.Header().Set("Content-Type", "application/json")
+      json.NewEncoder(w).Encode(GradioJoinResponse{EventID: evt})
+      return
+    }
+
+    if r.URL.Path == "/gradio_api/call/chat/evt_normal" {
+      w.Header().Set("Content-Type", "text/event-stream")
+      flusher, _ := w.(http.Flusher)
+      fmt.Fprintf(w, "event: complete\ndata: [\"Hello from multimodal space!\", null]\n\n")
+      flusher.Flush()
+      return
+    }
+
+    if r.URL.Path == "/gradio_api/call/chat/evt_tool" {
+      w.Header().Set("Content-Type", "text/event-stream")
+      flusher, _ := w.(http.Flusher)
+      fmt.Fprintf(w, "event: complete\ndata: [\"```json\\n[{\\\"name\\\": \\\"get_weather\\\", \\\"arguments\\\": {\\\"location\\\": \\\"Tokyo\\\"}}]\\n```\", null]\n\n")
+      flusher.Flush()
+      return
+    }
+
+    http.NotFound(w, r)
+  }))
+  defer ts.Close()
+
+  gw := NewGradioGateway(ts.URL, "", 10*time.Second)
+
+  // 1. Verify Space Inspection
+  disc, err := InspectSpace(gw.client, ts.URL, DefaultUserAgent)
+  if err != nil {
+    t.Fatalf("InspectSpace failed: %v", err)
+  }
+  if disc.TotalInputs != 9 {
+    t.Errorf("expected TotalInputs 9, got %d", disc.TotalInputs)
+  }
+  if disc.MessageIndex != 0 {
+    t.Errorf("expected MessageIndex 0, got %d", disc.MessageIndex)
+  }
+  if !disc.MessageIsMultimodal {
+    t.Errorf("expected MessageIsMultimodal true")
+  }
+  if disc.TempIndex != 5 {
+    t.Errorf("expected TempIndex 5, got %d", disc.TempIndex)
+  }
+  if disc.MaxTokensIndex != 4 {
+    t.Errorf("expected MaxTokensIndex 4, got %d", disc.MaxTokensIndex)
+  }
+  if disc.TopPIndex != 6 {
+    t.Errorf("expected TopPIndex 6, got %d", disc.TopPIndex)
+  }
+  if disc.StreamIndex != 7 {
+    t.Errorf("expected StreamIndex 7, got %d", disc.StreamIndex)
+  }
+
+  // 2. Normal Chat Completion
+  req1 := ChatCompletionRequest{
+    Model: "askcyph",
+    Messages: []ChatMessage{
+      {Role: "user", Content: "Hello world"},
+    },
+    Stream: false,
+  }
+  b1, _ := json.Marshal(req1)
+  httpReq1 := httptest.NewRequest("POST", "/v1/chat/completions", bytes.NewBuffer(b1))
+  rec1 := httptest.NewRecorder()
+
+  err = gw.ExecuteChatCompletion(rec1, httpReq1, req1)
+  if err != nil {
+    t.Fatalf("ExecuteChatCompletion failed: %v", err)
+  }
+  var resp1 ChatCompletionResponse
+  if err := json.NewDecoder(rec1.Body).Decode(&resp1); err != nil {
+    t.Fatalf("failed to decode response: %v", err)
+  }
+  if resp1.Choices[0].Message.Content != "Hello from multimodal space!" {
+    t.Errorf("unexpected content: %v", resp1.Choices[0].Message.Content)
+  }
+
+  // 3. Tool Calling Chat Completion
+  req2 := ChatCompletionRequest{
+    Model: "askcyph",
+    Messages: []ChatMessage{
+      {Role: "user", Content: "Weather in Tokyo?"},
+    },
+    Tools: []Tool{
+      {
+        Type: "function",
+        Function: map[string]interface{}{
+          "name":        "get_weather",
+          "description": "Get current weather",
+          "parameters": map[string]interface{}{
+            "type": "object",
+            "properties": map[string]interface{}{
+              "location": map[string]interface{}{"type": "string"},
+            },
+            "required": []string{"location"},
+          },
+        },
+      },
+    },
+    Stream: false,
+  }
+  b2, _ := json.Marshal(req2)
+  httpReq2 := httptest.NewRequest("POST", "/v1/chat/completions", bytes.NewBuffer(b2))
+  rec2 := httptest.NewRecorder()
+
+  err = gw.ExecuteChatCompletion(rec2, httpReq2, req2)
+  if err != nil {
+    t.Fatalf("ExecuteChatCompletion tool calling failed: %v", err)
+  }
+  var resp2 ChatCompletionResponse
+  if err := json.NewDecoder(rec2.Body).Decode(&resp2); err != nil {
+    t.Fatalf("failed to decode response: %v", err)
+  }
+  if resp2.Choices[0].FinishReason != "tool_calls" {
+    t.Fatalf("expected finish_reason 'tool_calls', got %q", resp2.Choices[0].FinishReason)
+  }
+  if len(resp2.Choices[0].Message.ToolCalls) != 1 {
+    t.Fatalf("expected 1 tool call, got %d", len(resp2.Choices[0].Message.ToolCalls))
+  }
+  if resp2.Choices[0].Message.ToolCalls[0].Function.Name != "get_weather" {
+    t.Errorf("expected get_weather, got %s", resp2.Choices[0].Message.ToolCalls[0].Function.Name)
+  }
+}
+
+