luxferre/gr2gw
1
0
Fork 0
Code Actions Projects Activity

Support multimodal textbox spaces with state inputs and default parameter mapping

Browse Source
This commit is contained in:
Luxferre
2026-09-07 11:38:37 +03:00
parent 9f2a5068f8
commit 491114b2bd
3 changed files with 479 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gr2gw.go
+277 -53
View File
@@ -1192,10 +1192,10 @@ func extractGradioErrorMessage(dataStr string) string {
}
}
clean := strings.TrimSpace(dataStr)
if clean != "" && clean != "null" {
return clean
if clean == "null" || clean == "" {
return "upstream Gradio space returned null error (space may have show_error=False or failed input validation)"
}
return "unknown upstream Gradio error"
return clean
}
type Streamer struct {
@@ -1594,21 +1594,23 @@ type HFSpaceInfoResponse struct {
type SpaceParamMapping struct {
InputIndex int
ComponentID int
ParamType string // "message", "history", "system_prompt", "temperature", "max_tokens", "top_p", "state", "other"
ParamType string // "message", "history", "system_prompt", "temperature", "max_tokens", "top_p", "think_level", "tools", "stream", "state", "other"
DefaultValue interface{}
}
type SpaceDiscovery struct {
SpaceURL string
Title string
Models []string
PrimaryModel string
APIPrefix string // e.g. "/gradio_api" or ""
Endpoint string // e.g. "/chat_fn" or "/chat"
CleanEndpoint string // e.g. "chat_fn" or "chat"
Protocol string // "call", "queue", "predict"
TotalInputs int
SpaceURL string
Title string
Models []string
PrimaryModel string
APIPrefix string // e.g. "/gradio_api" or ""
Endpoint string // e.g. "/chat_fn" or "/chat"
CleanEndpoint string // e.g. "chat_fn" or "chat"
Protocol string // "call", "queue", "predict"
TotalInputs int
ParamMappings []SpaceParamMapping
DefaultInputs []interface{}
MessageIsMultimodal bool
HistoryIndex int // -1 if none
MessageIndex int // index for user message text
SystemIndex int // -1 if none
@@ -1616,11 +1618,12 @@ type SpaceDiscovery struct {
TempIndex int // -1 if none
MaxTokensIndex int // -1 if none
TopPIndex int // -1 if none
StreamIndex int // -1 if none
ThinkLevelIndex int // -1 if none
FunctionsJSONIndex int // -1 if none
PreservedThinkingIndex int // -1 if none
IsHunyuan3 bool
HistoryFormat string // "messages", "pairs", "none"
HistoryFormat string // "messages", "pairs", "gradio_messages", "none"
LastDiscovered time.Time
}
@@ -1663,7 +1666,7 @@ func (d *SpaceDiscovery) GetModelList() []ModelItem {
if len(items) == 0 {
items = append(items, ModelItem{
ID: "default",
ID: "gradio-chat",
Object: "model",
Created: now,
OwnedBy: "gradio",
@@ -1673,6 +1676,33 @@ func (d *SpaceDiscovery) GetModelList() []ModelItem {
return items
}
func (d *SpaceDiscovery) MatchesModel(requested string) bool {
if requested == "" {
return true
}
cleanReq := strings.TrimPrefix(requested, "models/")
cleanReq = strings.TrimPrefix(cleanReq, "openai/")
cleanReq = strings.ToLower(cleanReq)
if ConfiguredModelName != "" && strings.ToLower(ConfiguredModelName) == cleanReq {
return true
}
for _, m := range d.Models {
mClean := strings.TrimPrefix(m, "models/")
mClean = strings.TrimPrefix(mClean, "openai/")
if strings.ToLower(mClean) == cleanReq || strings.ToLower(m) == cleanReq {
return true
}
}
if strings.ToLower(d.PrimaryModel) == cleanReq {
return true
}
return cleanReq == "default" || cleanReq == "gradio" || cleanReq == "gradio-chat"
}
func NewDefaultSpaceDiscovery(spaceURL string) *SpaceDiscovery {
cleanURL := strings.TrimRight(spaceURL, "/")
if cleanURL != "" && !strings.HasPrefix(cleanURL, "http://") && !strings.HasPrefix(cleanURL, "https://") {
@@ -1692,6 +1722,7 @@ func NewDefaultSpaceDiscovery(spaceURL string) *SpaceDiscovery {
TempIndex: -1,
MaxTokensIndex: -1,
TopPIndex: -1,
StreamIndex: -1,
ThinkLevelIndex: -1,
FunctionsJSONIndex: -1,
PreservedThinkingIndex: -1,
@@ -1863,11 +1894,12 @@ func InspectSpace(client *http.Client, rawURL, userAgent string) (*SpaceDiscover
for _, p := range epInfo.Parameters {
pLower := strings.ToLower(p.ParameterName)
pLabel := strings.ToLower(p.Label)
pComp := strings.ToLower(p.Component)
if strings.Contains(pLower, "message") || strings.Contains(pLower, "text") || strings.Contains(pLower, "prompt") || strings.Contains(pLower, "query") || strings.Contains(pLower, "question") || strings.Contains(pLower, "input") || pComp == "textbox" {
if strings.Contains(pLower, "message") || strings.Contains(pLabel, "message") || strings.Contains(pLower, "text") || strings.Contains(pLower, "prompt") || strings.Contains(pLower, "query") || strings.Contains(pLower, "question") || strings.Contains(pLower, "input") || pComp == "textbox" || pComp == "multimodaltextbox" {
score += 40
}
if strings.Contains(pLower, "history") || strings.Contains(pLower, "chat") || strings.Contains(pLower, "messages") || strings.Contains(pLower, "conversation") || pComp == "chatbot" {
if strings.Contains(pLower, "history") || strings.Contains(pLabel, "history") || strings.Contains(pLower, "chat") || strings.Contains(pLower, "messages") || strings.Contains(pLower, "conversation") || pComp == "chatbot" {
score += 30
}
}
@@ -1881,6 +1913,30 @@ func InspectSpace(client *http.Client, rawURL, userAgent string) (*SpaceDiscover
}
}
// Fallback: if no named endpoint from /info, inspect dependencies in config
if bestEndpoint == "" && configFetched && len(configResp.Dependencies) > 0 {
depScore := -1000
for _, dep := range configResp.Dependencies {
if depName, ok := dep.APIName.(string); ok && depName != "" {
cleanName := strings.TrimPrefix(depName, "/")
lowerName := strings.ToLower(cleanName)
if strings.Contains(lowerName, "clear") || strings.Contains(lowerName, "reset") || strings.Contains(lowerName, "save") || strings.Contains(lowerName, "delete") || strings.Contains(lowerName, "pop") || strings.Contains(lowerName, "lambda") {
continue
}
score := 0
if strings.Contains(lowerName, "chat") || strings.Contains(lowerName, "conversation") {
score += 100
} else if strings.Contains(lowerName, "generate") || strings.Contains(lowerName, "predict") || strings.Contains(lowerName, "submit") {
score += 50
}
if score > depScore {
depScore = score
bestEndpoint = "/" + cleanName
}
}
}
}
if bestEndpoint != "" {
discovery.Endpoint = bestEndpoint
discovery.CleanEndpoint = strings.TrimPrefix(bestEndpoint, "/")
@@ -1888,12 +1944,13 @@ func InspectSpace(client *http.Client, rawURL, userAgent string) (*SpaceDiscover
// 5. Correlate with config.dependencies to determine exact input count & state padding
compMap := make(map[int]GradioComponent)
var matchingDep *GradioDependency
if configFetched {
for _, comp := range configResp.Components {
compMap[comp.ID] = comp
}
var matchingDep *GradioDependency
cleanTarget := strings.TrimPrefix(discovery.Endpoint, "/")
for _, dep := range configResp.Dependencies {
@@ -1910,6 +1967,10 @@ func InspectSpace(client *http.Client, rawURL, userAgent string) (*SpaceDiscover
if matchingDep != nil {
discovery.TotalInputs = len(matchingDep.Inputs)
discovery.DefaultInputs = make([]interface{}, len(matchingDep.Inputs))
discovery.ParamMappings = nil
discovery.MessageIndex = -1
for idx, compID := range matchingDep.Inputs {
mapping := SpaceParamMapping{
InputIndex: idx,
@@ -1918,11 +1979,26 @@ func InspectSpace(client *http.Client, rawURL, userAgent string) (*SpaceDiscover
}
if comp, exists := compMap[compID]; exists {
cType := strings.ToLower(comp.Type)
cLabel := ""
if comp.Props != nil {
if l, ok := comp.Props["label"].(string); ok {
cLabel = strings.ToLower(l)
}
if val, ok := comp.Props["value"]; ok {
discovery.DefaultInputs[idx] = val
mapping.DefaultValue = val
}
}
switch cType {
case "textbox", "multimodaltextbox":
if discovery.MessageIndex == 0 && idx == 0 {
case "multimodaltextbox":
if discovery.MessageIndex == -1 || strings.Contains(cLabel, "message") || strings.Contains(cLabel, "prompt") || strings.Contains(cLabel, "input") {
mapping.ParamType = "message"
} else if discovery.SystemIndex == -1 {
discovery.MessageIndex = idx
discovery.MessageIsMultimodal = true
}
case "textbox":
if strings.Contains(cLabel, "system") || strings.Contains(cLabel, "instruction") {
mapping.ParamType = "system_prompt"
discovery.SystemIndex = idx
if comp.Props != nil {
@@ -1930,56 +2006,122 @@ func InspectSpace(client *http.Client, rawURL, userAgent string) (*SpaceDiscover
discovery.DefaultSystemPrompt = strings.TrimSpace(val)
}
}
} else if discovery.MessageIndex == -1 || strings.Contains(cLabel, "message") || strings.Contains(cLabel, "prompt") || strings.Contains(cLabel, "query") || strings.Contains(cLabel, "input") || strings.Contains(cLabel, "question") {
mapping.ParamType = "message"
discovery.MessageIndex = idx
discovery.MessageIsMultimodal = false
}
case "chatbot":
mapping.ParamType = "history"
discovery.HistoryIndex = idx
if strings.HasPrefix(configResp.Version, "5.") || strings.HasPrefix(configResp.Version, "6.") {
discovery.HistoryFormat = "gradio_messages"
} else {
discovery.HistoryFormat = "pairs"
}
case "state":
mapping.ParamType = "state"
if idx == 1 && len(matchingDep.Inputs) == 2 {
// Standard Gradio ChatInterface: [textbox, state]
// Component 13 is state
}
case "slider", "number":
label := ""
if comp.Props != nil {
if l, ok := comp.Props["label"].(string); ok {
label = strings.ToLower(l)
}
}
if strings.Contains(label, "temp") {
if strings.Contains(cLabel, "temp") {
mapping.ParamType = "temperature"
discovery.TempIndex = idx
} else if strings.Contains(label, "max") || strings.Contains(label, "token") {
} else if strings.Contains(cLabel, "max") || strings.Contains(cLabel, "token") {
mapping.ParamType = "max_tokens"
discovery.MaxTokensIndex = idx
} else if strings.Contains(label, "top_p") {
} else if strings.Contains(cLabel, "top_p") || strings.Contains(cLabel, "top-p") || strings.Contains(cLabel, "top p") {
mapping.ParamType = "top_p"
discovery.TopPIndex = idx
} else if strings.Contains(cLabel, "think") {
mapping.ParamType = "think_level"
discovery.ThinkLevelIndex = idx
}
case "checkbox":
if strings.Contains(cLabel, "stream") {
mapping.ParamType = "stream"
discovery.StreamIndex = idx
}
default:
if strings.Contains(cLabel, "tool") || strings.Contains(cLabel, "function") {
mapping.ParamType = "tools"
discovery.FunctionsJSONIndex = idx
}
}
}
discovery.ParamMappings = append(discovery.ParamMappings, mapping)
}
} else if bestEndpointInfo != nil {
discovery.TotalInputs = len(bestEndpointInfo.Parameters)
if discovery.MessageIndex == -1 {
discovery.MessageIndex = 0
if len(matchingDep.Inputs) > 0 {
if comp, exists := compMap[matchingDep.Inputs[0]]; exists {
if strings.ToLower(comp.Type) == "multimodaltextbox" {
discovery.MessageIsMultimodal = true
}
}
}
}
}
}
// Check parameters in bestEndpointInfo for input indices and history support
// Refine history format or discover tools from bestEndpointInfo
if bestEndpointInfo != nil {
if discovery.TotalInputs < len(bestEndpointInfo.Parameters) {
discovery.TotalInputs = len(bestEndpointInfo.Parameters)
for _, p := range bestEndpointInfo.Parameters {
pName := strings.ToLower(p.ParameterName)
pLabel := strings.ToLower(p.Label)
pComp := strings.ToLower(p.Component)
if strings.Contains(pName, "history") || strings.Contains(pLabel, "history") || strings.Contains(pName, "chat") || strings.Contains(pName, "messages") || pComp == "chatbot" {
bType, _ := json.Marshal(p.Type)
bPyType, _ := json.Marshal(p.PythonType)
pPyType := strings.ToLower(string(bPyType))
bTypeStr := strings.ToLower(string(bType))
if strings.Contains(pPyType, "list[tuple[") || strings.Contains(pPyType, "list[list[") || strings.Contains(bTypeStr, "tuple") {
discovery.HistoryFormat = "pairs"
} else if strings.Contains(pPyType, "textmessage") || strings.Contains(pPyType, "dict(text: str") || strings.Contains(bTypeStr, "textmessage") || strings.Contains(bTypeStr, "chatbotdatamessages") {
discovery.HistoryFormat = "gradio_messages"
}
}
}
}
// Fallback: If config.dependencies did not provide matchingDep, map directly from bestEndpointInfo.Parameters
if matchingDep == nil && bestEndpointInfo != nil {
discovery.TotalInputs = len(bestEndpointInfo.Parameters)
discovery.DefaultInputs = make([]interface{}, len(bestEndpointInfo.Parameters))
discovery.ParamMappings = nil
discovery.MessageIndex = -1
for idx, p := range bestEndpointInfo.Parameters {
pName := strings.ToLower(p.ParameterName)
pLabel := strings.ToLower(p.Label)
pComp := strings.ToLower(p.Component)
if strings.Contains(pName, "system") {
if p.ParameterDefault != nil {
discovery.DefaultInputs[idx] = p.ParameterDefault
}
mapping := SpaceParamMapping{
InputIndex: idx,
ParamType: "other",
DefaultValue: p.ParameterDefault,
}
if strings.Contains(pComp, "multimodal") {
if discovery.MessageIndex == -1 || strings.Contains(pLabel, "message") || strings.Contains(pName, "message") {
mapping.ParamType = "message"
discovery.MessageIndex = idx
discovery.MessageIsMultimodal = true
}
} else if strings.Contains(pName, "system") || strings.Contains(pLabel, "system") || strings.Contains(pLabel, "instruction") {
discovery.SystemIndex = idx
mapping.ParamType = "system_prompt"
if p.ParameterDefault != nil && discovery.DefaultSystemPrompt == "" {
if defStr, ok := p.ParameterDefault.(string); ok && strings.TrimSpace(defStr) != "" {
discovery.DefaultSystemPrompt = strings.TrimSpace(defStr)
}
}
} else if strings.Contains(pName, "history") || strings.Contains(pName, "chat") || strings.Contains(pName, "conversation") || strings.Contains(pName, "messages") || pComp == "chatbot" {
} else if strings.Contains(pName, "history") || strings.Contains(pLabel, "history") || strings.Contains(pName, "chat") || strings.Contains(pName, "messages") || strings.Contains(pName, "conversation") || pComp == "chatbot" {
discovery.HistoryIndex = idx
mapping.ParamType = "history"
bType, _ := json.Marshal(p.Type)
bPyType, _ := json.Marshal(p.PythonType)
pPyType := strings.ToLower(string(bPyType))
@@ -1991,21 +2133,37 @@ func InspectSpace(client *http.Client, rawURL, userAgent string) (*SpaceDiscover
} else if strings.HasPrefix(configResp.Version, "5.") || strings.HasPrefix(configResp.Version, "6.") {
discovery.HistoryFormat = "gradio_messages"
}
} else if strings.Contains(pName, "message") || (strings.Contains(pName, "prompt") && !strings.Contains(pName, "system")) || strings.Contains(pName, "text") || strings.Contains(pName, "query") || strings.Contains(pName, "question") || strings.Contains(pName, "input") || pComp == "textbox" {
} else if strings.Contains(pLabel, "message") || strings.Contains(pName, "message") || (strings.Contains(pLabel, "prompt") && !strings.Contains(pLabel, "system")) || (strings.Contains(pName, "prompt") && !strings.Contains(pName, "system")) || strings.Contains(pLabel, "query") || strings.Contains(pName, "query") || strings.Contains(pLabel, "question") || strings.Contains(pName, "question") || (discovery.MessageIndex == -1 && (pComp == "textbox" || idx == 0)) {
discovery.MessageIndex = idx
} else if strings.Contains(pName, "think_level") || strings.Contains(pName, "thinking_level") {
mapping.ParamType = "message"
discovery.MessageIsMultimodal = false
} else if strings.Contains(pName, "think_level") || strings.Contains(pName, "thinking_level") || strings.Contains(pLabel, "think") {
discovery.ThinkLevelIndex = idx
} else if strings.Contains(pName, "functions") || strings.Contains(pName, "tools") {
mapping.ParamType = "think_level"
} else if strings.Contains(pName, "functions") || strings.Contains(pName, "tools") || strings.Contains(pLabel, "tools") || strings.Contains(pLabel, "functions") {
discovery.FunctionsJSONIndex = idx
mapping.ParamType = "tools"
} else if strings.Contains(pName, "preserved") {
discovery.PreservedThinkingIndex = idx
} else if strings.Contains(pName, "temp") {
} else if strings.Contains(pName, "temp") || strings.Contains(pLabel, "temp") {
discovery.TempIndex = idx
} else if strings.Contains(pName, "token") {
mapping.ParamType = "temperature"
} else if strings.Contains(pName, "token") || strings.Contains(pLabel, "token") {
discovery.MaxTokensIndex = idx
} else if strings.Contains(pName, "top_p") {
mapping.ParamType = "max_tokens"
} else if strings.Contains(pName, "top_p") || strings.Contains(pLabel, "top_p") || strings.Contains(pLabel, "top p") {
discovery.TopPIndex = idx
mapping.ParamType = "top_p"
} else if strings.Contains(pName, "stream") || strings.Contains(pLabel, "stream") {
discovery.StreamIndex = idx
mapping.ParamType = "stream"
}
discovery.ParamMappings = append(discovery.ParamMappings, mapping)
}
if discovery.MessageIndex == -1 {
discovery.MessageIndex = 0
}
}
@@ -2022,6 +2180,9 @@ func InspectSpace(client *http.Client, rawURL, userAgent string) (*SpaceDiscover
if discovery.TotalInputs < 1 {
discovery.TotalInputs = 1
}
for len(discovery.DefaultInputs) < discovery.TotalInputs {
discovery.DefaultInputs = append(discovery.DefaultInputs, nil)
}
return discovery, nil
}
@@ -2320,10 +2481,56 @@ func (g *GradioGateway) BuildGradioPayload(disc *SpaceDiscovery, req ChatComplet
}
data := make([]interface{}, totalInputs)
// Initialize with space default inputs if available
if len(disc.DefaultInputs) == totalInputs {
for i := 0; i < totalInputs; i++ {
data[i] = disc.DefaultInputs[i]
}
}
// Extract multimodal files if message input is multimodal
var messageFiles []interface{}
if disc.MessageIsMultimodal && len(nonSystem) > 0 {
lastMsg := nonSystem[len(nonSystem)-1]
if parts, ok := lastMsg.Content.([]interface{}); ok {
for _, p := range parts {
if itemMap, ok := p.(map[string]interface{}); ok {
if itemMap["type"] == "image_url" {
imgURL := ""
if iuMap, ok := itemMap["image_url"].(map[string]interface{}); ok {
if u, ok := iuMap["url"].(string); ok {
imgURL = u
}
} else if iuStr, ok := itemMap["image_url"].(string); ok {
imgURL = iuStr
}
if imgURL != "" {
messageFiles = append(messageFiles, map[string]interface{}{
"path": imgURL,
"url": imgURL,
"meta": map[string]interface{}{"_type": "gradio.FileData"},
})
}
}
}
}
}
}
if messageFiles == nil {
messageFiles = []interface{}{}
}
// Populate mapped fields
msgIdx := disc.MessageIndex
if msgIdx >= 0 && msgIdx < len(data) {
data[msgIdx] = promptMessageText
if disc.MessageIsMultimodal {
data[msgIdx] = map[string]interface{}{
"text": promptMessageText,
"files": messageFiles,
}
} else {
data[msgIdx] = promptMessageText
}
}
if disc.HistoryIndex >= 0 && disc.HistoryIndex < len(data) {
@@ -2393,13 +2600,17 @@ func (g *GradioGateway) BuildGradioPayload(disc *SpaceDiscovery, req ChatComplet
data[disc.TempIndex] = *req.Temperature
} else if disc.IsHunyuan3 {
data[disc.TempIndex] = nil
} else {
} else if data[disc.TempIndex] == nil {
data[disc.TempIndex] = 0.7
}
}
if disc.MaxTokensIndex >= 0 && disc.MaxTokensIndex < len(data) {
data[disc.MaxTokensIndex] = ResolveMaxTokens(req)
if req.MaxTokens > 0 || req.MaxCompletionTokens > 0 {
data[disc.MaxTokensIndex] = ResolveMaxTokens(req)
} else if data[disc.MaxTokensIndex] == nil {
data[disc.MaxTokensIndex] = ResolveMaxTokens(req)
}
}
if disc.TopPIndex >= 0 && disc.TopPIndex < len(data) {
@@ -2407,11 +2618,15 @@ func (g *GradioGateway) BuildGradioPayload(disc *SpaceDiscovery, req ChatComplet
data[disc.TopPIndex] = *req.TopP
} else if disc.IsHunyuan3 {
data[disc.TopPIndex] = 0
} else {
} else if data[disc.TopPIndex] == nil {
data[disc.TopPIndex] = 1.0
}
}
if disc.StreamIndex > 0 && disc.StreamIndex != disc.MessageIndex && disc.StreamIndex < len(data) {
data[disc.StreamIndex] = false
}
if disc.FunctionsJSONIndex >= 0 && disc.FunctionsJSONIndex < len(data) {
functionsJSONStr := ""
if len(req.Tools) > 0 {
@@ -2435,8 +2650,17 @@ type GradioOutputFrame struct {
}
// ParseGradioStreamOutput extracts structured content, reasoning, and tool calls from Gradio output
func ParseGradioStreamOutput(rawJSON string) GradioOutputFrame {
var frame GradioOutputFrame
func ParseGradioStreamOutput(rawJSON string) (frame GradioOutputFrame) {
defer func() {
if frame.OK && len(frame.ToolCalls) == 0 && frame.Content != "" {
tcs, clean, has := DetectToolCalls(frame.Content)
if has && len(tcs) > 0 {
frame.ToolCalls = tcs
frame.Content = clean
}
}
}()
var val interface{}
if err := json.Unmarshal([]byte(rawJSON), &val); err != nil {
return frame