diff --git a/README.md b/README.md index 1b65327..090640a 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,13 @@ -# gr2gw: Universal Gradio to OpenAI LLM gateway +# gr2gw: universal Gradio to OpenAI LLM gateway A zero-dependency, high-performance Go proxy server that introspects any Gradio chat space (such as Hugging Face Spaces or custom deployments) and exposes a standards-compliant OpenAI `/v1/chat/completions` and `/v1/models` HTTP API. -Default demo space: `https://ghost2513-openai-gpt-oss-120b.hf.space` +Default demo space: `https://lucasmarchettidelima-digital-twin.hf.space` ## Features -- **Zero external dependencies**: Pure Go standard library (`net/http`, `encoding/json`, `bufio`, etc.). -- **Automatic space introspection**: Dynamically queries `/gradio_api/info`, `/config`, and Hugging Face space metadata to discover models, endpoints, and input parameter mappings. +- **Zero external dependencies**: pure Go standard library (`net/http`, `encoding/json`, `bufio`, etc.). +- **Automatic space introspection**: dynamically queries `/gradio_api/info`, `/config`, and Hugging Face space metadata to discover models, endpoints, and input parameter mappings. - **Native Tencent Hunyuan 3 (`tencent-hy3`) support**: - Full native zero-degradation handling for official spaces like `https://tencent-hy3.hf.space`. - Maps `functions_json_str` natively without polluting the system prompt. @@ -27,7 +27,8 @@ Default demo space: `https://ghost2513-openai-gpt-oss-120b.hf.space` - Keeps `content` clean without tag leakage. - **Full tool calling & function interception**: - Formats schemas into native `functions_json_str` (Hy3) or system prompts (standard spaces). - - **`StreamToolCallFilter`**: Stateful sliding-window filter that prevents `` tags from leaking into `delta.content`. Emits structured OpenAI `delta.tool_calls` chunks and sets `finish_reason: "tool_calls"`. + - Intercepts and recovers tool calls from upstream `tool_use_failed` errors containing `failed_generation`. + - **`StreamToolCallFilter`**: stateful sliding-window filter that prevents `` tags from leaking into `delta.content`. Emits structured OpenAI `delta.tool_calls` chunks and sets `finish_reason: "tool_calls"`. - Seamlessly maintains multi-turn context when tool results are submitted back via `role: "tool"`. - **Built-in SOCKS5 proxy client**: - Full RFC 1928 / RFC 1929 implementation with domain resolution (`socks5h://`), IPv4, IPv6, and username/password auth. @@ -59,7 +60,7 @@ make test ### Quick start -Run with the default space (`https://ghost2513-openai-gpt-oss-120b.hf.space`): +Run with the default space (`https://lucasmarchettidelima-digital-twin.hf.space`): ```bash ./bin/gr2gw -port 8080 ``` @@ -71,14 +72,14 @@ Target any other Gradio space: With SOCKS5 proxy: ```bash -./bin/gr2gw -space https://ghost2513-openai-gpt-oss-120b.hf.space -socks socks5://127.0.0.1:1080 +./bin/gr2gw -space https://lucasmarchettidelima-digital-twin.hf.space -socks socks5://127.0.0.1:1080 ``` ### CLI flags | Flag | Default | Description | |------|---------|-------------| -| `-space`, `-url` | `https://ghost2513-openai-gpt-oss-120b.hf.space` | Target Gradio space URL | +| `-space`, `-url` | `https://lucasmarchettidelima-digital-twin.hf.space` | Target Gradio space URL | | `-port` | `8080` | Port to listen on | | `-host` | `0.0.0.0` | Host interface to bind to | | `-socks`, `-proxy`, `-socks5` | `""` | SOCKS5 proxy URL (`socks5://user:pass@host:port`) | @@ -104,13 +105,7 @@ Response: "object": "list", "data": [ { - "id": "openai/gpt-oss-120b", - "object": "model", - "created": 1788756307, - "owned_by": "gradio" - }, - { - "id": "gpt-oss-120b", + "id": "digital-twin", "object": "model", "created": 1788756307, "owned_by": "gradio" @@ -125,7 +120,7 @@ Response: curl http://localhost:8080/v1/chat/completions \ -H "Content-Type: application/json" \ -d '{ - "model": "openai/gpt-oss-120b", + "model": "digital-twin", "messages": [ {"role": "user", "content": "What is the capital of France?"} ] @@ -138,7 +133,7 @@ Response: "id": "chatcmpl-16425f9d-c350-47a1-9a6d-e9ce10871545", "object": "chat.completion", "created": 1788756310, - "model": "openai/gpt-oss-120b", + "model": "digital-twin", "choices": [ { "index": 0, @@ -163,7 +158,7 @@ Response: curl -N http://localhost:8080/v1/chat/completions \ -H "Content-Type: application/json" \ -d '{ - "model": "openai/gpt-oss-120b", + "model": "digital-twin", "messages": [ {"role": "user", "content": "Count from 1 to 5."} ], @@ -177,7 +172,7 @@ curl -N http://localhost:8080/v1/chat/completions \ curl http://localhost:8080/v1/chat/completions \ -H "Content-Type: application/json" \ -d '{ - "model": "openai/gpt-oss-120b", + "model": "digital-twin", "messages": [ {"role": "user", "content": "What is the weather in Tokyo?"} ], @@ -204,7 +199,7 @@ Response: "id": "chatcmpl-32727c62-ef2a-4866-855b-f1c7ec2b8023", "object": "chat.completion", "created": 1788756322, - "model": "openai/gpt-oss-120b", + "model": "digital-twin", "choices": [ { "index": 0, @@ -239,7 +234,7 @@ Response: curl http://localhost:8080/v1/chat/completions \ -H "Content-Type: application/json" \ -d '{ - "model": "openai/gpt-oss-120b", + "model": "digital-twin", "messages": [ {"role": "user", "content": "What is the weather in Tokyo?"}, { @@ -284,7 +279,7 @@ Response: "id": "chatcmpl-4903ba12-f12b-4cd3-a801-7290bc91a421", "object": "chat.completion", "created": 1788756335, - "model": "openai/gpt-oss-120b", + "model": "digital-twin", "choices": [ { "index": 0, diff --git a/gr2gw.go b/gr2gw.go index ce58fcc..bf8437f 100644 --- a/gr2gw.go +++ b/gr2gw.go @@ -25,7 +25,7 @@ import ( ) var ( - DefaultSpaceURL = "https://ghost2513-openai-gpt-oss-120b.hf.space" + DefaultSpaceURL = "https://lucasmarchettidelima-digital-twin.hf.space" DefaultUserAgent = "Mozilla/5.0 (X11; Linux x86_64; rv:153.0) Gecko/20100101 Firefox/153.0" ConfiguredUserAgent string ) @@ -388,6 +388,9 @@ func DoWithFibonacciRetry(client *http.Client, makeReq func() (*http.Request, er respBody, _ := io.ReadAll(resp.Body) resp.Body.Close() lastErr = fmt.Errorf("HTTP status %d: %s", resp.StatusCode, string(respBody)) + if _, ok := extractFailedGeneration(string(respBody)); ok { + break + } } else { lastErr = err } @@ -1002,6 +1005,141 @@ func WriteCompletionResponse(w http.ResponseWriter, completionID string, created json.NewEncoder(w).Encode(resp) } +func extractFailedGeneration(raw string) (string, bool) { + var obj map[string]interface{} + if err := json.Unmarshal([]byte(raw), &obj); err == nil { + if fg, ok := obj["failed_generation"].(string); ok && fg != "" { + return strings.TrimSpace(fg), true + } + if errVal, ok := obj["error"]; ok { + if errMap, ok := errVal.(map[string]interface{}); ok { + if fg, ok := errMap["failed_generation"].(string); ok && fg != "" { + return strings.TrimSpace(fg), true + } + } else if errStr, ok := errVal.(string); ok { + if fg, ok := extractFailedGeneration(errStr); ok { + return fg, true + } + } + } + } + + keyIdx := strings.Index(raw, `"failed_generation"`) + if keyIdx == -1 { + keyIdx = strings.Index(raw, `'failed_generation'`) + } + if keyIdx == -1 { + keyIdx = strings.Index(raw, `failed_generation`) + } + if keyIdx == -1 { + return "", false + } + + colonIdx := strings.Index(raw[keyIdx:], ":") + if colonIdx == -1 { + return "", false + } + valStart := keyIdx + colonIdx + 1 + + for valStart < len(raw) && (raw[valStart] == ' ' || raw[valStart] == '\t' || raw[valStart] == '\r' || raw[valStart] == '\n') { + valStart++ + } + if valStart >= len(raw) { + return "", false + } + + firstChar := raw[valStart] + var candidate string + + if firstChar == '\'' || firstChar == '"' { + quoteChar := firstChar + var b strings.Builder + escaped := false + for i := valStart + 1; i < len(raw); i++ { + ch := raw[i] + if escaped { + switch ch { + case 'n': + b.WriteByte('\n') + case 'r': + b.WriteByte('\r') + case 't': + b.WriteByte('\t') + case '\\': + b.WriteByte('\\') + case '\'': + b.WriteByte('\'') + case '"': + b.WriteByte('"') + default: + b.WriteByte('\\') + b.WriteByte(ch) + } + escaped = false + } else if ch == '\\' { + escaped = true + } else if ch == quoteChar { + break + } else { + b.WriteByte(ch) + } + } + candidate = strings.TrimSpace(b.String()) + } else if firstChar == '{' || firstChar == '[' { + openChar := firstChar + closeChar := byte('}') + if openChar == '[' { + closeChar = ']' + } + depth := 0 + inStr := false + var strQuote byte + escaped := false + endIdx := -1 + + for i := valStart; i < len(raw); i++ { + ch := raw[i] + if inStr { + if escaped { + escaped = false + } else if ch == '\\' { + escaped = true + } else if ch == strQuote { + inStr = false + } + } else { + if ch == '"' || ch == '\'' { + inStr = true + strQuote = ch + } else if ch == openChar { + depth++ + } else if ch == closeChar { + depth-- + if depth == 0 { + endIdx = i + 1 + break + } + } + } + } + if endIdx != -1 { + candidate = strings.TrimSpace(raw[valStart:endIdx]) + } + } + + if candidate != "" { + if strings.Contains(candidate, `\"`) { + candidate = strings.ReplaceAll(candidate, `\"`, `"`) + } + if strings.Contains(candidate, `\n`) { + candidate = strings.ReplaceAll(candidate, `\n`, "\n") + } + return candidate, true + } + + return "", false +} + func extractGradioErrorMessage(dataStr string) string { var errObj map[string]interface{} if err := json.Unmarshal([]byte(dataStr), &errObj); err == nil { @@ -2339,6 +2477,9 @@ func (g *GradioGateway) ExecuteChatCompletion(w http.ResponseWriter, r *http.Req return fmt.Errorf("failed to encode request: %w", err) } + completionID := "chatcmpl-" + GenerateUUID() + createdTime := time.Now().Unix() + // 1. Submit to /call/{endpoint} callURL := fmt.Sprintf("%s%s/call/%s", disc.SpaceURL, disc.APIPrefix, disc.CleanEndpoint) makeCallReq := func() (*http.Request, error) { @@ -2353,6 +2494,27 @@ func (g *GradioGateway) ExecuteChatCompletion(w http.ResponseWriter, r *http.Req resp, err := DoWithFibonacciRetry(g.client, makeCallReq, 5) if err != nil { + if fg, ok := extractFailedGeneration(err.Error()); ok { + if tcs, _, has := DetectToolCalls(fg); has && len(tcs) > 0 { + if !req.Stream { + WriteCompletionResponse(w, completionID, createdTime, modelName, FinalOutput{ + ToolCalls: tcs, + FinishReason: "tool_calls", + }) + return nil + } + flusher, _ := w.(http.Flusher) + streamer := NewStreamer(w, flusher, completionID, createdTime, modelName) + for i, tc := range tcs { + iCopy := i + tc.Index = &iCopy + streamer.ToolCallDelta(tc) + } + streamer.Finish("tool_calls") + streamer.Done() + return nil + } + } // If call failed, try without APIPrefix or try /call/v2 altCallURL := fmt.Sprintf("%s/call/%s", disc.SpaceURL, disc.CleanEndpoint) makeAltReq := func() (*http.Request, error) { @@ -2366,6 +2528,27 @@ func (g *GradioGateway) ExecuteChatCompletion(w http.ResponseWriter, r *http.Req } resp, err = DoWithFibonacciRetry(g.client, makeAltReq, 3) if err != nil { + if fg, ok := extractFailedGeneration(err.Error()); ok { + if tcs, _, has := DetectToolCalls(fg); has && len(tcs) > 0 { + if !req.Stream { + WriteCompletionResponse(w, completionID, createdTime, modelName, FinalOutput{ + ToolCalls: tcs, + FinishReason: "tool_calls", + }) + return nil + } + flusher, _ := w.(http.Flusher) + streamer := NewStreamer(w, flusher, completionID, createdTime, modelName) + for i, tc := range tcs { + iCopy := i + tc.Index = &iCopy + streamer.ToolCallDelta(tc) + } + streamer.Finish("tool_calls") + streamer.Done() + return nil + } + } return fmt.Errorf("upstream Gradio call error: %w", err) } } @@ -2390,13 +2573,31 @@ func (g *GradioGateway) ExecuteChatCompletion(w http.ResponseWriter, r *http.Req streamResp, err := DoWithFibonacciRetry(g.client, makeStreamReq, 5) if err != nil { + if fg, ok := extractFailedGeneration(err.Error()); ok { + if tcs, _, has := DetectToolCalls(fg); has && len(tcs) > 0 { + if !req.Stream { + WriteCompletionResponse(w, completionID, createdTime, modelName, FinalOutput{ + ToolCalls: tcs, + FinishReason: "tool_calls", + }) + return nil + } + flusher, _ := w.(http.Flusher) + streamer := NewStreamer(w, flusher, completionID, createdTime, modelName) + for i, tc := range tcs { + iCopy := i + tc.Index = &iCopy + streamer.ToolCallDelta(tc) + } + streamer.Finish("tool_calls") + streamer.Done() + return nil + } + } return fmt.Errorf("upstream Gradio stream error: %w", err) } defer streamResp.Body.Close() - completionID := "chatcmpl-" + GenerateUUID() - createdTime := time.Now().Unix() - // 3. Handle Non-Streaming vs Streaming if !req.Stream { reader := bufio.NewReader(streamResp.Body) @@ -2418,6 +2619,15 @@ func (g *GradioGateway) ExecuteChatCompletion(w http.ResponseWriter, r *http.Req if strings.HasPrefix(line, "data: ") { dataStr := strings.TrimPrefix(line, "data: ") if currentEvent == "error" { + if fg, ok := extractFailedGeneration(dataStr); ok { + if tcs, _, has := DetectToolCalls(fg); has && len(tcs) > 0 { + latestFrame = GradioOutputFrame{ + ToolCalls: tcs, + OK: true, + } + break + } + } errMsg := extractGradioErrorMessage(dataStr) log.Printf("Upstream Gradio error: %s", errMsg) return fmt.Errorf("upstream Gradio error: %s", errMsg) @@ -2496,6 +2706,17 @@ func (g *GradioGateway) ExecuteChatCompletion(w http.ResponseWriter, r *http.Req if strings.HasPrefix(line, "data: ") { dataStr := strings.TrimPrefix(line, "data: ") if currentEvent == "error" { + if fg, ok := extractFailedGeneration(dataStr); ok { + if tcs, _, has := DetectToolCalls(fg); has && len(tcs) > 0 { + for i, tc := range tcs { + iCopy := i + tc.Index = &iCopy + streamer.ToolCallDelta(tc) + } + nativeToolCallsSeen = true + break + } + } errMsg := extractGradioErrorMessage(dataStr) log.Printf("Upstream Gradio error: %s", errMsg) if !streamer.started { diff --git a/gr2gw_test.go b/gr2gw_test.go index 35ccced..c0ab1ac 100644 --- a/gr2gw_test.go +++ b/gr2gw_test.go @@ -1024,4 +1024,184 @@ func TestUpstreamGradioErrorPropagation(t *testing.T) { } } +func TestExtractFailedGeneration(t *testing.T) { + cases := []struct { + name string + input string + expected string + }{ + { + name: "json wrapped", + input: `{"error": {"code": 400, "failed_generation": "{\"name\": \"test_fn\", \"arguments\": {\"a\": 1}}"}}`, + expected: `{"name": "test_fn", "arguments": {"a": 1}}`, + }, + { + name: "python repr with single quotes and escaped quotes and newlines", + input: `upstream Gradio error: {'message': 'Tool choice is none, but model called a tool', 'type': 'invalid_request_error', 'code': 'tool_use_failed', 'failed_generation': '{\"name\": \"repo_browser.print_tree\", \"arguments\": {\"path\": \"\", \"depth\": 2}\\n}', 'status_code': 400}`, + expected: "{\"name\": \"repo_browser.print_tree\", \"arguments\": {\"path\": \"\", \"depth\": 2}\n}", + }, + { + name: "python repr with standard single quotes", + input: `{'code': 'tool_use_failed', 'failed_generation': '{"name": "calc", "arguments": {"x": 5}}'}`, + expected: `{"name": "calc", "arguments": {"x": 5}}`, + }, + { + name: "raw object failed_generation", + input: `{"failed_generation": {"name": "calc", "arguments": {"x": 5}}}`, + expected: `{"name": "calc", "arguments": {"x": 5}}`, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got, ok := extractFailedGeneration(c.input) + if !ok { + t.Fatalf("extractFailedGeneration failed to find failed_generation in %q", c.input) + } + tcs, _, has := DetectToolCalls(got) + if !has || len(tcs) == 0 { + t.Fatalf("DetectToolCalls failed on extracted generation %q", got) + } + }) + } +} + +func TestToolUseFailedRecovery(t *testing.T) { + errPayload := `upstream Gradio error: {'message': 'Tool choice is none, but model called a tool', 'type': 'invalid_request_error', 'code': 'tool_use_failed', 'failed_generation': '{\"name\": \"repo_browser.print_tree\", \"arguments\": {\"path\": \"\", \"depth\": 2}\\n}', 'status_code': 400}` + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/gradio_api/info" { + resp := GradioAPIInfoResponse{ + NamedEndpoints: map[string]GradioEndpointInfo{ + "/chat_fn": { + Parameters: []GradioParamInfo{ + {ParameterName: "message", Component: "Textbox"}, + }, + }, + }, + } + json.NewEncoder(w).Encode(resp) + return + } + if r.URL.Path == "/gradio_api/call/chat_fn" { + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(GradioJoinResponse{EventID: "evt_fail"}) + return + } + if r.URL.Path == "/gradio_api/call/chat_fn/evt_fail" { + w.Header().Set("Content-Type", "text/event-stream") + flusher := w.(http.Flusher) + fmt.Fprintf(w, "event: error\ndata: %s\n\n", errPayload) + flusher.Flush() + return + } + http.NotFound(w, r) + })) + defer ts.Close() + + gw := NewGradioGateway(ts.URL, "", 10*time.Second) + + // 1. Non-streaming test + recNonStream := httptest.NewRecorder() + httpReqNonStream := httptest.NewRequest("POST", "/v1/chat/completions", nil) + err := gw.ExecuteChatCompletion(recNonStream, httpReqNonStream, ChatCompletionRequest{ + Model: "test-model", + Messages: []ChatMessage{{Role: "user", Content: "List files"}}, + Stream: false, + }) + if err != nil { + t.Fatalf("expected non-streaming to succeed by recovering tool call, got error: %v", err) + } + var nonStreamResp ChatCompletionResponse + if err := json.NewDecoder(recNonStream.Body).Decode(&nonStreamResp); err != nil { + t.Fatalf("failed to decode response: %v", err) + } + if nonStreamResp.Choices[0].FinishReason != "tool_calls" { + t.Fatalf("expected finish_reason 'tool_calls', got %q", nonStreamResp.Choices[0].FinishReason) + } + if len(nonStreamResp.Choices[0].Message.ToolCalls) != 1 { + t.Fatalf("expected 1 tool call, got %d", len(nonStreamResp.Choices[0].Message.ToolCalls)) + } + tc := nonStreamResp.Choices[0].Message.ToolCalls[0] + if tc.Function.Name != "repo_browser.print_tree" { + t.Errorf("expected function repo_browser.print_tree, got %q", tc.Function.Name) + } + + // 2. Streaming test + recStream := httptest.NewRecorder() + httpReqStream := httptest.NewRequest("POST", "/v1/chat/completions", nil) + err = gw.ExecuteChatCompletion(recStream, httpReqStream, ChatCompletionRequest{ + Model: "test-model", + Messages: []ChatMessage{{Role: "user", Content: "List files"}}, + Stream: true, + }) + if err != nil { + t.Fatalf("expected streaming to succeed by recovering tool call, got error: %v", err) + } + bodyStream := recStream.Body.String() + if !strings.Contains(bodyStream, "repo_browser.print_tree") { + t.Errorf("expected streaming body to contain repo_browser.print_tree, got: %s", bodyStream) + } + if !strings.Contains(bodyStream, `"finish_reason":"tool_calls"`) { + t.Errorf("expected streaming body to contain finish_reason tool_calls, got: %s", bodyStream) + } + if !strings.Contains(bodyStream, "[DONE]") { + t.Errorf("expected streaming body to contain [DONE], got: %s", bodyStream) + } +} + +func TestToolUseFailedImmediateCallRecovery(t *testing.T) { + errPayload := `{"error": {"code": 400, "message": "Tool choice is none, but model called a tool", "failed_generation": "{\"name\": \"calculator\", \"arguments\": {\"expr\": \"2+2\"}}"}}` + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/gradio_api/info" { + resp := GradioAPIInfoResponse{ + NamedEndpoints: map[string]GradioEndpointInfo{ + "/chat_fn": { + Parameters: []GradioParamInfo{ + {ParameterName: "message", Component: "Textbox"}, + }, + }, + }, + } + json.NewEncoder(w).Encode(resp) + return + } + if strings.HasPrefix(r.URL.Path, "/gradio_api/call/chat_fn") || strings.HasPrefix(r.URL.Path, "/call/chat_fn") { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte(errPayload)) + return + } + http.NotFound(w, r) + })) + defer ts.Close() + + gw := NewGradioGateway(ts.URL, "", 10*time.Second) + + rec := httptest.NewRecorder() + httpReq := httptest.NewRequest("POST", "/v1/chat/completions", nil) + err := gw.ExecuteChatCompletion(rec, httpReq, ChatCompletionRequest{ + Model: "test-model", + Messages: []ChatMessage{{Role: "user", Content: "Calculate"}}, + Stream: false, + }) + if err != nil { + t.Fatalf("expected call-level error recovery to succeed, got: %v", err) + } + var resp ChatCompletionResponse + if err := json.NewDecoder(rec.Body).Decode(&resp); err != nil { + t.Fatalf("failed to decode response: %v", err) + } + if resp.Choices[0].FinishReason != "tool_calls" { + t.Fatalf("expected finish_reason 'tool_calls', got %q", resp.Choices[0].FinishReason) + } + if len(resp.Choices[0].Message.ToolCalls) != 1 { + t.Fatalf("expected 1 tool call, got %d", len(resp.Choices[0].Message.ToolCalls)) + } + if resp.Choices[0].Message.ToolCalls[0].Function.Name != "calculator" { + t.Errorf("expected calculator function, got %q", resp.Choices[0].Message.ToolCalls[0].Function.Name) + } +} +