From deb1025bda30d155255cbafc977a14edb5101782 Mon Sep 17 00:00:00 2001 From: Luxferre Date: Fri, 31 Jul 2026 09:54:59 +0300 Subject: [PATCH] initial upl --- Makefile | 11 + README.md | 89 ++++++ go.mod | 3 + hygate_test.go | 307 ++++++++++++++++++++ main.go | 748 +++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 1158 insertions(+) create mode 100644 Makefile create mode 100644 README.md create mode 100644 go.mod create mode 100644 hygate_test.go create mode 100644 main.go diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..b18d42d --- /dev/null +++ b/Makefile @@ -0,0 +1,11 @@ +# Makefile for hygate (Hunyuan AI Gateway) + +hygate: + go build -trimpath -ldflags="-s -w" -o bin/hygate . + +all: hygate + +clean: + rm -rf bin/ + +.PHONY: all clean hygate diff --git a/README.md b/README.md new file mode 100644 index 0000000..27012d7 --- /dev/null +++ b/README.md @@ -0,0 +1,89 @@ +# Hygate + +## About + +Hygate is a standalone, single-binary gateway that exposes Tencent's Hunyuan 3 (hy3) Gradio space through an OpenAI-compatible API. It translates the standard `/v1/chat/completions` and `/v1/models` endpoints into Gradio's `/gradio_api/call/chat` request/stream protocol, so any OpenAI-compatible client can talk to Hunyuan 3 without modification. + +## Features + +- OpenAI-compatible chat completions (streaming and non-streaming) +- Reasoning content passthrough (`reasoning_content`) +- Tool/function calling, including incremental streaming of tool-call arguments +- Fibonacci backoff retry on upstream calls +- Configurable endpoint and User-Agent +- No external dependencies (standard library only) + +## Installation + +Build from source with Go 1.26 or newer: + +``` +make +``` + +This produces `bin/hygate` binary for your architecture. + +Install the latest release straight from the Git repository with `go install`: + +``` +go install code.luxferre.top/luxferre/hygate@latest +``` + +This fetches the module from `https://code.luxferre.top/luxferre/hygate.git` and places the `hygate` binary in `$(go env GOPATH)/bin`. Make sure that directory is on your `PATH`. + +If you prefer to build from a local checkout instead: + +``` +git clone https://code.luxferre.top/luxferre/hygate.git +cd hygate +go install . +``` + +## Usage + +Run the gateway: + +``` +./bin/hygate [-port 8080] [-endpoint https://tencent-hy3.hf.space] +``` + +Available flags: + +- `-port` — TCP port to listen on (default `8080`) +- `-endpoint` — root URL of the Hunyuan Gradio space (default `https://tencent-hy3.hf.space`) +- `-user-agent` / `-ua` — custom User-Agent sent to the upstream + +Endpoints served: + +- `GET /v1/models` +- `POST /v1/chat/completions` + +Example request with curl: + +``` +curl http://localhost:8080/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{"model":"hy3","messages":[{"role":"user","content":"Hello"}],"stream":false}' +``` + +## FAQ + +### Does this need an API key? + +No. The gateway proxies directly to the public Gradio space and does not require authentication. + +### Which models are reported? + +A single model, `hy3`, is advertised via `/v1/models`. + +### Are token usage counts returned? + +No. The `usage` field is always zero, since the upstream Gradio API does not expose token counts. + +### Can the upstream endpoint be changed? + +Yes, with the `-endpoint` flag. Any HuggingFace Gradio space hosting the compatible `chat` endpoint will work. + +## Credits + +Created by Luxferre in 2026, released into the public domain with no warranties. diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..d017322 --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module code.luxferre.top/luxferre/hygate + +go 1.26 diff --git a/hygate_test.go b/hygate_test.go new file mode 100644 index 0000000..90b0877 --- /dev/null +++ b/hygate_test.go @@ -0,0 +1,307 @@ +package main + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestHunyuanServiceEndpoint(t *testing.T) { + svc := NewHunyuanService("https://example.com/gradio/") + if svc.endpoint != "https://example.com/gradio" { + t.Errorf("expected trailing slash stripped, got %q", svc.endpoint) + } + + models := svc.ListModels() + if len(models) != 1 || models[0].ID != "hy3" { + t.Fatalf("expected model ID 'hy3', got %v", models) + } +} + +func TestParseGradioStreamData(t *testing.T) { + sampleJSON := `[["Hello world", "Thinking process...", null]]` + c, r, tc, ok := parseGradioStreamData(sampleJSON) + if !ok { + t.Fatalf("expected parseGradioStreamData to succeed") + } + if c != "Hello world" { + t.Errorf("content=%q, want 'Hello world'", c) + } + if r != "Thinking process..." { + t.Errorf("reasoning=%q, want 'Thinking process...'", r) + } + if len(tc) != 0 { + t.Errorf("expected 0 tool calls, got %d", len(tc)) + } +} + +func TestHunyuanMockedCompletion(t *testing.T) { + var receivedCallURL, receivedStreamURL string + + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/gradio_api/call/chat") && r.Method == http.MethodPost { + receivedCallURL = r.URL.Path + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"event_id":"test-event-123"}`) + return + } + if strings.Contains(r.URL.Path, "/gradio_api/call/chat/test-event-123") && r.Method == http.MethodGet { + receivedStreamURL = r.URL.Path + w.Header().Set("Content-Type", "text/event-stream") + fmt.Fprint(w, "data: [[\"Mocked answer\", \"Mocked reasoning\", null]]\n\n") + return + } + http.NotFound(w, r) + }) + + server := httptest.NewServer(handler) + defer server.Close() + + svc := NewHunyuanService(server.URL) + + rec := httptest.NewRecorder() + req, _ := http.NewRequest("POST", "/v1/chat/completions", nil) + + chatReq := ChatCompletionRequest{ + Model: "hy3", + Messages: []ChatMessage{ + {Role: "user", Content: "Hello!"}, + }, + Stream: false, + } + + err := svc.Chat(rec, req, chatReq) + if err != nil { + t.Fatalf("unexpected Chat error: %v", err) + } + + if receivedCallURL != "/gradio_api/call/chat" { + t.Errorf("expected call URL '/gradio_api/call/chat', got %q", receivedCallURL) + } + if receivedStreamURL != "/gradio_api/call/chat/test-event-123" { + t.Errorf("expected stream URL '/gradio_api/call/chat/test-event-123', got %q", receivedStreamURL) + } + + var resp ChatCompletionResponse + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("failed to decode completion response: %v", err) + } + + if len(resp.Choices) != 1 { + t.Fatalf("expected 1 choice, got %d", len(resp.Choices)) + } + if resp.Choices[0].Message.GetContentString() != "Mocked answer" { + t.Errorf("content=%q, want 'Mocked answer'", resp.Choices[0].Message.GetContentString()) + } + if resp.Choices[0].Message.ReasoningContent != "Mocked reasoning" { + t.Errorf("reasoning=%q, want 'Mocked reasoning'", resp.Choices[0].Message.ReasoningContent) + } +} + +func TestParseGradioStreamDataWithToolCalls(t *testing.T) { + sampleJSON := `[["", "Let me check the weather.", [{"id":"call_999","type":"function","function":{"name":"get_weather","arguments":"{\"location\":\"Tokyo\"}"}}]]]` + c, r, tc, ok := parseGradioStreamData(sampleJSON) + if !ok { + t.Fatalf("expected parseGradioStreamData to succeed") + } + if c != "" { + t.Errorf("content=%q, want empty", c) + } + if r != "Let me check the weather." { + t.Errorf("reasoning=%q, want 'Let me check the weather.'", r) + } + if len(tc) != 1 { + t.Fatalf("expected 1 tool call, got %d", len(tc)) + } + if tc[0].ID != "call_999" || tc[0].Function.Name != "get_weather" || tc[0].Function.Arguments != `{"location":"Tokyo"}` { + t.Errorf("unexpected tool call payload: %+v", tc[0]) + } +} + +func TestHunyuanToolCallingNonStreaming(t *testing.T) { + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/gradio_api/call/chat") && r.Method == http.MethodPost { + var body map[string]interface{} + json.NewDecoder(r.Body).Decode(&body) + dataArr, _ := body["data"].([]interface{}) + if len(dataArr) < 9 || dataArr[8] == nil || dataArr[8] == "" { + t.Errorf("expected functionsJSONStr in data[8], got %v", dataArr) + } + + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"event_id":"evt-tool-1"}`) + return + } + if strings.Contains(r.URL.Path, "/gradio_api/call/chat/evt-tool-1") && r.Method == http.MethodGet { + w.Header().Set("Content-Type", "text/event-stream") + fmt.Fprint(w, "data: [[\"\", \"I should call the search tool.\", [{\"id\":\"call_abc\",\"type\":\"function\",\"function\":{\"name\":\"search\",\"arguments\":\"{\\\"query\\\":\\\"golang\\\"}\"}}]]]\n\n") + return + } + http.NotFound(w, r) + }) + + server := httptest.NewServer(handler) + defer server.Close() + + svc := NewHunyuanService(server.URL) + rec := httptest.NewRecorder() + req, _ := http.NewRequest("POST", "/v1/chat/completions", nil) + + chatReq := ChatCompletionRequest{ + Model: "hy3", + Messages: []ChatMessage{ + {Role: "user", Content: "Search for golang news"}, + }, + Tools: []Tool{ + { + Type: "function", + Function: map[string]interface{}{ + "name": "search", + "description": "Search web", + }, + }, + }, + Stream: false, + } + + err := svc.Chat(rec, req, chatReq) + if err != nil { + t.Fatalf("unexpected Chat error: %v", err) + } + + var resp ChatCompletionResponse + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("failed to decode response: %v", err) + } + + if len(resp.Choices) != 1 { + t.Fatalf("expected 1 choice, got %d", len(resp.Choices)) + } + choice := resp.Choices[0] + if choice.FinishReason != "tool_calls" { + t.Errorf("finish_reason=%q, want 'tool_calls'", choice.FinishReason) + } + if choice.Message.ReasoningContent != "I should call the search tool." { + t.Errorf("reasoning_content=%q, want 'I should call the search tool.'", choice.Message.ReasoningContent) + } + if len(choice.Message.ToolCalls) != 1 { + t.Fatalf("expected 1 tool call, got %d", len(choice.Message.ToolCalls)) + } + tc := choice.Message.ToolCalls[0] + if tc.ID != "call_abc" || tc.Function.Name != "search" || tc.Function.Arguments != `{"query":"golang"}` { + t.Errorf("unexpected tool call: %+v", tc) + } +} + +func TestHunyuanReasoningAndToolCallingStreaming(t *testing.T) { + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/gradio_api/call/chat") && r.Method == http.MethodPost { + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"event_id":"evt-stream-1"}`) + return + } + if strings.Contains(r.URL.Path, "/gradio_api/call/chat/evt-stream-1") && r.Method == http.MethodGet { + w.Header().Set("Content-Type", "text/event-stream") + flusher, _ := w.(http.Flusher) + + fmt.Fprint(w, "data: [[\"\", \"Step 1 reasoning.\", null]]\n\n") + if flusher != nil { + flusher.Flush() + } + + fmt.Fprint(w, "data: [[\"Hello \", \"Step 1 reasoning. Step 2 reasoning.\", null]]\n\n") + if flusher != nil { + flusher.Flush() + } + + fmt.Fprint(w, "data: [[\"Hello \", \"Step 1 reasoning. Step 2 reasoning.\", [{\"id\":\"call_xyz\",\"type\":\"function\",\"function\":{\"name\":\"calculator\",\"arguments\":\"{\\\"expr\\\":\\\"2+2\\\"}\"}}]]]\n\n") + if flusher != nil { + flusher.Flush() + } + + return + } + http.NotFound(w, r) + }) + + server := httptest.NewServer(handler) + defer server.Close() + + svc := NewHunyuanService(server.URL) + rec := httptest.NewRecorder() + req, _ := http.NewRequest("POST", "/v1/chat/completions", nil) + + chatReq := ChatCompletionRequest{ + Model: "hy3", + Messages: []ChatMessage{ + {Role: "user", Content: "Calculate 2+2"}, + }, + Stream: true, + } + + err := svc.Chat(rec, req, chatReq) + if err != nil { + t.Fatalf("unexpected Chat error: %v", err) + } + + bodyStr := rec.Body.String() + lines := strings.Split(bodyStr, "\n") + + var receivedDeltas []StreamDelta + var finishReasons []string + var foundDone bool + + for _, line := range lines { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "data: ") { + payload := strings.TrimPrefix(line, "data: ") + if payload == "[DONE]" { + foundDone = true + continue + } + var streamResp StreamResponse + if err := json.Unmarshal([]byte(payload), &streamResp); err == nil && len(streamResp.Choices) > 0 { + choice := streamResp.Choices[0] + receivedDeltas = append(receivedDeltas, choice.Delta) + if choice.FinishReason != nil { + finishReasons = append(finishReasons, *choice.FinishReason) + } + } + } + } + + if !foundDone { + t.Errorf("expected 'data: [DONE]' in SSE stream") + } + + if len(receivedDeltas) < 1 || receivedDeltas[0].Role != "assistant" { + t.Errorf("expected first delta to set role='assistant', got %+v", receivedDeltas[0]) + } + + var combinedReasoning, combinedContent string + var toolCallsEmitted []ToolCall + for _, d := range receivedDeltas { + combinedReasoning += d.ReasoningContent + combinedContent += d.Content + if len(d.ToolCalls) > 0 { + toolCallsEmitted = append(toolCallsEmitted, d.ToolCalls...) + } + } + + if combinedReasoning != "Step 1 reasoning. Step 2 reasoning." { + t.Errorf("combined reasoning=%q, want 'Step 1 reasoning. Step 2 reasoning.'", combinedReasoning) + } + if combinedContent != "Hello " { + t.Errorf("combined content=%q, want 'Hello '", combinedContent) + } + if len(toolCallsEmitted) == 0 { + t.Fatalf("expected tool call deltas to be emitted") + } + if len(finishReasons) != 1 || finishReasons[0] != "tool_calls" { + t.Errorf("finish_reasons=%v, want ['tool_calls']", finishReasons) + } +} + diff --git a/main.go b/main.go new file mode 100644 index 0000000..527340e --- /dev/null +++ b/main.go @@ -0,0 +1,748 @@ +// hygate: Standalone OpenAI-compatible gateway for Hunyuan 3 Gradio spaces +// Created by Luxferre in 2026, released into the public domain + +package main + +import ( + "bufio" + "bytes" + "crypto/rand" + "encoding/json" + "flag" + "fmt" + "io" + "net/http" + "os" + "strings" + "time" +) + +var ( + DefaultUserAgent = "Mozilla/5.0 (X11; Linux x86_64; rv:153.0) Gecko/20100101 Firefox/153.0" + ConfiguredUserAgent string +) + +// --------------------------------------------------------------------------- +// OpenAI API Data Structures +// --------------------------------------------------------------------------- + +type ModelItem struct { + ID string `json:"id"` + Object string `json:"object"` + Created int64 `json:"created"` + OwnedBy string `json:"owned_by"` +} + +type ModelsResponse struct { + Object string `json:"object"` + Data []ModelItem `json:"data"` +} + +type ToolCallFunction struct { + Name string `json:"name"` + Arguments string `json:"arguments"` +} + +type ToolCall struct { + Index *int `json:"index,omitempty"` + ID string `json:"id,omitempty"` + Type string `json:"type,omitempty"` + Function ToolCallFunction `json:"function"` +} + +type Tool struct { + Type string `json:"type"` + Function interface{} `json:"function"` +} + +type ChatMessage struct { + Role string `json:"role"` + Content interface{} `json:"content"` + ReasoningContent string `json:"reasoning_content,omitempty"` + ToolCalls []ToolCall `json:"tool_calls,omitempty"` + ToolCallID string `json:"tool_call_id,omitempty"` + Name string `json:"name,omitempty"` +} + +func (m *ChatMessage) GetContentString() string { + if m.Content == nil { + return "" + } + if str, ok := m.Content.(string); ok { + return str + } + if parts, ok := m.Content.([]interface{}); ok { + var sb strings.Builder + for _, p := range parts { + if str, ok := p.(string); ok { + sb.WriteString(str) + } else if itemMap, ok := p.(map[string]interface{}); ok { + if textVal, ok := itemMap["text"].(string); ok { + sb.WriteString(textVal) + } + } + } + return sb.String() + } + b, err := json.Marshal(m.Content) + if err == nil { + return string(b) + } + return fmt.Sprintf("%v", m.Content) +} + +type ChatCompletionRequest struct { + Model string `json:"model"` + Messages []ChatMessage `json:"messages"` + Tools []Tool `json:"tools,omitempty"` + ToolChoice interface{} `json:"tool_choice,omitempty"` + Stream bool `json:"stream"` + MaxTokens int `json:"max_tokens"` + MaxCompletionTokens int `json:"max_completion_tokens"` + Temperature *float64 `json:"temperature,omitempty"` + TopP *float64 `json:"top_p,omitempty"` +} + +type ChatCompletionResponseChoice struct { + Index int `json:"index"` + Message ChatMessage `json:"message"` + FinishReason string `json:"finish_reason"` +} + +type Usage struct { + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` + TotalTokens int `json:"total_tokens"` +} + +type ChatCompletionResponse struct { + ID string `json:"id"` + Object string `json:"object"` + Created int64 `json:"created"` + Model string `json:"model"` + Choices []ChatCompletionResponseChoice `json:"choices"` + Usage Usage `json:"usage"` +} + +type StreamDelta struct { + Role string `json:"role,omitempty"` + Content string `json:"content,omitempty"` + ReasoningContent string `json:"reasoning_content,omitempty"` + ToolCalls []ToolCall `json:"tool_calls,omitempty"` +} + +type StreamChoice struct { + Index int `json:"index"` + Delta StreamDelta `json:"delta"` + FinishReason *string `json:"finish_reason,omitempty"` +} + +type StreamResponse struct { + ID string `json:"id"` + Object string `json:"object"` + Created int64 `json:"created"` + Model string `json:"model"` + Choices []StreamChoice `json:"choices"` +} + +type GradioJoinResponse struct { + EventID string `json:"event_id"` +} + +// --------------------------------------------------------------------------- +// Helper Utilities +// --------------------------------------------------------------------------- + +func GenerateUUID() string { + var b [16]byte + _, err := rand.Read(b[:]) + if err != nil { + return "00000000-0000-4000-8000-000000000000" + } + b[6] = (b[6] & 0x0f) | 0x40 + b[8] = (b[8] & 0x3f) | 0x80 + return fmt.Sprintf("%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:]) +} + +func FibonacciDelay(attempt int) time.Duration { + if attempt <= 0 { + return 1 * time.Second + } + a, b := 1, 1 + for i := 1; i < attempt; i++ { + a, b = b, a+b + } + return time.Duration(a) * time.Second +} + +func DoWithFibonacciRetry(client *http.Client, makeReq func() (*http.Request, error), maxRetries int) (*http.Response, error) { + var lastErr error + for attempt := 1; attempt <= maxRetries; attempt++ { + req, err := makeReq() + if err != nil { + return nil, err + } + resp, err := client.Do(req) + if err == nil && resp.StatusCode == http.StatusOK { + return resp, nil + } + + if resp != nil { + respBody, _ := io.ReadAll(resp.Body) + resp.Body.Close() + lastErr = fmt.Errorf("HTTP status %d: %s", resp.StatusCode, string(respBody)) + } else { + lastErr = err + } + + if attempt < maxRetries { + delay := FibonacciDelay(attempt) + time.Sleep(delay) + } + } + return nil, fmt.Errorf("request failed after %d retries: %v", maxRetries, lastErr) +} + +func EnableCORS(w http.ResponseWriter) { + w.Header().Set("Access-Control-Allow-Origin", "*") + w.Header().Set("Access-Control-Allow-Methods", "GET, POST, OPTIONS") + w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Authorization, api-key, X-User-Agent") +} + +func ResolveMaxTokens(req ChatCompletionRequest) int { + mt := req.MaxTokens + if mt == 0 && req.MaxCompletionTokens > 0 { + mt = req.MaxCompletionTokens + } + if mt <= 0 { + mt = 131072 + } + return mt +} + +func EffectiveUserAgent(r *http.Request) string { + if r != nil { + if c := r.Header.Get("X-User-Agent"); c != "" { + return c + } + } + if ConfiguredUserAgent != "" { + return ConfiguredUserAgent + } + return DefaultUserAgent +} + +// --------------------------------------------------------------------------- +// Response Framing +// --------------------------------------------------------------------------- + +type FinalOutput struct { + Content interface{} + ReasoningContent string + ToolCalls []ToolCall + FinishReason string +} + +func WriteCompletionResponse(w http.ResponseWriter, completionID string, created int64, model string, out FinalOutput) { + finish := out.FinishReason + if finish == "" { + finish = "stop" + } + resp := ChatCompletionResponse{ + ID: completionID, + Object: "chat.completion", + Created: created, + Model: model, + Choices: []ChatCompletionResponseChoice{ + { + Index: 0, + Message: ChatMessage{ + Role: "assistant", + Content: out.Content, + ReasoningContent: out.ReasoningContent, + ToolCalls: out.ToolCalls, + }, + FinishReason: finish, + }, + }, + Usage: Usage{ + PromptTokens: 0, + CompletionTokens: 0, + TotalTokens: 0, + }, + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(resp) +} + +type Streamer struct { + w http.ResponseWriter + flusher http.Flusher + id string + created int64 + model string +} + +func NewStreamer(w http.ResponseWriter, flusher http.Flusher, id string, created int64, model string) *Streamer { + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Cache-Control", "no-cache") + w.Header().Set("Connection", "keep-alive") + return &Streamer{w: w, flusher: flusher, id: id, created: created, model: model} +} + +func (s *Streamer) Role() { + sendStreamDelta(s.w, s.flusher, s.id, s.created, s.model, StreamDelta{Role: "assistant"}) +} + +func (s *Streamer) Reasoning(text string) { + sendStreamDelta(s.w, s.flusher, s.id, s.created, s.model, StreamDelta{ReasoningContent: text}) +} + +func (s *Streamer) Content(text string) { + sendStreamDelta(s.w, s.flusher, s.id, s.created, s.model, StreamDelta{Content: text}) +} + +func (s *Streamer) ToolCallDelta(tc ToolCall) { + sendStreamDelta(s.w, s.flusher, s.id, s.created, s.model, StreamDelta{ToolCalls: []ToolCall{tc}}) +} + +func (s *Streamer) Finish(reason string) { + sendStreamChunk(s.w, s.flusher, s.id, s.created, s.model, StreamDelta{}, &reason) +} + +func (s *Streamer) Done() { + fmt.Fprintf(s.w, "data: [DONE]\n\n") + if s.flusher != nil { + s.flusher.Flush() + } +} + +func sendStreamDelta(w http.ResponseWriter, flusher http.Flusher, completionID string, createdTime int64, modelName string, delta StreamDelta) { + sendStreamChunk(w, flusher, completionID, createdTime, modelName, delta, nil) +} + +func sendStreamChunk(w http.ResponseWriter, flusher http.Flusher, completionID string, createdTime int64, modelName string, delta StreamDelta, finishReason *string) { + chunk := StreamResponse{ + ID: completionID, + Object: "chat.completion.chunk", + Created: createdTime, + Model: modelName, + Choices: []StreamChoice{ + { + Index: 0, + Delta: delta, + FinishReason: finishReason, + }, + }, + } + b, _ := json.Marshal(chunk) + fmt.Fprintf(w, "data: %s\n\n", b) + if flusher != nil { + flusher.Flush() + } +} + +// --------------------------------------------------------------------------- +// Hunyuan 3 Provider & Stream Parsing +// --------------------------------------------------------------------------- + +func parseGradioStreamData(dataJSON string) (string, string, []ToolCall, bool) { + var raw []interface{} + if err := json.Unmarshal([]byte(dataJSON), &raw); err != nil || len(raw) == 0 { + return "", "", nil, false + } + + inner, ok := raw[0].([]interface{}) + if !ok || len(inner) < 2 { + return "", "", nil, false + } + + contentStr, _ := inner[0].(string) + reasoningStr, _ := inner[1].(string) + + var toolCalls []ToolCall + if len(inner) >= 3 && inner[2] != nil { + b, err := json.Marshal(inner[2]) + if err == nil { + var tcs []ToolCall + if json.Unmarshal(b, &tcs) == nil && len(tcs) > 0 { + toolCalls = tcs + } + } + } + + return contentStr, reasoningStr, toolCalls, true +} + +type HunyuanService struct { + endpoint string + client *http.Client +} + +func NewHunyuanService(endpoint string) *HunyuanService { + cleanEndpoint := strings.TrimRight(endpoint, "/") + return &HunyuanService{ + endpoint: cleanEndpoint, + client: &http.Client{Timeout: 300 * time.Second}, + } +} + +func (s *HunyuanService) ListModels() []ModelItem { + now := time.Now().Unix() + return []ModelItem{ + {ID: "hy3", Object: "model", Created: now, OwnedBy: "hunyuan"}, + } +} + +func (s *HunyuanService) Chat(w http.ResponseWriter, r *http.Request, req ChatCompletionRequest) error { + modelName := req.Model + if modelName == "" { + modelName = "hy3" + } + maxTokens := ResolveMaxTokens(req) + + var systemPromptStr string + var historyArray []map[string]interface{} + var messageStr string + + var nonSystemMsgs []ChatMessage + for _, msg := range req.Messages { + cStr := msg.GetContentString() + if msg.Role == "system" && systemPromptStr == "" { + systemPromptStr = cStr + } else { + nonSystemMsgs = append(nonSystemMsgs, msg) + } + } + + if len(nonSystemMsgs) > 0 { + for i := 0; i < len(nonSystemMsgs)-1; i++ { + m := nonSystemMsgs[i] + cStr := m.GetContentString() + item := map[string]interface{}{"role": m.Role} + switch m.Role { + case "assistant": + if cStr != "" { + item["content"] = cStr + } else { + item["content"] = nil + } + if m.ReasoningContent != "" { + item["reasoning_content"] = m.ReasoningContent + } + if len(m.ToolCalls) > 0 { + item["tool_calls"] = m.ToolCalls + } + case "tool", "function": + item["role"] = "tool" + item["content"] = cStr + toolID := m.ToolCallID + if toolID == "" { + toolID = m.Name + } + if toolID != "" { + item["tool_call_id"] = toolID + } + if m.Name != "" { + item["name"] = m.Name + } + default: + item["content"] = cStr + } + historyArray = append(historyArray, item) + } + + lastMsg := nonSystemMsgs[len(nonSystemMsgs)-1] + lastContent := lastMsg.GetContentString() + if lastMsg.Role == "tool" || lastMsg.Role == "function" { + toolName := lastMsg.Name + if toolName == "" { + toolName = lastMsg.ToolCallID + } + if toolName != "" { + messageStr = fmt.Sprintf("Tool result for %s: %s", toolName, lastContent) + } else { + messageStr = lastContent + } + } else { + messageStr = lastContent + } + } + + functionsJSONStr := "" + if len(req.Tools) > 0 { + b, err := json.Marshal(req.Tools) + if err == nil { + functionsJSONStr = string(b) + } + } + + var tempVal interface{} + if req.Temperature != nil { + tempVal = *req.Temperature + } + var topPVal interface{} + if req.TopP != nil { + topPVal = *req.TopP + } else { + topPVal = 0 + } + + gradioData := []interface{}{ + messageStr, + systemPromptStr, + historyArray, + "high", + tempVal, + maxTokens, + topPVal, + nil, + functionsJSONStr, + } + gradioPayload := map[string]interface{}{"data": gradioData} + jsonPayload, err := json.Marshal(gradioPayload) + if err != nil { + return fmt.Errorf("failed to encode request: %w", err) + } + + effUA := EffectiveUserAgent(r) + + callURL := s.endpoint + "/gradio_api/call/chat" + makeCallReq := func() (*http.Request, error) { + req, err := http.NewRequest("POST", callURL, bytes.NewBuffer(jsonPayload)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("User-Agent", effUA) + return req, nil + } + + resp, err := DoWithFibonacciRetry(s.client, makeCallReq, 5) + if err != nil { + return fmt.Errorf("upstream error: %w", err) + } + defer resp.Body.Close() + + var joinRes GradioJoinResponse + if err := json.NewDecoder(resp.Body).Decode(&joinRes); err != nil || joinRes.EventID == "" { + return fmt.Errorf("failed to parse Gradio event ID") + } + + streamURL := fmt.Sprintf("%s/gradio_api/call/chat/%s", s.endpoint, joinRes.EventID) + makeStreamReq := func() (*http.Request, error) { + req, err := http.NewRequest("GET", streamURL, nil) + if err != nil { + return nil, err + } + req.Header.Set("Accept", "text/event-stream") + req.Header.Set("User-Agent", effUA) + return req, nil + } + + streamResp, err := DoWithFibonacciRetry(s.client, makeStreamReq, 5) + if err != nil { + return fmt.Errorf("upstream stream error: %w", err) + } + defer streamResp.Body.Close() + + completionID := "chatcmpl-" + GenerateUUID() + createdTime := time.Now().Unix() + + if !req.Stream { + reader := bufio.NewReader(streamResp.Body) + var finalContent, finalReasoning string + var finalToolCalls []ToolCall + + for { + line, err := reader.ReadString('\n') + if err != nil { + break + } + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "data: ") { + dataJSON := strings.TrimPrefix(line, "data: ") + if c, r, tc, ok := parseGradioStreamData(dataJSON); ok { + finalContent = c + finalReasoning = r + if len(tc) > 0 { + finalToolCalls = tc + } + } + } + } + + finishReason := "stop" + var msgContent interface{} = finalContent + if len(finalToolCalls) > 0 { + finishReason = "tool_calls" + if finalContent == "" { + msgContent = nil + } + } + + WriteCompletionResponse(w, completionID, createdTime, modelName, FinalOutput{ + Content: msgContent, + ReasoningContent: finalReasoning, + ToolCalls: finalToolCalls, + FinishReason: finishReason, + }) + return nil + } + + flusher, _ := w.(http.Flusher) + streamer := NewStreamer(w, flusher, completionID, createdTime, modelName) + streamer.Role() + + reader := bufio.NewReader(streamResp.Body) + var emittedContent, emittedReasoning string + lastToolCallArgs := map[string]string{} + emittedToolCallIDs := map[string]bool{} + + for { + line, err := reader.ReadString('\n') + if err != nil { + break + } + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "data: ") { + dataJSON := strings.TrimPrefix(line, "data: ") + if c, r, tc, ok := parseGradioStreamData(dataJSON); ok { + if len(r) > len(emittedReasoning) { + rDelta := r[len(emittedReasoning):] + emittedReasoning = r + streamer.Reasoning(rDelta) + } + if len(c) > len(emittedContent) { + cDelta := c[len(emittedContent):] + emittedContent = c + streamer.Content(cDelta) + } + if len(tc) > 0 { + for idx, t := range tc { + key := t.ID + if key == "" { + key = fmt.Sprintf("idx_%d", idx) + } + prevArgs := lastToolCallArgs[key] + currArgs := t.Function.Arguments + if !emittedToolCallIDs[key] { + emittedToolCallIDs[key] = true + lastToolCallArgs[key] = currArgs + tCopy := ToolCall{ + Index: new(int), + ID: t.ID, + Type: "function", + Function: ToolCallFunction{Name: t.Function.Name, Arguments: currArgs}, + } + *tCopy.Index = idx + streamer.ToolCallDelta(tCopy) + } else if len(currArgs) > len(prevArgs) { + argDelta := currArgs[len(prevArgs):] + lastToolCallArgs[key] = currArgs + tCopy := ToolCall{ + Index: new(int), + Function: ToolCallFunction{Arguments: argDelta}, + } + *tCopy.Index = idx + streamer.ToolCallDelta(tCopy) + } + } + } + } + } + } + + if len(emittedToolCallIDs) > 0 { + streamer.Finish("tool_calls") + } else { + streamer.Finish("stop") + } + streamer.Done() + return nil +} + +// --------------------------------------------------------------------------- +// HTTP Server Wiring & Main Entry Point +// --------------------------------------------------------------------------- + +func ModelsHandler(service *HunyuanService) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + EnableCORS(w) + if r.Method == http.MethodOptions { + w.WriteHeader(http.StatusOK) + return + } + if r.Method != http.MethodGet { + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } + items := service.ListModels() + res := ModelsResponse{Object: "list", Data: items} + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(res) + } +} + +func ChatHandler(service *HunyuanService) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + EnableCORS(w) + if r.Method == http.MethodOptions { + w.WriteHeader(http.StatusOK) + return + } + if r.Method != http.MethodPost { + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } + var req ChatCompletionRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, "Invalid JSON payload: "+err.Error(), http.StatusBadRequest) + return + } + if err := service.Chat(w, r, req); err != nil { + if w.Header().Get("Content-Type") != "text/event-stream" { + http.Error(w, "Upstream error: "+err.Error(), http.StatusBadGateway) + } + } + } +} + +func NewMux(service *HunyuanService) *http.ServeMux { + mux := http.NewServeMux() + mh := ModelsHandler(service) + ch := ChatHandler(service) + mux.HandleFunc("/models", mh) + mux.HandleFunc("/v1/models", mh) + mux.HandleFunc("/chat/completions", ch) + mux.HandleFunc("/v1/chat/completions", ch) + return mux +} + +func main() { + portFlag := flag.String("port", "8080", "Port to listen on") + endpointFlag := flag.String("endpoint", "https://tencent-hy3.hf.space", "Root URL of the Hunyuan Gradio space") + uaFlag := flag.String("user-agent", DefaultUserAgent, "Custom User-Agent header") + uaShortFlag := flag.String("ua", "", "Alias for -user-agent") + flag.Parse() + + ConfiguredUserAgent = *uaFlag + if *uaShortFlag != "" { + ConfiguredUserAgent = *uaShortFlag + } + + service := NewHunyuanService(*endpointFlag) + mux := NewMux(service) + + fmt.Printf("hygate starting on port %s...\n", *portFlag) + fmt.Printf("Target Endpoint: %s\n", service.endpoint) + fmt.Printf("User-Agent: %s\n", ConfiguredUserAgent) + fmt.Printf("Endpoints:\n") + fmt.Printf(" GET http://localhost:%s/v1/models\n", *portFlag) + fmt.Printf(" POST http://localhost:%s/v1/chat/completions\n", *portFlag) + + if err := http.ListenAndServe(":"+*portFlag, mux); err != nil { + fmt.Fprintf(os.Stderr, "server failed: %v\n", err) + os.Exit(1) + } +}