initial upl
This commit is contained in:
Luxferre
@@ -0,0 +1,11 @@
|
||||
# Makefile for hygate (Hunyuan AI Gateway)
|
||||
|
||||
hygate:
|
||||
go build -trimpath -ldflags="-s -w" -o bin/hygate .
|
||||
|
||||
all: hygate
|
||||
|
||||
clean:
|
||||
rm -rf bin/
|
||||
|
||||
.PHONY: all clean hygate
|
||||
@@ -0,0 +1,89 @@
|
||||
# Hygate
|
||||
|
||||
## About
|
||||
|
||||
Hygate is a standalone, single-binary gateway that exposes Tencent's Hunyuan 3 (hy3) Gradio space through an OpenAI-compatible API. It translates the standard `/v1/chat/completions` and `/v1/models` endpoints into Gradio's `/gradio_api/call/chat` request/stream protocol, so any OpenAI-compatible client can talk to Hunyuan 3 without modification.
|
||||
|
||||
## Features
|
||||
|
||||
- OpenAI-compatible chat completions (streaming and non-streaming)
|
||||
- Reasoning content passthrough (`reasoning_content`)
|
||||
- Tool/function calling, including incremental streaming of tool-call arguments
|
||||
- Fibonacci backoff retry on upstream calls
|
||||
- Configurable endpoint and User-Agent
|
||||
- No external dependencies (standard library only)
|
||||
|
||||
## Installation
|
||||
|
||||
Build from source with Go 1.26 or newer:
|
||||
|
||||
```
|
||||
make
|
||||
```
|
||||
|
||||
This produces `bin/hygate` binary for your architecture.
|
||||
|
||||
Install the latest release straight from the Git repository with `go install`:
|
||||
|
||||
```
|
||||
go install code.luxferre.top/luxferre/hygate@latest
|
||||
```
|
||||
|
||||
This fetches the module from `https://code.luxferre.top/luxferre/hygate.git` and places the `hygate` binary in `$(go env GOPATH)/bin`. Make sure that directory is on your `PATH`.
|
||||
|
||||
If you prefer to build from a local checkout instead:
|
||||
|
||||
```
|
||||
git clone https://code.luxferre.top/luxferre/hygate.git
|
||||
cd hygate
|
||||
go install .
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
Run the gateway:
|
||||
|
||||
```
|
||||
./bin/hygate [-port 8080] [-endpoint https://tencent-hy3.hf.space]
|
||||
```
|
||||
|
||||
Available flags:
|
||||
|
||||
- `-port` — TCP port to listen on (default `8080`)
|
||||
- `-endpoint` — root URL of the Hunyuan Gradio space (default `https://tencent-hy3.hf.space`)
|
||||
- `-user-agent` / `-ua` — custom User-Agent sent to the upstream
|
||||
|
||||
Endpoints served:
|
||||
|
||||
- `GET /v1/models`
|
||||
- `POST /v1/chat/completions`
|
||||
|
||||
Example request with curl:
|
||||
|
||||
```
|
||||
curl http://localhost:8080/v1/chat/completions \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"model":"hy3","messages":[{"role":"user","content":"Hello"}],"stream":false}'
|
||||
```
|
||||
|
||||
## FAQ
|
||||
|
||||
### Does this need an API key?
|
||||
|
||||
No. The gateway proxies directly to the public Gradio space and does not require authentication.
|
||||
|
||||
### Which models are reported?
|
||||
|
||||
A single model, `hy3`, is advertised via `/v1/models`.
|
||||
|
||||
### Are token usage counts returned?
|
||||
|
||||
No. The `usage` field is always zero, since the upstream Gradio API does not expose token counts.
|
||||
|
||||
### Can the upstream endpoint be changed?
|
||||
|
||||
Yes, with the `-endpoint` flag. Any HuggingFace Gradio space hosting the compatible `chat` endpoint will work.
|
||||
|
||||
## Credits
|
||||
|
||||
Created by Luxferre in 2026, released into the public domain with no warranties.
|
||||
+307
@@ -0,0 +1,307 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestHunyuanServiceEndpoint(t *testing.T) {
|
||||
svc := NewHunyuanService("https://example.com/gradio/")
|
||||
if svc.endpoint != "https://example.com/gradio" {
|
||||
t.Errorf("expected trailing slash stripped, got %q", svc.endpoint)
|
||||
}
|
||||
|
||||
models := svc.ListModels()
|
||||
if len(models) != 1 || models[0].ID != "hy3" {
|
||||
t.Fatalf("expected model ID 'hy3', got %v", models)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseGradioStreamData(t *testing.T) {
|
||||
sampleJSON := `[["Hello world", "Thinking process...", null]]`
|
||||
c, r, tc, ok := parseGradioStreamData(sampleJSON)
|
||||
if !ok {
|
||||
t.Fatalf("expected parseGradioStreamData to succeed")
|
||||
}
|
||||
if c != "Hello world" {
|
||||
t.Errorf("content=%q, want 'Hello world'", c)
|
||||
}
|
||||
if r != "Thinking process..." {
|
||||
t.Errorf("reasoning=%q, want 'Thinking process...'", r)
|
||||
}
|
||||
if len(tc) != 0 {
|
||||
t.Errorf("expected 0 tool calls, got %d", len(tc))
|
||||
}
|
||||
}
|
||||
|
||||
func TestHunyuanMockedCompletion(t *testing.T) {
|
||||
var receivedCallURL, receivedStreamURL string
|
||||
|
||||
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if strings.HasSuffix(r.URL.Path, "/gradio_api/call/chat") && r.Method == http.MethodPost {
|
||||
receivedCallURL = r.URL.Path
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
fmt.Fprint(w, `{"event_id":"test-event-123"}`)
|
||||
return
|
||||
}
|
||||
if strings.Contains(r.URL.Path, "/gradio_api/call/chat/test-event-123") && r.Method == http.MethodGet {
|
||||
receivedStreamURL = r.URL.Path
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
fmt.Fprint(w, "data: [[\"Mocked answer\", \"Mocked reasoning\", null]]\n\n")
|
||||
return
|
||||
}
|
||||
http.NotFound(w, r)
|
||||
})
|
||||
|
||||
server := httptest.NewServer(handler)
|
||||
defer server.Close()
|
||||
|
||||
svc := NewHunyuanService(server.URL)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
req, _ := http.NewRequest("POST", "/v1/chat/completions", nil)
|
||||
|
||||
chatReq := ChatCompletionRequest{
|
||||
Model: "hy3",
|
||||
Messages: []ChatMessage{
|
||||
{Role: "user", Content: "Hello!"},
|
||||
},
|
||||
Stream: false,
|
||||
}
|
||||
|
||||
err := svc.Chat(rec, req, chatReq)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected Chat error: %v", err)
|
||||
}
|
||||
|
||||
if receivedCallURL != "/gradio_api/call/chat" {
|
||||
t.Errorf("expected call URL '/gradio_api/call/chat', got %q", receivedCallURL)
|
||||
}
|
||||
if receivedStreamURL != "/gradio_api/call/chat/test-event-123" {
|
||||
t.Errorf("expected stream URL '/gradio_api/call/chat/test-event-123', got %q", receivedStreamURL)
|
||||
}
|
||||
|
||||
var resp ChatCompletionResponse
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("failed to decode completion response: %v", err)
|
||||
}
|
||||
|
||||
if len(resp.Choices) != 1 {
|
||||
t.Fatalf("expected 1 choice, got %d", len(resp.Choices))
|
||||
}
|
||||
if resp.Choices[0].Message.GetContentString() != "Mocked answer" {
|
||||
t.Errorf("content=%q, want 'Mocked answer'", resp.Choices[0].Message.GetContentString())
|
||||
}
|
||||
if resp.Choices[0].Message.ReasoningContent != "Mocked reasoning" {
|
||||
t.Errorf("reasoning=%q, want 'Mocked reasoning'", resp.Choices[0].Message.ReasoningContent)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseGradioStreamDataWithToolCalls(t *testing.T) {
|
||||
sampleJSON := `[["", "Let me check the weather.", [{"id":"call_999","type":"function","function":{"name":"get_weather","arguments":"{\"location\":\"Tokyo\"}"}}]]]`
|
||||
c, r, tc, ok := parseGradioStreamData(sampleJSON)
|
||||
if !ok {
|
||||
t.Fatalf("expected parseGradioStreamData to succeed")
|
||||
}
|
||||
if c != "" {
|
||||
t.Errorf("content=%q, want empty", c)
|
||||
}
|
||||
if r != "Let me check the weather." {
|
||||
t.Errorf("reasoning=%q, want 'Let me check the weather.'", r)
|
||||
}
|
||||
if len(tc) != 1 {
|
||||
t.Fatalf("expected 1 tool call, got %d", len(tc))
|
||||
}
|
||||
if tc[0].ID != "call_999" || tc[0].Function.Name != "get_weather" || tc[0].Function.Arguments != `{"location":"Tokyo"}` {
|
||||
t.Errorf("unexpected tool call payload: %+v", tc[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestHunyuanToolCallingNonStreaming(t *testing.T) {
|
||||
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if strings.HasSuffix(r.URL.Path, "/gradio_api/call/chat") && r.Method == http.MethodPost {
|
||||
var body map[string]interface{}
|
||||
json.NewDecoder(r.Body).Decode(&body)
|
||||
dataArr, _ := body["data"].([]interface{})
|
||||
if len(dataArr) < 9 || dataArr[8] == nil || dataArr[8] == "" {
|
||||
t.Errorf("expected functionsJSONStr in data[8], got %v", dataArr)
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
fmt.Fprint(w, `{"event_id":"evt-tool-1"}`)
|
||||
return
|
||||
}
|
||||
if strings.Contains(r.URL.Path, "/gradio_api/call/chat/evt-tool-1") && r.Method == http.MethodGet {
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
fmt.Fprint(w, "data: [[\"\", \"I should call the search tool.\", [{\"id\":\"call_abc\",\"type\":\"function\",\"function\":{\"name\":\"search\",\"arguments\":\"{\\\"query\\\":\\\"golang\\\"}\"}}]]]\n\n")
|
||||
return
|
||||
}
|
||||
http.NotFound(w, r)
|
||||
})
|
||||
|
||||
server := httptest.NewServer(handler)
|
||||
defer server.Close()
|
||||
|
||||
svc := NewHunyuanService(server.URL)
|
||||
rec := httptest.NewRecorder()
|
||||
req, _ := http.NewRequest("POST", "/v1/chat/completions", nil)
|
||||
|
||||
chatReq := ChatCompletionRequest{
|
||||
Model: "hy3",
|
||||
Messages: []ChatMessage{
|
||||
{Role: "user", Content: "Search for golang news"},
|
||||
},
|
||||
Tools: []Tool{
|
||||
{
|
||||
Type: "function",
|
||||
Function: map[string]interface{}{
|
||||
"name": "search",
|
||||
"description": "Search web",
|
||||
},
|
||||
},
|
||||
},
|
||||
Stream: false,
|
||||
}
|
||||
|
||||
err := svc.Chat(rec, req, chatReq)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected Chat error: %v", err)
|
||||
}
|
||||
|
||||
var resp ChatCompletionResponse
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("failed to decode response: %v", err)
|
||||
}
|
||||
|
||||
if len(resp.Choices) != 1 {
|
||||
t.Fatalf("expected 1 choice, got %d", len(resp.Choices))
|
||||
}
|
||||
choice := resp.Choices[0]
|
||||
if choice.FinishReason != "tool_calls" {
|
||||
t.Errorf("finish_reason=%q, want 'tool_calls'", choice.FinishReason)
|
||||
}
|
||||
if choice.Message.ReasoningContent != "I should call the search tool." {
|
||||
t.Errorf("reasoning_content=%q, want 'I should call the search tool.'", choice.Message.ReasoningContent)
|
||||
}
|
||||
if len(choice.Message.ToolCalls) != 1 {
|
||||
t.Fatalf("expected 1 tool call, got %d", len(choice.Message.ToolCalls))
|
||||
}
|
||||
tc := choice.Message.ToolCalls[0]
|
||||
if tc.ID != "call_abc" || tc.Function.Name != "search" || tc.Function.Arguments != `{"query":"golang"}` {
|
||||
t.Errorf("unexpected tool call: %+v", tc)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHunyuanReasoningAndToolCallingStreaming(t *testing.T) {
|
||||
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if strings.HasSuffix(r.URL.Path, "/gradio_api/call/chat") && r.Method == http.MethodPost {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
fmt.Fprint(w, `{"event_id":"evt-stream-1"}`)
|
||||
return
|
||||
}
|
||||
if strings.Contains(r.URL.Path, "/gradio_api/call/chat/evt-stream-1") && r.Method == http.MethodGet {
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
flusher, _ := w.(http.Flusher)
|
||||
|
||||
fmt.Fprint(w, "data: [[\"\", \"Step 1 reasoning.\", null]]\n\n")
|
||||
if flusher != nil {
|
||||
flusher.Flush()
|
||||
}
|
||||
|
||||
fmt.Fprint(w, "data: [[\"Hello \", \"Step 1 reasoning. Step 2 reasoning.\", null]]\n\n")
|
||||
if flusher != nil {
|
||||
flusher.Flush()
|
||||
}
|
||||
|
||||
fmt.Fprint(w, "data: [[\"Hello \", \"Step 1 reasoning. Step 2 reasoning.\", [{\"id\":\"call_xyz\",\"type\":\"function\",\"function\":{\"name\":\"calculator\",\"arguments\":\"{\\\"expr\\\":\\\"2+2\\\"}\"}}]]]\n\n")
|
||||
if flusher != nil {
|
||||
flusher.Flush()
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
http.NotFound(w, r)
|
||||
})
|
||||
|
||||
server := httptest.NewServer(handler)
|
||||
defer server.Close()
|
||||
|
||||
svc := NewHunyuanService(server.URL)
|
||||
rec := httptest.NewRecorder()
|
||||
req, _ := http.NewRequest("POST", "/v1/chat/completions", nil)
|
||||
|
||||
chatReq := ChatCompletionRequest{
|
||||
Model: "hy3",
|
||||
Messages: []ChatMessage{
|
||||
{Role: "user", Content: "Calculate 2+2"},
|
||||
},
|
||||
Stream: true,
|
||||
}
|
||||
|
||||
err := svc.Chat(rec, req, chatReq)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected Chat error: %v", err)
|
||||
}
|
||||
|
||||
bodyStr := rec.Body.String()
|
||||
lines := strings.Split(bodyStr, "\n")
|
||||
|
||||
var receivedDeltas []StreamDelta
|
||||
var finishReasons []string
|
||||
var foundDone bool
|
||||
|
||||
for _, line := range lines {
|
||||
line = strings.TrimSpace(line)
|
||||
if strings.HasPrefix(line, "data: ") {
|
||||
payload := strings.TrimPrefix(line, "data: ")
|
||||
if payload == "[DONE]" {
|
||||
foundDone = true
|
||||
continue
|
||||
}
|
||||
var streamResp StreamResponse
|
||||
if err := json.Unmarshal([]byte(payload), &streamResp); err == nil && len(streamResp.Choices) > 0 {
|
||||
choice := streamResp.Choices[0]
|
||||
receivedDeltas = append(receivedDeltas, choice.Delta)
|
||||
if choice.FinishReason != nil {
|
||||
finishReasons = append(finishReasons, *choice.FinishReason)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !foundDone {
|
||||
t.Errorf("expected 'data: [DONE]' in SSE stream")
|
||||
}
|
||||
|
||||
if len(receivedDeltas) < 1 || receivedDeltas[0].Role != "assistant" {
|
||||
t.Errorf("expected first delta to set role='assistant', got %+v", receivedDeltas[0])
|
||||
}
|
||||
|
||||
var combinedReasoning, combinedContent string
|
||||
var toolCallsEmitted []ToolCall
|
||||
for _, d := range receivedDeltas {
|
||||
combinedReasoning += d.ReasoningContent
|
||||
combinedContent += d.Content
|
||||
if len(d.ToolCalls) > 0 {
|
||||
toolCallsEmitted = append(toolCallsEmitted, d.ToolCalls...)
|
||||
}
|
||||
}
|
||||
|
||||
if combinedReasoning != "Step 1 reasoning. Step 2 reasoning." {
|
||||
t.Errorf("combined reasoning=%q, want 'Step 1 reasoning. Step 2 reasoning.'", combinedReasoning)
|
||||
}
|
||||
if combinedContent != "Hello " {
|
||||
t.Errorf("combined content=%q, want 'Hello '", combinedContent)
|
||||
}
|
||||
if len(toolCallsEmitted) == 0 {
|
||||
t.Fatalf("expected tool call deltas to be emitted")
|
||||
}
|
||||
if len(finishReasons) != 1 || finishReasons[0] != "tool_calls" {
|
||||
t.Errorf("finish_reasons=%v, want ['tool_calls']", finishReasons)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,748 @@
|
||||
// hygate: Standalone OpenAI-compatible gateway for Hunyuan 3 Gradio spaces
|
||||
// Created by Luxferre in 2026, released into the public domain
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
DefaultUserAgent = "Mozilla/5.0 (X11; Linux x86_64; rv:153.0) Gecko/20100101 Firefox/153.0"
|
||||
ConfiguredUserAgent string
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// OpenAI API Data Structures
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type ModelItem struct {
|
||||
ID string `json:"id"`
|
||||
Object string `json:"object"`
|
||||
Created int64 `json:"created"`
|
||||
OwnedBy string `json:"owned_by"`
|
||||
}
|
||||
|
||||
type ModelsResponse struct {
|
||||
Object string `json:"object"`
|
||||
Data []ModelItem `json:"data"`
|
||||
}
|
||||
|
||||
type ToolCallFunction struct {
|
||||
Name string `json:"name"`
|
||||
Arguments string `json:"arguments"`
|
||||
}
|
||||
|
||||
type ToolCall struct {
|
||||
Index *int `json:"index,omitempty"`
|
||||
ID string `json:"id,omitempty"`
|
||||
Type string `json:"type,omitempty"`
|
||||
Function ToolCallFunction `json:"function"`
|
||||
}
|
||||
|
||||
type Tool struct {
|
||||
Type string `json:"type"`
|
||||
Function interface{} `json:"function"`
|
||||
}
|
||||
|
||||
type ChatMessage struct {
|
||||
Role string `json:"role"`
|
||||
Content interface{} `json:"content"`
|
||||
ReasoningContent string `json:"reasoning_content,omitempty"`
|
||||
ToolCalls []ToolCall `json:"tool_calls,omitempty"`
|
||||
ToolCallID string `json:"tool_call_id,omitempty"`
|
||||
Name string `json:"name,omitempty"`
|
||||
}
|
||||
|
||||
func (m *ChatMessage) GetContentString() string {
|
||||
if m.Content == nil {
|
||||
return ""
|
||||
}
|
||||
if str, ok := m.Content.(string); ok {
|
||||
return str
|
||||
}
|
||||
if parts, ok := m.Content.([]interface{}); ok {
|
||||
var sb strings.Builder
|
||||
for _, p := range parts {
|
||||
if str, ok := p.(string); ok {
|
||||
sb.WriteString(str)
|
||||
} else if itemMap, ok := p.(map[string]interface{}); ok {
|
||||
if textVal, ok := itemMap["text"].(string); ok {
|
||||
sb.WriteString(textVal)
|
||||
}
|
||||
}
|
||||
}
|
||||
return sb.String()
|
||||
}
|
||||
b, err := json.Marshal(m.Content)
|
||||
if err == nil {
|
||||
return string(b)
|
||||
}
|
||||
return fmt.Sprintf("%v", m.Content)
|
||||
}
|
||||
|
||||
type ChatCompletionRequest struct {
|
||||
Model string `json:"model"`
|
||||
Messages []ChatMessage `json:"messages"`
|
||||
Tools []Tool `json:"tools,omitempty"`
|
||||
ToolChoice interface{} `json:"tool_choice,omitempty"`
|
||||
Stream bool `json:"stream"`
|
||||
MaxTokens int `json:"max_tokens"`
|
||||
MaxCompletionTokens int `json:"max_completion_tokens"`
|
||||
Temperature *float64 `json:"temperature,omitempty"`
|
||||
TopP *float64 `json:"top_p,omitempty"`
|
||||
}
|
||||
|
||||
type ChatCompletionResponseChoice struct {
|
||||
Index int `json:"index"`
|
||||
Message ChatMessage `json:"message"`
|
||||
FinishReason string `json:"finish_reason"`
|
||||
}
|
||||
|
||||
type Usage struct {
|
||||
PromptTokens int `json:"prompt_tokens"`
|
||||
CompletionTokens int `json:"completion_tokens"`
|
||||
TotalTokens int `json:"total_tokens"`
|
||||
}
|
||||
|
||||
type ChatCompletionResponse struct {
|
||||
ID string `json:"id"`
|
||||
Object string `json:"object"`
|
||||
Created int64 `json:"created"`
|
||||
Model string `json:"model"`
|
||||
Choices []ChatCompletionResponseChoice `json:"choices"`
|
||||
Usage Usage `json:"usage"`
|
||||
}
|
||||
|
||||
type StreamDelta struct {
|
||||
Role string `json:"role,omitempty"`
|
||||
Content string `json:"content,omitempty"`
|
||||
ReasoningContent string `json:"reasoning_content,omitempty"`
|
||||
ToolCalls []ToolCall `json:"tool_calls,omitempty"`
|
||||
}
|
||||
|
||||
type StreamChoice struct {
|
||||
Index int `json:"index"`
|
||||
Delta StreamDelta `json:"delta"`
|
||||
FinishReason *string `json:"finish_reason,omitempty"`
|
||||
}
|
||||
|
||||
type StreamResponse struct {
|
||||
ID string `json:"id"`
|
||||
Object string `json:"object"`
|
||||
Created int64 `json:"created"`
|
||||
Model string `json:"model"`
|
||||
Choices []StreamChoice `json:"choices"`
|
||||
}
|
||||
|
||||
type GradioJoinResponse struct {
|
||||
EventID string `json:"event_id"`
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helper Utilities
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func GenerateUUID() string {
|
||||
var b [16]byte
|
||||
_, err := rand.Read(b[:])
|
||||
if err != nil {
|
||||
return "00000000-0000-4000-8000-000000000000"
|
||||
}
|
||||
b[6] = (b[6] & 0x0f) | 0x40
|
||||
b[8] = (b[8] & 0x3f) | 0x80
|
||||
return fmt.Sprintf("%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:])
|
||||
}
|
||||
|
||||
func FibonacciDelay(attempt int) time.Duration {
|
||||
if attempt <= 0 {
|
||||
return 1 * time.Second
|
||||
}
|
||||
a, b := 1, 1
|
||||
for i := 1; i < attempt; i++ {
|
||||
a, b = b, a+b
|
||||
}
|
||||
return time.Duration(a) * time.Second
|
||||
}
|
||||
|
||||
func DoWithFibonacciRetry(client *http.Client, makeReq func() (*http.Request, error), maxRetries int) (*http.Response, error) {
|
||||
var lastErr error
|
||||
for attempt := 1; attempt <= maxRetries; attempt++ {
|
||||
req, err := makeReq()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := client.Do(req)
|
||||
if err == nil && resp.StatusCode == http.StatusOK {
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
if resp != nil {
|
||||
respBody, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
lastErr = fmt.Errorf("HTTP status %d: %s", resp.StatusCode, string(respBody))
|
||||
} else {
|
||||
lastErr = err
|
||||
}
|
||||
|
||||
if attempt < maxRetries {
|
||||
delay := FibonacciDelay(attempt)
|
||||
time.Sleep(delay)
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("request failed after %d retries: %v", maxRetries, lastErr)
|
||||
}
|
||||
|
||||
func EnableCORS(w http.ResponseWriter) {
|
||||
w.Header().Set("Access-Control-Allow-Origin", "*")
|
||||
w.Header().Set("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
|
||||
w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Authorization, api-key, X-User-Agent")
|
||||
}
|
||||
|
||||
func ResolveMaxTokens(req ChatCompletionRequest) int {
|
||||
mt := req.MaxTokens
|
||||
if mt == 0 && req.MaxCompletionTokens > 0 {
|
||||
mt = req.MaxCompletionTokens
|
||||
}
|
||||
if mt <= 0 {
|
||||
mt = 131072
|
||||
}
|
||||
return mt
|
||||
}
|
||||
|
||||
func EffectiveUserAgent(r *http.Request) string {
|
||||
if r != nil {
|
||||
if c := r.Header.Get("X-User-Agent"); c != "" {
|
||||
return c
|
||||
}
|
||||
}
|
||||
if ConfiguredUserAgent != "" {
|
||||
return ConfiguredUserAgent
|
||||
}
|
||||
return DefaultUserAgent
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Response Framing
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type FinalOutput struct {
|
||||
Content interface{}
|
||||
ReasoningContent string
|
||||
ToolCalls []ToolCall
|
||||
FinishReason string
|
||||
}
|
||||
|
||||
func WriteCompletionResponse(w http.ResponseWriter, completionID string, created int64, model string, out FinalOutput) {
|
||||
finish := out.FinishReason
|
||||
if finish == "" {
|
||||
finish = "stop"
|
||||
}
|
||||
resp := ChatCompletionResponse{
|
||||
ID: completionID,
|
||||
Object: "chat.completion",
|
||||
Created: created,
|
||||
Model: model,
|
||||
Choices: []ChatCompletionResponseChoice{
|
||||
{
|
||||
Index: 0,
|
||||
Message: ChatMessage{
|
||||
Role: "assistant",
|
||||
Content: out.Content,
|
||||
ReasoningContent: out.ReasoningContent,
|
||||
ToolCalls: out.ToolCalls,
|
||||
},
|
||||
FinishReason: finish,
|
||||
},
|
||||
},
|
||||
Usage: Usage{
|
||||
PromptTokens: 0,
|
||||
CompletionTokens: 0,
|
||||
TotalTokens: 0,
|
||||
},
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(resp)
|
||||
}
|
||||
|
||||
type Streamer struct {
|
||||
w http.ResponseWriter
|
||||
flusher http.Flusher
|
||||
id string
|
||||
created int64
|
||||
model string
|
||||
}
|
||||
|
||||
func NewStreamer(w http.ResponseWriter, flusher http.Flusher, id string, created int64, model string) *Streamer {
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
w.Header().Set("Cache-Control", "no-cache")
|
||||
w.Header().Set("Connection", "keep-alive")
|
||||
return &Streamer{w: w, flusher: flusher, id: id, created: created, model: model}
|
||||
}
|
||||
|
||||
func (s *Streamer) Role() {
|
||||
sendStreamDelta(s.w, s.flusher, s.id, s.created, s.model, StreamDelta{Role: "assistant"})
|
||||
}
|
||||
|
||||
func (s *Streamer) Reasoning(text string) {
|
||||
sendStreamDelta(s.w, s.flusher, s.id, s.created, s.model, StreamDelta{ReasoningContent: text})
|
||||
}
|
||||
|
||||
func (s *Streamer) Content(text string) {
|
||||
sendStreamDelta(s.w, s.flusher, s.id, s.created, s.model, StreamDelta{Content: text})
|
||||
}
|
||||
|
||||
func (s *Streamer) ToolCallDelta(tc ToolCall) {
|
||||
sendStreamDelta(s.w, s.flusher, s.id, s.created, s.model, StreamDelta{ToolCalls: []ToolCall{tc}})
|
||||
}
|
||||
|
||||
func (s *Streamer) Finish(reason string) {
|
||||
sendStreamChunk(s.w, s.flusher, s.id, s.created, s.model, StreamDelta{}, &reason)
|
||||
}
|
||||
|
||||
func (s *Streamer) Done() {
|
||||
fmt.Fprintf(s.w, "data: [DONE]\n\n")
|
||||
if s.flusher != nil {
|
||||
s.flusher.Flush()
|
||||
}
|
||||
}
|
||||
|
||||
func sendStreamDelta(w http.ResponseWriter, flusher http.Flusher, completionID string, createdTime int64, modelName string, delta StreamDelta) {
|
||||
sendStreamChunk(w, flusher, completionID, createdTime, modelName, delta, nil)
|
||||
}
|
||||
|
||||
func sendStreamChunk(w http.ResponseWriter, flusher http.Flusher, completionID string, createdTime int64, modelName string, delta StreamDelta, finishReason *string) {
|
||||
chunk := StreamResponse{
|
||||
ID: completionID,
|
||||
Object: "chat.completion.chunk",
|
||||
Created: createdTime,
|
||||
Model: modelName,
|
||||
Choices: []StreamChoice{
|
||||
{
|
||||
Index: 0,
|
||||
Delta: delta,
|
||||
FinishReason: finishReason,
|
||||
},
|
||||
},
|
||||
}
|
||||
b, _ := json.Marshal(chunk)
|
||||
fmt.Fprintf(w, "data: %s\n\n", b)
|
||||
if flusher != nil {
|
||||
flusher.Flush()
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Hunyuan 3 Provider & Stream Parsing
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func parseGradioStreamData(dataJSON string) (string, string, []ToolCall, bool) {
|
||||
var raw []interface{}
|
||||
if err := json.Unmarshal([]byte(dataJSON), &raw); err != nil || len(raw) == 0 {
|
||||
return "", "", nil, false
|
||||
}
|
||||
|
||||
inner, ok := raw[0].([]interface{})
|
||||
if !ok || len(inner) < 2 {
|
||||
return "", "", nil, false
|
||||
}
|
||||
|
||||
contentStr, _ := inner[0].(string)
|
||||
reasoningStr, _ := inner[1].(string)
|
||||
|
||||
var toolCalls []ToolCall
|
||||
if len(inner) >= 3 && inner[2] != nil {
|
||||
b, err := json.Marshal(inner[2])
|
||||
if err == nil {
|
||||
var tcs []ToolCall
|
||||
if json.Unmarshal(b, &tcs) == nil && len(tcs) > 0 {
|
||||
toolCalls = tcs
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return contentStr, reasoningStr, toolCalls, true
|
||||
}
|
||||
|
||||
type HunyuanService struct {
|
||||
endpoint string
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
func NewHunyuanService(endpoint string) *HunyuanService {
|
||||
cleanEndpoint := strings.TrimRight(endpoint, "/")
|
||||
return &HunyuanService{
|
||||
endpoint: cleanEndpoint,
|
||||
client: &http.Client{Timeout: 300 * time.Second},
|
||||
}
|
||||
}
|
||||
|
||||
func (s *HunyuanService) ListModels() []ModelItem {
|
||||
now := time.Now().Unix()
|
||||
return []ModelItem{
|
||||
{ID: "hy3", Object: "model", Created: now, OwnedBy: "hunyuan"},
|
||||
}
|
||||
}
|
||||
|
||||
func (s *HunyuanService) Chat(w http.ResponseWriter, r *http.Request, req ChatCompletionRequest) error {
|
||||
modelName := req.Model
|
||||
if modelName == "" {
|
||||
modelName = "hy3"
|
||||
}
|
||||
maxTokens := ResolveMaxTokens(req)
|
||||
|
||||
var systemPromptStr string
|
||||
var historyArray []map[string]interface{}
|
||||
var messageStr string
|
||||
|
||||
var nonSystemMsgs []ChatMessage
|
||||
for _, msg := range req.Messages {
|
||||
cStr := msg.GetContentString()
|
||||
if msg.Role == "system" && systemPromptStr == "" {
|
||||
systemPromptStr = cStr
|
||||
} else {
|
||||
nonSystemMsgs = append(nonSystemMsgs, msg)
|
||||
}
|
||||
}
|
||||
|
||||
if len(nonSystemMsgs) > 0 {
|
||||
for i := 0; i < len(nonSystemMsgs)-1; i++ {
|
||||
m := nonSystemMsgs[i]
|
||||
cStr := m.GetContentString()
|
||||
item := map[string]interface{}{"role": m.Role}
|
||||
switch m.Role {
|
||||
case "assistant":
|
||||
if cStr != "" {
|
||||
item["content"] = cStr
|
||||
} else {
|
||||
item["content"] = nil
|
||||
}
|
||||
if m.ReasoningContent != "" {
|
||||
item["reasoning_content"] = m.ReasoningContent
|
||||
}
|
||||
if len(m.ToolCalls) > 0 {
|
||||
item["tool_calls"] = m.ToolCalls
|
||||
}
|
||||
case "tool", "function":
|
||||
item["role"] = "tool"
|
||||
item["content"] = cStr
|
||||
toolID := m.ToolCallID
|
||||
if toolID == "" {
|
||||
toolID = m.Name
|
||||
}
|
||||
if toolID != "" {
|
||||
item["tool_call_id"] = toolID
|
||||
}
|
||||
if m.Name != "" {
|
||||
item["name"] = m.Name
|
||||
}
|
||||
default:
|
||||
item["content"] = cStr
|
||||
}
|
||||
historyArray = append(historyArray, item)
|
||||
}
|
||||
|
||||
lastMsg := nonSystemMsgs[len(nonSystemMsgs)-1]
|
||||
lastContent := lastMsg.GetContentString()
|
||||
if lastMsg.Role == "tool" || lastMsg.Role == "function" {
|
||||
toolName := lastMsg.Name
|
||||
if toolName == "" {
|
||||
toolName = lastMsg.ToolCallID
|
||||
}
|
||||
if toolName != "" {
|
||||
messageStr = fmt.Sprintf("Tool result for %s: %s", toolName, lastContent)
|
||||
} else {
|
||||
messageStr = lastContent
|
||||
}
|
||||
} else {
|
||||
messageStr = lastContent
|
||||
}
|
||||
}
|
||||
|
||||
functionsJSONStr := ""
|
||||
if len(req.Tools) > 0 {
|
||||
b, err := json.Marshal(req.Tools)
|
||||
if err == nil {
|
||||
functionsJSONStr = string(b)
|
||||
}
|
||||
}
|
||||
|
||||
var tempVal interface{}
|
||||
if req.Temperature != nil {
|
||||
tempVal = *req.Temperature
|
||||
}
|
||||
var topPVal interface{}
|
||||
if req.TopP != nil {
|
||||
topPVal = *req.TopP
|
||||
} else {
|
||||
topPVal = 0
|
||||
}
|
||||
|
||||
gradioData := []interface{}{
|
||||
messageStr,
|
||||
systemPromptStr,
|
||||
historyArray,
|
||||
"high",
|
||||
tempVal,
|
||||
maxTokens,
|
||||
topPVal,
|
||||
nil,
|
||||
functionsJSONStr,
|
||||
}
|
||||
gradioPayload := map[string]interface{}{"data": gradioData}
|
||||
jsonPayload, err := json.Marshal(gradioPayload)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to encode request: %w", err)
|
||||
}
|
||||
|
||||
effUA := EffectiveUserAgent(r)
|
||||
|
||||
callURL := s.endpoint + "/gradio_api/call/chat"
|
||||
makeCallReq := func() (*http.Request, error) {
|
||||
req, err := http.NewRequest("POST", callURL, bytes.NewBuffer(jsonPayload))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("User-Agent", effUA)
|
||||
return req, nil
|
||||
}
|
||||
|
||||
resp, err := DoWithFibonacciRetry(s.client, makeCallReq, 5)
|
||||
if err != nil {
|
||||
return fmt.Errorf("upstream error: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
var joinRes GradioJoinResponse
|
||||
if err := json.NewDecoder(resp.Body).Decode(&joinRes); err != nil || joinRes.EventID == "" {
|
||||
return fmt.Errorf("failed to parse Gradio event ID")
|
||||
}
|
||||
|
||||
streamURL := fmt.Sprintf("%s/gradio_api/call/chat/%s", s.endpoint, joinRes.EventID)
|
||||
makeStreamReq := func() (*http.Request, error) {
|
||||
req, err := http.NewRequest("GET", streamURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Accept", "text/event-stream")
|
||||
req.Header.Set("User-Agent", effUA)
|
||||
return req, nil
|
||||
}
|
||||
|
||||
streamResp, err := DoWithFibonacciRetry(s.client, makeStreamReq, 5)
|
||||
if err != nil {
|
||||
return fmt.Errorf("upstream stream error: %w", err)
|
||||
}
|
||||
defer streamResp.Body.Close()
|
||||
|
||||
completionID := "chatcmpl-" + GenerateUUID()
|
||||
createdTime := time.Now().Unix()
|
||||
|
||||
if !req.Stream {
|
||||
reader := bufio.NewReader(streamResp.Body)
|
||||
var finalContent, finalReasoning string
|
||||
var finalToolCalls []ToolCall
|
||||
|
||||
for {
|
||||
line, err := reader.ReadString('\n')
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
line = strings.TrimSpace(line)
|
||||
if strings.HasPrefix(line, "data: ") {
|
||||
dataJSON := strings.TrimPrefix(line, "data: ")
|
||||
if c, r, tc, ok := parseGradioStreamData(dataJSON); ok {
|
||||
finalContent = c
|
||||
finalReasoning = r
|
||||
if len(tc) > 0 {
|
||||
finalToolCalls = tc
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
finishReason := "stop"
|
||||
var msgContent interface{} = finalContent
|
||||
if len(finalToolCalls) > 0 {
|
||||
finishReason = "tool_calls"
|
||||
if finalContent == "" {
|
||||
msgContent = nil
|
||||
}
|
||||
}
|
||||
|
||||
WriteCompletionResponse(w, completionID, createdTime, modelName, FinalOutput{
|
||||
Content: msgContent,
|
||||
ReasoningContent: finalReasoning,
|
||||
ToolCalls: finalToolCalls,
|
||||
FinishReason: finishReason,
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
flusher, _ := w.(http.Flusher)
|
||||
streamer := NewStreamer(w, flusher, completionID, createdTime, modelName)
|
||||
streamer.Role()
|
||||
|
||||
reader := bufio.NewReader(streamResp.Body)
|
||||
var emittedContent, emittedReasoning string
|
||||
lastToolCallArgs := map[string]string{}
|
||||
emittedToolCallIDs := map[string]bool{}
|
||||
|
||||
for {
|
||||
line, err := reader.ReadString('\n')
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
line = strings.TrimSpace(line)
|
||||
if strings.HasPrefix(line, "data: ") {
|
||||
dataJSON := strings.TrimPrefix(line, "data: ")
|
||||
if c, r, tc, ok := parseGradioStreamData(dataJSON); ok {
|
||||
if len(r) > len(emittedReasoning) {
|
||||
rDelta := r[len(emittedReasoning):]
|
||||
emittedReasoning = r
|
||||
streamer.Reasoning(rDelta)
|
||||
}
|
||||
if len(c) > len(emittedContent) {
|
||||
cDelta := c[len(emittedContent):]
|
||||
emittedContent = c
|
||||
streamer.Content(cDelta)
|
||||
}
|
||||
if len(tc) > 0 {
|
||||
for idx, t := range tc {
|
||||
key := t.ID
|
||||
if key == "" {
|
||||
key = fmt.Sprintf("idx_%d", idx)
|
||||
}
|
||||
prevArgs := lastToolCallArgs[key]
|
||||
currArgs := t.Function.Arguments
|
||||
if !emittedToolCallIDs[key] {
|
||||
emittedToolCallIDs[key] = true
|
||||
lastToolCallArgs[key] = currArgs
|
||||
tCopy := ToolCall{
|
||||
Index: new(int),
|
||||
ID: t.ID,
|
||||
Type: "function",
|
||||
Function: ToolCallFunction{Name: t.Function.Name, Arguments: currArgs},
|
||||
}
|
||||
*tCopy.Index = idx
|
||||
streamer.ToolCallDelta(tCopy)
|
||||
} else if len(currArgs) > len(prevArgs) {
|
||||
argDelta := currArgs[len(prevArgs):]
|
||||
lastToolCallArgs[key] = currArgs
|
||||
tCopy := ToolCall{
|
||||
Index: new(int),
|
||||
Function: ToolCallFunction{Arguments: argDelta},
|
||||
}
|
||||
*tCopy.Index = idx
|
||||
streamer.ToolCallDelta(tCopy)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(emittedToolCallIDs) > 0 {
|
||||
streamer.Finish("tool_calls")
|
||||
} else {
|
||||
streamer.Finish("stop")
|
||||
}
|
||||
streamer.Done()
|
||||
return nil
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// HTTP Server Wiring & Main Entry Point
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func ModelsHandler(service *HunyuanService) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
EnableCORS(w)
|
||||
if r.Method == http.MethodOptions {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
return
|
||||
}
|
||||
if r.Method != http.MethodGet {
|
||||
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
items := service.ListModels()
|
||||
res := ModelsResponse{Object: "list", Data: items}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(res)
|
||||
}
|
||||
}
|
||||
|
||||
func ChatHandler(service *HunyuanService) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
EnableCORS(w)
|
||||
if r.Method == http.MethodOptions {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
return
|
||||
}
|
||||
if r.Method != http.MethodPost {
|
||||
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
var req ChatCompletionRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
http.Error(w, "Invalid JSON payload: "+err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := service.Chat(w, r, req); err != nil {
|
||||
if w.Header().Get("Content-Type") != "text/event-stream" {
|
||||
http.Error(w, "Upstream error: "+err.Error(), http.StatusBadGateway)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func NewMux(service *HunyuanService) *http.ServeMux {
|
||||
mux := http.NewServeMux()
|
||||
mh := ModelsHandler(service)
|
||||
ch := ChatHandler(service)
|
||||
mux.HandleFunc("/models", mh)
|
||||
mux.HandleFunc("/v1/models", mh)
|
||||
mux.HandleFunc("/chat/completions", ch)
|
||||
mux.HandleFunc("/v1/chat/completions", ch)
|
||||
return mux
|
||||
}
|
||||
|
||||
func main() {
|
||||
portFlag := flag.String("port", "8080", "Port to listen on")
|
||||
endpointFlag := flag.String("endpoint", "https://tencent-hy3.hf.space", "Root URL of the Hunyuan Gradio space")
|
||||
uaFlag := flag.String("user-agent", DefaultUserAgent, "Custom User-Agent header")
|
||||
uaShortFlag := flag.String("ua", "", "Alias for -user-agent")
|
||||
flag.Parse()
|
||||
|
||||
ConfiguredUserAgent = *uaFlag
|
||||
if *uaShortFlag != "" {
|
||||
ConfiguredUserAgent = *uaShortFlag
|
||||
}
|
||||
|
||||
service := NewHunyuanService(*endpointFlag)
|
||||
mux := NewMux(service)
|
||||
|
||||
fmt.Printf("hygate starting on port %s...\n", *portFlag)
|
||||
fmt.Printf("Target Endpoint: %s\n", service.endpoint)
|
||||
fmt.Printf("User-Agent: %s\n", ConfiguredUserAgent)
|
||||
fmt.Printf("Endpoints:\n")
|
||||
fmt.Printf(" GET http://localhost:%s/v1/models\n", *portFlag)
|
||||
fmt.Printf(" POST http://localhost:%s/v1/chat/completions\n", *portFlag)
|
||||
|
||||
if err := http.ListenAndServe(":"+*portFlag, mux); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "server failed: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user