2115 lines
60 KiB
Go
2115 lines
60 KiB
Go
// qflash: Standalone OpenAI-compatible gateway for Qwen3.8-Flash-Next Gradio space
|
|
// Created by Luxferre in 2026, released into the public domain
|
|
|
|
package main
|
|
|
|
import (
|
|
"bufio"
|
|
"bytes"
|
|
"context"
|
|
"crypto/rand"
|
|
"encoding/json"
|
|
"flag"
|
|
"fmt"
|
|
"io"
|
|
"log"
|
|
"net"
|
|
"net/http"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
var (
|
|
DefaultUserAgent = "Mozilla/5.0 (X11; Linux x86_64; rv:153.0) Gecko/20100101 Firefox/153.0"
|
|
ConfiguredUserAgent string
|
|
ConfiguredToken string
|
|
ConfiguredAPIKey string
|
|
ConfiguredBaseURL string
|
|
ConfiguredModel string
|
|
EnableThinkingDefault = true
|
|
)
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// OpenAI API Data Structures
|
|
// ---------------------------------------------------------------------------
|
|
|
|
type ModelItem struct {
|
|
ID string `json:"id"`
|
|
Object string `json:"object"`
|
|
Created int64 `json:"created"`
|
|
OwnedBy string `json:"owned_by"`
|
|
}
|
|
|
|
type ModelsResponse struct {
|
|
Object string `json:"object"`
|
|
Data []ModelItem `json:"data"`
|
|
}
|
|
|
|
type ToolCallFunction struct {
|
|
Name string `json:"name"`
|
|
Arguments string `json:"arguments"`
|
|
}
|
|
|
|
type ToolCall struct {
|
|
Index *int `json:"index,omitempty"`
|
|
ID string `json:"id,omitempty"`
|
|
Type string `json:"type,omitempty"`
|
|
Function ToolCallFunction `json:"function"`
|
|
}
|
|
|
|
type Tool struct {
|
|
Type string `json:"type"`
|
|
Function interface{} `json:"function"`
|
|
}
|
|
|
|
type ChatMessage struct {
|
|
Role string `json:"role"`
|
|
Content interface{} `json:"content"`
|
|
ReasoningContent string `json:"reasoning_content,omitempty"`
|
|
ToolCalls []ToolCall `json:"tool_calls,omitempty"`
|
|
ToolCallID string `json:"tool_call_id,omitempty"`
|
|
Name string `json:"name,omitempty"`
|
|
}
|
|
|
|
func (m *ChatMessage) GetContentString() string {
|
|
if m.Content == nil {
|
|
return ""
|
|
}
|
|
if str, ok := m.Content.(string); ok {
|
|
return str
|
|
}
|
|
if parts, ok := m.Content.([]interface{}); ok {
|
|
var sb strings.Builder
|
|
for _, p := range parts {
|
|
if str, ok := p.(string); ok {
|
|
sb.WriteString(str)
|
|
} else if itemMap, ok := p.(map[string]interface{}); ok {
|
|
if textVal, ok := itemMap["text"].(string); ok {
|
|
sb.WriteString(textVal)
|
|
}
|
|
}
|
|
}
|
|
return sb.String()
|
|
}
|
|
b, err := json.Marshal(m.Content)
|
|
if err == nil {
|
|
return string(b)
|
|
}
|
|
return fmt.Sprintf("%v", m.Content)
|
|
}
|
|
|
|
type ChatCompletionRequest struct {
|
|
Model string `json:"model"`
|
|
Messages []ChatMessage `json:"messages"`
|
|
Tools []Tool `json:"tools,omitempty"`
|
|
ToolChoice interface{} `json:"tool_choice,omitempty"`
|
|
Stream bool `json:"stream"`
|
|
MaxTokens int `json:"max_tokens"`
|
|
MaxCompletionTokens int `json:"max_completion_tokens"`
|
|
Temperature *float64 `json:"temperature,omitempty"`
|
|
TopP *float64 `json:"top_p,omitempty"`
|
|
ReasoningEffort string `json:"reasoning_effort,omitempty"`
|
|
}
|
|
|
|
type ChatCompletionResponseChoice struct {
|
|
Index int `json:"index"`
|
|
Message ChatMessage `json:"message"`
|
|
FinishReason string `json:"finish_reason"`
|
|
}
|
|
|
|
type Usage struct {
|
|
PromptTokens int `json:"prompt_tokens"`
|
|
CompletionTokens int `json:"completion_tokens"`
|
|
TotalTokens int `json:"total_tokens"`
|
|
}
|
|
|
|
type ChatCompletionResponse struct {
|
|
ID string `json:"id"`
|
|
Object string `json:"object"`
|
|
Created int64 `json:"created"`
|
|
Model string `json:"model"`
|
|
Choices []ChatCompletionResponseChoice `json:"choices"`
|
|
Usage Usage `json:"usage"`
|
|
}
|
|
|
|
type StreamDelta struct {
|
|
Role string `json:"role,omitempty"`
|
|
Content string `json:"content,omitempty"`
|
|
ReasoningContent string `json:"reasoning_content,omitempty"`
|
|
ToolCalls []ToolCall `json:"tool_calls,omitempty"`
|
|
}
|
|
|
|
type StreamChoice struct {
|
|
Index int `json:"index"`
|
|
Delta StreamDelta `json:"delta"`
|
|
FinishReason *string `json:"finish_reason,omitempty"`
|
|
}
|
|
|
|
type StreamResponse struct {
|
|
ID string `json:"id"`
|
|
Object string `json:"object"`
|
|
Created int64 `json:"created"`
|
|
Model string `json:"model"`
|
|
Choices []StreamChoice `json:"choices"`
|
|
}
|
|
|
|
type GradioJoinResponse struct {
|
|
EventID string `json:"event_id"`
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Zero-Dependency SOCKS5 Proxy Client (RFC 1928 / RFC 1929)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
type SOCKS5Config struct {
|
|
Address string
|
|
Username string
|
|
Password string
|
|
}
|
|
|
|
func ParseSOCKS5URL(proxyURL string) (*SOCKS5Config, error) {
|
|
cleanURL := strings.TrimSpace(proxyURL)
|
|
if cleanURL == "" {
|
|
return nil, nil
|
|
}
|
|
if strings.HasPrefix(cleanURL, "socks5://") {
|
|
cleanURL = strings.TrimPrefix(cleanURL, "socks5://")
|
|
} else if strings.HasPrefix(cleanURL, "socks5h://") {
|
|
cleanURL = strings.TrimPrefix(cleanURL, "socks5h://")
|
|
}
|
|
|
|
cfg := &SOCKS5Config{}
|
|
if atIdx := strings.LastIndex(cleanURL, "@"); atIdx != -1 {
|
|
userPass := cleanURL[:atIdx]
|
|
cfg.Address = cleanURL[atIdx+1:]
|
|
if colonIdx := strings.Index(userPass, ":"); colonIdx != -1 {
|
|
cfg.Username = userPass[:colonIdx]
|
|
cfg.Password = userPass[colonIdx+1:]
|
|
} else {
|
|
cfg.Username = userPass
|
|
}
|
|
} else {
|
|
cfg.Address = cleanURL
|
|
}
|
|
|
|
if !strings.Contains(cfg.Address, ":") {
|
|
cfg.Address = cfg.Address + ":1080"
|
|
}
|
|
return cfg, nil
|
|
}
|
|
|
|
func DialSOCKS5(ctx context.Context, proxyURL, targetAddr string) (net.Conn, error) {
|
|
cfg, err := ParseSOCKS5URL(proxyURL)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("invalid socks5 proxy configuration: %w", err)
|
|
}
|
|
if cfg == nil {
|
|
var d net.Dialer
|
|
return d.DialContext(ctx, "tcp", targetAddr)
|
|
}
|
|
|
|
var d net.Dialer
|
|
conn, err := d.DialContext(ctx, "tcp", cfg.Address)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to connect to socks5 proxy at %s: %w", cfg.Address, err)
|
|
}
|
|
|
|
deadline, ok := ctx.Deadline()
|
|
if !ok {
|
|
deadline = time.Now().Add(30 * time.Second)
|
|
}
|
|
conn.SetDeadline(deadline)
|
|
defer conn.SetDeadline(time.Time{})
|
|
|
|
// 1. Negotiation Greeting (RFC 1928)
|
|
var greeting []byte
|
|
if cfg.Username != "" {
|
|
greeting = []byte{0x05, 0x02, 0x00, 0x02}
|
|
} else {
|
|
greeting = []byte{0x05, 0x01, 0x00}
|
|
}
|
|
|
|
if _, err := conn.Write(greeting); err != nil {
|
|
conn.Close()
|
|
return nil, fmt.Errorf("failed to write socks5 greeting: %w", err)
|
|
}
|
|
|
|
resp := make([]byte, 2)
|
|
if _, err := io.ReadFull(conn, resp); err != nil {
|
|
conn.Close()
|
|
return nil, fmt.Errorf("failed to read socks5 greeting response: %w", err)
|
|
}
|
|
|
|
if resp[0] != 0x05 {
|
|
conn.Close()
|
|
return nil, fmt.Errorf("unsupported socks version: 0x%02x", resp[0])
|
|
}
|
|
|
|
// 2. Authentication if required (RFC 1929)
|
|
if resp[1] == 0x02 {
|
|
if cfg.Username == "" {
|
|
conn.Close()
|
|
return nil, fmt.Errorf("socks5 proxy requires authentication, but no credentials provided")
|
|
}
|
|
uLen := byte(len(cfg.Username))
|
|
pLen := byte(len(cfg.Password))
|
|
authReq := []byte{0x01, uLen}
|
|
authReq = append(authReq, []byte(cfg.Username)...)
|
|
authReq = append(authReq, pLen)
|
|
authReq = append(authReq, []byte(cfg.Password)...)
|
|
|
|
if _, err := conn.Write(authReq); err != nil {
|
|
conn.Close()
|
|
return nil, fmt.Errorf("failed to send socks5 authentication: %w", err)
|
|
}
|
|
|
|
authResp := make([]byte, 2)
|
|
if _, err := io.ReadFull(conn, authResp); err != nil {
|
|
conn.Close()
|
|
return nil, fmt.Errorf("failed to read socks5 auth response: %w", err)
|
|
}
|
|
if authResp[1] != 0x00 {
|
|
conn.Close()
|
|
return nil, fmt.Errorf("socks5 authentication failed with status 0x%02x", authResp[1])
|
|
}
|
|
} else if resp[1] != 0x00 {
|
|
conn.Close()
|
|
return nil, fmt.Errorf("socks5 proxy rejected authentication methods: 0x%02x", resp[1])
|
|
}
|
|
|
|
// 3. Connection Request (CONNECT command)
|
|
host, portStr, err := net.SplitHostPort(targetAddr)
|
|
if err != nil {
|
|
conn.Close()
|
|
return nil, fmt.Errorf("invalid target address %s: %w", targetAddr, err)
|
|
}
|
|
|
|
port, err := strconv.Atoi(portStr)
|
|
if err != nil || port < 1 || port > 65535 {
|
|
conn.Close()
|
|
return nil, fmt.Errorf("invalid port in target address %s", targetAddr)
|
|
}
|
|
|
|
reqBuf := []byte{0x05, 0x01, 0x00}
|
|
ip := net.ParseIP(host)
|
|
if ip4 := ip.To4(); ip4 != nil {
|
|
reqBuf = append(reqBuf, 0x01)
|
|
reqBuf = append(reqBuf, ip4...)
|
|
} else if ip6 := ip.To16(); ip6 != nil {
|
|
reqBuf = append(reqBuf, 0x04)
|
|
reqBuf = append(reqBuf, ip6...)
|
|
} else {
|
|
reqBuf = append(reqBuf, 0x03, byte(len(host)))
|
|
reqBuf = append(reqBuf, []byte(host)...)
|
|
}
|
|
reqBuf = append(reqBuf, byte(port>>8), byte(port&0xFF))
|
|
|
|
if _, err := conn.Write(reqBuf); err != nil {
|
|
conn.Close()
|
|
return nil, fmt.Errorf("failed to send socks5 connect request: %w", err)
|
|
}
|
|
|
|
// 4. Connection Response
|
|
respHdr := make([]byte, 4)
|
|
if _, err := io.ReadFull(conn, respHdr); err != nil {
|
|
conn.Close()
|
|
return nil, fmt.Errorf("failed to read socks5 connect response: %w", err)
|
|
}
|
|
|
|
if respHdr[1] != 0x00 {
|
|
conn.Close()
|
|
return nil, fmt.Errorf("socks5 connect failed with reply code 0x%02x", respHdr[1])
|
|
}
|
|
|
|
switch respHdr[3] {
|
|
case 0x01:
|
|
bnd := make([]byte, 6)
|
|
io.ReadFull(conn, bnd)
|
|
case 0x03:
|
|
lenBuf := make([]byte, 1)
|
|
io.ReadFull(conn, lenBuf)
|
|
bnd := make([]byte, int(lenBuf[0])+2)
|
|
io.ReadFull(conn, bnd)
|
|
case 0x04:
|
|
bnd := make([]byte, 18)
|
|
io.ReadFull(conn, bnd)
|
|
}
|
|
|
|
return conn, nil
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Helper Utilities
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func GenerateUUID() string {
|
|
var b [16]byte
|
|
_, err := rand.Read(b[:])
|
|
if err != nil {
|
|
return "00000000-0000-4000-8000-000000000000"
|
|
}
|
|
b[6] = (b[6] & 0x0f) | 0x40
|
|
b[8] = (b[8] & 0x3f) | 0x80
|
|
return fmt.Sprintf("%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:])
|
|
}
|
|
|
|
func FibonacciDelay(attempt int) time.Duration {
|
|
if attempt <= 0 {
|
|
return 1 * time.Second
|
|
}
|
|
a, b := 1, 1
|
|
for i := 1; i < attempt; i++ {
|
|
a, b = b, a+b
|
|
}
|
|
return time.Duration(a) * time.Second
|
|
}
|
|
|
|
func DoWithFibonacciRetry(client *http.Client, makeReq func() (*http.Request, error), maxRetries int) (*http.Response, error) {
|
|
var lastErr error
|
|
for attempt := 1; attempt <= maxRetries; attempt++ {
|
|
req, err := makeReq()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
resp, err := client.Do(req)
|
|
if err == nil && resp.StatusCode == http.StatusOK {
|
|
return resp, nil
|
|
}
|
|
|
|
if resp != nil {
|
|
respBody, _ := io.ReadAll(resp.Body)
|
|
resp.Body.Close()
|
|
lastErr = fmt.Errorf("HTTP status %d: %s", resp.StatusCode, string(respBody))
|
|
} else {
|
|
lastErr = err
|
|
}
|
|
|
|
if attempt < maxRetries {
|
|
delay := FibonacciDelay(attempt)
|
|
time.Sleep(delay)
|
|
}
|
|
}
|
|
return nil, fmt.Errorf("request failed after %d retries: %v", maxRetries, lastErr)
|
|
}
|
|
|
|
func EnableCORS(w http.ResponseWriter) {
|
|
w.Header().Set("Access-Control-Allow-Origin", "*")
|
|
w.Header().Set("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
|
|
w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Authorization, api-key, X-User-Agent, X-HF-Token, X-Base-URL, X-Model-ID")
|
|
}
|
|
|
|
func ResolveMaxTokens(req ChatCompletionRequest) int {
|
|
mt := req.MaxTokens
|
|
if mt == 0 && req.MaxCompletionTokens > 0 {
|
|
mt = req.MaxCompletionTokens
|
|
}
|
|
if mt <= 0 {
|
|
mt = 8192
|
|
}
|
|
if mt > 32768 {
|
|
mt = 32768
|
|
}
|
|
return mt
|
|
}
|
|
|
|
func EffectiveUserAgent(r *http.Request) string {
|
|
if r != nil {
|
|
if c := r.Header.Get("X-User-Agent"); c != "" {
|
|
return c
|
|
}
|
|
}
|
|
if ConfiguredUserAgent != "" {
|
|
return ConfiguredUserAgent
|
|
}
|
|
return DefaultUserAgent
|
|
}
|
|
|
|
func EffectiveHFToken(r *http.Request) string {
|
|
if r != nil {
|
|
if hf := r.Header.Get("X-HF-Token"); hf != "" {
|
|
return hf
|
|
}
|
|
if auth := r.Header.Get("Authorization"); auth != "" {
|
|
if strings.HasPrefix(strings.ToLower(auth), "bearer ") {
|
|
tok := strings.TrimSpace(auth[7:])
|
|
if strings.HasPrefix(tok, "hf_") {
|
|
return tok
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if ConfiguredToken != "" {
|
|
return ConfiguredToken
|
|
}
|
|
if envTok := os.Getenv("HF_TOKEN"); envTok != "" {
|
|
return envTok
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func EffectiveUpstreamKey(r *http.Request) string {
|
|
if r != nil {
|
|
if auth := r.Header.Get("Authorization"); auth != "" {
|
|
if strings.HasPrefix(strings.ToLower(auth), "bearer ") {
|
|
tok := strings.TrimSpace(auth[7:])
|
|
if tok != "" && tok != "-" && !strings.HasPrefix(tok, "hf_") && tok != "sk-dummy" {
|
|
return tok
|
|
}
|
|
}
|
|
}
|
|
if key := r.Header.Get("api-key"); key != "" {
|
|
return key
|
|
}
|
|
}
|
|
if ConfiguredAPIKey != "" {
|
|
return ConfiguredAPIKey
|
|
}
|
|
if envKey := os.Getenv("OPENAI_API_KEY"); envKey != "" {
|
|
return envKey
|
|
}
|
|
if envKey := os.Getenv("QWEN_API_KEY"); envKey != "" {
|
|
return envKey
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func EffectiveUpstreamBaseURL(r *http.Request) string {
|
|
if r != nil {
|
|
if bu := r.Header.Get("X-Base-URL"); bu != "" {
|
|
return bu
|
|
}
|
|
}
|
|
if ConfiguredBaseURL != "" {
|
|
return ConfiguredBaseURL
|
|
}
|
|
if envBU := os.Getenv("OPENAI_BASE_URL"); envBU != "" {
|
|
return envBU
|
|
}
|
|
if envBU := os.Getenv("QWEN_BASE_URL"); envBU != "" {
|
|
return envBU
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func EffectiveModelID(reqModel string, defaultModel string) string {
|
|
clean := strings.TrimSpace(reqModel)
|
|
if clean == "" {
|
|
return defaultModel
|
|
}
|
|
switch strings.ToLower(clean) {
|
|
case "qwen/qwen3.8-flash-next", "qwen3.8-flash-next", "qwen-flash-next", "qwen-flash", "qwen3.8-flash":
|
|
return "Qwen/Qwen3.8-Flash-Next"
|
|
case "qwen/qwen3.8-27b-uncensored", "qwen3.8-27b-uncensored", "qwen-27b", "qwen3.8-27b":
|
|
return "Qwen/Qwen3.8-27B-Uncensored"
|
|
case "qwen":
|
|
if defaultModel != "" {
|
|
return defaultModel
|
|
}
|
|
return "Qwen/Qwen3.8-Flash-Next"
|
|
default:
|
|
return clean
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Response Framing
|
|
// ---------------------------------------------------------------------------
|
|
|
|
type FinalOutput struct {
|
|
Content interface{}
|
|
ReasoningContent string
|
|
ToolCalls []ToolCall
|
|
FinishReason string
|
|
}
|
|
|
|
func WriteCompletionResponse(w http.ResponseWriter, completionID string, created int64, model string, out FinalOutput) {
|
|
finish := out.FinishReason
|
|
if finish == "" {
|
|
finish = "stop"
|
|
}
|
|
resp := ChatCompletionResponse{
|
|
ID: completionID,
|
|
Object: "chat.completion",
|
|
Created: created,
|
|
Model: model,
|
|
Choices: []ChatCompletionResponseChoice{
|
|
{
|
|
Index: 0,
|
|
Message: ChatMessage{
|
|
Role: "assistant",
|
|
Content: out.Content,
|
|
ReasoningContent: out.ReasoningContent,
|
|
ToolCalls: out.ToolCalls,
|
|
},
|
|
FinishReason: finish,
|
|
},
|
|
},
|
|
Usage: Usage{
|
|
PromptTokens: 0,
|
|
CompletionTokens: 0,
|
|
TotalTokens: 0,
|
|
},
|
|
}
|
|
w.Header().Set("Content-Type", "application/json")
|
|
json.NewEncoder(w).Encode(resp)
|
|
}
|
|
|
|
type Streamer struct {
|
|
w http.ResponseWriter
|
|
flusher http.Flusher
|
|
id string
|
|
created int64
|
|
model string
|
|
}
|
|
|
|
func NewStreamer(w http.ResponseWriter, flusher http.Flusher, id string, created int64, model string) *Streamer {
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
w.Header().Set("Cache-Control", "no-cache")
|
|
w.Header().Set("Connection", "keep-alive")
|
|
return &Streamer{w: w, flusher: flusher, id: id, created: created, model: model}
|
|
}
|
|
|
|
func (s *Streamer) Role() {
|
|
sendStreamDelta(s.w, s.flusher, s.id, s.created, s.model, StreamDelta{Role: "assistant"})
|
|
}
|
|
|
|
func (s *Streamer) Reasoning(text string) {
|
|
sendStreamDelta(s.w, s.flusher, s.id, s.created, s.model, StreamDelta{ReasoningContent: text})
|
|
}
|
|
|
|
func (s *Streamer) Content(text string) {
|
|
sendStreamDelta(s.w, s.flusher, s.id, s.created, s.model, StreamDelta{Content: text})
|
|
}
|
|
|
|
func (s *Streamer) ToolCallDelta(tc ToolCall) {
|
|
sendStreamDelta(s.w, s.flusher, s.id, s.created, s.model, StreamDelta{ToolCalls: []ToolCall{tc}})
|
|
}
|
|
|
|
func (s *Streamer) Finish(reason string) {
|
|
sendStreamChunk(s.w, s.flusher, s.id, s.created, s.model, StreamDelta{}, &reason)
|
|
}
|
|
|
|
func (s *Streamer) Done() {
|
|
fmt.Fprintf(s.w, "data: [DONE]\n\n")
|
|
if s.flusher != nil {
|
|
s.flusher.Flush()
|
|
}
|
|
}
|
|
|
|
func sendStreamDelta(w http.ResponseWriter, flusher http.Flusher, completionID string, createdTime int64, modelName string, delta StreamDelta) {
|
|
sendStreamChunk(w, flusher, completionID, createdTime, modelName, delta, nil)
|
|
}
|
|
|
|
func sendStreamChunk(w http.ResponseWriter, flusher http.Flusher, completionID string, createdTime int64, modelName string, delta StreamDelta, finishReason *string) {
|
|
chunk := StreamResponse{
|
|
ID: completionID,
|
|
Object: "chat.completion.chunk",
|
|
Created: createdTime,
|
|
Model: modelName,
|
|
Choices: []StreamChoice{
|
|
{
|
|
Index: 0,
|
|
Delta: delta,
|
|
FinishReason: finishReason,
|
|
},
|
|
},
|
|
}
|
|
b, _ := json.Marshal(chunk)
|
|
fmt.Fprintf(w, "data: %s\n\n", b)
|
|
if flusher != nil {
|
|
flusher.Flush()
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Tool Calling and Reasoning Extraction
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func cleanJSONBlock(input string) string {
|
|
s := strings.TrimSpace(input)
|
|
if strings.HasPrefix(s, "```") {
|
|
lines := strings.Split(s, "\n")
|
|
if len(lines) >= 2 {
|
|
if strings.HasPrefix(lines[len(lines)-1], "```") {
|
|
lines = lines[1 : len(lines)-1]
|
|
} else {
|
|
lines = lines[1:]
|
|
}
|
|
s = strings.TrimSpace(strings.Join(lines, "\n"))
|
|
}
|
|
}
|
|
return s
|
|
}
|
|
|
|
func sanitizeJSONValue(v interface{}) interface{} {
|
|
switch val := v.(type) {
|
|
case string:
|
|
return strings.TrimSpace(val)
|
|
case map[string]interface{}:
|
|
cleanMap := make(map[string]interface{})
|
|
for k, childV := range val {
|
|
cleanKey := strings.TrimSpace(k)
|
|
cleanMap[cleanKey] = sanitizeJSONValue(childV)
|
|
}
|
|
return cleanMap
|
|
case []interface{}:
|
|
cleanSlice := make([]interface{}, len(val))
|
|
for i, childV := range val {
|
|
cleanSlice[i] = sanitizeJSONValue(childV)
|
|
}
|
|
return cleanSlice
|
|
default:
|
|
return v
|
|
}
|
|
}
|
|
|
|
func parseSingleToolCall(jsonStr string) (ToolCall, bool) {
|
|
cleaned := cleanJSONBlock(jsonStr)
|
|
var raw map[string]interface{}
|
|
if err := json.Unmarshal([]byte(cleaned), &raw); err == nil {
|
|
sanitizedRaw, ok := sanitizeJSONValue(raw).(map[string]interface{})
|
|
if !ok {
|
|
sanitizedRaw = raw
|
|
}
|
|
|
|
for _, wrapperKey := range []string{"function", "function_call", "tool_call"} {
|
|
if fnObj, ok := sanitizedRaw[wrapperKey].(map[string]interface{}); ok {
|
|
if nameVal, ok := fnObj["name"].(string); ok && nameVal != "" {
|
|
argsStr := "{}"
|
|
var argsVal interface{}
|
|
if a, hasA := fnObj["arguments"]; hasA {
|
|
argsVal = a
|
|
} else if p, hasP := fnObj["parameters"]; hasP {
|
|
argsVal = p
|
|
} else if args, hasArgs := fnObj["args"]; hasArgs {
|
|
argsVal = args
|
|
}
|
|
if argsVal != nil {
|
|
if s, isStr := argsVal.(string); isStr {
|
|
var innerObj interface{}
|
|
if json.Unmarshal([]byte(s), &innerObj) == nil {
|
|
b, _ := json.Marshal(sanitizeJSONValue(innerObj))
|
|
argsStr = string(b)
|
|
} else {
|
|
argsStr = strings.TrimSpace(s)
|
|
}
|
|
} else {
|
|
b, _ := json.Marshal(sanitizeJSONValue(argsVal))
|
|
argsStr = string(b)
|
|
}
|
|
}
|
|
return ToolCall{
|
|
ID: "call_" + GenerateUUID()[:8],
|
|
Type: "function",
|
|
Function: ToolCallFunction{
|
|
Name: nameVal,
|
|
Arguments: argsStr,
|
|
},
|
|
}, true
|
|
}
|
|
}
|
|
}
|
|
|
|
nameVal := ""
|
|
for _, key := range []string{"name", "function", "action", "call"} {
|
|
if n, ok := sanitizedRaw[key].(string); ok && n != "" {
|
|
nameVal = n
|
|
break
|
|
}
|
|
}
|
|
|
|
if nameVal != "" {
|
|
argsStr := "{}"
|
|
var argsVal interface{}
|
|
for _, key := range []string{"arguments", "parameters", "args", "input"} {
|
|
if a, ok := sanitizedRaw[key]; ok {
|
|
argsVal = a
|
|
break
|
|
}
|
|
}
|
|
if argsVal != nil {
|
|
if s, isStr := argsVal.(string); isStr {
|
|
var innerObj interface{}
|
|
if json.Unmarshal([]byte(s), &innerObj) == nil {
|
|
b, _ := json.Marshal(sanitizeJSONValue(innerObj))
|
|
argsStr = string(b)
|
|
} else {
|
|
argsStr = strings.TrimSpace(s)
|
|
}
|
|
} else {
|
|
b, _ := json.Marshal(sanitizeJSONValue(argsVal))
|
|
argsStr = string(b)
|
|
}
|
|
} else {
|
|
argsMap := make(map[string]interface{})
|
|
for k, v := range sanitizedRaw {
|
|
if k != "name" && k != "function" && k != "type" && k != "action" && k != "call" {
|
|
argsMap[k] = v
|
|
}
|
|
}
|
|
if len(argsMap) > 0 {
|
|
b, _ := json.Marshal(sanitizeJSONValue(argsMap))
|
|
argsStr = string(b)
|
|
}
|
|
}
|
|
return ToolCall{
|
|
ID: "call_" + GenerateUUID()[:8],
|
|
Type: "function",
|
|
Function: ToolCallFunction{
|
|
Name: nameVal,
|
|
Arguments: argsStr,
|
|
},
|
|
}, true
|
|
}
|
|
}
|
|
|
|
return ToolCall{}, false
|
|
}
|
|
|
|
func parseXMLToolCall(block string) (ToolCall, bool) {
|
|
inner := strings.TrimSpace(block)
|
|
if strings.HasPrefix(inner, "<tool_call>") {
|
|
inner = strings.TrimPrefix(inner, "<tool_call>")
|
|
}
|
|
if strings.HasSuffix(inner, "</tool_call>") {
|
|
inner = strings.TrimSuffix(inner, "</tool_call>")
|
|
}
|
|
inner = cleanJSONBlock(inner)
|
|
|
|
if tc, ok := parseSingleToolCall(inner); ok {
|
|
return tc, true
|
|
}
|
|
|
|
var fnName string
|
|
if strings.Contains(inner, "<name>") && strings.Contains(inner, "</name>") {
|
|
nStart := strings.Index(inner, "<name>") + len("<name>")
|
|
nEnd := strings.Index(inner, "</name>")
|
|
if nStart < nEnd {
|
|
fnName = strings.TrimSpace(inner[nStart:nEnd])
|
|
}
|
|
} else if strings.Contains(inner, "<function_name>") && strings.Contains(inner, "</function_name>") {
|
|
nStart := strings.Index(inner, "<function_name>") + len("<function_name>")
|
|
nEnd := strings.Index(inner, "</function_name>")
|
|
if nStart < nEnd {
|
|
fnName = strings.TrimSpace(inner[nStart:nEnd])
|
|
}
|
|
} else if strings.Contains(inner, "<function>") && strings.Contains(inner, "</function>") {
|
|
nStart := strings.Index(inner, "<function>") + len("<function>")
|
|
nEnd := strings.Index(inner, "</function>")
|
|
if nStart < nEnd {
|
|
fnName = strings.TrimSpace(inner[nStart:nEnd])
|
|
}
|
|
}
|
|
|
|
var argsStr string
|
|
if strings.Contains(inner, "<arguments>") && strings.Contains(inner, "</arguments>") {
|
|
aStart := strings.Index(inner, "<arguments>") + len("<arguments>")
|
|
aEnd := strings.Index(inner, "</arguments>")
|
|
if aStart < aEnd {
|
|
argsStr = strings.TrimSpace(inner[aStart:aEnd])
|
|
}
|
|
} else if strings.Contains(inner, "<parameters>") && strings.Contains(inner, "</parameters>") {
|
|
pStart := strings.Index(inner, "<parameters>") + len("<parameters>")
|
|
pEnd := strings.Index(inner, "</parameters>")
|
|
if pStart < pEnd {
|
|
argsStr = strings.TrimSpace(inner[pStart:pEnd])
|
|
}
|
|
}
|
|
|
|
if fnName != "" {
|
|
if argsStr == "" {
|
|
argsStr = "{}"
|
|
}
|
|
return ToolCall{
|
|
ID: "call_" + GenerateUUID()[:8],
|
|
Type: "function",
|
|
Function: ToolCallFunction{
|
|
Name: fnName,
|
|
Arguments: argsStr,
|
|
},
|
|
}, true
|
|
}
|
|
|
|
return ToolCall{}, false
|
|
}
|
|
|
|
func ExtractToolCallBlocks(content string) (blocks []string, remaining string) {
|
|
s := content
|
|
remaining = content
|
|
|
|
for strings.Contains(s, "<tool_call>") {
|
|
sIdx := strings.Index(s, "<tool_call>")
|
|
rest := s[sIdx+len("<tool_call>"):]
|
|
|
|
relNextSIdx := strings.Index(rest, "<tool_call>")
|
|
var nextSIdx int
|
|
if relNextSIdx != -1 {
|
|
nextSIdx = sIdx + len("<tool_call>") + relNextSIdx
|
|
} else {
|
|
nextSIdx = -1
|
|
}
|
|
|
|
relEIdx := strings.Index(rest, "</tool_call>")
|
|
var eIdx int
|
|
if relEIdx != -1 {
|
|
eIdx = sIdx + len("<tool_call>") + relEIdx
|
|
} else {
|
|
eIdx = -1
|
|
}
|
|
|
|
var blockText string
|
|
var blockEndPos int
|
|
|
|
if eIdx != -1 && (nextSIdx == -1 || eIdx < nextSIdx) {
|
|
blockEndPos = eIdx + len("</tool_call>")
|
|
blockText = s[sIdx:blockEndPos]
|
|
s = s[blockEndPos:]
|
|
} else if nextSIdx != -1 {
|
|
blockEndPos = nextSIdx
|
|
blockText = s[sIdx:blockEndPos]
|
|
s = s[blockEndPos:]
|
|
} else {
|
|
blockText = s[sIdx:]
|
|
s = ""
|
|
}
|
|
|
|
blocks = append(blocks, blockText)
|
|
}
|
|
|
|
for strings.Contains(remaining, "<tool_call>") {
|
|
st := strings.Index(remaining, "<tool_call>")
|
|
rest := remaining[st+len("<tool_call>"):]
|
|
|
|
relNext := strings.Index(rest, "<tool_call>")
|
|
var nextSt int
|
|
if relNext != -1 {
|
|
nextSt = st + len("<tool_call>") + relNext
|
|
} else {
|
|
nextSt = -1
|
|
}
|
|
|
|
relEn := strings.Index(rest, "</tool_call>")
|
|
var en int
|
|
if relEn != -1 {
|
|
en = st + len("<tool_call>") + relEn
|
|
} else {
|
|
en = -1
|
|
}
|
|
|
|
if en != -1 && (nextSt == -1 || en < nextSt) {
|
|
remaining = strings.TrimSpace(remaining[:st] + remaining[en+len("</tool_call>"):])
|
|
} else if nextSt != -1 {
|
|
remaining = strings.TrimSpace(remaining[:st] + remaining[nextSt:])
|
|
} else {
|
|
remaining = strings.TrimSpace(remaining[:st])
|
|
}
|
|
}
|
|
|
|
return blocks, remaining
|
|
}
|
|
|
|
func DetectToolCalls(content string) ([]ToolCall, string, bool) {
|
|
blocks, remaining := ExtractToolCallBlocks(content)
|
|
var calls []ToolCall
|
|
|
|
for _, block := range blocks {
|
|
if toolCall, ok := parseXMLToolCall(block); ok {
|
|
calls = append(calls, toolCall)
|
|
}
|
|
}
|
|
|
|
if len(calls) > 0 {
|
|
return calls, remaining, true
|
|
}
|
|
|
|
if tc, ok := parseSingleToolCall(strings.TrimSpace(content)); ok {
|
|
return []ToolCall{tc}, "", true
|
|
}
|
|
|
|
return nil, content, false
|
|
}
|
|
|
|
// CleanGradioThought strips Markdown blockquote indicators ('> ') and header lines
|
|
// from the playground's reasoning trace.
|
|
func CleanGradioThought(raw string) string {
|
|
lines := strings.Split(raw, "\n")
|
|
var cleaned []string
|
|
skipHeader := true
|
|
|
|
for _, line := range lines {
|
|
trimmed := strings.TrimSpace(line)
|
|
if skipHeader {
|
|
low := strings.ToLower(trimmed)
|
|
if strings.Contains(low, "thinking process") || trimmed == ">" || trimmed == "" {
|
|
continue
|
|
}
|
|
skipHeader = false
|
|
}
|
|
|
|
if strings.HasPrefix(line, "> ") {
|
|
cleaned = append(cleaned, strings.TrimPrefix(line, "> "))
|
|
} else if line == ">" {
|
|
cleaned = append(cleaned, "")
|
|
} else if strings.HasPrefix(line, ">") {
|
|
cleaned = append(cleaned, strings.TrimPrefix(line, ">"))
|
|
} else {
|
|
cleaned = append(cleaned, line)
|
|
}
|
|
}
|
|
|
|
return strings.TrimSpace(strings.Join(cleaned, "\n"))
|
|
}
|
|
|
|
// SeparateReasoningAndContent separates thinking/reasoning traces from actual response content.
|
|
// Handles both standard <think>...</think> tags and the HF Gradio playground blockquote format.
|
|
func SeparateReasoningAndContent(text string) (string, string) {
|
|
// 1. Standard <think> tags
|
|
if strings.Contains(text, "<think>") {
|
|
sIdx := strings.Index(text, "<think>")
|
|
if eIdx := strings.Index(text, "</think>"); eIdx != -1 && eIdx > sIdx {
|
|
reasoning := text[sIdx+len("<think>") : eIdx]
|
|
content := text[:sIdx] + text[eIdx+len("</think>"):]
|
|
return strings.TrimSpace(reasoning), strings.TrimSpace(content)
|
|
} else {
|
|
reasoning := text[sIdx+len("<think>"):]
|
|
content := text[:sIdx]
|
|
return strings.TrimSpace(reasoning), strings.TrimSpace(content)
|
|
}
|
|
}
|
|
|
|
// 1b. Output starting with reasoning and ending in </think> (without opening <think>)
|
|
if strings.Contains(text, "</think>") {
|
|
eIdx := strings.Index(text, "</think>")
|
|
reasoning := text[:eIdx]
|
|
content := text[eIdx+len("</think>"):]
|
|
return strings.TrimSpace(reasoning), strings.TrimSpace(content)
|
|
}
|
|
|
|
// 2. HF Gradio Playground blockquote format: "> ... Thinking Process"
|
|
low := strings.ToLower(text)
|
|
if strings.Contains(low, "thinking process") && (strings.HasPrefix(strings.TrimSpace(text), ">") || strings.Contains(text, "\n>")) {
|
|
// Check if completion divider has arrived: "\n\n---\n\n"
|
|
if divIdx := strings.Index(text, "\n\n---\n\n"); divIdx != -1 {
|
|
rawThought := text[:divIdx]
|
|
content := text[divIdx+len("\n\n---\n\n"):]
|
|
return CleanGradioThought(rawThought), content
|
|
}
|
|
// Check if intermediate streaming divider is present: "\n\n---\n*Generating response...*"
|
|
if divIdx := strings.Index(text, "\n\n---\n*Generating response...*"); divIdx != -1 {
|
|
rawThought := text[:divIdx]
|
|
return CleanGradioThought(rawThought), ""
|
|
}
|
|
// If still in thought generation phase
|
|
if strings.HasPrefix(strings.TrimSpace(text), ">") {
|
|
return CleanGradioThought(text), ""
|
|
}
|
|
}
|
|
|
|
return "", text
|
|
}
|
|
|
|
func FormatToolsPrompt(tools []Tool) string {
|
|
if len(tools) == 0 {
|
|
return ""
|
|
}
|
|
b, err := json.MarshalIndent(tools, "", " ")
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
var sb strings.Builder
|
|
sb.WriteString("\n\n[Available Tools]\nYou have access to the following tools:\n```json\n")
|
|
sb.WriteString(string(b))
|
|
sb.WriteString("\n```\n")
|
|
sb.WriteString("If you choose to invoke one or more tools, respond ONLY with the tool invocation formatted as:\n")
|
|
sb.WriteString("<tool_call>\n{\"name\": \"function_name\", \"arguments\": {\"param\": \"value\"}}\n</tool_call>\n")
|
|
sb.WriteString("Do not add conversational preamble around the tool call when invoking a tool.\n")
|
|
return sb.String()
|
|
}
|
|
|
|
// StreamToolCallFilter is a stateful streaming filter that intercepts <tool_call> tags
|
|
// in real-time, preventing control tags from leaking to delta.content while emitting delta.tool_calls.
|
|
type StreamToolCallFilter struct {
|
|
inToolCall bool
|
|
toolCallBuf string
|
|
buf string
|
|
toolIndex int
|
|
emittedCall bool
|
|
}
|
|
|
|
func (f *StreamToolCallFilter) Feed(chunk string, onContent func(string), onToolCall func(ToolCall)) {
|
|
f.buf += chunk
|
|
for len(f.buf) > 0 {
|
|
if !f.inToolCall {
|
|
idx := strings.Index(f.buf, "<tool_call>")
|
|
if idx != -1 {
|
|
if idx > 0 {
|
|
onContent(f.buf[:idx])
|
|
}
|
|
f.inToolCall = true
|
|
f.toolCallBuf = ""
|
|
f.buf = f.buf[idx+len("<tool_call>"):]
|
|
} else {
|
|
matchLen := 0
|
|
tag := "<tool_call>"
|
|
for i := 1; i < len(tag) && i <= len(f.buf); i++ {
|
|
if strings.HasSuffix(f.buf, tag[:i]) {
|
|
matchLen = i
|
|
}
|
|
}
|
|
if matchLen > 0 {
|
|
onContent(f.buf[:len(f.buf)-matchLen])
|
|
f.buf = f.buf[len(f.buf)-matchLen:]
|
|
break
|
|
} else {
|
|
onContent(f.buf)
|
|
f.buf = ""
|
|
break
|
|
}
|
|
}
|
|
} else {
|
|
idx := strings.Index(f.buf, "</tool_call>")
|
|
if idx != -1 {
|
|
f.toolCallBuf += f.buf[:idx]
|
|
f.inToolCall = false
|
|
f.buf = f.buf[idx+len("</tool_call>"):]
|
|
|
|
if tc, ok := parseSingleToolCall(f.toolCallBuf); ok {
|
|
idxCopy := f.toolIndex
|
|
tc.Index = &idxCopy
|
|
f.toolIndex++
|
|
f.emittedCall = true
|
|
onToolCall(tc)
|
|
} else if tc2, ok2 := parseXMLToolCall("<tool_call>" + f.toolCallBuf + "</tool_call>"); ok2 {
|
|
idxCopy := f.toolIndex
|
|
tc2.Index = &idxCopy
|
|
f.toolIndex++
|
|
f.emittedCall = true
|
|
onToolCall(tc2)
|
|
} else {
|
|
onContent("<tool_call>" + f.toolCallBuf + "</tool_call>")
|
|
}
|
|
f.toolCallBuf = ""
|
|
} else {
|
|
matchLen := 0
|
|
tag := "</tool_call>"
|
|
for i := 1; i < len(tag) && i <= len(f.buf); i++ {
|
|
if strings.HasSuffix(f.buf, tag[:i]) {
|
|
matchLen = i
|
|
}
|
|
}
|
|
if matchLen > 0 {
|
|
f.toolCallBuf += f.buf[:len(f.buf)-matchLen]
|
|
f.buf = f.buf[len(f.buf)-matchLen:]
|
|
break
|
|
} else {
|
|
f.toolCallBuf += f.buf
|
|
f.buf = ""
|
|
break
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (f *StreamToolCallFilter) Flush(onContent func(string), onToolCall func(ToolCall)) {
|
|
if f.inToolCall && len(f.toolCallBuf) > 0 {
|
|
if tc, ok := parseSingleToolCall(f.toolCallBuf); ok {
|
|
idxCopy := f.toolIndex
|
|
tc.Index = &idxCopy
|
|
f.emittedCall = true
|
|
onToolCall(tc)
|
|
} else if tc2, ok2 := parseXMLToolCall("<tool_call>" + f.toolCallBuf + "</tool_call>"); ok2 {
|
|
idxCopy := f.toolIndex
|
|
tc2.Index = &idxCopy
|
|
f.emittedCall = true
|
|
onToolCall(tc2)
|
|
} else {
|
|
onContent("<tool_call>" + f.toolCallBuf)
|
|
}
|
|
f.toolCallBuf = ""
|
|
}
|
|
if len(f.buf) > 0 {
|
|
onContent(f.buf)
|
|
f.buf = ""
|
|
}
|
|
}
|
|
|
|
func (f *StreamToolCallFilter) HasEmittedCalls() bool {
|
|
return f.emittedCall
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Qwen3.8-Flash Service & Gradio Stream Parsing
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// parseAssistantText extracts the latest assistant message text from the Gradio output array.
|
|
func parseAssistantText(dataJSON string) (string, bool) {
|
|
var raw []interface{}
|
|
if err := json.Unmarshal([]byte(dataJSON), &raw); err != nil || len(raw) == 0 {
|
|
return "", false
|
|
}
|
|
|
|
// Check if raw[0] is directly a string (MicroHERO /respond format)
|
|
if str, ok := raw[0].(string); ok {
|
|
return str, true
|
|
}
|
|
|
|
// The first element is the chatbot message list
|
|
msgList, ok := raw[0].([]interface{})
|
|
if !ok || len(msgList) == 0 {
|
|
return "", false
|
|
}
|
|
|
|
// Find the last assistant message
|
|
for i := len(msgList) - 1; i >= 0; i-- {
|
|
msgMap, ok := msgList[i].(map[string]interface{})
|
|
if !ok {
|
|
continue
|
|
}
|
|
role, _ := msgMap["role"].(string)
|
|
if role != "assistant" {
|
|
continue
|
|
}
|
|
|
|
contentVal := msgMap["content"]
|
|
if contentStr, ok := contentVal.(string); ok {
|
|
return contentStr, true
|
|
}
|
|
if contentSlice, ok := contentVal.([]interface{}); ok {
|
|
var sb strings.Builder
|
|
for _, item := range contentSlice {
|
|
if s, ok := item.(string); ok {
|
|
sb.WriteString(s)
|
|
} else if m, ok := item.(map[string]interface{}); ok {
|
|
if textVal, ok := m["text"].(string); ok {
|
|
sb.WriteString(textVal)
|
|
}
|
|
}
|
|
}
|
|
return sb.String(), true
|
|
}
|
|
}
|
|
|
|
return "", false
|
|
}
|
|
|
|
type QwenService struct {
|
|
endpoint string
|
|
modelName string
|
|
mode string
|
|
token string
|
|
apiKey string
|
|
baseURL string
|
|
enableThinking bool
|
|
client *http.Client
|
|
}
|
|
|
|
func NewQwenService(endpoint, modelName, mode, token, apiKey, baseURL, socksProxy string, enableThinking bool) *QwenService {
|
|
cleanEndpoint := strings.TrimRight(endpoint, "/")
|
|
if modelName == "" {
|
|
modelName = "Qwen/Qwen3.8-27B-Uncensored"
|
|
}
|
|
if mode == "" {
|
|
mode = "auto"
|
|
}
|
|
|
|
transport := &http.Transport{
|
|
MaxIdleConns: 100,
|
|
IdleConnTimeout: 90 * time.Second,
|
|
TLSHandshakeTimeout: 10 * time.Second,
|
|
}
|
|
|
|
if socksProxy != "" {
|
|
transport.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
|
|
return DialSOCKS5(ctx, socksProxy, addr)
|
|
}
|
|
}
|
|
|
|
return &QwenService{
|
|
endpoint: cleanEndpoint,
|
|
modelName: modelName,
|
|
mode: mode,
|
|
token: token,
|
|
apiKey: apiKey,
|
|
baseURL: baseURL,
|
|
enableThinking: enableThinking,
|
|
client: &http.Client{Transport: transport, Timeout: 300 * time.Second},
|
|
}
|
|
}
|
|
|
|
func (s *QwenService) ListModels() []ModelItem {
|
|
now := time.Now().Unix()
|
|
primaryID := s.modelName
|
|
if primaryID == "" {
|
|
primaryID = "Qwen/Qwen3.8-27B-Uncensored"
|
|
}
|
|
|
|
candidates := []string{
|
|
primaryID,
|
|
"Qwen/Qwen3.8-27B-Uncensored",
|
|
"Qwen/Qwen3.8-Flash-Next",
|
|
"qwen3.8-27b-uncensored",
|
|
"qwen3.8-flash-next",
|
|
"qwen-flash-next",
|
|
"qwen-flash",
|
|
"qwen",
|
|
}
|
|
|
|
seen := make(map[string]bool)
|
|
var models []ModelItem
|
|
for _, id := range candidates {
|
|
if !seen[id] {
|
|
seen[id] = true
|
|
models = append(models, ModelItem{
|
|
ID: id,
|
|
Object: "model",
|
|
Created: now,
|
|
OwnedBy: "qwen",
|
|
})
|
|
}
|
|
}
|
|
|
|
return models
|
|
}
|
|
|
|
func (s *QwenService) detectEndpointMode() string {
|
|
if s.mode != "" && s.mode != "auto" {
|
|
return s.mode
|
|
}
|
|
|
|
ep := strings.ToLower(s.endpoint)
|
|
if strings.Contains(ep, "microhero") || strings.HasSuffix(ep, "/respond") {
|
|
return "respond"
|
|
}
|
|
if strings.Contains(ep, "halvo78") || strings.HasSuffix(ep, "/chat_response") {
|
|
return "chat_response"
|
|
}
|
|
if strings.Contains(ep, "apathy-exe") || strings.HasSuffix(ep, "/v1") {
|
|
return "openai"
|
|
}
|
|
|
|
// Probe /gradio_api/info
|
|
infoURL := s.endpoint + "/gradio_api/info"
|
|
req, err := http.NewRequest("GET", infoURL, nil)
|
|
if err == nil {
|
|
req.Header.Set("User-Agent", DefaultUserAgent)
|
|
if s.token != "" {
|
|
req.Header.Set("Authorization", "Bearer "+s.token)
|
|
}
|
|
probeClient := &http.Client{Timeout: 3 * time.Second}
|
|
resp, err := probeClient.Do(req)
|
|
if err == nil && resp.StatusCode == http.StatusOK {
|
|
defer resp.Body.Close()
|
|
var info struct {
|
|
NamedEndpoints map[string]interface{} `json:"named_endpoints"`
|
|
}
|
|
if err := json.NewDecoder(resp.Body).Decode(&info); err == nil {
|
|
if _, ok := info.NamedEndpoints["/respond"]; ok {
|
|
return "respond"
|
|
}
|
|
if _, ok := info.NamedEndpoints["/chat_response"]; ok {
|
|
return "chat_response"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return "respond"
|
|
}
|
|
|
|
func (s *QwenService) Chat(w http.ResponseWriter, r *http.Request, req ChatCompletionRequest) error {
|
|
resolvedModel := EffectiveModelID(req.Model, s.modelName)
|
|
maxTokens := ResolveMaxTokens(req)
|
|
mode := s.detectEndpointMode()
|
|
|
|
switch mode {
|
|
case "openai":
|
|
return s.chatDirectOpenAI(w, r, req, resolvedModel, maxTokens)
|
|
case "chat_response":
|
|
return s.chatGradioChatResponse(w, r, req, resolvedModel, maxTokens)
|
|
case "respond":
|
|
fallthrough
|
|
default:
|
|
return s.chatGradioRespond(w, r, req, resolvedModel, maxTokens)
|
|
}
|
|
}
|
|
|
|
func (s *QwenService) chatDirectOpenAI(w http.ResponseWriter, r *http.Request, req ChatCompletionRequest, resolvedModel string, maxTokens int) error {
|
|
targetURL := s.endpoint
|
|
if !strings.HasSuffix(targetURL, "/v1/chat/completions") && !strings.HasSuffix(targetURL, "/chat/completions") {
|
|
if strings.HasSuffix(targetURL, "/v1") {
|
|
targetURL += "/chat/completions"
|
|
} else {
|
|
targetURL += "/v1/chat/completions"
|
|
}
|
|
}
|
|
|
|
reqCopy := req
|
|
reqCopy.Model = resolvedModel
|
|
if reqCopy.MaxTokens == 0 && maxTokens > 0 {
|
|
reqCopy.MaxTokens = maxTokens
|
|
}
|
|
|
|
reqJSON, err := json.Marshal(reqCopy)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to encode upstream request: %w", err)
|
|
}
|
|
|
|
outReq, err := http.NewRequest("POST", targetURL, bytes.NewBuffer(reqJSON))
|
|
if err != nil {
|
|
return fmt.Errorf("failed to create upstream request: %w", err)
|
|
}
|
|
outReq.Header.Set("Content-Type", "application/json")
|
|
outReq.Header.Set("User-Agent", EffectiveUserAgent(r))
|
|
if effKey := EffectiveUpstreamKey(r); effKey != "" {
|
|
outReq.Header.Set("Authorization", "Bearer "+effKey)
|
|
} else if effHF := EffectiveHFToken(r); effHF != "" {
|
|
outReq.Header.Set("Authorization", "Bearer "+effHF)
|
|
}
|
|
|
|
resp, err := s.client.Do(outReq)
|
|
if err != nil {
|
|
return fmt.Errorf("upstream error: %w", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
for k, vv := range resp.Header {
|
|
for _, v := range vv {
|
|
w.Header().Add(k, v)
|
|
}
|
|
}
|
|
w.WriteHeader(resp.StatusCode)
|
|
|
|
if req.Stream {
|
|
flusher, _ := w.(http.Flusher)
|
|
reader := bufio.NewReader(resp.Body)
|
|
for {
|
|
line, err := reader.ReadBytes('\n')
|
|
if len(line) > 0 {
|
|
w.Write(line)
|
|
if flusher != nil {
|
|
flusher.Flush()
|
|
}
|
|
}
|
|
if err != nil {
|
|
break
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
_, err = io.Copy(w, resp.Body)
|
|
return err
|
|
}
|
|
|
|
func (s *QwenService) chatGradioRespond(w http.ResponseWriter, r *http.Request, req ChatCompletionRequest, resolvedModel string, maxTokens int) error {
|
|
var promptText string
|
|
toolsPrompt := FormatToolsPrompt(req.Tools)
|
|
|
|
if len(req.Messages) == 1 && req.Messages[0].Role == "user" && len(req.Tools) == 0 {
|
|
promptText = req.Messages[0].GetContentString()
|
|
} else {
|
|
var sb strings.Builder
|
|
var sysPrompt string
|
|
for _, m := range req.Messages {
|
|
if m.Role == "system" {
|
|
if sysPrompt != "" {
|
|
sysPrompt += "\n"
|
|
}
|
|
sysPrompt += m.GetContentString()
|
|
}
|
|
}
|
|
if toolsPrompt != "" {
|
|
if sysPrompt != "" {
|
|
sysPrompt += "\n" + toolsPrompt
|
|
} else {
|
|
sysPrompt = strings.TrimSpace(toolsPrompt)
|
|
}
|
|
}
|
|
if sysPrompt != "" {
|
|
sb.WriteString("<|im_start|>system\n")
|
|
sb.WriteString(sysPrompt)
|
|
sb.WriteString("<|im_end|>\n")
|
|
}
|
|
for _, m := range req.Messages {
|
|
if m.Role == "system" {
|
|
continue
|
|
}
|
|
sb.WriteString("<|im_start|>")
|
|
sb.WriteString(m.Role)
|
|
sb.WriteString("\n")
|
|
if m.Role == "assistant" && len(m.ToolCalls) > 0 {
|
|
for _, tc := range m.ToolCalls {
|
|
sb.WriteString(fmt.Sprintf("<tool_call>\n{\"name\": \"%s\", \"arguments\": %s}\n</tool_call>\n", tc.Function.Name, tc.Function.Arguments))
|
|
}
|
|
}
|
|
if m.Role == "tool" || m.Role == "function" {
|
|
toolName := m.Name
|
|
if toolName == "" {
|
|
toolName = m.ToolCallID
|
|
}
|
|
sb.WriteString(fmt.Sprintf("<tool_response name=\"%s\">\n%s\n</tool_response>\n", toolName, m.GetContentString()))
|
|
} else {
|
|
c := m.GetContentString()
|
|
if c != "" {
|
|
sb.WriteString(c)
|
|
sb.WriteString("\n")
|
|
}
|
|
}
|
|
sb.WriteString("<|im_end|>\n")
|
|
}
|
|
sb.WriteString("<|im_start|>assistant\n")
|
|
promptText = sb.String()
|
|
}
|
|
|
|
tempVal := 0.7
|
|
if req.Temperature != nil {
|
|
tempVal = *req.Temperature
|
|
}
|
|
|
|
topPVal := 0.95
|
|
if req.TopP != nil {
|
|
topPVal = *req.TopP
|
|
}
|
|
|
|
effUA := EffectiveUserAgent(r)
|
|
effHFToken := EffectiveHFToken(r)
|
|
if effHFToken == "" {
|
|
effHFToken = s.token
|
|
}
|
|
|
|
gradioData := []interface{}{
|
|
map[string]interface{}{
|
|
"text": promptText,
|
|
"files": []interface{}{},
|
|
},
|
|
maxTokens,
|
|
tempVal,
|
|
topPVal,
|
|
}
|
|
|
|
gradioPayload := map[string]interface{}{"data": gradioData}
|
|
jsonPayload, err := json.Marshal(gradioPayload)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to encode request: %w", err)
|
|
}
|
|
|
|
callURL := s.endpoint + "/gradio_api/call/respond"
|
|
makeCallReq := func() (*http.Request, error) {
|
|
reqObj, err := http.NewRequest("POST", callURL, bytes.NewBuffer(jsonPayload))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
reqObj.Header.Set("Content-Type", "application/json")
|
|
reqObj.Header.Set("User-Agent", effUA)
|
|
if effHFToken != "" {
|
|
reqObj.Header.Set("Authorization", "Bearer "+effHFToken)
|
|
}
|
|
return reqObj, nil
|
|
}
|
|
|
|
resp, err := DoWithFibonacciRetry(s.client, makeCallReq, 5)
|
|
if err != nil {
|
|
return fmt.Errorf("upstream join error: %w", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
var joinRes GradioJoinResponse
|
|
if err := json.NewDecoder(resp.Body).Decode(&joinRes); err != nil || joinRes.EventID == "" {
|
|
return fmt.Errorf("failed to parse Gradio event ID")
|
|
}
|
|
|
|
streamURL := fmt.Sprintf("%s/gradio_api/call/respond/%s", s.endpoint, joinRes.EventID)
|
|
makeStreamReq := func() (*http.Request, error) {
|
|
reqObj, err := http.NewRequest("GET", streamURL, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
reqObj.Header.Set("Accept", "text/event-stream")
|
|
reqObj.Header.Set("User-Agent", effUA)
|
|
if effHFToken != "" {
|
|
reqObj.Header.Set("Authorization", "Bearer "+effHFToken)
|
|
}
|
|
return reqObj, nil
|
|
}
|
|
|
|
streamResp, err := DoWithFibonacciRetry(s.client, makeStreamReq, 5)
|
|
if err != nil {
|
|
return fmt.Errorf("upstream stream error: %w", err)
|
|
}
|
|
defer streamResp.Body.Close()
|
|
|
|
completionID := "chatcmpl-" + GenerateUUID()
|
|
createdTime := time.Now().Unix()
|
|
|
|
reader := bufio.NewReader(streamResp.Body)
|
|
var currentEvent string
|
|
var finalRawText string
|
|
|
|
for {
|
|
line, err := reader.ReadString('\n')
|
|
if err != nil {
|
|
break
|
|
}
|
|
line = strings.TrimSpace(line)
|
|
if line == "" {
|
|
continue
|
|
}
|
|
|
|
if strings.HasPrefix(line, "event: ") {
|
|
currentEvent = strings.TrimPrefix(line, "event: ")
|
|
continue
|
|
}
|
|
|
|
if strings.HasPrefix(line, "data: ") {
|
|
dataJSON := strings.TrimPrefix(line, "data: ")
|
|
if currentEvent == "error" {
|
|
if strings.Contains(dataJSON, "ZeroGPU") || strings.Contains(dataJSON, "quota") {
|
|
return fmt.Errorf("ZeroGPU quota exceeded: authenticate with a Hugging Face token (set HF_TOKEN env var, -hf-token flag, or Authorization: Bearer hf_... header): %s", dataJSON)
|
|
}
|
|
return fmt.Errorf("gradio upstream error: %s", dataJSON)
|
|
}
|
|
if text, ok := parseAssistantText(dataJSON); ok {
|
|
finalRawText = text
|
|
}
|
|
}
|
|
}
|
|
|
|
cleanedReasoning, cleanedContent := SeparateReasoningAndContent(finalRawText)
|
|
toolCalls, remContent, hasToolCalls := DetectToolCalls(cleanedContent)
|
|
|
|
finishReason := "stop"
|
|
var msgContent interface{} = cleanedContent
|
|
|
|
if hasToolCalls && len(toolCalls) > 0 {
|
|
finishReason = "tool_calls"
|
|
if remContent == "" {
|
|
msgContent = nil
|
|
} else {
|
|
msgContent = remContent
|
|
}
|
|
}
|
|
|
|
if !req.Stream {
|
|
WriteCompletionResponse(w, completionID, createdTime, resolvedModel, FinalOutput{
|
|
Content: msgContent,
|
|
ReasoningContent: cleanedReasoning,
|
|
ToolCalls: toolCalls,
|
|
FinishReason: finishReason,
|
|
})
|
|
return nil
|
|
}
|
|
|
|
// Streaming response
|
|
flusher, _ := w.(http.Flusher)
|
|
streamer := NewStreamer(w, flusher, completionID, createdTime, resolvedModel)
|
|
streamer.Role()
|
|
|
|
if cleanedReasoning != "" {
|
|
streamer.Reasoning(cleanedReasoning)
|
|
}
|
|
|
|
if hasToolCalls && len(toolCalls) > 0 {
|
|
for _, tc := range toolCalls {
|
|
streamer.ToolCallDelta(tc)
|
|
}
|
|
} else if cleanedContent != "" {
|
|
streamer.Content(cleanedContent)
|
|
}
|
|
|
|
streamer.Finish(finishReason)
|
|
streamer.Done()
|
|
return nil
|
|
}
|
|
|
|
func (s *QwenService) chatGradioChatResponse(w http.ResponseWriter, r *http.Request, req ChatCompletionRequest, resolvedModel string, maxTokens int) error {
|
|
var systemPromptStr string
|
|
var historyArray []map[string]interface{}
|
|
var messageStr string
|
|
|
|
var nonSystemMsgs []ChatMessage
|
|
for _, msg := range req.Messages {
|
|
cStr := msg.GetContentString()
|
|
if msg.Role == "system" && systemPromptStr == "" {
|
|
systemPromptStr = cStr
|
|
} else {
|
|
nonSystemMsgs = append(nonSystemMsgs, msg)
|
|
}
|
|
}
|
|
|
|
toolsPrompt := FormatToolsPrompt(req.Tools)
|
|
|
|
if len(nonSystemMsgs) > 0 {
|
|
for i := 0; i < len(nonSystemMsgs)-1; i++ {
|
|
m := nonSystemMsgs[i]
|
|
cStr := m.GetContentString()
|
|
itemRole := m.Role
|
|
|
|
switch m.Role {
|
|
case "assistant":
|
|
contentBlocks := []interface{}{}
|
|
if cStr != "" {
|
|
contentBlocks = append(contentBlocks, map[string]interface{}{"text": cStr, "type": "text"})
|
|
}
|
|
item := map[string]interface{}{
|
|
"role": "assistant",
|
|
"metadata": nil,
|
|
"content": contentBlocks,
|
|
"options": nil,
|
|
}
|
|
historyArray = append(historyArray, item)
|
|
case "tool", "function":
|
|
toolName := m.Name
|
|
if toolName == "" {
|
|
toolName = m.ToolCallID
|
|
}
|
|
formatted := fmt.Sprintf("<tool_response name=\"%s\">\n%s\n</tool_response>", toolName, cStr)
|
|
item := map[string]interface{}{
|
|
"role": "user",
|
|
"metadata": nil,
|
|
"content": []interface{}{map[string]interface{}{"text": formatted, "type": "text"}},
|
|
"options": nil,
|
|
}
|
|
historyArray = append(historyArray, item)
|
|
default:
|
|
item := map[string]interface{}{
|
|
"role": itemRole,
|
|
"metadata": nil,
|
|
"content": []interface{}{map[string]interface{}{"text": cStr, "type": "text"}},
|
|
"options": nil,
|
|
}
|
|
historyArray = append(historyArray, item)
|
|
}
|
|
}
|
|
|
|
lastMsg := nonSystemMsgs[len(nonSystemMsgs)-1]
|
|
lastContent := lastMsg.GetContentString()
|
|
if lastMsg.Role == "tool" || lastMsg.Role == "function" {
|
|
toolName := lastMsg.Name
|
|
if toolName == "" {
|
|
toolName = lastMsg.ToolCallID
|
|
}
|
|
messageStr = fmt.Sprintf("<tool_response name=\"%s\">\n%s\n</tool_response>", toolName, lastContent)
|
|
} else {
|
|
messageStr = lastContent
|
|
}
|
|
}
|
|
|
|
if toolsPrompt != "" {
|
|
if systemPromptStr != "" {
|
|
systemPromptStr = systemPromptStr + "\n" + toolsPrompt
|
|
} else {
|
|
systemPromptStr = strings.TrimSpace(toolsPrompt)
|
|
}
|
|
}
|
|
|
|
// Determine thinking mode:
|
|
// Can be controlled by reasoning_effort ("none" disables thinking) or service default
|
|
enableThinking := s.enableThinking
|
|
if req.ReasoningEffort != "" {
|
|
if strings.EqualFold(req.ReasoningEffort, "none") {
|
|
enableThinking = false
|
|
} else {
|
|
enableThinking = true
|
|
}
|
|
}
|
|
|
|
tempVal := 1.0
|
|
if req.Temperature != nil {
|
|
tempVal = *req.Temperature
|
|
} else if !enableThinking {
|
|
tempVal = 0.7
|
|
}
|
|
|
|
topPVal := 0.95
|
|
if req.TopP != nil {
|
|
topPVal = *req.TopP
|
|
} else if !enableThinking {
|
|
topPVal = 0.80
|
|
}
|
|
|
|
topKVal := 20
|
|
presenceVal := 0.0
|
|
if !enableThinking {
|
|
presenceVal = 1.5
|
|
}
|
|
|
|
customAPIKey := EffectiveUpstreamKey(r)
|
|
if customAPIKey == "" {
|
|
customAPIKey = s.apiKey
|
|
}
|
|
|
|
customBaseURL := EffectiveUpstreamBaseURL(r)
|
|
if customBaseURL == "" {
|
|
customBaseURL = s.baseURL
|
|
}
|
|
|
|
effUA := EffectiveUserAgent(r)
|
|
effHFToken := EffectiveHFToken(r)
|
|
if effHFToken == "" {
|
|
effHFToken = s.token
|
|
}
|
|
|
|
// Gradio parameter list for /chat_response:
|
|
// 0: message (dict: text, files)
|
|
// 1: history (list of Message objects)
|
|
// 2: enable_thinking (bool)
|
|
// 3: preserve_thinking (bool)
|
|
// 4: temperature (float)
|
|
// 5: top_p (float)
|
|
// 6: top_k (float)
|
|
// 7: presence_penalty (float)
|
|
// 8: max_tokens (float)
|
|
// 9: system_prompt (str)
|
|
// 10: custom_base_url (str)
|
|
// 11: custom_api_key (str)
|
|
// 12: custom_model_id (str)
|
|
gradioData := []interface{}{
|
|
map[string]interface{}{
|
|
"text": messageStr,
|
|
"files": []interface{}{},
|
|
},
|
|
historyArray,
|
|
enableThinking,
|
|
false, // preserve_thinking
|
|
tempVal,
|
|
topPVal,
|
|
topKVal,
|
|
presenceVal,
|
|
maxTokens,
|
|
systemPromptStr,
|
|
customBaseURL,
|
|
customAPIKey,
|
|
resolvedModel,
|
|
}
|
|
|
|
gradioPayload := map[string]interface{}{"data": gradioData}
|
|
jsonPayload, err := json.Marshal(gradioPayload)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to encode request: %w", err)
|
|
}
|
|
|
|
callURL := s.endpoint + "/gradio_api/call/chat_response"
|
|
makeCallReq := func() (*http.Request, error) {
|
|
reqObj, err := http.NewRequest("POST", callURL, bytes.NewBuffer(jsonPayload))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
reqObj.Header.Set("Content-Type", "application/json")
|
|
reqObj.Header.Set("User-Agent", effUA)
|
|
if effHFToken != "" {
|
|
reqObj.Header.Set("Authorization", "Bearer "+effHFToken)
|
|
}
|
|
return reqObj, nil
|
|
}
|
|
|
|
resp, err := DoWithFibonacciRetry(s.client, makeCallReq, 5)
|
|
if err != nil {
|
|
return fmt.Errorf("upstream join error: %w", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
var joinRes GradioJoinResponse
|
|
if err := json.NewDecoder(resp.Body).Decode(&joinRes); err != nil || joinRes.EventID == "" {
|
|
return fmt.Errorf("failed to parse Gradio event ID")
|
|
}
|
|
|
|
streamURL := fmt.Sprintf("%s/gradio_api/call/chat_response/%s", s.endpoint, joinRes.EventID)
|
|
makeStreamReq := func() (*http.Request, error) {
|
|
reqObj, err := http.NewRequest("GET", streamURL, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
reqObj.Header.Set("Accept", "text/event-stream")
|
|
reqObj.Header.Set("User-Agent", effUA)
|
|
if effHFToken != "" {
|
|
reqObj.Header.Set("Authorization", "Bearer "+effHFToken)
|
|
}
|
|
return reqObj, nil
|
|
}
|
|
|
|
streamResp, err := DoWithFibonacciRetry(s.client, makeStreamReq, 5)
|
|
if err != nil {
|
|
return fmt.Errorf("upstream stream error: %w", err)
|
|
}
|
|
defer streamResp.Body.Close()
|
|
|
|
completionID := "chatcmpl-" + GenerateUUID()
|
|
createdTime := time.Now().Unix()
|
|
|
|
// Non-streaming completion
|
|
if !req.Stream {
|
|
reader := bufio.NewReader(streamResp.Body)
|
|
var currentEvent string
|
|
var finalRawText string
|
|
|
|
for {
|
|
line, err := reader.ReadString('\n')
|
|
if err != nil {
|
|
break
|
|
}
|
|
line = strings.TrimSpace(line)
|
|
if line == "" {
|
|
continue
|
|
}
|
|
|
|
if strings.HasPrefix(line, "event: ") {
|
|
currentEvent = strings.TrimPrefix(line, "event: ")
|
|
continue
|
|
}
|
|
|
|
if strings.HasPrefix(line, "data: ") {
|
|
dataJSON := strings.TrimPrefix(line, "data: ")
|
|
if currentEvent == "error" {
|
|
return fmt.Errorf("gradio upstream error: %s", dataJSON)
|
|
}
|
|
if text, ok := parseAssistantText(dataJSON); ok {
|
|
finalRawText = text
|
|
}
|
|
}
|
|
}
|
|
|
|
cleanedReasoning, cleanedContent := SeparateReasoningAndContent(finalRawText)
|
|
toolCalls, remContent, hasToolCalls := DetectToolCalls(cleanedContent)
|
|
|
|
finishReason := "stop"
|
|
var msgContent interface{} = cleanedContent
|
|
|
|
if hasToolCalls && len(toolCalls) > 0 {
|
|
finishReason = "tool_calls"
|
|
if remContent == "" {
|
|
msgContent = nil
|
|
} else {
|
|
msgContent = remContent
|
|
}
|
|
}
|
|
|
|
WriteCompletionResponse(w, completionID, createdTime, resolvedModel, FinalOutput{
|
|
Content: msgContent,
|
|
ReasoningContent: cleanedReasoning,
|
|
ToolCalls: toolCalls,
|
|
FinishReason: finishReason,
|
|
})
|
|
return nil
|
|
}
|
|
|
|
// Streaming completion
|
|
flusher, _ := w.(http.Flusher)
|
|
streamer := NewStreamer(w, flusher, completionID, createdTime, resolvedModel)
|
|
streamer.Role()
|
|
|
|
reader := bufio.NewReader(streamResp.Body)
|
|
var currentEvent string
|
|
var emittedReasoning string
|
|
var emittedContent string
|
|
|
|
toolFilter := &StreamToolCallFilter{}
|
|
|
|
onContentChunk := func(text string) {
|
|
if text != "" {
|
|
streamer.Content(text)
|
|
}
|
|
}
|
|
|
|
onToolCallChunk := func(tc ToolCall) {
|
|
streamer.ToolCallDelta(tc)
|
|
}
|
|
|
|
for {
|
|
line, err := reader.ReadString('\n')
|
|
if err != nil {
|
|
break
|
|
}
|
|
line = strings.TrimSpace(line)
|
|
if line == "" {
|
|
continue
|
|
}
|
|
|
|
if strings.HasPrefix(line, "event: ") {
|
|
currentEvent = strings.TrimPrefix(line, "event: ")
|
|
continue
|
|
}
|
|
|
|
if strings.HasPrefix(line, "data: ") {
|
|
dataJSON := strings.TrimPrefix(line, "data: ")
|
|
if currentEvent == "error" {
|
|
break
|
|
}
|
|
|
|
if fullAssistantText, ok := parseAssistantText(dataJSON); ok {
|
|
currentReasoning, currentContent := SeparateReasoningAndContent(fullAssistantText)
|
|
|
|
// Stream reasoning tokens incrementally
|
|
if len(currentReasoning) > len(emittedReasoning) {
|
|
rDelta := currentReasoning[len(emittedReasoning):]
|
|
emittedReasoning = currentReasoning
|
|
streamer.Reasoning(rDelta)
|
|
}
|
|
|
|
// Stream content tokens incrementally through tool filter
|
|
if len(currentContent) > len(emittedContent) {
|
|
cDelta := currentContent[len(emittedContent):]
|
|
emittedContent = currentContent
|
|
toolFilter.Feed(cDelta, onContentChunk, onToolCallChunk)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Flush remaining buffer in tool filter
|
|
toolFilter.Flush(onContentChunk, onToolCallChunk)
|
|
|
|
finishReason := "stop"
|
|
if toolFilter.HasEmittedCalls() {
|
|
finishReason = "tool_calls"
|
|
}
|
|
streamer.Finish(finishReason)
|
|
streamer.Done()
|
|
|
|
return nil
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Main Server & Handlers
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func main() {
|
|
port := flag.Int("port", 8080, "Port to listen on")
|
|
defaultEndpoint := "https://microhero-qwen3-8-27b-uncensored-chat.hf.space"
|
|
if envEP := os.Getenv("QFLASH_ENDPOINT"); envEP != "" {
|
|
defaultEndpoint = envEP
|
|
}
|
|
endpoint := flag.String("endpoint", defaultEndpoint, "Upstream Hugging Face Space or OpenAI URL")
|
|
defaultModelVal := "Qwen/Qwen3.8-27B-Uncensored"
|
|
if envModel := os.Getenv("QFLASH_MODEL"); envModel != "" {
|
|
defaultModelVal = envModel
|
|
}
|
|
defaultModel := flag.String("model", defaultModelVal, "Default model ID")
|
|
mode := flag.String("mode", "auto", "Endpoint protocol mode: auto, respond, chat_response, openai")
|
|
thinking := flag.Bool("thinking", true, "Enable thinking/reasoning mode by default")
|
|
flag.BoolVar(thinking, "enable-thinking", true, "Alias for -thinking")
|
|
hfToken := flag.String("hf-token", "", "Optional Hugging Face token for ZeroGPU quota or private spaces")
|
|
flag.StringVar(hfToken, "token", "", "Alias for -hf-token")
|
|
apiKey := flag.String("api-key", "", "Optional upstream API key for BYOK inference")
|
|
baseURL := flag.String("base-url", "", "Optional upstream base URL for BYOK inference")
|
|
userAgent := flag.String("user-agent", "", "Custom User-Agent header")
|
|
flag.StringVar(userAgent, "ua", "", "Alias for -user-agent")
|
|
socksProxy := flag.String("socks", "", "SOCKS5 proxy URL (e.g. socks5://127.0.0.1:1080)")
|
|
flag.StringVar(socksProxy, "proxy", "", "Alias for -socks")
|
|
flag.StringVar(socksProxy, "socks5", "", "Alias for -socks")
|
|
|
|
flag.Parse()
|
|
|
|
if envMode := os.Getenv("QFLASH_MODE"); envMode != "" && *mode == "auto" {
|
|
*mode = envMode
|
|
}
|
|
|
|
if *userAgent != "" {
|
|
ConfiguredUserAgent = *userAgent
|
|
}
|
|
if *hfToken != "" {
|
|
ConfiguredToken = *hfToken
|
|
}
|
|
if *apiKey != "" {
|
|
ConfiguredAPIKey = *apiKey
|
|
}
|
|
if *baseURL != "" {
|
|
ConfiguredBaseURL = *baseURL
|
|
}
|
|
ConfiguredModel = *defaultModel
|
|
EnableThinkingDefault = *thinking
|
|
|
|
proxyURL := *socksProxy
|
|
if proxyURL == "" {
|
|
for _, envKey := range []string{"ALL_PROXY", "all_proxy", "SOCKS5_PROXY", "socks5_proxy", "SOCKS_PROXY", "socks_proxy"} {
|
|
if v := os.Getenv(envKey); v != "" {
|
|
proxyURL = v
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
svc := NewQwenService(*endpoint, *defaultModel, *mode, *hfToken, *apiKey, *baseURL, proxyURL, *thinking)
|
|
|
|
mux := http.NewServeMux()
|
|
|
|
handleModels := func(w http.ResponseWriter, r *http.Request) {
|
|
EnableCORS(w)
|
|
if r.Method == http.MethodOptions {
|
|
w.WriteHeader(http.StatusNoContent)
|
|
return
|
|
}
|
|
if r.Method != http.MethodGet {
|
|
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
|
return
|
|
}
|
|
models := svc.ListModels()
|
|
w.Header().Set("Content-Type", "application/json")
|
|
json.NewEncoder(w).Encode(ModelsResponse{
|
|
Object: "list",
|
|
Data: models,
|
|
})
|
|
}
|
|
|
|
handleChatCompletions := func(w http.ResponseWriter, r *http.Request) {
|
|
EnableCORS(w)
|
|
if r.Method == http.MethodOptions {
|
|
w.WriteHeader(http.StatusNoContent)
|
|
return
|
|
}
|
|
if r.Method != http.MethodPost {
|
|
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
|
return
|
|
}
|
|
|
|
var req ChatCompletionRequest
|
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
|
http.Error(w, fmt.Sprintf("Invalid JSON request: %v", err), http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
if len(req.Messages) == 0 {
|
|
http.Error(w, "messages array must not be empty", http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
if err := svc.Chat(w, r, req); err != nil {
|
|
log.Printf("Chat completion error: %v", err)
|
|
http.Error(w, fmt.Sprintf("Upstream gateway error: %v", err), http.StatusBadGateway)
|
|
return
|
|
}
|
|
}
|
|
|
|
mux.HandleFunc("/models", handleModels)
|
|
mux.HandleFunc("/v1/models", handleModels)
|
|
mux.HandleFunc("/chat/completions", handleChatCompletions)
|
|
mux.HandleFunc("/v1/chat/completions", handleChatCompletions)
|
|
|
|
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
|
EnableCORS(w)
|
|
if r.URL.Path == "/" || r.URL.Path == "/healthz" {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
fmt.Fprintf(w, "{\"status\":\"ok\",\"service\":\"qflash\",\"model\":\"%s\"}\n", *defaultModel)
|
|
return
|
|
}
|
|
http.NotFound(w, r)
|
|
})
|
|
|
|
addr := fmt.Sprintf(":%d", *port)
|
|
log.Printf("Starting qflash gateway on %s -> %s", addr, *endpoint)
|
|
if proxyURL != "" {
|
|
log.Printf("Routing through SOCKS5 proxy: %s", proxyURL)
|
|
}
|
|
|
|
server := &http.Server{
|
|
Addr: addr,
|
|
Handler: mux,
|
|
}
|
|
|
|
if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
|
log.Fatalf("Server failed: %v", err)
|
|
}
|
|
}
|