From bb56840d59bcd133bada5fe330c247427cc8d6da Mon Sep 17 00:00:00 2001 From: Luxferre Date: Sat, 5 Sep 2026 16:15:15 +0300 Subject: [PATCH] feat: add multi-space support with MicroHERO ZeroGPU default and auto-detection --- README.md | 85 +++++++--- main.go | 430 +++++++++++++++++++++++++++++++++++++++++++++++-- qflash_test.go | 128 ++++++++++++++- 3 files changed, 608 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index df88246..2f20ccc 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,47 @@ -# Qflash: Free Qwen3.8-Flash-Next LLM gateway +# Qflash: OpenAI-compatible gateway for Qwen3.8 models ## About -Qflash is a standalone, single-binary gateway that exposes the Qwen3.8-Flash-Next Gradio space (`https://halvo78-qwen3-8-flash-next-playground.hf.space`) through an OpenAI-compatible API. It translates the standard `/v1/chat/completions` and `/v1/models` endpoints into Gradio's `/gradio_api/call/chat_response` request and Server-Sent Events (SSE) stream protocol, allowing any standard OpenAI-compatible client, agent, or IDE to interface with Qwen3.8-Flash-Next without modification. +Qflash is a standalone, single-binary gateway that exposes Qwen3.8 model spaces on Hugging Face through a standard OpenAI-compatible API. It automatically detects and supports multiple upstream protocols: + +- Gradio `/respond` endpoints (such as `https://microhero-qwen3-8-27b-uncensored-chat.hf.space`, running live on ZeroGPU) +- Direct OpenAI `/v1/chat/completions` endpoints (such as `https://apathy-exe-qwen3-8-flash-next.hf.space`, running `llama-server`) +- Gradio `/chat_response` endpoints (such as `https://halvo78-qwen3-8-flash-next-playground.hf.space`) + +It translates standard `/v1/chat/completions` and `/v1/models` requests and Server-Sent Events (SSE) stream protocols into upstream formats, allowing any standard OpenAI-compatible client, agent, or IDE to interface with Qwen3.8 models without modification. ## Features -- OpenAI-compatible chat completions (streaming and non-streaming) -- Deep reasoning extraction with thinking trace passthrough (`` tags and blockquotes mapped to `reasoning_content`) -- Stateful streaming tool call interception (`StreamToolCallFilter`) with zero XML/JSON leakage into `delta.content` -- Real-time token streaming with incremental SSE delivery -- Support for instruct mode via standard `reasoning_effort: "none"` -- Zero-dependency SOCKS5 proxy client (RFC 1928 / RFC 1929) with domain resolution (`socks5h://`), IPv4, IPv6, and auth -- Bring Your Own Key (BYOK) pass-through support via `Authorization: Bearer` or CLI flags -- Fibonacci backoff retry on transient upstream errors -- Zero external dependencies (Go standard library only) +- openai-compatible chat completions (streaming and non-streaming) +- automatic upstream endpoint detection (`respond`, `openai`, `chat_response`) +- deep reasoning extraction with thinking trace passthrough (`` tags and blockquotes mapped to `reasoning_content`) +- stateful streaming tool call interception (`StreamToolCallFilter`) with zero XML or JSON leakage into `delta.content` +- real-time token streaming with incremental SSE delivery +- support for instruct mode via standard `reasoning_effort: "none"` +- zero-dependency SOCKS5 proxy client (RFC 1928 / RFC 1929) with domain resolution (`socks5h://`), IPv4, IPv6, and auth +- bring your own key (BYOK) pass-through support via `Authorization: Bearer` or CLI flags +- zero-gpu quota authentication via free Hugging Face personal tokens (`HF_TOKEN`) +- fibonacci backoff retry on transient upstream errors +- zero external dependencies (Go standard library only) + +## Supported upstream spaces + +| Space | Model | Hardware | Protocol | Notes | +|---|---|---|---|---| +| `MicroHERO/qwen3.8-27b-uncensored-chat` *(default)* | Qwen3.8-27B Uncensored | ZeroGPU (A10G) | Gradio `/respond` | Fast live inference (~2-5s), vision projector, uncensored build | +| `apathy-exe/Qwen3.8-Flash-Next` | Qwen3.8-Flash-Next (~177B) | CPU (OpenMP/AVX512) | Native OpenAI `/v1` | Full 177B model, single execution slot | +| `apathy-exe/Qwen3.8-27B` | Qwen3.8-27B (MTP) | CPU (OpenMP/AVX512) | Native OpenAI `/v1` | 27B model with speculative decoding, single execution slot | +| `Halvo78/qwen3-8-flash-next-playground` | Qwen3.8-Flash-Next | CPU basic | Gradio `/chat_response` | Sandbox client; requires BYOK API key/base URL | + +### Hugging Face ZeroGPU quota note + +Spaces running on ZeroGPU provide free compute. Anonymous requests share a small pool per IP address. To avoid quota limits, set a free Hugging Face token: + +```bash +export HF_TOKEN="hf_..." +``` + +Or pass it via the `-hf-token` flag, or via the `Authorization: Bearer hf_...` header in your client requests. Personal access tokens can be created for free at `https://huggingface.co/settings/tokens`. ## Installation @@ -50,10 +77,13 @@ The gateway advertises the following models under `/v1/models`: | Model ID | Target model | Description | |---|---|---| -| `Qwen/Qwen3.8-Flash-Next` | `Qwen/Qwen3.8-Flash-Next` | Primary playground model (125B MoE, 6B activated) | -| `qwen3.8-flash-next` | `Qwen/Qwen3.8-Flash-Next` | Standard lowercase alias | +| `Qwen/Qwen3.8-27B-Uncensored` | `Qwen/Qwen3.8-27B-Uncensored` | Default primary live model (ZeroGPU) | +| `Qwen/Qwen3.8-Flash-Next` | `Qwen/Qwen3.8-Flash-Next` | Flash-Next model identifier | +| `qwen3.8-27b-uncensored` | `Qwen/Qwen3.8-27B-Uncensored` | Standard lowercase alias | +| `qwen3.8-flash-next` | `Qwen/Qwen3.8-Flash-Next` | Flash-Next lowercase alias | | `qwen-flash-next` | `Qwen/Qwen3.8-Flash-Next` | Shorthand alias | | `qwen-flash` | `Qwen/Qwen3.8-Flash-Next` | Quick convenience alias | +| `qwen` | Default model | Generic shorthand alias | Any unlisted custom model name requested by the client is passed through directly. @@ -62,16 +92,19 @@ Any unlisted custom model name requested by the client is passed through directl Run the gateway: ```bash -qflash [-port 8080] [-endpoint https://halvo78-qwen3-8-flash-next-playground.hf.space] [-model Qwen/Qwen3.8-Flash-Next] +qflash ``` +By default, this listens on `http://127.0.0.1:8080` and connects to `https://microhero-qwen3-8-27b-uncensored-chat.hf.space`. + Available flags: - `-port` — TCP port to listen on (default `8080`) -- `-endpoint` — root URL of the Gradio space (default `https://halvo78-qwen3-8-flash-next-playground.hf.space`) -- `-model` — exposed model name (default `Qwen/Qwen3.8-Flash-Next`) +- `-endpoint` — root URL of upstream space (default `https://microhero-qwen3-8-27b-uncensored-chat.hf.space`, `QFLASH_ENDPOINT` env) +- `-mode` — upstream protocol mode: `auto`, `respond`, `chat_response`, `openai` (default `auto`, `QFLASH_MODE` env) +- `-model` — exposed model name (default `Qwen/Qwen3.8-27B-Uncensored`, `QFLASH_MODEL` env) - `-thinking` / `-enable-thinking` — enable chain-of-thought reasoning by default (default `true`) -- `-hf-token` — optional Hugging Face API token for authenticated spaces (`HF_TOKEN` env) +- `-hf-token` / `-token` — Hugging Face API token for ZeroGPU quota or private spaces (`HF_TOKEN` env) - `-api-key` — upstream inference engine API key for BYOK mode (`OPENAI_API_KEY` / `QWEN_API_KEY` env) - `-base-url` — upstream inference engine base URL for BYOK mode (`OPENAI_BASE_URL` / `QWEN_BASE_URL` env) - `-socks` / `-proxy` / `-socks5` — SOCKS5 proxy URL, e.g. `socks5://127.0.0.1:1080` (`ALL_PROXY` env) @@ -87,7 +120,7 @@ Example request with curl: ```bash curl http://localhost:8080/v1/chat/completions \ -H "Content-Type: application/json" \ - -d '{"model":"qwen3.8-flash-next","messages":[{"role":"user","content":"Explain QSA micro-blocks in one sentence."}],"stream":false}' + -d '{"model":"qwen","messages":[{"role":"user","content":"Explain quantum superposition in one sentence."}],"stream":false}' ``` Streaming example: @@ -95,7 +128,7 @@ Streaming example: ```bash curl -N http://localhost:8080/v1/chat/completions \ -H "Content-Type: application/json" \ - -d '{"model":"qwen-flash","messages":[{"role":"user","content":"Count from 1 to 5."}],"stream":true}' + -d '{"model":"qwen","messages":[{"role":"user","content":"Count from 1 to 5."}],"stream":true}' ``` Instruct mode example (disables thinking): @@ -103,7 +136,17 @@ Instruct mode example (disables thinking): ```bash curl http://localhost:8080/v1/chat/completions \ -H "Content-Type: application/json" \ - -d '{"model":"qwen-flash","messages":[{"role":"user","content":"Hello!"}],"reasoning_effort":"none"}' + -d '{"model":"qwen","messages":[{"role":"user","content":"Hello!"}],"reasoning_effort":"none"}' +``` + +Using an alternative upstream space: + +```bash +# Point to apathy-exe CPU llama-server +qflash -endpoint https://apathy-exe-qwen3-8-flash-next.hf.space -mode openai + +# Point to Halvo78 playground with your own API credentials +qflash -endpoint https://halvo78-qwen3-8-flash-next-playground.hf.space -api-key "sk-..." -base-url "https://api.openai.com/v1" ``` Python OpenAI SDK integration: @@ -117,7 +160,7 @@ client = OpenAI( ) stream = client.chat.completions.create( - model="qwen3.8-flash-next", + model="qwen", messages=[ {"role": "user", "content": "Prove that the sum of the first n odd numbers is n^2."} ], diff --git a/main.go b/main.go index 6eb95d7..196c0bd 100644 --- a/main.go +++ b/main.go @@ -499,7 +499,14 @@ func EffectiveModelID(reqModel string, defaultModel string) string { return defaultModel } switch strings.ToLower(clean) { - case "qwen/qwen3.8-flash-next", "qwen3.8-flash-next", "qwen-flash-next", "qwen-flash", "qwen3.8-flash", "qwen": + case "qwen/qwen3.8-flash-next", "qwen3.8-flash-next", "qwen-flash-next", "qwen-flash", "qwen3.8-flash": + return "Qwen/Qwen3.8-Flash-Next" + case "qwen/qwen3.8-27b-uncensored", "qwen3.8-27b-uncensored", "qwen-27b", "qwen3.8-27b": + return "Qwen/Qwen3.8-27B-Uncensored" + case "qwen": + if defaultModel != "" { + return defaultModel + } return "Qwen/Qwen3.8-Flash-Next" default: return clean @@ -972,6 +979,14 @@ func SeparateReasoningAndContent(text string) (string, string) { } } + // 1b. Output starting with reasoning and ending in (without opening ) + if strings.Contains(text, "") { + eIdx := strings.Index(text, "") + reasoning := text[:eIdx] + content := text[eIdx+len(""):] + return strings.TrimSpace(reasoning), strings.TrimSpace(content) + } + // 2. HF Gradio Playground blockquote format: "> ... Thinking Process" low := strings.ToLower(text) if strings.Contains(low, "thinking process") && (strings.HasPrefix(strings.TrimSpace(text), ">") || strings.Contains(text, "\n>")) { @@ -1136,6 +1151,11 @@ func parseAssistantText(dataJSON string) (string, bool) { return "", false } + // Check if raw[0] is directly a string (MicroHERO /respond format) + if str, ok := raw[0].(string); ok { + return str, true + } + // The first element is the chatbot message list msgList, ok := raw[0].([]interface{}) if !ok || len(msgList) == 0 { @@ -1178,6 +1198,7 @@ func parseAssistantText(dataJSON string) (string, bool) { type QwenService struct { endpoint string modelName string + mode string token string apiKey string baseURL string @@ -1185,10 +1206,13 @@ type QwenService struct { client *http.Client } -func NewQwenService(endpoint, modelName, token, apiKey, baseURL, socksProxy string, enableThinking bool) *QwenService { +func NewQwenService(endpoint, modelName, mode, token, apiKey, baseURL, socksProxy string, enableThinking bool) *QwenService { cleanEndpoint := strings.TrimRight(endpoint, "/") if modelName == "" { - modelName = "Qwen/Qwen3.8-Flash-Next" + modelName = "Qwen/Qwen3.8-27B-Uncensored" + } + if mode == "" { + mode = "auto" } transport := &http.Transport{ @@ -1206,6 +1230,7 @@ func NewQwenService(endpoint, modelName, token, apiKey, baseURL, socksProxy stri return &QwenService{ endpoint: cleanEndpoint, modelName: modelName, + mode: mode, token: token, apiKey: apiKey, baseURL: baseURL, @@ -1218,23 +1243,388 @@ func (s *QwenService) ListModels() []ModelItem { now := time.Now().Unix() primaryID := s.modelName if primaryID == "" { - primaryID = "Qwen/Qwen3.8-Flash-Next" + primaryID = "Qwen/Qwen3.8-27B-Uncensored" } - models := []ModelItem{ - {ID: primaryID, Object: "model", Created: now, OwnedBy: "qwen"}, - {ID: "qwen3.8-flash-next", Object: "model", Created: now, OwnedBy: "qwen"}, - {ID: "qwen-flash-next", Object: "model", Created: now, OwnedBy: "qwen"}, - {ID: "qwen-flash", Object: "model", Created: now, OwnedBy: "qwen"}, + candidates := []string{ + primaryID, + "Qwen/Qwen3.8-27B-Uncensored", + "Qwen/Qwen3.8-Flash-Next", + "qwen3.8-27b-uncensored", + "qwen3.8-flash-next", + "qwen-flash-next", + "qwen-flash", + "qwen", + } + + seen := make(map[string]bool) + var models []ModelItem + for _, id := range candidates { + if !seen[id] { + seen[id] = true + models = append(models, ModelItem{ + ID: id, + Object: "model", + Created: now, + OwnedBy: "qwen", + }) + } } return models } +func (s *QwenService) detectEndpointMode() string { + if s.mode != "" && s.mode != "auto" { + return s.mode + } + + ep := strings.ToLower(s.endpoint) + if strings.Contains(ep, "microhero") || strings.HasSuffix(ep, "/respond") { + return "respond" + } + if strings.Contains(ep, "halvo78") || strings.HasSuffix(ep, "/chat_response") { + return "chat_response" + } + if strings.Contains(ep, "apathy-exe") || strings.HasSuffix(ep, "/v1") { + return "openai" + } + + // Probe /gradio_api/info + infoURL := s.endpoint + "/gradio_api/info" + req, err := http.NewRequest("GET", infoURL, nil) + if err == nil { + req.Header.Set("User-Agent", DefaultUserAgent) + if s.token != "" { + req.Header.Set("Authorization", "Bearer "+s.token) + } + probeClient := &http.Client{Timeout: 3 * time.Second} + resp, err := probeClient.Do(req) + if err == nil && resp.StatusCode == http.StatusOK { + defer resp.Body.Close() + var info struct { + NamedEndpoints map[string]interface{} `json:"named_endpoints"` + } + if err := json.NewDecoder(resp.Body).Decode(&info); err == nil { + if _, ok := info.NamedEndpoints["/respond"]; ok { + return "respond" + } + if _, ok := info.NamedEndpoints["/chat_response"]; ok { + return "chat_response" + } + } + } + } + + return "respond" +} + func (s *QwenService) Chat(w http.ResponseWriter, r *http.Request, req ChatCompletionRequest) error { resolvedModel := EffectiveModelID(req.Model, s.modelName) maxTokens := ResolveMaxTokens(req) + mode := s.detectEndpointMode() + switch mode { + case "openai": + return s.chatDirectOpenAI(w, r, req, resolvedModel, maxTokens) + case "chat_response": + return s.chatGradioChatResponse(w, r, req, resolvedModel, maxTokens) + case "respond": + fallthrough + default: + return s.chatGradioRespond(w, r, req, resolvedModel, maxTokens) + } +} + +func (s *QwenService) chatDirectOpenAI(w http.ResponseWriter, r *http.Request, req ChatCompletionRequest, resolvedModel string, maxTokens int) error { + targetURL := s.endpoint + if !strings.HasSuffix(targetURL, "/v1/chat/completions") && !strings.HasSuffix(targetURL, "/chat/completions") { + if strings.HasSuffix(targetURL, "/v1") { + targetURL += "/chat/completions" + } else { + targetURL += "/v1/chat/completions" + } + } + + reqCopy := req + reqCopy.Model = resolvedModel + if reqCopy.MaxTokens == 0 && maxTokens > 0 { + reqCopy.MaxTokens = maxTokens + } + + reqJSON, err := json.Marshal(reqCopy) + if err != nil { + return fmt.Errorf("failed to encode upstream request: %w", err) + } + + outReq, err := http.NewRequest("POST", targetURL, bytes.NewBuffer(reqJSON)) + if err != nil { + return fmt.Errorf("failed to create upstream request: %w", err) + } + outReq.Header.Set("Content-Type", "application/json") + outReq.Header.Set("User-Agent", EffectiveUserAgent(r)) + if effKey := EffectiveUpstreamKey(r); effKey != "" { + outReq.Header.Set("Authorization", "Bearer "+effKey) + } else if effHF := EffectiveHFToken(r); effHF != "" { + outReq.Header.Set("Authorization", "Bearer "+effHF) + } + + resp, err := s.client.Do(outReq) + if err != nil { + return fmt.Errorf("upstream error: %w", err) + } + defer resp.Body.Close() + + for k, vv := range resp.Header { + for _, v := range vv { + w.Header().Add(k, v) + } + } + w.WriteHeader(resp.StatusCode) + + if req.Stream { + flusher, _ := w.(http.Flusher) + reader := bufio.NewReader(resp.Body) + for { + line, err := reader.ReadBytes('\n') + if len(line) > 0 { + w.Write(line) + if flusher != nil { + flusher.Flush() + } + } + if err != nil { + break + } + } + return nil + } + + _, err = io.Copy(w, resp.Body) + return err +} + +func (s *QwenService) chatGradioRespond(w http.ResponseWriter, r *http.Request, req ChatCompletionRequest, resolvedModel string, maxTokens int) error { + var promptText string + toolsPrompt := FormatToolsPrompt(req.Tools) + + if len(req.Messages) == 1 && req.Messages[0].Role == "user" && len(req.Tools) == 0 { + promptText = req.Messages[0].GetContentString() + } else { + var sb strings.Builder + var sysPrompt string + for _, m := range req.Messages { + if m.Role == "system" { + if sysPrompt != "" { + sysPrompt += "\n" + } + sysPrompt += m.GetContentString() + } + } + if toolsPrompt != "" { + if sysPrompt != "" { + sysPrompt += "\n" + toolsPrompt + } else { + sysPrompt = strings.TrimSpace(toolsPrompt) + } + } + if sysPrompt != "" { + sb.WriteString("<|im_start|>system\n") + sb.WriteString(sysPrompt) + sb.WriteString("<|im_end|>\n") + } + for _, m := range req.Messages { + if m.Role == "system" { + continue + } + sb.WriteString("<|im_start|>") + sb.WriteString(m.Role) + sb.WriteString("\n") + if m.Role == "assistant" && len(m.ToolCalls) > 0 { + for _, tc := range m.ToolCalls { + sb.WriteString(fmt.Sprintf("\n{\"name\": \"%s\", \"arguments\": %s}\n\n", tc.Function.Name, tc.Function.Arguments)) + } + } + if m.Role == "tool" || m.Role == "function" { + toolName := m.Name + if toolName == "" { + toolName = m.ToolCallID + } + sb.WriteString(fmt.Sprintf("\n%s\n\n", toolName, m.GetContentString())) + } else { + c := m.GetContentString() + if c != "" { + sb.WriteString(c) + sb.WriteString("\n") + } + } + sb.WriteString("<|im_end|>\n") + } + sb.WriteString("<|im_start|>assistant\n") + promptText = sb.String() + } + + tempVal := 0.7 + if req.Temperature != nil { + tempVal = *req.Temperature + } + + topPVal := 0.95 + if req.TopP != nil { + topPVal = *req.TopP + } + + effUA := EffectiveUserAgent(r) + effHFToken := EffectiveHFToken(r) + if effHFToken == "" { + effHFToken = s.token + } + + gradioData := []interface{}{ + map[string]interface{}{ + "text": promptText, + "files": []interface{}{}, + }, + maxTokens, + tempVal, + topPVal, + } + + gradioPayload := map[string]interface{}{"data": gradioData} + jsonPayload, err := json.Marshal(gradioPayload) + if err != nil { + return fmt.Errorf("failed to encode request: %w", err) + } + + callURL := s.endpoint + "/gradio_api/call/respond" + makeCallReq := func() (*http.Request, error) { + reqObj, err := http.NewRequest("POST", callURL, bytes.NewBuffer(jsonPayload)) + if err != nil { + return nil, err + } + reqObj.Header.Set("Content-Type", "application/json") + reqObj.Header.Set("User-Agent", effUA) + if effHFToken != "" { + reqObj.Header.Set("Authorization", "Bearer "+effHFToken) + } + return reqObj, nil + } + + resp, err := DoWithFibonacciRetry(s.client, makeCallReq, 5) + if err != nil { + return fmt.Errorf("upstream join error: %w", err) + } + defer resp.Body.Close() + + var joinRes GradioJoinResponse + if err := json.NewDecoder(resp.Body).Decode(&joinRes); err != nil || joinRes.EventID == "" { + return fmt.Errorf("failed to parse Gradio event ID") + } + + streamURL := fmt.Sprintf("%s/gradio_api/call/respond/%s", s.endpoint, joinRes.EventID) + makeStreamReq := func() (*http.Request, error) { + reqObj, err := http.NewRequest("GET", streamURL, nil) + if err != nil { + return nil, err + } + reqObj.Header.Set("Accept", "text/event-stream") + reqObj.Header.Set("User-Agent", effUA) + if effHFToken != "" { + reqObj.Header.Set("Authorization", "Bearer "+effHFToken) + } + return reqObj, nil + } + + streamResp, err := DoWithFibonacciRetry(s.client, makeStreamReq, 5) + if err != nil { + return fmt.Errorf("upstream stream error: %w", err) + } + defer streamResp.Body.Close() + + completionID := "chatcmpl-" + GenerateUUID() + createdTime := time.Now().Unix() + + reader := bufio.NewReader(streamResp.Body) + var currentEvent string + var finalRawText string + + for { + line, err := reader.ReadString('\n') + if err != nil { + break + } + line = strings.TrimSpace(line) + if line == "" { + continue + } + + if strings.HasPrefix(line, "event: ") { + currentEvent = strings.TrimPrefix(line, "event: ") + continue + } + + if strings.HasPrefix(line, "data: ") { + dataJSON := strings.TrimPrefix(line, "data: ") + if currentEvent == "error" { + if strings.Contains(dataJSON, "ZeroGPU") || strings.Contains(dataJSON, "quota") { + return fmt.Errorf("ZeroGPU quota exceeded: authenticate with a Hugging Face token (set HF_TOKEN env var, -hf-token flag, or Authorization: Bearer hf_... header): %s", dataJSON) + } + return fmt.Errorf("gradio upstream error: %s", dataJSON) + } + if text, ok := parseAssistantText(dataJSON); ok { + finalRawText = text + } + } + } + + cleanedReasoning, cleanedContent := SeparateReasoningAndContent(finalRawText) + toolCalls, remContent, hasToolCalls := DetectToolCalls(cleanedContent) + + finishReason := "stop" + var msgContent interface{} = cleanedContent + + if hasToolCalls && len(toolCalls) > 0 { + finishReason = "tool_calls" + if remContent == "" { + msgContent = nil + } else { + msgContent = remContent + } + } + + if !req.Stream { + WriteCompletionResponse(w, completionID, createdTime, resolvedModel, FinalOutput{ + Content: msgContent, + ReasoningContent: cleanedReasoning, + ToolCalls: toolCalls, + FinishReason: finishReason, + }) + return nil + } + + // Streaming response + flusher, _ := w.(http.Flusher) + streamer := NewStreamer(w, flusher, completionID, createdTime, resolvedModel) + streamer.Role() + + if cleanedReasoning != "" { + streamer.Reasoning(cleanedReasoning) + } + + if hasToolCalls && len(toolCalls) > 0 { + for _, tc := range toolCalls { + streamer.ToolCallDelta(tc) + } + } else if cleanedContent != "" { + streamer.Content(cleanedContent) + } + + streamer.Finish(finishReason) + streamer.Done() + return nil +} + +func (s *QwenService) chatGradioChatResponse(w http.ResponseWriter, r *http.Request, req ChatCompletionRequest, resolvedModel string, maxTokens int) error { var systemPromptStr string var historyArray []map[string]interface{} var messageStr string @@ -1587,11 +1977,21 @@ func (s *QwenService) Chat(w http.ResponseWriter, r *http.Request, req ChatCompl func main() { port := flag.Int("port", 8080, "Port to listen on") - endpoint := flag.String("endpoint", "https://halvo78-qwen3-8-flash-next-playground.hf.space", "Upstream HuggingFace Space URL") - defaultModel := flag.String("model", "Qwen/Qwen3.8-Flash-Next", "Default model ID") + defaultEndpoint := "https://microhero-qwen3-8-27b-uncensored-chat.hf.space" + if envEP := os.Getenv("QFLASH_ENDPOINT"); envEP != "" { + defaultEndpoint = envEP + } + endpoint := flag.String("endpoint", defaultEndpoint, "Upstream Hugging Face Space or OpenAI URL") + defaultModelVal := "Qwen/Qwen3.8-27B-Uncensored" + if envModel := os.Getenv("QFLASH_MODEL"); envModel != "" { + defaultModelVal = envModel + } + defaultModel := flag.String("model", defaultModelVal, "Default model ID") + mode := flag.String("mode", "auto", "Endpoint protocol mode: auto, respond, chat_response, openai") thinking := flag.Bool("thinking", true, "Enable thinking/reasoning mode by default") flag.BoolVar(thinking, "enable-thinking", true, "Alias for -thinking") - hfToken := flag.String("hf-token", "", "Optional HuggingFace Token for private/gated spaces") + hfToken := flag.String("hf-token", "", "Optional Hugging Face token for ZeroGPU quota or private spaces") + flag.StringVar(hfToken, "token", "", "Alias for -hf-token") apiKey := flag.String("api-key", "", "Optional upstream API key for BYOK inference") baseURL := flag.String("base-url", "", "Optional upstream base URL for BYOK inference") userAgent := flag.String("user-agent", "", "Custom User-Agent header") @@ -1602,6 +2002,10 @@ func main() { flag.Parse() + if envMode := os.Getenv("QFLASH_MODE"); envMode != "" && *mode == "auto" { + *mode = envMode + } + if *userAgent != "" { ConfiguredUserAgent = *userAgent } @@ -1627,7 +2031,7 @@ func main() { } } - svc := NewQwenService(*endpoint, *defaultModel, *hfToken, *apiKey, *baseURL, proxyURL, *thinking) + svc := NewQwenService(*endpoint, *defaultModel, *mode, *hfToken, *apiKey, *baseURL, proxyURL, *thinking) mux := http.NewServeMux() diff --git a/qflash_test.go b/qflash_test.go index 7fbbb7e..7f93708 100644 --- a/qflash_test.go +++ b/qflash_test.go @@ -232,7 +232,7 @@ func TestQwenServiceChatMock(t *testing.T) { })) defer mockServer.Close() - svc := NewQwenService(mockServer.URL, "Qwen/Qwen3.8-Flash-Next", "", "", "", "", true) + svc := NewQwenService(mockServer.URL, "Qwen/Qwen3.8-Flash-Next", "chat_response", "", "", "", "", true) // 1. Test Non-streaming completion rec := httptest.NewRecorder() @@ -319,3 +319,129 @@ func TestQwenServiceChatMock(t *testing.T) { t.Fatalf("expected streamed content, got %q", receivedContent.String()) } } + +func TestParseAssistantTextDirectString(t *testing.T) { + dataJSON := `["Step-by-step reasoning\n\n\nFinal answer here", null]` + text, ok := parseAssistantText(dataJSON) + if !ok { + t.Fatalf("expected successful parse of direct string assistant text") + } + if text != "Step-by-step reasoning\n\n\nFinal answer here" { + t.Fatalf("unexpected text: %q", text) + } + + reasoning, content := SeparateReasoningAndContent(text) + if reasoning != "Step-by-step reasoning" { + t.Fatalf("unexpected reasoning: %q", reasoning) + } + if content != "Final answer here" { + t.Fatalf("unexpected content: %q", content) + } +} + +func TestQwenServiceChatRespondMock(t *testing.T) { + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/gradio_api/call/respond" && r.Method == http.MethodPost { + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(`{"event_id": "respond_event_456"}`)) + return + } + + if r.URL.Path == "/gradio_api/call/respond/respond_event_456" && r.Method == http.MethodGet { + w.Header().Set("Content-Type", "text/event-stream") + flusher, _ := w.(http.Flusher) + + chunk := `event: complete` + "\n" + + `data: ["Analyzing the problem.\n\n\nThe solution is 42.", null]` + "\n\n" + w.Write([]byte(chunk)) + flusher.Flush() + return + } + + http.NotFound(w, r) + })) + defer mockServer.Close() + + svc := NewQwenService(mockServer.URL, "Qwen/Qwen3.8-27B-Uncensored", "respond", "", "", "", "", true) + + // Non-streaming test + rec := httptest.NewRecorder() + req := ChatCompletionRequest{ + Model: "qwen3.8-27b", + Messages: []ChatMessage{ + {Role: "user", Content: "What is the answer?"}, + }, + Stream: false, + } + + err := svc.Chat(rec, nil, req) + if err != nil { + t.Fatalf("unexpected error in respond non-streaming: %v", err) + } + if rec.Code != http.StatusOK { + t.Fatalf("expected HTTP 200, got %d", rec.Code) + } + + var resp ChatCompletionResponse + if err := json.NewDecoder(rec.Body).Decode(&resp); err != nil { + t.Fatalf("failed to decode response: %v", err) + } + if resp.Choices[0].Message.Content != "The solution is 42." { + t.Fatalf("unexpected content: %v", resp.Choices[0].Message.Content) + } + if resp.Choices[0].Message.ReasoningContent != "Analyzing the problem." { + t.Fatalf("unexpected reasoning: %v", resp.Choices[0].Message.ReasoningContent) + } + + // Streaming test + recStream := httptest.NewRecorder() + reqStream := ChatCompletionRequest{ + Model: "qwen3.8-27b", + Messages: []ChatMessage{ + {Role: "user", Content: "What is the answer?"}, + }, + Stream: true, + } + + errStream := svc.Chat(recStream, nil, reqStream) + if errStream != nil { + t.Fatalf("unexpected error in respond streaming: %v", errStream) + } + + scanner := bufio.NewScanner(recStream.Body) + var receivedContent strings.Builder + var receivedReasoning strings.Builder + var sawDone bool + + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, "data: ") { + payload := strings.TrimPrefix(line, "data: ") + if payload == "[DONE]" { + sawDone = true + continue + } + var sResp StreamResponse + if err := json.Unmarshal([]byte(payload), &sResp); err == nil && len(sResp.Choices) > 0 { + delta := sResp.Choices[0].Delta + if delta.ReasoningContent != "" { + receivedReasoning.WriteString(delta.ReasoningContent) + } + if delta.Content != "" { + receivedContent.WriteString(delta.Content) + } + } + } + } + + if !sawDone { + t.Fatalf("expected [DONE] in stream") + } + if receivedReasoning.String() != "Analyzing the problem." { + t.Fatalf("unexpected streamed reasoning: %q", receivedReasoning.String()) + } + if receivedContent.String() != "The solution is 42." { + t.Fatalf("unexpected streamed content: %q", receivedContent.String()) + } +} +