diff --git a/README.md b/README.md index f14b5b3..fd79c89 100644 --- a/README.md +++ b/README.md @@ -5,15 +5,15 @@ Qflash is a standalone, single-binary gateway that exposes Qwen3.8 model spaces on Hugging Face through a standard OpenAI-compatible API. It automatically detects and supports multiple upstream protocols: - gradio `/chat_response` endpoints (such as `https://halvo78-qwen3-8-flash-next-playground.hf.space`) -- direct OpenAI `/v1/chat/completions` endpoints (such as `https://wanyamaelis-qwen3-8-27b.hf.space` and `https://apathy-exe-qwen3-8-flash-next.hf.space`, running persistent `llama-server` instances on CPU with zero quota limits) -- gradio `/respond` endpoints (supported for custom spaces such as `https://microhero-qwen3-8-27b-uncensored-chat.hf.space`) +- direct OpenAI `/v1/chat/completions` endpoints (such as `https://wanyamaelis-qwen3-8-27b.hf.space`, `https://apathy-exe-qwen3-8-27b.hf.space`, and `https://mikelalda-qwen3-8-euskera-gguf-q4.hf.space`, running persistent `llama-server` instances on CPU with zero quota limits) +- gradio `/respond` endpoints (supported for spaces such as `https://jickman125-qwen3-8-27b-uncensored-demo.hf.space` and `https://microhero-qwen3-8-27b-uncensored-chat.hf.space`) It translates standard `/v1/chat/completions` and `/v1/models` requests and Server-Sent Events (SSE) stream protocols into upstream formats, allowing any standard OpenAI-compatible client, agent, or IDE to interface with Qwen3.8 models without modification. ## Features - openai-compatible chat completions (streaming and non-streaming) -- automatic upstream failover from primary playground to non-ZeroGPU endpoints +- automatic upstream failover across persistent non-ZeroGPU endpoints - smart endpoint cooldown (5 minutes on quota exhaustion, 30 seconds on network errors) - automatic upstream endpoint detection (`respond`, `openai`, `chat_response`) - deep reasoning extraction with thinking trace passthrough (`` tags and blockquotes mapped to `reasoning_content`) @@ -30,15 +30,17 @@ It translates standard `/v1/chat/completions` and `/v1/models` requests and Serv | Space | Model | Hardware | Protocol | Notes | |---|---|---|---|---| -| `Halvo78/qwen3-8-flash-next-playground` *(primary)* | Qwen3.8-Flash-Next | CPU basic | Gradio `/chat_response` | Default endpoint; auto-fails over to live endpoints if unavailable | -| `Wanyamaelis/Qwen3.8-27B` *(fallback)* | Qwen3.8-27B (MTP) | CPU basic (8 vCPU) | Native OpenAI `/v1` | Non-ZeroGPU, no quota limits, fast speculative decoding (~5s) | -| `apathy-exe/Qwen3.8-27B` *(fallback)* | Qwen3.8-27B (MTP) | CPU (OpenMP/AVX512) | Native OpenAI `/v1` | Non-ZeroGPU, no quota limits, speculative decoding | -| `apathy-exe/Qwen3.8-Flash-Next` *(fallback)* | Qwen3.8-Flash-Next (~177B) | CPU (OpenMP/AVX512) | Native OpenAI `/v1` | Non-ZeroGPU, no quota limits, full 177B Flash-Next model | +| `Wanyamaelis/Qwen3.8-27B` *(primary)* | Qwen3.8-27B (MTP) | CPU basic (8 vCPU) | Native OpenAI `/v1` | Default endpoint; non-ZeroGPU, no quota limits, fast speculative decoding | +| `apathy-exe/Qwen3.8-27B` *(fallback)* | Qwen3.8-27B | CPU (AVX-512) | Native OpenAI `/v1` | Non-ZeroGPU, no quota limits, persistent `llama-server` | +| `apathy-exe/Qwen3.8-Flash-Next` *(fallback)* | Qwen3.8-Flash-Next (~177B) | CPU (AVX-512) | Native OpenAI `/v1` | Non-ZeroGPU, no quota limits, full 177B Flash-Next model | +| `mikelalda/Qwen3.8-euskera-GGUF-Q4` *(fallback)* | Qwen3.8-27B | CPU basic | Native OpenAI `/v1` | Non-ZeroGPU, no quota limits, persistent `llama-server` (Q4_K_M) | +| `jickman125/Qwen3.8-27B-Uncensored-Demo` | Qwen3.8-27B Uncensored | ZeroGPU (A10G) | Gradio `/respond` | Fast live GPU inference, can be configured via custom `-endpoints` | | `MicroHERO/qwen3.8-27b-uncensored-chat` | Qwen3.8-27B Uncensored | ZeroGPU (A10G) | Gradio `/respond` | Fast live GPU inference, can be configured via custom `-endpoints` | +| `Halvo78/qwen3-8-flash-next-playground` | Qwen3.8-Flash-Next | CPU basic | Gradio `/chat_response` | Sandbox playground, supports BYOK upstream inference credentials | ### Auto-failover and quota handling -By default, Qflash attempts the primary `Halvo78` playground endpoint. If the primary space is unavailable, offline, or lacks live inference credentials, it automatically fails over to the persistent CPU endpoints (`wanyamaelis` and `apathy-exe`), ensuring uninterrupted service: +By default, Qflash attempts the primary `wanyamaelis` endpoint with CPU MTP speculative decoding. If any endpoint is unavailable, offline, or exhausts resources, it automatically fails over across the persistent CPU endpoints (`apathy-exe` and `mikelalda`), ensuring uninterrupted service: - with auto-failover enabled (default), when an endpoint encounters an error, timeout, or unavailable backend, it is placed on cooldown and the gateway automatically fails over to the next configured endpoint - transient network errors trigger a 30-second cooldown, while quota limits or unavailable sandbox spaces trigger a 5-minute cooldown @@ -78,13 +80,13 @@ The gateway advertises the following models under `/v1/models`: | Model ID | Target model | Description | |---|---|---| -| `Qwen/Qwen3.8-Flash-Next` | `Qwen/Qwen3.8-Flash-Next` | Default primary model identifier | -| `Qwen/Qwen3.8-27B` | `Qwen/Qwen3.8-27B` | 27B model identifier | +| `Qwen/Qwen3.8-27B` | `Qwen/Qwen3.8-27B` | Default primary model identifier | +| `Qwen/Qwen3.8-Flash-Next` | `Qwen/Qwen3.8-Flash-Next` | Flash-Next model identifier | | `Qwen/Qwen3.8-27B-Uncensored` | `Qwen/Qwen3.8-27B-Uncensored` | Uncensored model identifier | +| `qwen3.8-27b` | `Qwen/Qwen3.8-27B` | Standard 27B lowercase alias | | `qwen3.8-flash-next` | `Qwen/Qwen3.8-Flash-Next` | Flash-Next lowercase alias | | `qwen-flash-next` | `Qwen/Qwen3.8-Flash-Next` | Shorthand alias | | `qwen-flash` | `Qwen/Qwen3.8-Flash-Next` | Quick convenience alias | -| `qwen3.8-27b` | `Qwen/Qwen3.8-27B` | Standard 27B lowercase alias | | `qwen3.8-27b-uncensored` | `Qwen/Qwen3.8-27B-Uncensored` | Uncensored lowercase alias | | `qwen` | Default model | Generic shorthand alias | @@ -98,15 +100,15 @@ Run the gateway with default auto-failover endpoints: qflash ``` -By default, this listens on `http://127.0.0.1:8080` targeting `Halvo78` by default with automatic failover to `wanyamaelis` and `apathy-exe` persistent endpoints. +By default, this listens on `http://127.0.0.1:8080` targeting `wanyamaelis` by default with automatic failover to `apathy-exe` and `mikelalda` persistent endpoints. Available flags: - `-port` — tcp port to listen on (default `8080`) -- `-endpoints` / `-endpoint` — comma-separated upstream space or OpenAI URLs (default list of 4 endpoints, `QFLASH_ENDPOINTS` / `QFLASH_ENDPOINT` env) +- `-endpoints` / `-endpoint` — comma-separated upstream space or OpenAI URLs (default list of 4 persistent endpoints, `QFLASH_ENDPOINTS` / `QFLASH_ENDPOINT` env) - `-failover` / `-auto-failover` — enable automatic failover across endpoints on quota exhaustion or error (default `true`, `QFLASH_FAILOVER` env) - `-mode` — upstream protocol mode: `auto`, `respond`, `chat_response`, `openai` (default `auto`, `QFLASH_MODE` env) -- `-model` — exposed model name (default `Qwen/Qwen3.8-Flash-Next`, `QFLASH_MODEL` env) +- `-model` — exposed model name (default `Qwen/Qwen3.8-27B`, `QFLASH_MODEL` env) - `-thinking` / `-enable-thinking` — enable chain-of-thought reasoning by default (default `true`) - `-hf-token` / `-token` — Hugging Face API token for ZeroGPU quota or private spaces (`HF_TOKEN` env) - `-api-key` — upstream inference engine API key for BYOK mode (`OPENAI_API_KEY` / `QWEN_API_KEY` env) diff --git a/main.go b/main.go index aa6d617..dda5dce 100644 --- a/main.go +++ b/main.go @@ -32,10 +32,10 @@ var ( ConfiguredModel string EnableThinkingDefault = true DefaultEndpoints = []string{ - "https://halvo78-qwen3-8-flash-next-playground.hf.space", "https://wanyamaelis-qwen3-8-27b.hf.space", "https://apathy-exe-qwen3-8-27b.hf.space", "https://apathy-exe-qwen3-8-flash-next.hf.space", + "https://mikelalda-qwen3-8-euskera-gguf-q4.hf.space", } ) @@ -517,7 +517,7 @@ func EffectiveModelID(reqModel string, defaultModel string) string { if defaultModel != "" { return defaultModel } - return "Qwen/Qwen3.8-Flash-Next" + return "Qwen/Qwen3.8-27B" default: return clean } @@ -1154,6 +1154,27 @@ func (f *StreamToolCallFilter) HasEmittedCalls() bool { // Qwen3.8-Flash Service & Gradio Stream Parsing // --------------------------------------------------------------------------- +func extractMessageContent(msgMap map[string]interface{}) string { + contentVal := msgMap["content"] + if contentStr, ok := contentVal.(string); ok { + return contentStr + } + if contentSlice, ok := contentVal.([]interface{}); ok { + var sb strings.Builder + for _, item := range contentSlice { + if s, ok := item.(string); ok { + sb.WriteString(s) + } else if m, ok := item.(map[string]interface{}); ok { + if textVal, ok := m["text"].(string); ok { + sb.WriteString(textVal) + } + } + } + return sb.String() + } + return "" +} + // parseAssistantText extracts the latest assistant message text from the Gradio output array. func parseAssistantText(dataJSON string) (string, bool) { var raw []interface{} @@ -1172,9 +1193,12 @@ func parseAssistantText(dataJSON string) (string, bool) { return "", false } - // Find the last assistant message - for i := len(msgList) - 1; i >= 0; i-- { - msgMap, ok := msgList[i].(map[string]interface{}) + var reasoningText string + var answerText string + var foundAssistant bool + + for _, item := range msgList { + msgMap, ok := item.(map[string]interface{}) if !ok { continue } @@ -1182,27 +1206,35 @@ func parseAssistantText(dataJSON string) (string, bool) { if role != "assistant" { continue } + foundAssistant = true + text := extractMessageContent(msgMap) - contentVal := msgMap["content"] - if contentStr, ok := contentVal.(string); ok { - return contentStr, true - } - if contentSlice, ok := contentVal.([]interface{}); ok { - var sb strings.Builder - for _, item := range contentSlice { - if s, ok := item.(string); ok { - sb.WriteString(s) - } else if m, ok := item.(map[string]interface{}); ok { - if textVal, ok := m["text"].(string); ok { - sb.WriteString(textVal) - } - } + isReasoning := false + if meta, ok := msgMap["metadata"].(map[string]interface{}); ok { + if title, ok := meta["title"].(string); ok && strings.EqualFold(title, "reasoning") { + isReasoning = true } - return sb.String(), true + } + + if isReasoning { + reasoningText = text + } else { + answerText = text } } - return "", false + if !foundAssistant { + return "", false + } + + if reasoningText != "" { + if answerText != "" { + return fmt.Sprintf("\n%s\n\n\n%s", reasoningText, answerText), true + } + return fmt.Sprintf("\n%s\n", reasoningText), true + } + + return answerText, true } type EndpointNode struct { @@ -1252,7 +1284,7 @@ func parseEndpointList(rawList []string) []*EndpointNode { func NewQwenService(endpoints []string, modelName, mode, token, apiKey, baseURL, socksProxy string, enableThinking, autoFailover bool) *QwenService { if modelName == "" { - modelName = "Qwen/Qwen3.8-Flash-Next" + modelName = "Qwen/Qwen3.8-27B" } if mode == "" { mode = "auto" @@ -1289,18 +1321,18 @@ func (s *QwenService) ListModels() []ModelItem { now := time.Now().Unix() primaryID := s.modelName if primaryID == "" { - primaryID = "Qwen/Qwen3.8-Flash-Next" + primaryID = "Qwen/Qwen3.8-27B" } candidates := []string{ primaryID, - "Qwen/Qwen3.8-Flash-Next", "Qwen/Qwen3.8-27B", + "Qwen/Qwen3.8-Flash-Next", "Qwen/Qwen3.8-27B-Uncensored", + "qwen3.8-27b", "qwen3.8-flash-next", "qwen-flash-next", "qwen-flash", - "qwen3.8-27b", "qwen3.8-27b-uncensored", "qwen", } @@ -1314,7 +1346,7 @@ func (s *QwenService) ListModels() []ModelItem { ID: id, Object: "model", Created: now, - OwnedBy: "qwen", + OwnedBy: "qwen-community", }) } } @@ -1328,13 +1360,13 @@ func (s *QwenService) detectEndpointModeFor(epURL, defaultMode string) string { } ep := strings.ToLower(epURL) - if strings.Contains(ep, "microhero") || strings.HasSuffix(ep, "/respond") { + if strings.Contains(ep, "microhero") || strings.Contains(ep, "jickman") || strings.HasSuffix(ep, "/respond") { return "respond" } if strings.Contains(ep, "halvo78") || strings.HasSuffix(ep, "/chat_response") { return "chat_response" } - if strings.Contains(ep, "wanyamaelis") || strings.Contains(ep, "apathy-exe") || strings.HasSuffix(ep, "/v1") { + if strings.Contains(ep, "wanyamaelis") || strings.Contains(ep, "apathy-exe") || strings.Contains(ep, "mikelalda") || strings.HasSuffix(ep, "/v1") { return "openai" } @@ -1587,13 +1619,13 @@ func (s *QwenService) chatDirectOpenAI(endpointURL string, w http.ResponseWriter func (s *QwenService) chatGradioRespond(endpointURL string, w http.ResponseWriter, r *http.Request, req ChatCompletionRequest, resolvedModel string, maxTokens int) error { var promptText string + var sysPrompt string toolsPrompt := FormatToolsPrompt(req.Tools) if len(req.Messages) == 1 && req.Messages[0].Role == "user" && len(req.Tools) == 0 { promptText = req.Messages[0].GetContentString() } else { var sb strings.Builder - var sysPrompt string for _, m := range req.Messages { if m.Role == "system" { if sysPrompt != "" { @@ -1679,14 +1711,34 @@ func (s *QwenService) chatGradioRespond(endpointURL string, w http.ResponseWrite effHFToken = s.token } - gradioData := []interface{}{ - map[string]interface{}{ - "text": promptText, - "files": []interface{}{}, - }, - clampedTokens, - tempVal, - topPVal, + var gradioData []interface{} + if strings.Contains(strings.ToLower(endpointURL), "jickman") { + jickmanSys := sysPrompt + if jickmanSys == "" { + jickmanSys = "You are a helpful assistant." + } + gradioData = []interface{}{ + map[string]interface{}{ + "text": promptText, + "files": []interface{}{}, + }, + "low", + tempVal, + topPVal, + 40, + clampedTokens, + jickmanSys, + } + } else { + gradioData = []interface{}{ + map[string]interface{}{ + "text": promptText, + "files": []interface{}{}, + }, + clampedTokens, + tempVal, + topPVal, + } } gradioPayload := map[string]interface{}{"data": gradioData} @@ -2232,7 +2284,7 @@ func main() { autoFailover := flag.Bool("failover", true, "Enable automatic failover across endpoints on error or quota limit") flag.BoolVar(autoFailover, "auto-failover", true, "Alias for -failover") - defaultModelVal := "Qwen/Qwen3.8-Flash-Next" + defaultModelVal := "Qwen/Qwen3.8-27B" if envModel := os.Getenv("QFLASH_MODEL"); envModel != "" { defaultModelVal = envModel } diff --git a/qflash_test.go b/qflash_test.go index d63b4ca..f67ce16 100644 --- a/qflash_test.go +++ b/qflash_test.go @@ -66,19 +66,20 @@ func TestSOCKS5Parsing(t *testing.T) { } func TestEffectiveModelID(t *testing.T) { - def := "Qwen/Qwen3.8-Flash-Next" + def := "Qwen/Qwen3.8-27B" cases := map[string]string{ - "": def, - "qwen3.8-flash-next": def, - "qwen-flash-next": def, - "qwen-flash": def, - "Qwen/Qwen3.8-Flash-Next": def, - "qwen3.8-27b": "Qwen/Qwen3.8-27B", - "qwen-27b": "Qwen/Qwen3.8-27B", - "Qwen/Qwen3.8-27B": "Qwen/Qwen3.8-27B", - "qwen3.8-27b-uncensored": "Qwen/Qwen3.8-27B-Uncensored", + "": def, + "qwen": def, + "qwen3.8-flash-next": "Qwen/Qwen3.8-Flash-Next", + "qwen-flash-next": "Qwen/Qwen3.8-Flash-Next", + "qwen-flash": "Qwen/Qwen3.8-Flash-Next", + "Qwen/Qwen3.8-Flash-Next": "Qwen/Qwen3.8-Flash-Next", + "qwen3.8-27b": "Qwen/Qwen3.8-27B", + "qwen-27b": "Qwen/Qwen3.8-27B", + "Qwen/Qwen3.8-27B": "Qwen/Qwen3.8-27B", + "qwen3.8-27b-uncensored": "Qwen/Qwen3.8-27B-Uncensored", "Qwen/Qwen3.8-27B-Uncensored": "Qwen/Qwen3.8-27B-Uncensored", - "custom-org/my-model": "custom-org/my-model", + "custom-org/my-model": "custom-org/my-model", } for in, exp := range cases { @@ -206,6 +207,27 @@ func TestParseAssistantText(t *testing.T) { } } +func TestParseAssistantTextMultiMessageReasoning(t *testing.T) { + dataJSON := `[[ + {"role": "user", "metadata": null, "content": [{"text": "Say hi", "type": "text"}]}, + {"role": "assistant", "metadata": {"title": "Reasoning"}, "content": [{"text": "First, consider greeting politely.", "type": "text"}]}, + {"role": "assistant", "metadata": null, "content": [{"text": "Hi there!", "type": "text"}]} + ]]` + + text, ok := parseAssistantText(dataJSON) + if !ok { + t.Fatalf("expected successful parse of multi-message assistant text") + } + + reasoning, content := SeparateReasoningAndContent(text) + if reasoning != "First, consider greeting politely." { + t.Fatalf("expected reasoning 'First, consider greeting politely.', got %q", reasoning) + } + if content != "Hi there!" { + t.Fatalf("expected content 'Hi there!', got %q", content) + } +} + func TestQwenServiceChatMock(t *testing.T) { // Mock upstream Gradio space server mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {