feat: add multi-space support with MicroHERO ZeroGPU default and auto-detection

2026-09-05 16:15:15 +03:00
parent 0da31212b4
commit bb56840d59
3 changed files with 608 additions and 35 deletions
@@ -499,7 +499,14 @@ func EffectiveModelID(reqModel string, defaultModel string) string {
    return defaultModel
  }
  switch strings.ToLower(clean) {
-  case "qwen/qwen3.8-flash-next", "qwen3.8-flash-next", "qwen-flash-next", "qwen-flash", "qwen3.8-flash", "qwen":
+  case "qwen/qwen3.8-flash-next", "qwen3.8-flash-next", "qwen-flash-next", "qwen-flash", "qwen3.8-flash":
+    return "Qwen/Qwen3.8-Flash-Next"
+  case "qwen/qwen3.8-27b-uncensored", "qwen3.8-27b-uncensored", "qwen-27b", "qwen3.8-27b":
+    return "Qwen/Qwen3.8-27B-Uncensored"
+  case "qwen":
+    if defaultModel != "" {
+      return defaultModel
+    }
    return "Qwen/Qwen3.8-Flash-Next"
  default:
    return clean
@@ -972,6 +979,14 @@ func SeparateReasoningAndContent(text string) (string, string) {
    }
  }

+  // 1b. Output starting with reasoning and ending in </think> (without opening <think>)
+  if strings.Contains(text, "</think>") {
+    eIdx := strings.Index(text, "</think>")
+    reasoning := text[:eIdx]
+    content := text[eIdx+len("</think>"):]
+    return strings.TrimSpace(reasoning), strings.TrimSpace(content)
+  }
+
  // 2. HF Gradio Playground blockquote format: "> ... Thinking Process"
  low := strings.ToLower(text)
  if strings.Contains(low, "thinking process") && (strings.HasPrefix(strings.TrimSpace(text), ">") || strings.Contains(text, "\n>")) {
@@ -1136,6 +1151,11 @@ func parseAssistantText(dataJSON string) (string, bool) {
    return "", false
  }

+  // Check if raw[0] is directly a string (MicroHERO /respond format)
+  if str, ok := raw[0].(string); ok {
+    return str, true
+  }
+
  // The first element is the chatbot message list
  msgList, ok := raw[0].([]interface{})
  if !ok || len(msgList) == 0 {
@@ -1178,6 +1198,7 @@ func parseAssistantText(dataJSON string) (string, bool) {
 type QwenService struct {
  endpoint       string
  modelName      string
+  mode           string
  token          string
  apiKey         string
  baseURL        string
@@ -1185,10 +1206,13 @@ type QwenService struct {
  client         *http.Client
 }

-func NewQwenService(endpoint, modelName, token, apiKey, baseURL, socksProxy string, enableThinking bool) *QwenService {
+func NewQwenService(endpoint, modelName, mode, token, apiKey, baseURL, socksProxy string, enableThinking bool) *QwenService {
  cleanEndpoint := strings.TrimRight(endpoint, "/")
  if modelName == "" {
-    modelName = "Qwen/Qwen3.8-Flash-Next"
+    modelName = "Qwen/Qwen3.8-27B-Uncensored"
+  }
+  if mode == "" {
+    mode = "auto"
  }

  transport := &http.Transport{
@@ -1206,6 +1230,7 @@ func NewQwenService(endpoint, modelName, token, apiKey, baseURL, socksProxy stri
  return &QwenService{
    endpoint:       cleanEndpoint,
    modelName:      modelName,
+    mode:           mode,
    token:          token,
    apiKey:         apiKey,
    baseURL:        baseURL,
@@ -1218,23 +1243,388 @@ func (s *QwenService) ListModels() []ModelItem {
  now := time.Now().Unix()
  primaryID := s.modelName
  if primaryID == "" {
-    primaryID = "Qwen/Qwen3.8-Flash-Next"
+    primaryID = "Qwen/Qwen3.8-27B-Uncensored"
  }

-  models := []ModelItem{
-    {ID: primaryID, Object: "model", Created: now, OwnedBy: "qwen"},
-    {ID: "qwen3.8-flash-next", Object: "model", Created: now, OwnedBy: "qwen"},
-    {ID: "qwen-flash-next", Object: "model", Created: now, OwnedBy: "qwen"},
-    {ID: "qwen-flash", Object: "model", Created: now, OwnedBy: "qwen"},
+  candidates := []string{
+    primaryID,
+    "Qwen/Qwen3.8-27B-Uncensored",
+    "Qwen/Qwen3.8-Flash-Next",
+    "qwen3.8-27b-uncensored",
+    "qwen3.8-flash-next",
+    "qwen-flash-next",
+    "qwen-flash",
+    "qwen",
+  }
+
+  seen := make(map[string]bool)
+  var models []ModelItem
+  for _, id := range candidates {
+    if !seen[id] {
+      seen[id] = true
+      models = append(models, ModelItem{
+        ID:      id,
+        Object:  "model",
+        Created: now,
+        OwnedBy: "qwen",
+      })
+    }
  }

  return models
 }

+func (s *QwenService) detectEndpointMode() string {
+  if s.mode != "" && s.mode != "auto" {
+    return s.mode
+  }
+
+  ep := strings.ToLower(s.endpoint)
+  if strings.Contains(ep, "microhero") || strings.HasSuffix(ep, "/respond") {
+    return "respond"
+  }
+  if strings.Contains(ep, "halvo78") || strings.HasSuffix(ep, "/chat_response") {
+    return "chat_response"
+  }
+  if strings.Contains(ep, "apathy-exe") || strings.HasSuffix(ep, "/v1") {
+    return "openai"
+  }
+
+  // Probe /gradio_api/info
+  infoURL := s.endpoint + "/gradio_api/info"
+  req, err := http.NewRequest("GET", infoURL, nil)
+  if err == nil {
+    req.Header.Set("User-Agent", DefaultUserAgent)
+    if s.token != "" {
+      req.Header.Set("Authorization", "Bearer "+s.token)
+    }
+    probeClient := &http.Client{Timeout: 3 * time.Second}
+    resp, err := probeClient.Do(req)
+    if err == nil && resp.StatusCode == http.StatusOK {
+      defer resp.Body.Close()
+      var info struct {
+        NamedEndpoints map[string]interface{} `json:"named_endpoints"`
+      }
+      if err := json.NewDecoder(resp.Body).Decode(&info); err == nil {
+        if _, ok := info.NamedEndpoints["/respond"]; ok {
+          return "respond"
+        }
+        if _, ok := info.NamedEndpoints["/chat_response"]; ok {
+          return "chat_response"
+        }
+      }
+    }
+  }
+
+  return "respond"
+}
+
 func (s *QwenService) Chat(w http.ResponseWriter, r *http.Request, req ChatCompletionRequest) error {
  resolvedModel := EffectiveModelID(req.Model, s.modelName)
  maxTokens := ResolveMaxTokens(req)
+  mode := s.detectEndpointMode()

+  switch mode {
+  case "openai":
+    return s.chatDirectOpenAI(w, r, req, resolvedModel, maxTokens)
+  case "chat_response":
+    return s.chatGradioChatResponse(w, r, req, resolvedModel, maxTokens)
+  case "respond":
+    fallthrough
+  default:
+    return s.chatGradioRespond(w, r, req, resolvedModel, maxTokens)
+  }
+}
+
+func (s *QwenService) chatDirectOpenAI(w http.ResponseWriter, r *http.Request, req ChatCompletionRequest, resolvedModel string, maxTokens int) error {
+  targetURL := s.endpoint
+  if !strings.HasSuffix(targetURL, "/v1/chat/completions") && !strings.HasSuffix(targetURL, "/chat/completions") {
+    if strings.HasSuffix(targetURL, "/v1") {
+      targetURL += "/chat/completions"
+    } else {
+      targetURL += "/v1/chat/completions"
+    }
+  }
+
+  reqCopy := req
+  reqCopy.Model = resolvedModel
+  if reqCopy.MaxTokens == 0 && maxTokens > 0 {
+    reqCopy.MaxTokens = maxTokens
+  }
+
+  reqJSON, err := json.Marshal(reqCopy)
+  if err != nil {
+    return fmt.Errorf("failed to encode upstream request: %w", err)
+  }
+
+  outReq, err := http.NewRequest("POST", targetURL, bytes.NewBuffer(reqJSON))
+  if err != nil {
+    return fmt.Errorf("failed to create upstream request: %w", err)
+  }
+  outReq.Header.Set("Content-Type", "application/json")
+  outReq.Header.Set("User-Agent", EffectiveUserAgent(r))
+  if effKey := EffectiveUpstreamKey(r); effKey != "" {
+    outReq.Header.Set("Authorization", "Bearer "+effKey)
+  } else if effHF := EffectiveHFToken(r); effHF != "" {
+    outReq.Header.Set("Authorization", "Bearer "+effHF)
+  }
+
+  resp, err := s.client.Do(outReq)
+  if err != nil {
+    return fmt.Errorf("upstream error: %w", err)
+  }
+  defer resp.Body.Close()
+
+  for k, vv := range resp.Header {
+    for _, v := range vv {
+      w.Header().Add(k, v)
+    }
+  }
+  w.WriteHeader(resp.StatusCode)
+
+  if req.Stream {
+    flusher, _ := w.(http.Flusher)
+    reader := bufio.NewReader(resp.Body)
+    for {
+      line, err := reader.ReadBytes('\n')
+      if len(line) > 0 {
+        w.Write(line)
+        if flusher != nil {
+          flusher.Flush()
+        }
+      }
+      if err != nil {
+        break
+      }
+    }
+    return nil
+  }
+
+  _, err = io.Copy(w, resp.Body)
+  return err
+}
+
+func (s *QwenService) chatGradioRespond(w http.ResponseWriter, r *http.Request, req ChatCompletionRequest, resolvedModel string, maxTokens int) error {
+  var promptText string
+  toolsPrompt := FormatToolsPrompt(req.Tools)
+
+  if len(req.Messages) == 1 && req.Messages[0].Role == "user" && len(req.Tools) == 0 {
+    promptText = req.Messages[0].GetContentString()
+  } else {
+    var sb strings.Builder
+    var sysPrompt string
+    for _, m := range req.Messages {
+      if m.Role == "system" {
+        if sysPrompt != "" {
+          sysPrompt += "\n"
+        }
+        sysPrompt += m.GetContentString()
+      }
+    }
+    if toolsPrompt != "" {
+      if sysPrompt != "" {
+        sysPrompt += "\n" + toolsPrompt
+      } else {
+        sysPrompt = strings.TrimSpace(toolsPrompt)
+      }
+    }
+    if sysPrompt != "" {
+      sb.WriteString("<|im_start|>system\n")
+      sb.WriteString(sysPrompt)
+      sb.WriteString("<|im_end|>\n")
+    }
+    for _, m := range req.Messages {
+      if m.Role == "system" {
+        continue
+      }
+      sb.WriteString("<|im_start|>")
+      sb.WriteString(m.Role)
+      sb.WriteString("\n")
+      if m.Role == "assistant" && len(m.ToolCalls) > 0 {
+        for _, tc := range m.ToolCalls {
+          sb.WriteString(fmt.Sprintf("<tool_call>\n{\"name\": \"%s\", \"arguments\": %s}\n</tool_call>\n", tc.Function.Name, tc.Function.Arguments))
+        }
+      }
+      if m.Role == "tool" || m.Role == "function" {
+        toolName := m.Name
+        if toolName == "" {
+          toolName = m.ToolCallID
+        }
+        sb.WriteString(fmt.Sprintf("<tool_response name=\"%s\">\n%s\n</tool_response>\n", toolName, m.GetContentString()))
+      } else {
+        c := m.GetContentString()
+        if c != "" {
+          sb.WriteString(c)
+          sb.WriteString("\n")
+        }
+      }
+      sb.WriteString("<|im_end|>\n")
+    }
+    sb.WriteString("<|im_start|>assistant\n")
+    promptText = sb.String()
+  }
+
+  tempVal := 0.7
+  if req.Temperature != nil {
+    tempVal = *req.Temperature
+  }
+
+  topPVal := 0.95
+  if req.TopP != nil {
+    topPVal = *req.TopP
+  }
+
+  effUA := EffectiveUserAgent(r)
+  effHFToken := EffectiveHFToken(r)
+  if effHFToken == "" {
+    effHFToken = s.token
+  }
+
+  gradioData := []interface{}{
+    map[string]interface{}{
+      "text":  promptText,
+      "files": []interface{}{},
+    },
+    maxTokens,
+    tempVal,
+    topPVal,
+  }
+
+  gradioPayload := map[string]interface{}{"data": gradioData}
+  jsonPayload, err := json.Marshal(gradioPayload)
+  if err != nil {
+    return fmt.Errorf("failed to encode request: %w", err)
+  }
+
+  callURL := s.endpoint + "/gradio_api/call/respond"
+  makeCallReq := func() (*http.Request, error) {
+    reqObj, err := http.NewRequest("POST", callURL, bytes.NewBuffer(jsonPayload))
+    if err != nil {
+      return nil, err
+    }
+    reqObj.Header.Set("Content-Type", "application/json")
+    reqObj.Header.Set("User-Agent", effUA)
+    if effHFToken != "" {
+      reqObj.Header.Set("Authorization", "Bearer "+effHFToken)
+    }
+    return reqObj, nil
+  }
+
+  resp, err := DoWithFibonacciRetry(s.client, makeCallReq, 5)
+  if err != nil {
+    return fmt.Errorf("upstream join error: %w", err)
+  }
+  defer resp.Body.Close()
+
+  var joinRes GradioJoinResponse
+  if err := json.NewDecoder(resp.Body).Decode(&joinRes); err != nil || joinRes.EventID == "" {
+    return fmt.Errorf("failed to parse Gradio event ID")
+  }
+
+  streamURL := fmt.Sprintf("%s/gradio_api/call/respond/%s", s.endpoint, joinRes.EventID)
+  makeStreamReq := func() (*http.Request, error) {
+    reqObj, err := http.NewRequest("GET", streamURL, nil)
+    if err != nil {
+      return nil, err
+    }
+    reqObj.Header.Set("Accept", "text/event-stream")
+    reqObj.Header.Set("User-Agent", effUA)
+    if effHFToken != "" {
+      reqObj.Header.Set("Authorization", "Bearer "+effHFToken)
+    }
+    return reqObj, nil
+  }
+
+  streamResp, err := DoWithFibonacciRetry(s.client, makeStreamReq, 5)
+  if err != nil {
+    return fmt.Errorf("upstream stream error: %w", err)
+  }
+  defer streamResp.Body.Close()
+
+  completionID := "chatcmpl-" + GenerateUUID()
+  createdTime := time.Now().Unix()
+
+  reader := bufio.NewReader(streamResp.Body)
+  var currentEvent string
+  var finalRawText string
+
+  for {
+    line, err := reader.ReadString('\n')
+    if err != nil {
+      break
+    }
+    line = strings.TrimSpace(line)
+    if line == "" {
+      continue
+    }
+
+    if strings.HasPrefix(line, "event: ") {
+      currentEvent = strings.TrimPrefix(line, "event: ")
+      continue
+    }
+
+    if strings.HasPrefix(line, "data: ") {
+      dataJSON := strings.TrimPrefix(line, "data: ")
+      if currentEvent == "error" {
+        if strings.Contains(dataJSON, "ZeroGPU") || strings.Contains(dataJSON, "quota") {
+          return fmt.Errorf("ZeroGPU quota exceeded: authenticate with a Hugging Face token (set HF_TOKEN env var, -hf-token flag, or Authorization: Bearer hf_... header): %s", dataJSON)
+        }
+        return fmt.Errorf("gradio upstream error: %s", dataJSON)
+      }
+      if text, ok := parseAssistantText(dataJSON); ok {
+        finalRawText = text
+      }
+    }
+  }
+
+  cleanedReasoning, cleanedContent := SeparateReasoningAndContent(finalRawText)
+  toolCalls, remContent, hasToolCalls := DetectToolCalls(cleanedContent)
+
+  finishReason := "stop"
+  var msgContent interface{} = cleanedContent
+
+  if hasToolCalls && len(toolCalls) > 0 {
+    finishReason = "tool_calls"
+    if remContent == "" {
+      msgContent = nil
+    } else {
+      msgContent = remContent
+    }
+  }
+
+  if !req.Stream {
+    WriteCompletionResponse(w, completionID, createdTime, resolvedModel, FinalOutput{
+      Content:          msgContent,
+      ReasoningContent: cleanedReasoning,
+      ToolCalls:        toolCalls,
+      FinishReason:     finishReason,
+    })
+    return nil
+  }
+
+  // Streaming response
+  flusher, _ := w.(http.Flusher)
+  streamer := NewStreamer(w, flusher, completionID, createdTime, resolvedModel)
+  streamer.Role()
+
+  if cleanedReasoning != "" {
+    streamer.Reasoning(cleanedReasoning)
+  }
+
+  if hasToolCalls && len(toolCalls) > 0 {
+    for _, tc := range toolCalls {
+      streamer.ToolCallDelta(tc)
+    }
+  } else if cleanedContent != "" {
+    streamer.Content(cleanedContent)
+  }
+
+  streamer.Finish(finishReason)
+  streamer.Done()
+  return nil
+}
+
+func (s *QwenService) chatGradioChatResponse(w http.ResponseWriter, r *http.Request, req ChatCompletionRequest, resolvedModel string, maxTokens int) error {
  var systemPromptStr string
  var historyArray []map[string]interface{}
  var messageStr string
@@ -1587,11 +1977,21 @@ func (s *QwenService) Chat(w http.ResponseWriter, r *http.Request, req ChatCompl

 func main() {
  port := flag.Int("port", 8080, "Port to listen on")
-  endpoint := flag.String("endpoint", "https://halvo78-qwen3-8-flash-next-playground.hf.space", "Upstream HuggingFace Space URL")
-  defaultModel := flag.String("model", "Qwen/Qwen3.8-Flash-Next", "Default model ID")
+  defaultEndpoint := "https://microhero-qwen3-8-27b-uncensored-chat.hf.space"
+  if envEP := os.Getenv("QFLASH_ENDPOINT"); envEP != "" {
+    defaultEndpoint = envEP
+  }
+  endpoint := flag.String("endpoint", defaultEndpoint, "Upstream Hugging Face Space or OpenAI URL")
+  defaultModelVal := "Qwen/Qwen3.8-27B-Uncensored"
+  if envModel := os.Getenv("QFLASH_MODEL"); envModel != "" {
+    defaultModelVal = envModel
+  }
+  defaultModel := flag.String("model", defaultModelVal, "Default model ID")
+  mode := flag.String("mode", "auto", "Endpoint protocol mode: auto, respond, chat_response, openai")
  thinking := flag.Bool("thinking", true, "Enable thinking/reasoning mode by default")
  flag.BoolVar(thinking, "enable-thinking", true, "Alias for -thinking")
-  hfToken := flag.String("hf-token", "", "Optional HuggingFace Token for private/gated spaces")
+  hfToken := flag.String("hf-token", "", "Optional Hugging Face token for ZeroGPU quota or private spaces")
+  flag.StringVar(hfToken, "token", "", "Alias for -hf-token")
  apiKey := flag.String("api-key", "", "Optional upstream API key for BYOK inference")
  baseURL := flag.String("base-url", "", "Optional upstream base URL for BYOK inference")
  userAgent := flag.String("user-agent", "", "Custom User-Agent header")
@@ -1602,6 +2002,10 @@ func main() {

  flag.Parse()

+  if envMode := os.Getenv("QFLASH_MODE"); envMode != "" && *mode == "auto" {
+    *mode = envMode
+  }
+
  if *userAgent != "" {
    ConfiguredUserAgent = *userAgent
  }
@@ -1627,7 +2031,7 @@ func main() {
    }
  }

-  svc := NewQwenService(*endpoint, *defaultModel, *hfToken, *apiKey, *baseURL, proxyURL, *thinking)
+  svc := NewQwenService(*endpoint, *defaultModel, *mode, *hfToken, *apiKey, *baseURL, proxyURL, *thinking)

  mux := http.NewServeMux()