feat: add OpenAI-compatible prompt token caching details and stream_options support
This commit is contained in:
Luxferre
@@ -98,18 +98,23 @@ func (m *ChatMessage) GetContentString() string {
|
||||
return fmt.Sprintf("%v", m.Content)
|
||||
}
|
||||
|
||||
type StreamOptions struct {
|
||||
IncludeUsage bool `json:"include_usage,omitempty"`
|
||||
}
|
||||
|
||||
type ChatCompletionRequest struct {
|
||||
Model string `json:"model"`
|
||||
Messages []ChatMessage `json:"messages"`
|
||||
Tools []Tool `json:"tools,omitempty"`
|
||||
ToolChoice interface{} `json:"tool_choice,omitempty"`
|
||||
Stream bool `json:"stream"`
|
||||
MaxTokens int `json:"max_tokens"`
|
||||
MaxCompletionTokens int `json:"max_completion_tokens"`
|
||||
Temperature *float64 `json:"temperature,omitempty"`
|
||||
TopP *float64 `json:"top_p,omitempty"`
|
||||
ReasoningEffort string `json:"reasoning_effort,omitempty"`
|
||||
Thinking interface{} `json:"thinking,omitempty"`
|
||||
Model string `json:"model"`
|
||||
Messages []ChatMessage `json:"messages"`
|
||||
Tools []Tool `json:"tools,omitempty"`
|
||||
ToolChoice interface{} `json:"tool_choice,omitempty"`
|
||||
Stream bool `json:"stream"`
|
||||
StreamOptions *StreamOptions `json:"stream_options,omitempty"`
|
||||
MaxTokens int `json:"max_tokens"`
|
||||
MaxCompletionTokens int `json:"max_completion_tokens"`
|
||||
Temperature *float64 `json:"temperature,omitempty"`
|
||||
TopP *float64 `json:"top_p,omitempty"`
|
||||
ReasoningEffort string `json:"reasoning_effort,omitempty"`
|
||||
Thinking interface{} `json:"thinking,omitempty"`
|
||||
}
|
||||
|
||||
type ChatCompletionResponseChoice struct {
|
||||
@@ -118,10 +123,20 @@ type ChatCompletionResponseChoice struct {
|
||||
FinishReason string `json:"finish_reason"`
|
||||
}
|
||||
|
||||
type PromptTokensDetails struct {
|
||||
CachedTokens int `json:"cached_tokens"`
|
||||
}
|
||||
|
||||
type CompletionTokensDetails struct {
|
||||
ReasoningTokens int `json:"reasoning_tokens,omitempty"`
|
||||
}
|
||||
|
||||
type Usage struct {
|
||||
PromptTokens int `json:"prompt_tokens"`
|
||||
CompletionTokens int `json:"completion_tokens"`
|
||||
TotalTokens int `json:"total_tokens"`
|
||||
PromptTokens int `json:"prompt_tokens"`
|
||||
CompletionTokens int `json:"completion_tokens"`
|
||||
TotalTokens int `json:"total_tokens"`
|
||||
PromptTokensDetails *PromptTokensDetails `json:"prompt_tokens_details,omitempty"`
|
||||
CompletionTokensDetails *CompletionTokensDetails `json:"completion_tokens_details,omitempty"`
|
||||
}
|
||||
|
||||
type ChatCompletionResponse struct {
|
||||
@@ -152,6 +167,7 @@ type StreamResponse struct {
|
||||
Created int64 `json:"created"`
|
||||
Model string `json:"model"`
|
||||
Choices []StreamChoice `json:"choices"`
|
||||
Usage *Usage `json:"usage,omitempty"`
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -402,6 +418,23 @@ type FinalOutput struct {
|
||||
FinishReason string
|
||||
PromptTokens int
|
||||
CompletionTokens int
|
||||
CachedTokens int
|
||||
ReasoningTokens int
|
||||
}
|
||||
|
||||
func BuildUsage(promptTokens, completionTokens, cachedTokens, reasoningTokens int) Usage {
|
||||
totalTokens := promptTokens + completionTokens
|
||||
return Usage{
|
||||
PromptTokens: promptTokens,
|
||||
CompletionTokens: completionTokens,
|
||||
TotalTokens: totalTokens,
|
||||
PromptTokensDetails: &PromptTokensDetails{
|
||||
CachedTokens: cachedTokens,
|
||||
},
|
||||
CompletionTokensDetails: &CompletionTokensDetails{
|
||||
ReasoningTokens: reasoningTokens,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func WriteCompletionResponse(w http.ResponseWriter, completionID string, created int64, model string, out FinalOutput) {
|
||||
@@ -409,7 +442,6 @@ func WriteCompletionResponse(w http.ResponseWriter, completionID string, created
|
||||
if finish == "" {
|
||||
finish = "stop"
|
||||
}
|
||||
totalTokens := out.PromptTokens + out.CompletionTokens
|
||||
resp := ChatCompletionResponse{
|
||||
ID: completionID,
|
||||
Object: "chat.completion",
|
||||
@@ -427,11 +459,7 @@ func WriteCompletionResponse(w http.ResponseWriter, completionID string, created
|
||||
FinishReason: finish,
|
||||
},
|
||||
},
|
||||
Usage: Usage{
|
||||
PromptTokens: out.PromptTokens,
|
||||
CompletionTokens: out.CompletionTokens,
|
||||
TotalTokens: totalTokens,
|
||||
},
|
||||
Usage: BuildUsage(out.PromptTokens, out.CompletionTokens, out.CachedTokens, out.ReasoningTokens),
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(resp)
|
||||
@@ -472,6 +500,22 @@ func (s *Streamer) Finish(reason string) {
|
||||
sendStreamChunk(s.w, s.flusher, s.id, s.created, s.model, StreamDelta{}, &reason)
|
||||
}
|
||||
|
||||
func (s *Streamer) EmitUsage(usage Usage) {
|
||||
chunk := StreamResponse{
|
||||
ID: s.id,
|
||||
Object: "chat.completion.chunk",
|
||||
Created: s.created,
|
||||
Model: s.model,
|
||||
Choices: []StreamChoice{},
|
||||
Usage: &usage,
|
||||
}
|
||||
b, _ := json.Marshal(chunk)
|
||||
fmt.Fprintf(s.w, "data: %s\n\n", b)
|
||||
if s.flusher != nil {
|
||||
s.flusher.Flush()
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Streamer) Done() {
|
||||
fmt.Fprintf(s.w, "data: [DONE]\n\n")
|
||||
if s.flusher != nil {
|
||||
@@ -1397,10 +1441,14 @@ func (gw *QoronaGateway) HandleChatCompletions(w http.ResponseWriter, r *http.Re
|
||||
Cursor int `json:"cursor"`
|
||||
Done bool `json:"done"`
|
||||
FinalStatus struct {
|
||||
Status string `json:"status"`
|
||||
PromptTokens int `json:"prompt_tokens"`
|
||||
CompletionTokens int `json:"completion_tokens"`
|
||||
Error string `json:"error"`
|
||||
Status string `json:"status"`
|
||||
PromptTokens int `json:"prompt_tokens"`
|
||||
CompletionTokens int `json:"completion_tokens"`
|
||||
CachedTokens int `json:"cached_tokens"`
|
||||
PromptTokensDetails struct {
|
||||
CachedTokens int `json:"cached_tokens"`
|
||||
} `json:"prompt_tokens_details"`
|
||||
Error string `json:"error"`
|
||||
} `json:"final_status"`
|
||||
} `json:"result"`
|
||||
}
|
||||
@@ -1418,8 +1466,19 @@ func (gw *QoronaGateway) HandleChatCompletions(w http.ResponseWriter, r *http.Re
|
||||
finishReason = "tool_calls"
|
||||
}
|
||||
streamer.Finish(finishReason)
|
||||
|
||||
promptTokens := sResult.Result.FinalStatus.PromptTokens
|
||||
completionTokens := sResult.Result.FinalStatus.CompletionTokens
|
||||
cachedTokens := sResult.Result.FinalStatus.CachedTokens
|
||||
if cachedTokens == 0 {
|
||||
cachedTokens = sResult.Result.FinalStatus.PromptTokensDetails.CachedTokens
|
||||
}
|
||||
|
||||
if req.StreamOptions != nil && req.StreamOptions.IncludeUsage {
|
||||
streamer.EmitUsage(BuildUsage(promptTokens, completionTokens, cachedTokens, 0))
|
||||
}
|
||||
streamer.Done()
|
||||
log.Printf("[Stream] Completed run_id=%s finish_reason=%s", runID, finishReason)
|
||||
log.Printf("[Stream] Completed run_id=%s finish_reason=%s tokens=%d/%d", runID, finishReason, promptTokens, completionTokens)
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -1523,10 +1582,14 @@ func (gw *QoronaGateway) HandleChatCompletions(w http.ResponseWriter, r *http.Re
|
||||
Cursor int `json:"cursor"`
|
||||
Done bool `json:"done"`
|
||||
FinalStatus struct {
|
||||
Status string `json:"status"`
|
||||
PromptTokens int `json:"prompt_tokens"`
|
||||
CompletionTokens int `json:"completion_tokens"`
|
||||
Error string `json:"error"`
|
||||
Status string `json:"status"`
|
||||
PromptTokens int `json:"prompt_tokens"`
|
||||
CompletionTokens int `json:"completion_tokens"`
|
||||
CachedTokens int `json:"cached_tokens"`
|
||||
PromptTokensDetails struct {
|
||||
CachedTokens int `json:"cached_tokens"`
|
||||
} `json:"prompt_tokens_details"`
|
||||
Error string `json:"error"`
|
||||
} `json:"final_status"`
|
||||
} `json:"result"`
|
||||
}
|
||||
@@ -1539,6 +1602,10 @@ func (gw *QoronaGateway) HandleChatCompletions(w http.ResponseWriter, r *http.Re
|
||||
sResp.Body.Close()
|
||||
promptTokens = sResult.Result.FinalStatus.PromptTokens
|
||||
completionTokens = sResult.Result.FinalStatus.CompletionTokens
|
||||
cachedTokens := sResult.Result.FinalStatus.CachedTokens
|
||||
if cachedTokens == 0 {
|
||||
cachedTokens = sResult.Result.FinalStatus.PromptTokensDetails.CachedTokens
|
||||
}
|
||||
|
||||
fullContent := accumulatedContent.String()
|
||||
rawThinking := accumulatedThinking.String()
|
||||
@@ -1572,6 +1639,8 @@ func (gw *QoronaGateway) HandleChatCompletions(w http.ResponseWriter, r *http.Re
|
||||
FinishReason: finishReason,
|
||||
PromptTokens: promptTokens,
|
||||
CompletionTokens: completionTokens,
|
||||
CachedTokens: cachedTokens,
|
||||
ReasoningTokens: 0,
|
||||
}
|
||||
|
||||
WriteCompletionResponse(w, completionID, createdTime, modelName, out)
|
||||
|
||||
Reference in New Issue
Block a user