fixed thinking token logic
This commit is contained in:
@@ -684,6 +684,7 @@ type streamDelta struct {
|
|||||||
Delta struct {
|
Delta struct {
|
||||||
ReasoningContent string `json:"reasoning_content"`
|
ReasoningContent string `json:"reasoning_content"`
|
||||||
Reasoning string `json:"reasoning"`
|
Reasoning string `json:"reasoning"`
|
||||||
|
Thought string `json:"thought"`
|
||||||
Content string `json:"content"`
|
Content string `json:"content"`
|
||||||
ToolCalls []struct {
|
ToolCalls []struct {
|
||||||
Index int `json:"index"`
|
Index int `json:"index"`
|
||||||
@@ -698,6 +699,19 @@ type streamDelta struct {
|
|||||||
Usage *Usage `json:"usage"`
|
Usage *Usage `json:"usage"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func cleanMessagesForLLM(msgs []Message) []Message {
|
||||||
|
out := make([]Message, len(msgs))
|
||||||
|
for i, m := range msgs {
|
||||||
|
out[i] = Message{
|
||||||
|
Role: m.Role,
|
||||||
|
Content: m.Content,
|
||||||
|
ToolCalls: m.ToolCalls,
|
||||||
|
ToolCallID: m.ToolCallID,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
func sanitizeMessages(msgs []Message) {
|
func sanitizeMessages(msgs []Message) {
|
||||||
for i := range msgs {
|
for i := range msgs {
|
||||||
if msgs[i].Role == "assistant" && len(msgs[i].ToolCalls) > 0 {
|
if msgs[i].Role == "assistant" && len(msgs[i].ToolCalls) > 0 {
|
||||||
@@ -721,8 +735,9 @@ func isInvalidAssistantErr(err error) bool {
|
|||||||
|
|
||||||
func llm(ctx context.Context, cfg *Cfg, msgs []Message, tools []map[string]any) (Message, Usage, error) {
|
func llm(ctx context.Context, cfg *Cfg, msgs []Message, tools []map[string]any) (Message, Usage, error) {
|
||||||
if err := ctx.Err(); err != nil { return Message{}, Usage{}, err }
|
if err := ctx.Err(); err != nil { return Message{}, Usage{}, err }
|
||||||
sanitizeMessages(msgs)
|
cleanMsgs := cleanMessagesForLLM(msgs)
|
||||||
p := map[string]any{"model": cfg.Model, "temperature": cfg.Temperature, "messages": msgs, "stream": cfg.Stream}
|
sanitizeMessages(cleanMsgs)
|
||||||
|
p := map[string]any{"model": cfg.Model, "temperature": cfg.Temperature, "messages": cleanMsgs, "stream": cfg.Stream}
|
||||||
if tools != nil { p["tools"] = tools }
|
if tools != nil { p["tools"] = tools }
|
||||||
if cfg.Stream { p["stream_options"] = map[string]any{"include_usage": true} }
|
if cfg.Stream { p["stream_options"] = map[string]any{"include_usage": true} }
|
||||||
for k, v := range cfg.Raw {
|
for k, v := range cfg.Raw {
|
||||||
@@ -798,6 +813,7 @@ func llm(ctx context.Context, cfg *Cfg, msgs []Message, tools []map[string]any)
|
|||||||
Message struct {
|
Message struct {
|
||||||
Message
|
Message
|
||||||
Reasoning string `json:"reasoning"`
|
Reasoning string `json:"reasoning"`
|
||||||
|
Thought string `json:"thought"`
|
||||||
} `json:"message"`
|
} `json:"message"`
|
||||||
} `json:"choices"`
|
} `json:"choices"`
|
||||||
Usage Usage `json:"usage"`
|
Usage Usage `json:"usage"`
|
||||||
@@ -809,6 +825,7 @@ func llm(ctx context.Context, cfg *Cfg, msgs []Message, tools []map[string]any)
|
|||||||
if len(cr.Choices) == 0 { return Message{}, Usage{}, errors.New("empty choices in LLM response") }
|
if len(cr.Choices) == 0 { return Message{}, Usage{}, errors.New("empty choices in LLM response") }
|
||||||
m := cr.Choices[0].Message.Message
|
m := cr.Choices[0].Message.Message
|
||||||
if m.ReasoningContent == "" { m.ReasoningContent = cr.Choices[0].Message.Reasoning }
|
if m.ReasoningContent == "" { m.ReasoningContent = cr.Choices[0].Message.Reasoning }
|
||||||
|
if m.ReasoningContent == "" { m.ReasoningContent = cr.Choices[0].Message.Thought }
|
||||||
u := cr.Usage
|
u := cr.Usage
|
||||||
if u.PromptTokens == 0 {
|
if u.PromptTokens == 0 {
|
||||||
u.PromptTokens = estTokens(msgs)
|
u.PromptTokens = estTokens(msgs)
|
||||||
@@ -828,7 +845,7 @@ func llm(ctx context.Context, cfg *Cfg, msgs []Message, tools []map[string]any)
|
|||||||
|
|
||||||
func parseStream(ctx context.Context, r io.Reader) (Message, Usage, error) {
|
func parseStream(ctx context.Context, r io.Reader) (Message, Usage, error) {
|
||||||
var content, reas string
|
var content, reas string
|
||||||
var rh, ch bool
|
var inReasoning bool
|
||||||
var lineBuf string
|
var lineBuf string
|
||||||
var mdSt mdState
|
var mdSt mdState
|
||||||
var tblBuf []string
|
var tblBuf []string
|
||||||
@@ -860,14 +877,25 @@ func parseStream(ctx context.Context, r io.Reader) (Message, Usage, error) {
|
|||||||
dl := d.Choices[0].Delta
|
dl := d.Choices[0].Delta
|
||||||
rc := dl.ReasoningContent
|
rc := dl.ReasoningContent
|
||||||
if rc == "" { rc = dl.Reasoning }
|
if rc == "" { rc = dl.Reasoning }
|
||||||
|
if rc == "" { rc = dl.Thought }
|
||||||
if rc != "" {
|
if rc != "" {
|
||||||
if !rh { fmt.Println(c("--- reasoning start ---", 36)); rh = true }
|
if !inReasoning {
|
||||||
|
if content != "" {
|
||||||
|
flushTable()
|
||||||
|
if lineBuf != "" { fmt.Println(renderMDLine(lineBuf, &mdSt)); lineBuf = "" }
|
||||||
|
fmt.Println()
|
||||||
|
}
|
||||||
|
fmt.Println(c("--- reasoning start ---", 36))
|
||||||
|
inReasoning = true
|
||||||
|
}
|
||||||
fmt.Print(c(rc, 2))
|
fmt.Print(c(rc, 2))
|
||||||
reas += rc
|
reas += rc
|
||||||
}
|
}
|
||||||
if dl.Content != "" {
|
if dl.Content != "" {
|
||||||
if rh && !ch { fmt.Print("\n" + c("--- reasoning end ---", 36) + "\n\n") }
|
if inReasoning {
|
||||||
ch = true
|
fmt.Print("\n" + c("--- reasoning end ---", 36) + "\n\n")
|
||||||
|
inReasoning = false
|
||||||
|
}
|
||||||
content += dl.Content
|
content += dl.Content
|
||||||
lineBuf += dl.Content
|
lineBuf += dl.Content
|
||||||
for {
|
for {
|
||||||
@@ -911,7 +939,7 @@ func parseStream(ctx context.Context, r io.Reader) (Message, Usage, error) {
|
|||||||
fmt.Println(renderMDLine(lineBuf, &mdSt))
|
fmt.Println(renderMDLine(lineBuf, &mdSt))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if rh && !ch {
|
if inReasoning {
|
||||||
fmt.Print("\n" + c("--- reasoning end ---", 36) + "\n")
|
fmt.Print("\n" + c("--- reasoning end ---", 36) + "\n")
|
||||||
}
|
}
|
||||||
m := Message{Role: "assistant"}
|
m := Message{Role: "assistant"}
|
||||||
|
|||||||
Reference in New Issue
Block a user