luxferre/hygate-c
1
0
Fork 0
Code Actions Activity

tok/mkf upd

Browse Source
This commit is contained in:
Luxferre
2026-10-15 23:13:11 +03:00
parent 561d90629f
commit 18ebe25d34
4 changed files with 92 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
jsonapi.c
+75 -18
View File
@@ -39,6 +39,35 @@ static int count_content_tokens(json_object *content) {
return s ? tok_count_str(s) : 0;
}
/* Count tokens for a tool-call arguments JSON string as rendered by
* tencent/Hy3's chat_template.jinja: each key is emitted raw inside
* <arg_key:opensource>...</arg_key:opensource> and each value is emitted
* as a raw unescaped string (when string) or JSON (when non-string)
* inside <arg_value:opensource>...</arg_value:opensource>. */
static int count_tool_args_tokens(const char *ar_str, int ar_len) {
if (!ar_str || ar_len <= 0) return 0;
json_object *parsed = json_tokener_parse(ar_str);
if (parsed && json_object_get_type(parsed) == json_type_object) {
int total = 0;
json_object_object_foreach(parsed, key, val) {
if (key && key[0]) total += tok_count_str(key);
if (val) {
if (json_object_get_type(val) == json_type_string) {
total += tok_count(json_object_get_string(val),
json_object_get_string_len(val));
} else {
const char *vs = json_object_to_json_string(val);
if (vs) total += tok_count_str(vs);
}
}
}
json_object_put(parsed);
return total;
}
if (parsed) json_object_put(parsed);
return tok_count(ar_str, ar_len);
}
/* ------------------------------------------------------------------ */
/* Token counting (embedded Tencent Hy3 BPE) */
/* ------------------------------------------------------------------ */
@@ -47,12 +76,32 @@ int count_messages_tokens(void *req_obj) {
if (!req) return 0;
int total = 0;
bool has_tools = false;
json_object *tools = NULL;
if (json_object_object_get_ex(req, "tools", &tools) &&
json_object_get_type(tools) == json_type_array &&
json_object_array_length(tools) > 0) {
const char *ts = json_object_to_json_string(tools);
if (ts) total += tok_count_str(ts);
json_object_get_type(tools) == json_type_array) {
int tn = json_object_array_length(tools);
if (tn > 0) {
has_tools = true;
for (int i = 0; i < tn; i++) {
json_object *ti = json_object_array_get_idx(tools, i);
if (!ti) continue;
const char *ts = json_object_to_json_string(ti);
if (ts) total += tok_count_str(ts);
}
}
}
/* In tencent/Hy3's chat_template.jinja, preserved_thinking defaults to
* true when tools are present and false when tools are absent (unless
* explicitly overridden). Since spaces/tencent/Hy3 appends the final
* turn as a user message, historical assistant reasoning_content is
* only included in the prompt when preserved_thinking is active. */
bool preserved_thinking = has_tools;
json_object *pt_obj = NULL;
if (json_object_object_get_ex(req, "preserved_thinking", &pt_obj) &&
json_object_get_type(pt_obj) == json_type_boolean) {
preserved_thinking = json_object_get_boolean(pt_obj) ? true : false;
}
json_object *messages = NULL;
@@ -114,20 +163,26 @@ int count_messages_tokens(void *req_obj) {
total += count_content_tokens(content);
/* Historical tool messages in tencent/Hy3 chat_template.jinja render
* only visible_text(message['content']); tool_call_id and name are not
* emitted into the prompt. */
if (strcmp(roleStr, "tool") == 0 || strcmp(roleStr, "function") == 0) {
const char *toolID = get_str(m, "tool_call_id");
const char *nm = get_str(m, "name");
if (toolID[0] != 0) total += tok_count_str(toolID);
if (nm[0] != 0) total += tok_count_str(nm);
continue;
}
if (strcmp(roleStr, "assistant") == 0) {
json_object *rc = NULL;
if (json_object_object_get_ex(m, "reasoning_content", &rc) &&
json_object_get_type(rc) == json_type_string)
total += tok_count(json_object_get_string(rc),
json_object_get_string_len(rc));
if (preserved_thinking) {
json_object *rc = NULL;
if (json_object_object_get_ex(m, "reasoning_content", &rc) &&
json_object_get_type(rc) == json_type_string) {
total += tok_count(json_object_get_string(rc),
json_object_get_string_len(rc));
} else if (json_object_object_get_ex(m, "reasoning", &rc) &&
json_object_get_type(rc) == json_type_string) {
total += tok_count(json_object_get_string(rc),
json_object_get_string_len(rc));
}
}
json_object *tcs = NULL;
if (json_object_object_get_ex(m, "tool_calls", &tcs) &&
json_object_get_type(tcs) == json_type_array) {
@@ -145,8 +200,8 @@ int count_messages_tokens(void *req_obj) {
json_object_get_string_len(nm));
if (json_object_object_get_ex(fn, "arguments", &ar) &&
json_object_get_type(ar) == json_type_string)
total += tok_count(json_object_get_string(ar),
json_object_get_string_len(ar));
total += count_tool_args_tokens(json_object_get_string(ar),
json_object_get_string_len(ar));
}
}
}
@@ -160,8 +215,8 @@ int count_messages_tokens(void *req_obj) {
json_object_get_string_len(nm));
if (json_object_object_get_ex(fc, "arguments", &ar) &&
json_object_get_type(ar) == json_type_string)
total += tok_count(json_object_get_string(ar),
json_object_get_string_len(ar));
total += count_tool_args_tokens(json_object_get_string(ar),
json_object_get_string_len(ar));
}
}
}
@@ -175,7 +230,9 @@ int count_completion_tokens(const char *content, const char *reasoning,
if (reasoning && reasoning[0]) total += tok_count_str(reasoning);
for (int i = 0; i < n_tc; i++) {
if (tc[i].name && tc[i].name[0]) total += tok_count_str(tc[i].name);
if (tc[i].arguments && tc[i].arguments[0]) total += tok_count_str(tc[i].arguments);
if (tc[i].arguments && tc[i].arguments[0])
total += count_tool_args_tokens(tc[i].arguments,
(int)strlen(tc[i].arguments));
}
return total;
}