#include #include "hygate.h" #include #include #include /* forward declarations (defined later in this file) */ static char *content_to_string(json_object *content); static const char *get_str(json_object *obj, const char *key); static int count_content_tokens(json_object *content) { if (!content) return 0; enum json_type t = json_object_get_type(content); if (t == json_type_null) return 0; if (t == json_type_string) return tok_count(json_object_get_string(content), json_object_get_string_len(content)); if (t == json_type_array) { int total = 0; int n = json_object_array_length(content); for (int i = 0; i < n; i++) { json_object *part = json_object_array_get_idx(content, i); if (!part) continue; enum json_type pt = json_object_get_type(part); if (pt == json_type_string) { total += tok_count(json_object_get_string(part), json_object_get_string_len(part)); } else if (pt == json_type_object) { json_object *txt = NULL; if (json_object_object_get_ex(part, "text", &txt) && json_object_get_type(txt) == json_type_string) total += tok_count(json_object_get_string(txt), json_object_get_string_len(txt)); } } return total; } const char *s = json_object_to_json_string(content); return s ? tok_count_str(s) : 0; } /* Count tokens for a tool-call arguments JSON string as rendered by * tencent/Hy3's chat_template.jinja: each key is emitted raw inside * ... and each value is emitted * as a raw unescaped string (when string) or JSON (when non-string) * inside .... */ static int count_tool_args_tokens(const char *ar_str, int ar_len) { if (!ar_str || ar_len <= 0) return 0; json_object *parsed = json_tokener_parse(ar_str); if (parsed && json_object_get_type(parsed) == json_type_object) { int total = 0; json_object_object_foreach(parsed, key, val) { if (key && key[0]) total += tok_count_str(key); if (val) { if (json_object_get_type(val) == json_type_string) { total += tok_count(json_object_get_string(val), json_object_get_string_len(val)); } else { const char *vs = json_object_to_json_string(val); if (vs) total += tok_count_str(vs); } } } json_object_put(parsed); return total; } if (parsed) json_object_put(parsed); return tok_count(ar_str, ar_len); } /* ------------------------------------------------------------------ */ /* Token counting (embedded Tencent Hy3 BPE) */ /* ------------------------------------------------------------------ */ int count_messages_tokens(void *req_obj) { json_object *req = (json_object *)req_obj; if (!req) return 0; int total = 0; bool has_tools = false; json_object *tools = NULL; if (json_object_object_get_ex(req, "tools", &tools) && json_object_get_type(tools) == json_type_array) { int tn = json_object_array_length(tools); if (tn > 0) { has_tools = true; for (int i = 0; i < tn; i++) { json_object *ti = json_object_array_get_idx(tools, i); if (!ti) continue; const char *ts = json_object_to_json_string(ti); if (ts) total += tok_count_str(ts); } } } /* In tencent/Hy3's chat_template.jinja, preserved_thinking defaults to * true when tools are present and false when tools are absent (unless * explicitly overridden). Since spaces/tencent/Hy3 appends the final * turn as a user message, historical assistant reasoning_content is * only included in the prompt when preserved_thinking is active. */ bool preserved_thinking = has_tools; json_object *pt_obj = NULL; if (json_object_object_get_ex(req, "preserved_thinking", &pt_obj) && json_object_get_type(pt_obj) == json_type_boolean) { preserved_thinking = json_object_get_boolean(pt_obj) ? true : false; } json_object *messages = NULL; if (!json_object_object_get_ex(req, "messages", &messages) || json_object_get_type(messages) != json_type_array) return total; int n = json_object_array_length(messages); int sys_idx = -1; int last_nonsys_idx = -1; bool sys_has_content = false; for (int i = 0; i < n; i++) { json_object *m = json_object_array_get_idx(messages, i); if (!m) continue; const char *roleStr = get_str(m, "role"); if (strcmp(roleStr, "system") == 0 && !sys_has_content) { sys_idx = i; if (count_content_tokens(json_object_object_get(m, "content")) > 0) sys_has_content = true; } else { last_nonsys_idx = i; } } for (int i = 0; i < n; i++) { json_object *m = json_object_array_get_idx(messages, i); if (!m) continue; const char *roleStr = get_str(m, "role"); json_object *content = json_object_object_get(m, "content"); if (i == sys_idx) { total += count_content_tokens(content); continue; } if (i == last_nonsys_idx && (strcmp(roleStr, "tool") == 0 || strcmp(roleStr, "function") == 0)) { const char *toolName = get_str(m, "name"); if (toolName[0] == 0) toolName = get_str(m, "tool_call_id"); if (toolName[0] != 0) { char *lastContent = content_to_string(content); const char *lc = lastContent ? lastContent : ""; size_t need = strlen("Tool result for : ") + strlen(toolName) + strlen(lc) + 1; char *s = (char *)malloc(need); if (s) { snprintf(s, need, "Tool result for %s: %s", toolName, lc); total += tok_count_str(s); free(s); } else { total += tok_count_str(toolName) + count_content_tokens(content); } free(lastContent); } else { total += count_content_tokens(content); } continue; } total += count_content_tokens(content); /* Historical tool messages in tencent/Hy3 chat_template.jinja render * only visible_text(message['content']); tool_call_id and name are not * emitted into the prompt. */ if (strcmp(roleStr, "tool") == 0 || strcmp(roleStr, "function") == 0) { continue; } if (strcmp(roleStr, "assistant") == 0) { if (preserved_thinking) { json_object *rc = NULL; if (json_object_object_get_ex(m, "reasoning_content", &rc) && json_object_get_type(rc) == json_type_string) { total += tok_count(json_object_get_string(rc), json_object_get_string_len(rc)); } else if (json_object_object_get_ex(m, "reasoning", &rc) && json_object_get_type(rc) == json_type_string) { total += tok_count(json_object_get_string(rc), json_object_get_string_len(rc)); } } json_object *tcs = NULL; if (json_object_object_get_ex(m, "tool_calls", &tcs) && json_object_get_type(tcs) == json_type_array) { int tn = json_object_array_length(tcs); for (int j = 0; j < tn; j++) { json_object *tc = json_object_array_get_idx(tcs, j); if (!tc) continue; json_object *fn = NULL; if (json_object_object_get_ex(tc, "function", &fn) && json_object_get_type(fn) == json_type_object) { json_object *nm = NULL, *ar = NULL; if (json_object_object_get_ex(fn, "name", &nm) && json_object_get_type(nm) == json_type_string) total += tok_count(json_object_get_string(nm), json_object_get_string_len(nm)); if (json_object_object_get_ex(fn, "arguments", &ar) && json_object_get_type(ar) == json_type_string) total += count_tool_args_tokens(json_object_get_string(ar), json_object_get_string_len(ar)); } } } json_object *fc = NULL; if (json_object_object_get_ex(m, "function_call", &fc) && json_object_get_type(fc) == json_type_object) { json_object *nm = NULL, *ar = NULL; if (json_object_object_get_ex(fc, "name", &nm) && json_object_get_type(nm) == json_type_string) total += tok_count(json_object_get_string(nm), json_object_get_string_len(nm)); if (json_object_object_get_ex(fc, "arguments", &ar) && json_object_get_type(ar) == json_type_string) total += count_tool_args_tokens(json_object_get_string(ar), json_object_get_string_len(ar)); } } } return total; } int count_completion_tokens(const char *content, const char *reasoning, tool_call_t *tc, int n_tc) { int total = 0; if (content && content[0]) total += tok_count_str(content); if (reasoning && reasoning[0]) total += tok_count_str(reasoning); for (int i = 0; i < n_tc; i++) { if (tc[i].name && tc[i].name[0]) total += tok_count_str(tc[i].name); if (tc[i].arguments && tc[i].arguments[0]) total += count_tool_args_tokens(tc[i].arguments, (int)strlen(tc[i].arguments)); } return total; } /* Convert an OpenAI "content" field to a flat string. Caller frees. */ static char *content_to_string(json_object *content) { if (!content) return strdup(""); enum json_type t = json_object_get_type(content); if (t == json_type_null) return strdup(""); if (t == json_type_string) return strdup(json_object_get_string(content)); if (t == json_type_array) { strbuf sb; sb_init(&sb); if (!sb.buf) return strdup(""); int n = json_object_array_length(content); for (int i = 0; i < n; i++) { json_object *part = json_object_array_get_idx(content, i); if (!part) continue; enum json_type pt = json_object_get_type(part); if (pt == json_type_string) { sb_append_str(&sb, json_object_get_string(part)); } else if (pt == json_type_object) { json_object *txt = NULL; if (json_object_object_get_ex(part, "text", &txt) && json_object_get_type(txt) == json_type_string) sb_append_str(&sb, json_object_get_string(txt)); } } return sb.buf; } const char *s = json_object_to_json_string(content); return s ? strdup(s) : strdup(""); } static const char *get_str(json_object *obj, const char *key) { json_object *v = NULL; if (json_object_object_get_ex(obj, key, &v) && json_object_get_type(v) == json_type_string) return json_object_get_string(v); return ""; } static bool has_nonnull(json_object *req, const char *key) { json_object *o = NULL; if (!json_object_object_get_ex(req, key, &o)) return false; if (json_object_get_type(o) == json_type_null) return false; return true; } static double get_double(json_object *req, const char *key) { json_object *o = NULL; if (json_object_object_get_ex(req, key, &o)) { enum json_type t = json_object_get_type(o); if (t == json_type_double) return json_object_get_double(o); if (t == json_type_int) return (double)json_object_get_int(o); } return 0.0; } /* ------------------------------------------------------------------ */ /* Build Gradio payload */ /* ------------------------------------------------------------------ */ char *build_gradio_payload(void *req_obj) { json_object *req = (json_object *)req_obj; json_object *messages = NULL; if (!json_object_object_get_ex(req, "messages", &messages) || json_object_get_type(messages) != json_type_array) return NULL; int n = json_object_array_length(messages); int *nonSys = (int *)malloc(sizeof(int) * (size_t)(n > 0 ? n : 1)); if (!nonSys) return NULL; json_object *history_arr = json_object_new_array(); if (!history_arr) { free(nonSys); return NULL; } char *system_str = NULL; char *message_str = NULL; /* First pass: pick first system message as system prompt, collect the * rest as non-system messages. */ int nonSysN = 0; for (int i = 0; i < n; i++) { json_object *m = json_object_array_get_idx(messages, i); if (!m) continue; const char *roleStr = get_str(m, "role"); if (strcmp(roleStr, "system") == 0 && (!system_str || system_str[0] == 0)) { char *cs = content_to_string(json_object_object_get(m, "content")); free(system_str); system_str = cs; } else { nonSys[nonSysN++] = i; } } /* Build history from all but the last non-system message. */ for (int k = 0; k < nonSysN - 1; k++) { int i = nonSys[k]; json_object *m = json_object_array_get_idx(messages, i); const char *roleStr = get_str(m, "role"); json_object *item = json_object_new_object(); if (!item) continue; json_object_object_add(item, "role", json_object_new_string(roleStr)); json_object *content = json_object_object_get(m, "content"); char *cs = content_to_string(content); const char *cstr = cs ? cs : ""; if (strcmp(roleStr, "assistant") == 0) { if (cstr[0] != 0) json_object_object_add(item, "content", json_object_new_string(cstr)); else json_object_object_add(item, "content", json_object_new_null()); } else { json_object_object_add(item, "content", json_object_new_string(cstr)); } free(cs); if (strcmp(roleStr, "assistant") == 0) { const char *rc = get_str(m, "reasoning_content"); if (rc[0] != 0) json_object_object_add(item, "reasoning_content", json_object_new_string(rc)); json_object *tc = NULL; if (json_object_object_get_ex(m, "tool_calls", &tc) && json_object_get_type(tc) == json_type_array && json_object_array_length(tc) > 0) { /* Deep-copy the request's tool_calls into a fresh object so the * output tree does not alias a subtree owned by the caller's * request object. Aliasing caused a double-free / heap corruption * when json_object_put(root) and json_object_put(jreq) both tried * to release the shared node under concurrent worker threads. */ json_object *tc_copy = NULL; if (json_object_deep_copy(tc, &tc_copy, NULL) == 0 && tc_copy) json_object_object_add(item, "tool_calls", tc_copy); } } else if (strcmp(roleStr, "tool") == 0 || strcmp(roleStr, "function") == 0) { json_object_object_del(item, "role"); json_object_object_add(item, "role", json_object_new_string("tool")); const char *toolID = get_str(m, "tool_call_id"); if (toolID[0] == 0) toolID = get_str(m, "name"); if (toolID[0] != 0) json_object_object_add(item, "tool_call_id", json_object_new_string(toolID)); const char *nm = get_str(m, "name"); if (nm[0] != 0) json_object_object_add(item, "name", json_object_new_string(nm)); } json_object_array_add(history_arr, item); } /* Last non-system message becomes the message string. */ if (nonSysN > 0) { int i = nonSys[nonSysN - 1]; json_object *m = json_object_array_get_idx(messages, i); const char *roleStr = get_str(m, "role"); char *lastContent = content_to_string(json_object_object_get(m, "content")); const char *lc = lastContent ? lastContent : ""; if (strcmp(roleStr, "tool") == 0 || strcmp(roleStr, "function") == 0) { const char *toolName = get_str(m, "name"); if (toolName[0] == 0) toolName = get_str(m, "tool_call_id"); if (toolName[0] != 0) { size_t need = strlen("Tool result for : ") + strlen(toolName) + strlen(lc) + 1; char *s = (char *)malloc(need); if (s) { snprintf(s, need, "Tool result for %s: %s", toolName, lc); message_str = s; } free(lastContent); } else { message_str = lastContent; } } else { message_str = lastContent; } } free(nonSys); /* max_tokens resolution (mirrors ResolveMaxTokens). */ long mt = 0; if (has_nonnull(req, "max_tokens")) mt = (long)get_double(req, "max_tokens"); if (mt == 0 && has_nonnull(req, "max_completion_tokens")) mt = (long)get_double(req, "max_completion_tokens"); if (mt <= 0) mt = 131072; bool temp_present = has_nonnull(req, "temperature"); double temp = get_double(req, "temperature"); bool topp_present = has_nonnull(req, "top_p"); double topp = topp_present ? get_double(req, "top_p") : 0.0; const char *functions_json = ""; json_object *tools = NULL; if (json_object_object_get_ex(req, "tools", &tools) && json_object_get_type(tools) == json_type_array) { const char *s = json_object_to_json_string(tools); if (s) functions_json = s; } /* Assemble the Gradio data array. */ json_object *root = json_object_new_object(); json_object *data = json_object_new_array(); if (!root || !data) { if (root) json_object_put(root); if (data) json_object_put(data); json_object_put(history_arr); free(system_str); free(message_str); return NULL; } json_object_array_add(data, json_object_new_string(message_str ? message_str : "")); json_object_array_add(data, json_object_new_string(system_str ? system_str : "")); json_object_array_add(data, history_arr); json_object_array_add(data, json_object_new_string("high")); if (temp_present) json_object_array_add(data, json_object_new_double(temp)); else json_object_array_add(data, json_object_new_null()); json_object_array_add(data, json_object_new_int((int)mt)); if (topp_present) json_object_array_add(data, json_object_new_double(topp)); else json_object_array_add(data, json_object_new_int(0)); json_object_array_add(data, json_object_new_null()); json_object_array_add(data, json_object_new_string(functions_json)); json_object_object_add(root, "data", data); const char *s = json_object_to_json_string(root); char *payload = s ? strdup(s) : NULL; json_object_put(root); free(system_str); free(message_str); return payload; } /* ------------------------------------------------------------------ */ /* Parse Gradio stream payload */ /* ------------------------------------------------------------------ */ int parse_gradio_stream(const char *data_json, char **out_content, char **out_reasoning, tool_call_t **out_tc, int *out_n_tc) { *out_content = NULL; *out_reasoning = NULL; *out_tc = NULL; *out_n_tc = 0; json_object *root = json_tokener_parse(data_json); if (!root) return 0; if (json_object_get_type(root) != json_type_array) { json_object_put(root); return 0; } int n = json_object_array_length(root); if (n == 0) { json_object_put(root); return 0; } /* Gradio emits a double-wrapped array: [[content, reasoning, tools]]. Tolerate a single-wrapped [content, reasoning, tools] as well. */ json_object *first = json_object_array_get_idx(root, 0); json_object *inner; if (json_object_get_type(first) == json_type_array) { inner = first; } else if (n >= 2) { inner = root; } else { json_object_put(root); return 0; } int m = json_object_array_length(inner); if (m < 2) { json_object_put(root); return 0; } json_object *c = json_object_array_get_idx(inner, 0); json_object *r = json_object_array_get_idx(inner, 1); *out_content = (c && json_object_get_type(c) == json_type_string) ? strdup(json_object_get_string(c)) : strdup(""); *out_reasoning = (r && json_object_get_type(r) == json_type_string) ? strdup(json_object_get_string(r)) : strdup(""); if (!*out_content || !*out_reasoning) { free(*out_content); free(*out_reasoning); *out_content = NULL; *out_reasoning = NULL; json_object_put(root); return 0; } if (m >= 3) { json_object *tcobj = json_object_array_get_idx(inner, 2); if (tcobj && json_object_get_type(tcobj) == json_type_array) { int len = json_object_array_length(tcobj); if (len > 0) { tool_call_t *arr = (tool_call_t *)calloc((size_t)len, sizeof(tool_call_t)); if (arr) { int cnt = 0; for (int i = 0; i < len; i++) { json_object *e = json_object_array_get_idx(tcobj, i); if (json_object_get_type(e) != json_type_object) continue; json_object *fid = NULL, *ftype = NULL, *ffn = NULL, *fname = NULL, *fargs = NULL; json_object_object_get_ex(e, "id", &fid); json_object_object_get_ex(e, "type", &ftype); json_object_object_get_ex(e, "function", &ffn); if (ffn && json_object_get_type(ffn) == json_type_object) { json_object_object_get_ex(ffn, "name", &fname); json_object_object_get_ex(ffn, "arguments", &fargs); } arr[cnt].id = (fid && json_object_get_type(fid) == json_type_string) ? strdup(json_object_get_string(fid)) : strdup(""); arr[cnt].type = (ftype && json_object_get_type(ftype) == json_type_string) ? strdup(json_object_get_string(ftype)) : strdup(""); arr[cnt].name = (fname && json_object_get_type(fname) == json_type_string) ? strdup(json_object_get_string(fname)) : strdup(""); arr[cnt].arguments = (fargs && json_object_get_type(fargs) == json_type_string) ? strdup(json_object_get_string(fargs)) : strdup(""); if (!arr[cnt].id || !arr[cnt].type || !arr[cnt].name || !arr[cnt].arguments) { free(arr[cnt].id); free(arr[cnt].type); free(arr[cnt].name); free(arr[cnt].arguments); continue; } cnt++; } if (cnt > 0) { *out_tc = arr; *out_n_tc = cnt; } else { free(arr); } } } } } json_object_put(root); return 1; } /* ------------------------------------------------------------------ */ /* Build completion responses */ /* ------------------------------------------------------------------ */ char *build_completion_response(const char *id, long created, const char *model, const char *content, const char *reasoning, tool_call_t *tc, int n_tc, int prompt_tokens) { bool has_tc = n_tc > 0; json_object *root = json_object_new_object(); json_object_object_add(root, "id", json_object_new_string(id)); json_object_object_add(root, "object", json_object_new_string("chat.completion")); json_object_object_add(root, "created", json_object_new_int64(created)); json_object_object_add(root, "model", json_object_new_string(model)); json_object *choices = json_object_new_array(); json_object *choice = json_object_new_object(); json_object_object_add(choice, "index", json_object_new_int(0)); json_object *msg = json_object_new_object(); json_object_object_add(msg, "role", json_object_new_string("assistant")); if (has_tc && (content == NULL || content[0] == 0)) json_object_object_add(msg, "content", json_object_new_null()); else json_object_object_add(msg, "content", json_object_new_string(content ? content : "")); if (reasoning && reasoning[0] != 0) json_object_object_add(msg, "reasoning_content", json_object_new_string(reasoning)); if (has_tc) { json_object *tcs = json_object_new_array(); for (int i = 0; i < n_tc; i++) { json_object *t = json_object_new_object(); if (tc[i].id && tc[i].id[0]) json_object_object_add(t, "id", json_object_new_string(tc[i].id)); if (tc[i].type && tc[i].type[0]) json_object_object_add(t, "type", json_object_new_string(tc[i].type)); json_object *fn = json_object_new_object(); if (tc[i].name && tc[i].name[0]) json_object_object_add(fn, "name", json_object_new_string(tc[i].name)); json_object_object_add(fn, "arguments", json_object_new_string(tc[i].arguments ? tc[i].arguments : "")); json_object_object_add(t, "function", fn); json_object_array_add(tcs, t); } json_object_object_add(msg, "tool_calls", tcs); } json_object_object_add(choice, "message", msg); json_object_object_add(choice, "finish_reason", json_object_new_string(has_tc ? "tool_calls" : "stop")); json_object_array_add(choices, choice); json_object_object_add(root, "choices", choices); int completion_tokens = count_completion_tokens(content, reasoning, tc, n_tc); int reasoning_tokens = (reasoning && reasoning[0]) ? tok_count_str(reasoning) : 0; int total_tokens = prompt_tokens + completion_tokens; json_object *usage = json_object_new_object(); json_object_object_add(usage, "prompt_tokens", json_object_new_int(prompt_tokens)); json_object_object_add(usage, "completion_tokens", json_object_new_int(completion_tokens)); json_object_object_add(usage, "total_tokens", json_object_new_int(total_tokens)); json_object *details = json_object_new_object(); json_object_object_add(details, "reasoning_tokens", json_object_new_int(reasoning_tokens)); json_object_object_add(usage, "completion_tokens_details", details); json_object_object_add(root, "usage", usage); const char *s = json_object_to_json_string(root); char *out = s ? strdup(s) : NULL; json_object_put(root); return out; } char *toolcall_delta_json(tool_call_t *tc, int idx) { json_object *arr = json_object_new_array(); json_object *t = json_object_new_object(); json_object_object_add(t, "index", json_object_new_int(idx)); if (tc->id && tc->id[0]) json_object_object_add(t, "id", json_object_new_string(tc->id)); if (tc->type && tc->type[0]) json_object_object_add(t, "type", json_object_new_string(tc->type)); json_object *fn = json_object_new_object(); if (tc->name && tc->name[0]) json_object_object_add(fn, "name", json_object_new_string(tc->name)); json_object_object_add(fn, "arguments", json_object_new_string(tc->arguments ? tc->arguments : "")); json_object_object_add(t, "function", fn); json_object_array_add(arr, t); const char *s = json_object_to_json_string(arr); char *out = s ? strdup(s) : NULL; json_object_put(arr); return out; } char *toolcall_delta_argdelta(int idx, const char *delta) { json_object *arr = json_object_new_array(); json_object *t = json_object_new_object(); json_object_object_add(t, "index", json_object_new_int(idx)); json_object *fn = json_object_new_object(); json_object_object_add(fn, "arguments", json_object_new_string(delta ? delta : "")); json_object_object_add(t, "function", fn); json_object_array_add(arr, t); const char *s = json_object_to_json_string(arr); char *out = s ? strdup(s) : NULL; json_object_put(arr); return out; }