From 18ebe25d34f9e83507fdf83902933f894d3e69eb Mon Sep 17 00:00:00 2001 From: Luxferre Date: Thu, 15 Oct 2026 23:13:11 +0300 Subject: [PATCH] tok/mkf upd --- Makefile | 5 ++- README.md | 12 ++++--- jsonapi.c | 93 ++++++++++++++++++++++++++++++++++++++++++----------- tokenizer.c | 12 +++---- 4 files changed, 92 insertions(+), 30 deletions(-) diff --git a/Makefile b/Makefile index 4eb552e..b75538d 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ LIBS = $$(pkg-config --cflags --libs libcurl json-c 2>/dev/null || echo '-lcurl # pkg-config --static provides (nghttp2, ssl, crypto, z, ...). STATICLIBS = $$(pkg-config --static --cflags --libs libcurl json-c 2>/dev/null || echo '-lcurl -ljson-c') -.PHONY: all static clean +.PHONY: all static clean install all: $(BIN) @@ -31,5 +31,8 @@ $(STATICBIN): $(SRCS) $(HDRS) @test -d $(BINDIR) || mkdir $(BINDIR) $(CC) $(CFLAGS) -static $(SRCS) -o $@ $(STATICLIBS) +install: $(BIN) + install -m 0755 $(BIN) $(DESTDIR)/usr/local/bin/hygate + clean: rm -rf $(BINDIR) \ No newline at end of file diff --git a/README.md b/README.md index e7bf0fe..eea1d21 100644 --- a/README.md +++ b/README.md @@ -77,12 +77,14 @@ pre-tokenizer and BPE logic in `tokenizer.c`) — no upstream token counts are available from the Gradio backend. - `prompt_tokens` sums the active `system` prompt, each message's `content` - (or individual multi-part content parts), `reasoning_content`, `tool_calls` - / `function_call` (`name` and `arguments`), `tool` result wrappers, and any - request `tools` function definitions (`name`, `description`, `parameters`). + (or individual multi-part content parts), `reasoning_content` when + `preserved_thinking` is active (enabled by default when `tools` are + present, matching `tencent/Hy3`'s `chat_template.jinja`), `tool_calls` / + `function_call` (`name` and unpacked `arguments` key/value pairs), the + final `tool` result wrapper, and any request `tools` function definitions. - `completion_tokens` sums the assistant `content`, `reasoning_content`, and - any emitted `tool_calls` (`name` and `arguments`), with `reasoning_tokens` - broken out under `completion_tokens_details`. + any emitted `tool_calls` (`name` and unpacked `arguments` key/value pairs), + with `reasoning_tokens` broken out under `completion_tokens_details`. - `total_tokens = prompt_tokens + completion_tokens`. The embedded tokenizer implements `tencent/Hy3`'s exact 4-stage `Sequence` diff --git a/jsonapi.c b/jsonapi.c index af70c87..87b3de5 100644 --- a/jsonapi.c +++ b/jsonapi.c @@ -39,6 +39,35 @@ static int count_content_tokens(json_object *content) { return s ? tok_count_str(s) : 0; } +/* Count tokens for a tool-call arguments JSON string as rendered by + * tencent/Hy3's chat_template.jinja: each key is emitted raw inside + * ... and each value is emitted + * as a raw unescaped string (when string) or JSON (when non-string) + * inside .... */ +static int count_tool_args_tokens(const char *ar_str, int ar_len) { + if (!ar_str || ar_len <= 0) return 0; + json_object *parsed = json_tokener_parse(ar_str); + if (parsed && json_object_get_type(parsed) == json_type_object) { + int total = 0; + json_object_object_foreach(parsed, key, val) { + if (key && key[0]) total += tok_count_str(key); + if (val) { + if (json_object_get_type(val) == json_type_string) { + total += tok_count(json_object_get_string(val), + json_object_get_string_len(val)); + } else { + const char *vs = json_object_to_json_string(val); + if (vs) total += tok_count_str(vs); + } + } + } + json_object_put(parsed); + return total; + } + if (parsed) json_object_put(parsed); + return tok_count(ar_str, ar_len); +} + /* ------------------------------------------------------------------ */ /* Token counting (embedded Tencent Hy3 BPE) */ /* ------------------------------------------------------------------ */ @@ -47,12 +76,32 @@ int count_messages_tokens(void *req_obj) { if (!req) return 0; int total = 0; + bool has_tools = false; json_object *tools = NULL; if (json_object_object_get_ex(req, "tools", &tools) && - json_object_get_type(tools) == json_type_array && - json_object_array_length(tools) > 0) { - const char *ts = json_object_to_json_string(tools); - if (ts) total += tok_count_str(ts); + json_object_get_type(tools) == json_type_array) { + int tn = json_object_array_length(tools); + if (tn > 0) { + has_tools = true; + for (int i = 0; i < tn; i++) { + json_object *ti = json_object_array_get_idx(tools, i); + if (!ti) continue; + const char *ts = json_object_to_json_string(ti); + if (ts) total += tok_count_str(ts); + } + } + } + + /* In tencent/Hy3's chat_template.jinja, preserved_thinking defaults to + * true when tools are present and false when tools are absent (unless + * explicitly overridden). Since spaces/tencent/Hy3 appends the final + * turn as a user message, historical assistant reasoning_content is + * only included in the prompt when preserved_thinking is active. */ + bool preserved_thinking = has_tools; + json_object *pt_obj = NULL; + if (json_object_object_get_ex(req, "preserved_thinking", &pt_obj) && + json_object_get_type(pt_obj) == json_type_boolean) { + preserved_thinking = json_object_get_boolean(pt_obj) ? true : false; } json_object *messages = NULL; @@ -114,20 +163,26 @@ int count_messages_tokens(void *req_obj) { total += count_content_tokens(content); + /* Historical tool messages in tencent/Hy3 chat_template.jinja render + * only visible_text(message['content']); tool_call_id and name are not + * emitted into the prompt. */ if (strcmp(roleStr, "tool") == 0 || strcmp(roleStr, "function") == 0) { - const char *toolID = get_str(m, "tool_call_id"); - const char *nm = get_str(m, "name"); - if (toolID[0] != 0) total += tok_count_str(toolID); - if (nm[0] != 0) total += tok_count_str(nm); continue; } if (strcmp(roleStr, "assistant") == 0) { - json_object *rc = NULL; - if (json_object_object_get_ex(m, "reasoning_content", &rc) && - json_object_get_type(rc) == json_type_string) - total += tok_count(json_object_get_string(rc), - json_object_get_string_len(rc)); + if (preserved_thinking) { + json_object *rc = NULL; + if (json_object_object_get_ex(m, "reasoning_content", &rc) && + json_object_get_type(rc) == json_type_string) { + total += tok_count(json_object_get_string(rc), + json_object_get_string_len(rc)); + } else if (json_object_object_get_ex(m, "reasoning", &rc) && + json_object_get_type(rc) == json_type_string) { + total += tok_count(json_object_get_string(rc), + json_object_get_string_len(rc)); + } + } json_object *tcs = NULL; if (json_object_object_get_ex(m, "tool_calls", &tcs) && json_object_get_type(tcs) == json_type_array) { @@ -145,8 +200,8 @@ int count_messages_tokens(void *req_obj) { json_object_get_string_len(nm)); if (json_object_object_get_ex(fn, "arguments", &ar) && json_object_get_type(ar) == json_type_string) - total += tok_count(json_object_get_string(ar), - json_object_get_string_len(ar)); + total += count_tool_args_tokens(json_object_get_string(ar), + json_object_get_string_len(ar)); } } } @@ -160,8 +215,8 @@ int count_messages_tokens(void *req_obj) { json_object_get_string_len(nm)); if (json_object_object_get_ex(fc, "arguments", &ar) && json_object_get_type(ar) == json_type_string) - total += tok_count(json_object_get_string(ar), - json_object_get_string_len(ar)); + total += count_tool_args_tokens(json_object_get_string(ar), + json_object_get_string_len(ar)); } } } @@ -175,7 +230,9 @@ int count_completion_tokens(const char *content, const char *reasoning, if (reasoning && reasoning[0]) total += tok_count_str(reasoning); for (int i = 0; i < n_tc; i++) { if (tc[i].name && tc[i].name[0]) total += tok_count_str(tc[i].name); - if (tc[i].arguments && tc[i].arguments[0]) total += tok_count_str(tc[i].arguments); + if (tc[i].arguments && tc[i].arguments[0]) + total += count_tool_args_tokens(tc[i].arguments, + (int)strlen(tc[i].arguments)); } return total; } diff --git a/tokenizer.c b/tokenizer.c index 49b9348..de2845e 100644 --- a/tokenizer.c +++ b/tokenizer.c @@ -30,11 +30,11 @@ static int *tok_ranks = NULL; static uint16_t *tok_klen = NULL; static unsigned int tok_mask = 0; -static unsigned long tok_fnv1a(const unsigned char *s, int len) { - unsigned long h = 1469598103934665603UL; +static uint64_t tok_fnv1a(const unsigned char *s, int len) { + uint64_t h = 1469598103934665603ULL; for (int i = 0; i < len; i++) { - h ^= (unsigned long)s[i]; - h *= 1099511628211UL; + h ^= (uint64_t)s[i]; + h *= 1099511628211ULL; } return h; } @@ -56,7 +56,7 @@ static void tok_init(void) { for (int i = 0; i < tok_ntok; i++) { uint16_t kl = tok_len[i]; const char *key = (const char *)(tok_blob + offset); - unsigned long h = tok_fnv1a((const unsigned char *)key, kl) & mask; + unsigned int h = (unsigned int)tok_fnv1a((const unsigned char *)key, kl) & mask; while (keys[h]) { h = (h + 1) & mask; } @@ -73,7 +73,7 @@ static void tok_init(void) { static int tok_lookup(const unsigned char *s, int len) { if (!tok_keys) return -1; - unsigned long h = tok_fnv1a(s, len) & tok_mask; + unsigned int h = (unsigned int)tok_fnv1a(s, len) & tok_mask; while (tok_keys[h]) { if ((int)tok_klen[h] == len && memcmp(tok_keys[h], s, (size_t)len) == 0) return tok_ranks[h];