tok/mkf upd

Makefile

@@ -17,7 +17,7 @@ LIBS = $$(pkg-config --cflags --libs libcurl json-c 2>/dev/null || echo '-lcurl
 # pkg-config --static provides (nghttp2, ssl, crypto, z, ...).
 STATICLIBS = $$(pkg-config --static --cflags --libs libcurl json-c 2>/dev/null || echo '-lcurl -ljson-c')
 
-.PHONY: all static clean
+.PHONY: all static clean install
 
 all: $(BIN)
 
@@ -31,5 +31,8 @@ $(STATICBIN): $(SRCS) $(HDRS)
 	@test -d $(BINDIR) || mkdir $(BINDIR)
 	$(CC) $(CFLAGS) -static $(SRCS) -o $@ $(STATICLIBS)
 
+install: $(BIN)
+	install -m 0755 $(BIN) $(DESTDIR)/usr/local/bin/hygate
+
 clean:
 	rm -rf $(BINDIR)

README.md

@@ -77,12 +77,14 @@ pre-tokenizer and BPE logic in `tokenizer.c`) — no upstream token counts are
 available from the Gradio backend.
 
 - `prompt_tokens` sums the active `system` prompt, each message's `content`
-  (or individual multi-part content parts), `reasoning_content`, `tool_calls`
+  (or individual multi-part content parts), `reasoning_content` when
-  / `function_call` (`name` and `arguments`), `tool` result wrappers, and any
+  `preserved_thinking` is active (enabled by default when `tools` are
-  request `tools` function definitions (`name`, `description`, `parameters`).
+  present, matching `tencent/Hy3`'s `chat_template.jinja`), `tool_calls` /
+  `function_call` (`name` and unpacked `arguments` key/value pairs), the
+  final `tool` result wrapper, and any request `tools` function definitions.
 - `completion_tokens` sums the assistant `content`, `reasoning_content`, and
-  any emitted `tool_calls` (`name` and `arguments`), with `reasoning_tokens`
+  any emitted `tool_calls` (`name` and unpacked `arguments` key/value pairs),
-  broken out under `completion_tokens_details`.
+  with `reasoning_tokens` broken out under `completion_tokens_details`.
 - `total_tokens = prompt_tokens + completion_tokens`.
 
 The embedded tokenizer implements `tencent/Hy3`'s exact 4-stage `Sequence`

jsonapi.c

@@ -39,6 +39,35 @@ static int count_content_tokens(json_object *content) {
   return s ? tok_count_str(s) : 0;
 }
 
+/* Count tokens for a tool-call arguments JSON string as rendered by
+ * tencent/Hy3's chat_template.jinja: each key is emitted raw inside
+ * <arg_key:opensource>...</arg_key:opensource> and each value is emitted
+ * as a raw unescaped string (when string) or JSON (when non-string)
+ * inside <arg_value:opensource>...</arg_value:opensource>. */
+static int count_tool_args_tokens(const char *ar_str, int ar_len) {
+  if (!ar_str || ar_len <= 0) return 0;
+  json_object *parsed = json_tokener_parse(ar_str);
+  if (parsed && json_object_get_type(parsed) == json_type_object) {
+    int total = 0;
+    json_object_object_foreach(parsed, key, val) {
+      if (key && key[0]) total += tok_count_str(key);
+      if (val) {
+        if (json_object_get_type(val) == json_type_string) {
+          total += tok_count(json_object_get_string(val),
+                             json_object_get_string_len(val));
+        } else {
+          const char *vs = json_object_to_json_string(val);
+          if (vs) total += tok_count_str(vs);
+        }
+      }
+    }
+    json_object_put(parsed);
+    return total;
+  }
+  if (parsed) json_object_put(parsed);
+  return tok_count(ar_str, ar_len);
+}
+
 /* ------------------------------------------------------------------ */
 /* Token counting (embedded Tencent Hy3 BPE)                           */
 /* ------------------------------------------------------------------ */
@@ -47,12 +76,32 @@ int count_messages_tokens(void *req_obj) {
   if (!req) return 0;
   int total = 0;
 
+  bool has_tools = false;
   json_object *tools = NULL;
   if (json_object_object_get_ex(req, "tools", &tools) &&
-      json_object_get_type(tools) == json_type_array &&
+      json_object_get_type(tools) == json_type_array) {
-      json_object_array_length(tools) > 0) {
+    int tn = json_object_array_length(tools);
-    const char *ts = json_object_to_json_string(tools);
+    if (tn > 0) {
-    if (ts) total += tok_count_str(ts);
+      has_tools = true;
+      for (int i = 0; i < tn; i++) {
+        json_object *ti = json_object_array_get_idx(tools, i);
+        if (!ti) continue;
+        const char *ts = json_object_to_json_string(ti);
+        if (ts) total += tok_count_str(ts);
+      }
+    }
+  }
+
+  /* In tencent/Hy3's chat_template.jinja, preserved_thinking defaults to
+   * true when tools are present and false when tools are absent (unless
+   * explicitly overridden). Since spaces/tencent/Hy3 appends the final
+   * turn as a user message, historical assistant reasoning_content is
+   * only included in the prompt when preserved_thinking is active. */
+  bool preserved_thinking = has_tools;
+  json_object *pt_obj = NULL;
+  if (json_object_object_get_ex(req, "preserved_thinking", &pt_obj) &&
+      json_object_get_type(pt_obj) == json_type_boolean) {
+    preserved_thinking = json_object_get_boolean(pt_obj) ? true : false;
   }
 
   json_object *messages = NULL;
@@ -114,20 +163,26 @@ int count_messages_tokens(void *req_obj) {
 
     total += count_content_tokens(content);
 
+    /* Historical tool messages in tencent/Hy3 chat_template.jinja render
+     * only visible_text(message['content']); tool_call_id and name are not
+     * emitted into the prompt. */
     if (strcmp(roleStr, "tool") == 0 || strcmp(roleStr, "function") == 0) {
-      const char *toolID = get_str(m, "tool_call_id");
-      const char *nm = get_str(m, "name");
-      if (toolID[0] != 0) total += tok_count_str(toolID);
-      if (nm[0] != 0) total += tok_count_str(nm);
       continue;
     }
 
     if (strcmp(roleStr, "assistant") == 0) {
-      json_object *rc = NULL;
+      if (preserved_thinking) {
-      if (json_object_object_get_ex(m, "reasoning_content", &rc) &&
+        json_object *rc = NULL;
-          json_object_get_type(rc) == json_type_string)
+        if (json_object_object_get_ex(m, "reasoning_content", &rc) &&
-        total += tok_count(json_object_get_string(rc),
+            json_object_get_type(rc) == json_type_string) {
-                           json_object_get_string_len(rc));
+          total += tok_count(json_object_get_string(rc),
+                             json_object_get_string_len(rc));
+        } else if (json_object_object_get_ex(m, "reasoning", &rc) &&
+                   json_object_get_type(rc) == json_type_string) {
+          total += tok_count(json_object_get_string(rc),
+                             json_object_get_string_len(rc));
+        }
+      }
       json_object *tcs = NULL;
       if (json_object_object_get_ex(m, "tool_calls", &tcs) &&
           json_object_get_type(tcs) == json_type_array) {
@@ -145,8 +200,8 @@ int count_messages_tokens(void *req_obj) {
                                  json_object_get_string_len(nm));
             if (json_object_object_get_ex(fn, "arguments", &ar) &&
                 json_object_get_type(ar) == json_type_string)
-              total += tok_count(json_object_get_string(ar),
+              total += count_tool_args_tokens(json_object_get_string(ar),
-                                 json_object_get_string_len(ar));
+                                              json_object_get_string_len(ar));
           }
         }
       }
@@ -160,8 +215,8 @@ int count_messages_tokens(void *req_obj) {
                              json_object_get_string_len(nm));
         if (json_object_object_get_ex(fc, "arguments", &ar) &&
             json_object_get_type(ar) == json_type_string)
-          total += tok_count(json_object_get_string(ar),
+          total += count_tool_args_tokens(json_object_get_string(ar),
-                             json_object_get_string_len(ar));
+                                          json_object_get_string_len(ar));
       }
     }
   }
@@ -175,7 +230,9 @@ int count_completion_tokens(const char *content, const char *reasoning,
   if (reasoning && reasoning[0]) total += tok_count_str(reasoning);
   for (int i = 0; i < n_tc; i++) {
     if (tc[i].name && tc[i].name[0]) total += tok_count_str(tc[i].name);
-    if (tc[i].arguments && tc[i].arguments[0]) total += tok_count_str(tc[i].arguments);
+    if (tc[i].arguments && tc[i].arguments[0])
+      total += count_tool_args_tokens(tc[i].arguments,
+                                      (int)strlen(tc[i].arguments));
   }
   return total;
 }

tokenizer.c

@@ -30,11 +30,11 @@ static int *tok_ranks = NULL;
 static uint16_t *tok_klen = NULL;
 static unsigned int tok_mask = 0;
 
-static unsigned long tok_fnv1a(const unsigned char *s, int len) {
+static uint64_t tok_fnv1a(const unsigned char *s, int len) {
-  unsigned long h = 1469598103934665603UL;
+  uint64_t h = 1469598103934665603ULL;
   for (int i = 0; i < len; i++) {
-    h ^= (unsigned long)s[i];
+    h ^= (uint64_t)s[i];
-    h *= 1099511628211UL;
+    h *= 1099511628211ULL;
   }
   return h;
 }
@@ -56,7 +56,7 @@ static void tok_init(void) {
   for (int i = 0; i < tok_ntok; i++) {
     uint16_t kl = tok_len[i];
     const char *key = (const char *)(tok_blob + offset);
-    unsigned long h = tok_fnv1a((const unsigned char *)key, kl) & mask;
+    unsigned int h = (unsigned int)tok_fnv1a((const unsigned char *)key, kl) & mask;
     while (keys[h]) {
       h = (h + 1) & mask;
     }
@@ -73,7 +73,7 @@ static void tok_init(void) {
 
 static int tok_lookup(const unsigned char *s, int len) {
   if (!tok_keys) return -1;
-  unsigned long h = tok_fnv1a(s, len) & tok_mask;
+  unsigned int h = (unsigned int)tok_fnv1a(s, len) & tok_mask;
   while (tok_keys[h]) {
     if ((int)tok_klen[h] == len && memcmp(tok_keys[h], s, (size_t)len) == 0)
       return tok_ranks[h];