diff --git a/Makefile b/Makefile
index 4eb552e..b75538d 100644
--- a/Makefile
+++ b/Makefile
@@ -17,7 +17,7 @@ LIBS = $$(pkg-config --cflags --libs libcurl json-c 2>/dev/null || echo '-lcurl
# pkg-config --static provides (nghttp2, ssl, crypto, z, ...).
STATICLIBS = $$(pkg-config --static --cflags --libs libcurl json-c 2>/dev/null || echo '-lcurl -ljson-c')
-.PHONY: all static clean
+.PHONY: all static clean install
all: $(BIN)
@@ -31,5 +31,8 @@ $(STATICBIN): $(SRCS) $(HDRS)
@test -d $(BINDIR) || mkdir $(BINDIR)
$(CC) $(CFLAGS) -static $(SRCS) -o $@ $(STATICLIBS)
+install: $(BIN)
+ install -m 0755 $(BIN) $(DESTDIR)/usr/local/bin/hygate
+
clean:
rm -rf $(BINDIR)
\ No newline at end of file
diff --git a/README.md b/README.md
index e7bf0fe..eea1d21 100644
--- a/README.md
+++ b/README.md
@@ -77,12 +77,14 @@ pre-tokenizer and BPE logic in `tokenizer.c`) — no upstream token counts are
available from the Gradio backend.
- `prompt_tokens` sums the active `system` prompt, each message's `content`
- (or individual multi-part content parts), `reasoning_content`, `tool_calls`
- / `function_call` (`name` and `arguments`), `tool` result wrappers, and any
- request `tools` function definitions (`name`, `description`, `parameters`).
+ (or individual multi-part content parts), `reasoning_content` when
+ `preserved_thinking` is active (enabled by default when `tools` are
+ present, matching `tencent/Hy3`'s `chat_template.jinja`), `tool_calls` /
+ `function_call` (`name` and unpacked `arguments` key/value pairs), the
+ final `tool` result wrapper, and any request `tools` function definitions.
- `completion_tokens` sums the assistant `content`, `reasoning_content`, and
- any emitted `tool_calls` (`name` and `arguments`), with `reasoning_tokens`
- broken out under `completion_tokens_details`.
+ any emitted `tool_calls` (`name` and unpacked `arguments` key/value pairs),
+ with `reasoning_tokens` broken out under `completion_tokens_details`.
- `total_tokens = prompt_tokens + completion_tokens`.
The embedded tokenizer implements `tencent/Hy3`'s exact 4-stage `Sequence`
diff --git a/jsonapi.c b/jsonapi.c
index af70c87..87b3de5 100644
--- a/jsonapi.c
+++ b/jsonapi.c
@@ -39,6 +39,35 @@ static int count_content_tokens(json_object *content) {
return s ? tok_count_str(s) : 0;
}
+/* Count tokens for a tool-call arguments JSON string as rendered by
+ * tencent/Hy3's chat_template.jinja: each key is emitted raw inside
+ * ... and each value is emitted
+ * as a raw unescaped string (when string) or JSON (when non-string)
+ * inside .... */
+static int count_tool_args_tokens(const char *ar_str, int ar_len) {
+ if (!ar_str || ar_len <= 0) return 0;
+ json_object *parsed = json_tokener_parse(ar_str);
+ if (parsed && json_object_get_type(parsed) == json_type_object) {
+ int total = 0;
+ json_object_object_foreach(parsed, key, val) {
+ if (key && key[0]) total += tok_count_str(key);
+ if (val) {
+ if (json_object_get_type(val) == json_type_string) {
+ total += tok_count(json_object_get_string(val),
+ json_object_get_string_len(val));
+ } else {
+ const char *vs = json_object_to_json_string(val);
+ if (vs) total += tok_count_str(vs);
+ }
+ }
+ }
+ json_object_put(parsed);
+ return total;
+ }
+ if (parsed) json_object_put(parsed);
+ return tok_count(ar_str, ar_len);
+}
+
/* ------------------------------------------------------------------ */
/* Token counting (embedded Tencent Hy3 BPE) */
/* ------------------------------------------------------------------ */
@@ -47,12 +76,32 @@ int count_messages_tokens(void *req_obj) {
if (!req) return 0;
int total = 0;
+ bool has_tools = false;
json_object *tools = NULL;
if (json_object_object_get_ex(req, "tools", &tools) &&
- json_object_get_type(tools) == json_type_array &&
- json_object_array_length(tools) > 0) {
- const char *ts = json_object_to_json_string(tools);
- if (ts) total += tok_count_str(ts);
+ json_object_get_type(tools) == json_type_array) {
+ int tn = json_object_array_length(tools);
+ if (tn > 0) {
+ has_tools = true;
+ for (int i = 0; i < tn; i++) {
+ json_object *ti = json_object_array_get_idx(tools, i);
+ if (!ti) continue;
+ const char *ts = json_object_to_json_string(ti);
+ if (ts) total += tok_count_str(ts);
+ }
+ }
+ }
+
+ /* In tencent/Hy3's chat_template.jinja, preserved_thinking defaults to
+ * true when tools are present and false when tools are absent (unless
+ * explicitly overridden). Since spaces/tencent/Hy3 appends the final
+ * turn as a user message, historical assistant reasoning_content is
+ * only included in the prompt when preserved_thinking is active. */
+ bool preserved_thinking = has_tools;
+ json_object *pt_obj = NULL;
+ if (json_object_object_get_ex(req, "preserved_thinking", &pt_obj) &&
+ json_object_get_type(pt_obj) == json_type_boolean) {
+ preserved_thinking = json_object_get_boolean(pt_obj) ? true : false;
}
json_object *messages = NULL;
@@ -114,20 +163,26 @@ int count_messages_tokens(void *req_obj) {
total += count_content_tokens(content);
+ /* Historical tool messages in tencent/Hy3 chat_template.jinja render
+ * only visible_text(message['content']); tool_call_id and name are not
+ * emitted into the prompt. */
if (strcmp(roleStr, "tool") == 0 || strcmp(roleStr, "function") == 0) {
- const char *toolID = get_str(m, "tool_call_id");
- const char *nm = get_str(m, "name");
- if (toolID[0] != 0) total += tok_count_str(toolID);
- if (nm[0] != 0) total += tok_count_str(nm);
continue;
}
if (strcmp(roleStr, "assistant") == 0) {
- json_object *rc = NULL;
- if (json_object_object_get_ex(m, "reasoning_content", &rc) &&
- json_object_get_type(rc) == json_type_string)
- total += tok_count(json_object_get_string(rc),
- json_object_get_string_len(rc));
+ if (preserved_thinking) {
+ json_object *rc = NULL;
+ if (json_object_object_get_ex(m, "reasoning_content", &rc) &&
+ json_object_get_type(rc) == json_type_string) {
+ total += tok_count(json_object_get_string(rc),
+ json_object_get_string_len(rc));
+ } else if (json_object_object_get_ex(m, "reasoning", &rc) &&
+ json_object_get_type(rc) == json_type_string) {
+ total += tok_count(json_object_get_string(rc),
+ json_object_get_string_len(rc));
+ }
+ }
json_object *tcs = NULL;
if (json_object_object_get_ex(m, "tool_calls", &tcs) &&
json_object_get_type(tcs) == json_type_array) {
@@ -145,8 +200,8 @@ int count_messages_tokens(void *req_obj) {
json_object_get_string_len(nm));
if (json_object_object_get_ex(fn, "arguments", &ar) &&
json_object_get_type(ar) == json_type_string)
- total += tok_count(json_object_get_string(ar),
- json_object_get_string_len(ar));
+ total += count_tool_args_tokens(json_object_get_string(ar),
+ json_object_get_string_len(ar));
}
}
}
@@ -160,8 +215,8 @@ int count_messages_tokens(void *req_obj) {
json_object_get_string_len(nm));
if (json_object_object_get_ex(fc, "arguments", &ar) &&
json_object_get_type(ar) == json_type_string)
- total += tok_count(json_object_get_string(ar),
- json_object_get_string_len(ar));
+ total += count_tool_args_tokens(json_object_get_string(ar),
+ json_object_get_string_len(ar));
}
}
}
@@ -175,7 +230,9 @@ int count_completion_tokens(const char *content, const char *reasoning,
if (reasoning && reasoning[0]) total += tok_count_str(reasoning);
for (int i = 0; i < n_tc; i++) {
if (tc[i].name && tc[i].name[0]) total += tok_count_str(tc[i].name);
- if (tc[i].arguments && tc[i].arguments[0]) total += tok_count_str(tc[i].arguments);
+ if (tc[i].arguments && tc[i].arguments[0])
+ total += count_tool_args_tokens(tc[i].arguments,
+ (int)strlen(tc[i].arguments));
}
return total;
}
diff --git a/tokenizer.c b/tokenizer.c
index 49b9348..de2845e 100644
--- a/tokenizer.c
+++ b/tokenizer.c
@@ -30,11 +30,11 @@ static int *tok_ranks = NULL;
static uint16_t *tok_klen = NULL;
static unsigned int tok_mask = 0;
-static unsigned long tok_fnv1a(const unsigned char *s, int len) {
- unsigned long h = 1469598103934665603UL;
+static uint64_t tok_fnv1a(const unsigned char *s, int len) {
+ uint64_t h = 1469598103934665603ULL;
for (int i = 0; i < len; i++) {
- h ^= (unsigned long)s[i];
- h *= 1099511628211UL;
+ h ^= (uint64_t)s[i];
+ h *= 1099511628211ULL;
}
return h;
}
@@ -56,7 +56,7 @@ static void tok_init(void) {
for (int i = 0; i < tok_ntok; i++) {
uint16_t kl = tok_len[i];
const char *key = (const char *)(tok_blob + offset);
- unsigned long h = tok_fnv1a((const unsigned char *)key, kl) & mask;
+ unsigned int h = (unsigned int)tok_fnv1a((const unsigned char *)key, kl) & mask;
while (keys[h]) {
h = (h + 1) & mask;
}
@@ -73,7 +73,7 @@ static void tok_init(void) {
static int tok_lookup(const unsigned char *s, int len) {
if (!tok_keys) return -1;
- unsigned long h = tok_fnv1a(s, len) & tok_mask;
+ unsigned int h = (unsigned int)tok_fnv1a(s, len) & tok_mask;
while (tok_keys[h]) {
if ((int)tok_klen[h] == len && memcmp(tok_keys[h], s, (size_t)len) == 0)
return tok_ranks[h];