luxferre/hygate-c
1
0
Fork 0
Code Actions Activity

tok/mkf upd

Browse Source
This commit is contained in:
Luxferre
2026-10-15 23:13:11 +03:00
parent 561d90629f
commit 18ebe25d34
4 changed files with 92 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile
+4 -1
View File
@@ -17,7 +17,7 @@ LIBS = $$(pkg-config --cflags --libs libcurl json-c 2>/dev/null || echo '-lcurl
# pkg-config --static provides (nghttp2, ssl, crypto, z, ...).
STATICLIBS = $$(pkg-config --static --cflags --libs libcurl json-c 2>/dev/null || echo '-lcurl -ljson-c')
.PHONY: all static clean
.PHONY: all static clean install
all: $(BIN)
@@ -31,5 +31,8 @@ $(STATICBIN): $(SRCS) $(HDRS)
@test -d $(BINDIR) || mkdir $(BINDIR)
$(CC) $(CFLAGS) -static $(SRCS) -o $@ $(STATICLIBS)
install: $(BIN)
install -m 0755 $(BIN) $(DESTDIR)/usr/local/bin/hygate
clean:
rm -rf $(BINDIR)
README.md
+7 -5
View File
@@ -77,12 +77,14 @@ pre-tokenizer and BPE logic in `tokenizer.c`) — no upstream token counts are
available from the Gradio backend.
- `prompt_tokens` sums the active `system` prompt, each message's `content`
(or individual multi-part content parts), `reasoning_content`, `tool_calls`
/ `function_call` (`name` and `arguments`), `tool` result wrappers, and any
request `tools` function definitions (`name`, `description`, `parameters`).
(or individual multi-part content parts), `reasoning_content` when
`preserved_thinking` is active (enabled by default when `tools` are
present, matching `tencent/Hy3`'s `chat_template.jinja`), `tool_calls` /
`function_call` (`name` and unpacked `arguments` key/value pairs), the
final `tool` result wrapper, and any request `tools` function definitions.
- `completion_tokens` sums the assistant `content`, `reasoning_content`, and
any emitted `tool_calls` (`name` and `arguments`), with `reasoning_tokens`
broken out under `completion_tokens_details`.
any emitted `tool_calls` (`name` and unpacked `arguments` key/value pairs),
with `reasoning_tokens` broken out under `completion_tokens_details`.
- `total_tokens = prompt_tokens + completion_tokens`.
The embedded tokenizer implements `tencent/Hy3`'s exact 4-stage `Sequence`
jsonapi.c
+68 -11
View File
@@ -39,6 +39,35 @@ static int count_content_tokens(json_object *content) {
return s ? tok_count_str(s) : 0;
}
/* Count tokens for a tool-call arguments JSON string as rendered by
* tencent/Hy3's chat_template.jinja: each key is emitted raw inside
* <arg_key:opensource>...</arg_key:opensource> and each value is emitted
* as a raw unescaped string (when string) or JSON (when non-string)
* inside <arg_value:opensource>...</arg_value:opensource>. */
static int count_tool_args_tokens(const char *ar_str, int ar_len) {
if (!ar_str || ar_len <= 0) return 0;
json_object *parsed = json_tokener_parse(ar_str);
if (parsed && json_object_get_type(parsed) == json_type_object) {
int total = 0;
json_object_object_foreach(parsed, key, val) {
if (key && key[0]) total += tok_count_str(key);
if (val) {
if (json_object_get_type(val) == json_type_string) {
total += tok_count(json_object_get_string(val),
json_object_get_string_len(val));
} else {
const char *vs = json_object_to_json_string(val);
if (vs) total += tok_count_str(vs);
}
}
}
json_object_put(parsed);
return total;
}
if (parsed) json_object_put(parsed);
return tok_count(ar_str, ar_len);
}
/* ------------------------------------------------------------------ */
/* Token counting (embedded Tencent Hy3 BPE) */
/* ------------------------------------------------------------------ */
@@ -47,13 +76,33 @@ int count_messages_tokens(void *req_obj) {
if (!req) return 0;
int total = 0;
bool has_tools = false;
json_object *tools = NULL;
if (json_object_object_get_ex(req, "tools", &tools) &&
json_object_get_type(tools) == json_type_array &&
json_object_array_length(tools) > 0) {
const char *ts = json_object_to_json_string(tools);
json_object_get_type(tools) == json_type_array) {
int tn = json_object_array_length(tools);
if (tn > 0) {
has_tools = true;
for (int i = 0; i < tn; i++) {
json_object *ti = json_object_array_get_idx(tools, i);
if (!ti) continue;
const char *ts = json_object_to_json_string(ti);
if (ts) total += tok_count_str(ts);
}
}
}
/* In tencent/Hy3's chat_template.jinja, preserved_thinking defaults to
* true when tools are present and false when tools are absent (unless
* explicitly overridden). Since spaces/tencent/Hy3 appends the final
* turn as a user message, historical assistant reasoning_content is
* only included in the prompt when preserved_thinking is active. */
bool preserved_thinking = has_tools;
json_object *pt_obj = NULL;
if (json_object_object_get_ex(req, "preserved_thinking", &pt_obj) &&
json_object_get_type(pt_obj) == json_type_boolean) {
preserved_thinking = json_object_get_boolean(pt_obj) ? true : false;
}
json_object *messages = NULL;
if (!json_object_object_get_ex(req, "messages", &messages) ||
@@ -114,20 +163,26 @@ int count_messages_tokens(void *req_obj) {
total += count_content_tokens(content);
/* Historical tool messages in tencent/Hy3 chat_template.jinja render
* only visible_text(message['content']); tool_call_id and name are not
* emitted into the prompt. */
if (strcmp(roleStr, "tool") == 0 || strcmp(roleStr, "function") == 0) {
const char *toolID = get_str(m, "tool_call_id");
const char *nm = get_str(m, "name");
if (toolID[0] != 0) total += tok_count_str(toolID);
if (nm[0] != 0) total += tok_count_str(nm);
continue;
}
if (strcmp(roleStr, "assistant") == 0) {
if (preserved_thinking) {
json_object *rc = NULL;
if (json_object_object_get_ex(m, "reasoning_content", &rc) &&
json_object_get_type(rc) == json_type_string)
json_object_get_type(rc) == json_type_string) {
total += tok_count(json_object_get_string(rc),
json_object_get_string_len(rc));
} else if (json_object_object_get_ex(m, "reasoning", &rc) &&
json_object_get_type(rc) == json_type_string) {
total += tok_count(json_object_get_string(rc),
json_object_get_string_len(rc));
}
}
json_object *tcs = NULL;
if (json_object_object_get_ex(m, "tool_calls", &tcs) &&
json_object_get_type(tcs) == json_type_array) {
@@ -145,7 +200,7 @@ int count_messages_tokens(void *req_obj) {
json_object_get_string_len(nm));
if (json_object_object_get_ex(fn, "arguments", &ar) &&
json_object_get_type(ar) == json_type_string)
total += tok_count(json_object_get_string(ar),
total += count_tool_args_tokens(json_object_get_string(ar),
json_object_get_string_len(ar));
}
}
@@ -160,7 +215,7 @@ int count_messages_tokens(void *req_obj) {
json_object_get_string_len(nm));
if (json_object_object_get_ex(fc, "arguments", &ar) &&
json_object_get_type(ar) == json_type_string)
total += tok_count(json_object_get_string(ar),
total += count_tool_args_tokens(json_object_get_string(ar),
json_object_get_string_len(ar));
}
}
@@ -175,7 +230,9 @@ int count_completion_tokens(const char *content, const char *reasoning,
if (reasoning && reasoning[0]) total += tok_count_str(reasoning);
for (int i = 0; i < n_tc; i++) {
if (tc[i].name && tc[i].name[0]) total += tok_count_str(tc[i].name);
if (tc[i].arguments && tc[i].arguments[0]) total += tok_count_str(tc[i].arguments);
if (tc[i].arguments && tc[i].arguments[0])
total += count_tool_args_tokens(tc[i].arguments,
(int)strlen(tc[i].arguments));
}
return total;
}
tokenizer.c
+6 -6
View File
@@ -30,11 +30,11 @@ static int *tok_ranks = NULL;
static uint16_t *tok_klen = NULL;
static unsigned int tok_mask = 0;
static unsigned long tok_fnv1a(const unsigned char *s, int len) {
unsigned long h = 1469598103934665603UL;
static uint64_t tok_fnv1a(const unsigned char *s, int len) {
uint64_t h = 1469598103934665603ULL;
for (int i = 0; i < len; i++) {
h ^= (unsigned long)s[i];
h *= 1099511628211UL;
h ^= (uint64_t)s[i];
h *= 1099511628211ULL;
}
return h;
}
@@ -56,7 +56,7 @@ static void tok_init(void) {
for (int i = 0; i < tok_ntok; i++) {
uint16_t kl = tok_len[i];
const char *key = (const char *)(tok_blob + offset);
unsigned long h = tok_fnv1a((const unsigned char *)key, kl) & mask;
unsigned int h = (unsigned int)tok_fnv1a((const unsigned char *)key, kl) & mask;
while (keys[h]) {
h = (h + 1) & mask;
}
@@ -73,7 +73,7 @@ static void tok_init(void) {
static int tok_lookup(const unsigned char *s, int len) {
if (!tok_keys) return -1;
unsigned long h = tok_fnv1a(s, len) & tok_mask;
unsigned int h = (unsigned int)tok_fnv1a(s, len) & tok_mask;
while (tok_keys[h]) {
if ((int)tok_klen[h] == len && memcmp(tok_keys[h], s, (size_t)len) == 0)
return tok_ranks[h];