luxferre/hygate-c
1
0
Fork 0
Code Actions Activity

Tokenizer implemented

Browse Source
This commit is contained in:
Luxferre
2026-10-13 20:36:29 +03:00
parent 41cfd6f25e
commit 2d21b6f77e
5 changed files with 150 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile
+1 -1
View File
@@ -3,7 +3,7 @@
CC ?= cc
CFLAGS ?= -O3 -flto -pipe -std=c11 -Wall -D_POSIX_C_SOURCE=200809L -pthread
SRCS = main.c server.c jsonapi.c util.c
SRCS = main.c server.c jsonapi.c util.c tokenizer.c tokenizer_data.c
HDRS = hygate.h
BINDIR = bin
BIN = $(BINDIR)/hygate
README.md
+22
View File
@@ -66,8 +66,30 @@ CORS is enabled for all origins (`Access-Control-Allow-Origin: *`).
- Upstream calls use Fibonacci-backoff retries (up to `MAX_RETRIES = 5`,
300 s timeout).
## Token usage counting
Responses include an OpenAI-style `usage` object with `prompt_tokens`,
`completion_tokens`, and `total_tokens`. These are computed locally by an
**embedded cl100k_base BPE tokenizer** (vocabulary data in
`tokenizer_data.c`, logic in `tokenizer.c`) — no upstream token counts are
available from the Gradio backend.
- `prompt_tokens` sums each message's `content` (or multi-part content
parts), `reasoning_content`, and any `tool_calls` function `name` /
`arguments`.
- `completion_tokens` sums the assistant `content`, `reasoning_content`, and
any emitted `tool_calls`.
- `total_tokens = prompt_tokens + completion_tokens`.
cl100k_base is OpenAI's canonical multilingual tokenizer and is used as the
best available approximation for Hunyuan; the resulting counts are
deterministic subword counts and may differ slightly from the exact Hunyuan
vocabulary. No per-message formatting overhead (the `+3`/priming tokens
OpenAI applies) is added.
## Security considerations
- The server binds all interfaces (`0.0.0.0`) and currently enforces **no
authentication** — it is an open relay to the configured upstream. Do not
expose it to untrusted networks without a fronting proxy or auth layer.
hygate.h
+13 -1
View File
@@ -68,8 +68,20 @@ char *build_gradio_payload(void *req_obj, const char *model_default);
* frees the result. */
char *build_completion_response(const char *id, long created, const char *model,
const char *content, const char *reasoning,
tool_call_t *tc, int n_tc);
tool_call_t *tc, int n_tc, int prompt_tokens);
/* Token counting (embedded cl100k_base BPE tokenizer). Counts are
* deterministic subword counts; the served model (Hunyuan) uses a
* different tokenizer, so these are an approximation, not exact. */
int tok_count(const char *text, int len);
int tok_count_str(const char *text);
/* Count tokens across an OpenAI request's messages (prompt side). */
int count_messages_tokens(void *req_obj);
/* Count tokens of a generated completion (content + reasoning + tool calls). */
int count_completion_tokens(const char *content, const char *reasoning,
tool_call_t *tc, int n_tc);
/* Build a streaming tool-call delta JSON array (first emission: full
* tool call with index). Caller frees the result. */
char *toolcall_delta_json(tool_call_t *tc, int idx);
jsonapi.c
+67 -4
View File
@@ -4,6 +4,67 @@
#include <string.h>
#include <json-c/json.h>
/* forward declaration (defined later in this file) */
static char *content_to_string(json_object *content);
/* ------------------------------------------------------------------ */
/* Token counting (embedded cl100k_base BPE) */
/* ------------------------------------------------------------------ */
int count_messages_tokens(void *req_obj) {
json_object *req = (json_object *)req_obj;
json_object *messages = NULL;
if (!json_object_object_get_ex(req, "messages", &messages) ||
json_object_get_type(messages) != json_type_array)
return 0;
int total = 0;
int n = json_object_array_length(messages);
for (int i = 0; i < n; i++) {
json_object *m = json_object_array_get_idx(messages, i);
if (!m) continue;
json_object *content = json_object_object_get(m, "content");
char *cs = content_to_string(content);
total += tok_count_str(cs);
free(cs);
json_object *rc = NULL;
if (json_object_object_get_ex(m, "reasoning_content", &rc) &&
json_object_get_type(rc) == json_type_string)
total += tok_count_str(json_object_get_string(rc));
json_object *tcs = NULL;
if (json_object_object_get_ex(m, "tool_calls", &tcs) &&
json_object_get_type(tcs) == json_type_array) {
int tn = json_object_array_length(tcs);
for (int j = 0; j < tn; j++) {
json_object *tc = json_object_array_get_idx(tcs, j);
if (!tc) continue;
json_object *fn = NULL;
if (json_object_object_get_ex(tc, "function", &fn) &&
json_object_get_type(fn) == json_type_object) {
json_object *nm = NULL, *ar = NULL;
if (json_object_object_get_ex(fn, "name", &nm) &&
json_object_get_type(nm) == json_type_string)
total += tok_count_str(json_object_get_string(nm));
if (json_object_object_get_ex(fn, "arguments", &ar) &&
json_object_get_type(ar) == json_type_string)
total += tok_count_str(json_object_get_string(ar));
}
}
}
}
return total;
}
int count_completion_tokens(const char *content, const char *reasoning,
tool_call_t *tc, int n_tc) {
int total = 0;
if (content && content[0]) total += tok_count_str(content);
if (reasoning && reasoning[0]) total += tok_count_str(reasoning);
for (int i = 0; i < n_tc; i++) {
if (tc[i].name && tc[i].name[0]) total += tok_count_str(tc[i].name);
if (tc[i].arguments && tc[i].arguments[0]) total += tok_count_str(tc[i].arguments);
}
return total;
}
/* Convert an OpenAI "content" field to a flat string. Caller frees. */
static char *content_to_string(json_object *content) {
if (!content) return strdup("");
@@ -328,7 +389,7 @@ int parse_gradio_stream(const char *data_json,
/* ------------------------------------------------------------------ */
char *build_completion_response(const char *id, long created, const char *model,
const char *content, const char *reasoning,
tool_call_t *tc, int n_tc) {
tool_call_t *tc, int n_tc, int prompt_tokens) {
bool has_tc = n_tc > 0;
json_object *root = json_object_new_object();
json_object_object_add(root, "id", json_object_new_string(id));
@@ -374,10 +435,12 @@ char *build_completion_response(const char *id, long created, const char *model,
json_object_array_add(choices, choice);
json_object_object_add(root, "choices", choices);
int completion_tokens = count_completion_tokens(content, reasoning, tc, n_tc);
int total_tokens = prompt_tokens + completion_tokens;
json_object *usage = json_object_new_object();
json_object_object_add(usage, "prompt_tokens", json_object_new_int(0));
json_object_object_add(usage, "completion_tokens", json_object_new_int(0));
json_object_object_add(usage, "total_tokens", json_object_new_int(0));
json_object_object_add(usage, "prompt_tokens", json_object_new_int(prompt_tokens));
json_object_object_add(usage, "completion_tokens", json_object_new_int(completion_tokens));
json_object_object_add(usage, "total_tokens", json_object_new_int(total_tokens));
json_object_object_add(root, "usage", usage);
const char *s = json_object_to_json_string(root);
server.c
+47 -1
View File
@@ -30,6 +30,7 @@ typedef struct {
int tc_state_n;
int tc_state_cap;
bool any_toolcall;
int include_usage;
char *final_content; /* non-streaming accumulation */
char *final_reasoning;
tool_call_t *final_tc;
@@ -132,6 +133,31 @@ static void emit_chunk(sse_ctx_t *ctx, const char *role, const char *content,
json_object_put(root);
}
static void emit_usage_chunk(sse_ctx_t *ctx, int prompt_tokens, int completion_tokens) {
json_object *root = json_object_new_object();
json_object_object_add(root, "id", json_object_new_string(ctx->completion_id));
json_object_object_add(root, "object", json_object_new_string("chat.completion.chunk"));
json_object_object_add(root, "created", json_object_new_int64(ctx->created));
json_object_object_add(root, "model", json_object_new_string(ctx->model));
json_object *choices = json_object_new_array();
json_object_object_add(root, "choices", choices);
json_object *usage = json_object_new_object();
json_object_object_add(usage, "prompt_tokens", json_object_new_int(prompt_tokens));
json_object_object_add(usage, "completion_tokens", json_object_new_int(completion_tokens));
json_object_object_add(usage, "total_tokens",
json_object_new_int(prompt_tokens + completion_tokens));
json_object_object_add(root, "usage", usage);
const char *j = json_object_to_json_string(root);
if (j) {
size_t need = strlen(j) + 32;
char *buf = (char *)malloc(need);
int bl = snprintf(buf, need, "data: %s\n\n", j);
write_all(ctx->client_fd, buf, (size_t)bl);
free(buf);
}
json_object_put(root);
}
/* ------------------------------------------------------------------ */
/* SSE line processing */
/* ------------------------------------------------------------------ */
@@ -396,6 +422,18 @@ static void handle_chat(int fd, http_req_t *req, const config_t *cfg) {
}
char *payload = build_gradio_payload(jreq, model);
int prompt_tokens = count_messages_tokens(jreq);
bool include_usage = false;
json_object *so = NULL;
if (json_object_object_get_ex(jreq, "stream_options", &so) &&
json_object_get_type(so) == json_type_object) {
json_object *iu = NULL;
if (json_object_object_get_ex(so, "include_usage", &iu) &&
json_object_get_type(iu) == json_type_boolean)
include_usage = json_object_get_boolean(iu);
}
json_object_put(jreq);
if (!payload) {
send_simple(fd, 400, "Bad Request", "Invalid request: missing messages");
@@ -457,6 +495,7 @@ static void handle_chat(int fd, http_req_t *req, const config_t *cfg) {
ctx.client_fd = fd;
ctx.streaming = stream;
ctx.model = model;
ctx.include_usage = include_usage;
ctx.created = (long)time(NULL);
char uuid[37];
generate_uuid(uuid);
@@ -479,12 +518,19 @@ static void handle_chat(int fd, http_req_t *req, const config_t *cfg) {
if (stream) {
emit_chunk(&ctx, NULL, NULL, NULL, NULL, ctx.any_toolcall ? "tool_calls" : "stop");
if (ctx.include_usage) {
int completion_tokens = tok_count_str(ctx.emitted_content.buf);
completion_tokens += tok_count_str(ctx.emitted_reasoning.buf);
for (int i = 0; i < ctx.tc_state_n; i++)
completion_tokens += tok_count_str(ctx.tc_state[i].prev_args);
emit_usage_chunk(&ctx, prompt_tokens, completion_tokens);
}
const char *done = "data: [DONE]\n\n";
write_all(fd, done, strlen(done));
} else {
char *resp = build_completion_response(ctx.completion_id, ctx.created, ctx.model,
ctx.final_content, ctx.final_reasoning,
ctx.final_tc, ctx.final_n_tc);
ctx.final_tc, ctx.final_n_tc, prompt_tokens);
if (resp) {
send_json(fd, resp);
free(resp);