luxferre/hygate-c
1
0
Fork 0
Code Actions Activity

Tokenizer implemented

Browse Source
This commit is contained in:
Luxferre
2026-10-13 20:36:29 +03:00
parent 41cfd6f25e
commit 2d21b6f77e
5 changed files with 150 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
jsonapi.c
+67 -4
View File
@@ -4,6 +4,67 @@
#include <string.h>
#include <json-c/json.h>
/* forward declaration (defined later in this file) */
static char *content_to_string(json_object *content);
/* ------------------------------------------------------------------ */
/* Token counting (embedded cl100k_base BPE) */
/* ------------------------------------------------------------------ */
int count_messages_tokens(void *req_obj) {
json_object *req = (json_object *)req_obj;
json_object *messages = NULL;
if (!json_object_object_get_ex(req, "messages", &messages) ||
json_object_get_type(messages) != json_type_array)
return 0;
int total = 0;
int n = json_object_array_length(messages);
for (int i = 0; i < n; i++) {
json_object *m = json_object_array_get_idx(messages, i);
if (!m) continue;
json_object *content = json_object_object_get(m, "content");
char *cs = content_to_string(content);
total += tok_count_str(cs);
free(cs);
json_object *rc = NULL;
if (json_object_object_get_ex(m, "reasoning_content", &rc) &&
json_object_get_type(rc) == json_type_string)
total += tok_count_str(json_object_get_string(rc));
json_object *tcs = NULL;
if (json_object_object_get_ex(m, "tool_calls", &tcs) &&
json_object_get_type(tcs) == json_type_array) {
int tn = json_object_array_length(tcs);
for (int j = 0; j < tn; j++) {
json_object *tc = json_object_array_get_idx(tcs, j);
if (!tc) continue;
json_object *fn = NULL;
if (json_object_object_get_ex(tc, "function", &fn) &&
json_object_get_type(fn) == json_type_object) {
json_object *nm = NULL, *ar = NULL;
if (json_object_object_get_ex(fn, "name", &nm) &&
json_object_get_type(nm) == json_type_string)
total += tok_count_str(json_object_get_string(nm));
if (json_object_object_get_ex(fn, "arguments", &ar) &&
json_object_get_type(ar) == json_type_string)
total += tok_count_str(json_object_get_string(ar));
}
}
}
}
return total;
}
int count_completion_tokens(const char *content, const char *reasoning,
tool_call_t *tc, int n_tc) {
int total = 0;
if (content && content[0]) total += tok_count_str(content);
if (reasoning && reasoning[0]) total += tok_count_str(reasoning);
for (int i = 0; i < n_tc; i++) {
if (tc[i].name && tc[i].name[0]) total += tok_count_str(tc[i].name);
if (tc[i].arguments && tc[i].arguments[0]) total += tok_count_str(tc[i].arguments);
}
return total;
}
/* Convert an OpenAI "content" field to a flat string. Caller frees. */
static char *content_to_string(json_object *content) {
if (!content) return strdup("");
@@ -328,7 +389,7 @@ int parse_gradio_stream(const char *data_json,
/* ------------------------------------------------------------------ */
char *build_completion_response(const char *id, long created, const char *model,
const char *content, const char *reasoning,
tool_call_t *tc, int n_tc) {
tool_call_t *tc, int n_tc, int prompt_tokens) {
bool has_tc = n_tc > 0;
json_object *root = json_object_new_object();
json_object_object_add(root, "id", json_object_new_string(id));
@@ -374,10 +435,12 @@ char *build_completion_response(const char *id, long created, const char *model,
json_object_array_add(choices, choice);
json_object_object_add(root, "choices", choices);
int completion_tokens = count_completion_tokens(content, reasoning, tc, n_tc);
int total_tokens = prompt_tokens + completion_tokens;
json_object *usage = json_object_new_object();
json_object_object_add(usage, "prompt_tokens", json_object_new_int(0));
json_object_object_add(usage, "completion_tokens", json_object_new_int(0));
json_object_object_add(usage, "total_tokens", json_object_new_int(0));
json_object_object_add(usage, "prompt_tokens", json_object_new_int(prompt_tokens));
json_object_object_add(usage, "completion_tokens", json_object_new_int(completion_tokens));
json_object_object_add(usage, "total_tokens", json_object_new_int(total_tokens));
json_object_object_add(root, "usage", usage);
const char *s = json_object_to_json_string(root);