luxferre/hygate-c
1
0
Fork 0
Code Actions Activity

recent changes

Browse Source
This commit is contained in:
Luxferre
2026-10-14 00:00:02 +03:00
parent 8a0c1d0ec0
commit 80ad504d16
7 changed files with 550 additions and 251 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+10 -8
View File
@@ -69,16 +69,18 @@ CORS is enabled for all origins (`Access-Control-Allow-Origin: *`).
## Token usage counting ## Token usage counting
Responses include an OpenAI-style `usage` object with `prompt_tokens`, Responses include an OpenAI-style `usage` object with `prompt_tokens`,
`completion_tokens`, and `total_tokens`. These are computed locally by an `completion_tokens`, `total_tokens`, and `completion_tokens_details`
**embedded cl100k_base BPE tokenizer** (vocabulary data in (`reasoning_tokens`). These are computed locally by an **embedded cl100k_base
`tokenizer_data.c`, logic in `tokenizer.c`) — no upstream token counts are BPE tokenizer** (vocabulary data in `tokenizer_data.c`, logic in `tokenizer.c`)
available from the Gradio backend. — no upstream token counts are available from the Gradio backend.
- `prompt_tokens` sums each message's `content` (or multi-part content - `prompt_tokens` sums the active `system` prompt, each message's `content`
parts), `reasoning_content`, and any `tool_calls` function `name` / (or individual multi-part content parts), `reasoning_content`, `tool_calls`
`arguments`. / `function_call` (`name` and `arguments`), `tool` result wrappers, and any
request `tools` function definitions (`name`, `description`, `parameters`).
- `completion_tokens` sums the assistant `content`, `reasoning_content`, and - `completion_tokens` sums the assistant `content`, `reasoning_content`, and
any emitted `tool_calls`. any emitted `tool_calls` (`name` and `arguments`), with `reasoning_tokens`
broken out under `completion_tokens_details`.
- `total_tokens = prompt_tokens + completion_tokens`. - `total_tokens = prompt_tokens + completion_tokens`.
cl100k_base is OpenAI's canonical multilingual tokenizer and is used as the cl100k_base is OpenAI's canonical multilingual tokenizer and is used as the
hygate.h
+1 -1
View File
@@ -62,7 +62,7 @@ int parse_gradio_stream(const char *data_json,
/* Build the Gradio call payload {"data":[...]} from an OpenAI chat /* Build the Gradio call payload {"data":[...]} from an OpenAI chat
* completion request object. Caller frees the result. Returns NULL on * completion request object. Caller frees the result. Returns NULL on
* malformed input. */ * malformed input. */
char *build_gradio_payload(void *req_obj, const char *model_default); char *build_gradio_payload(void *req_obj);
/* Build a non-streaming OpenAI completion response JSON string. Caller /* Build a non-streaming OpenAI completion response JSON string. Caller
* frees the result. */ * frees the result. */
jsonapi.c
+190 -41
View File
@@ -4,31 +4,130 @@
#include <string.h> #include <string.h>
#include <json-c/json.h> #include <json-c/json.h>
/* forward declaration (defined later in this file) */ /* forward declarations (defined later in this file) */
static char *content_to_string(json_object *content); static char *content_to_string(json_object *content);
static const char *get_str(json_object *obj, const char *key);
static int count_content_tokens(json_object *content) {
if (!content) return 0;
enum json_type t = json_object_get_type(content);
if (t == json_type_null) return 0;
if (t == json_type_string)
return tok_count(json_object_get_string(content),
json_object_get_string_len(content));
if (t == json_type_array) {
int total = 0;
int n = json_object_array_length(content);
for (int i = 0; i < n; i++) {
json_object *part = json_object_array_get_idx(content, i);
if (!part) continue;
enum json_type pt = json_object_get_type(part);
if (pt == json_type_string) {
total += tok_count(json_object_get_string(part),
json_object_get_string_len(part));
} else if (pt == json_type_object) {
json_object *txt = NULL;
if (json_object_object_get_ex(part, "text", &txt) &&
json_object_get_type(txt) == json_type_string)
total += tok_count(json_object_get_string(txt),
json_object_get_string_len(txt));
}
}
return total;
}
const char *s = json_object_to_json_string(content);
return s ? tok_count_str(s) : 0;
}
/* ------------------------------------------------------------------ */ /* ------------------------------------------------------------------ */
/* Token counting (embedded cl100k_base BPE) */ /* Token counting (embedded cl100k_base BPE) */
/* ------------------------------------------------------------------ */ /* ------------------------------------------------------------------ */
int count_messages_tokens(void *req_obj) { int count_messages_tokens(void *req_obj) {
json_object *req = (json_object *)req_obj; json_object *req = (json_object *)req_obj;
if (!req) return 0;
int total = 0;
json_object *tools = NULL;
if (json_object_object_get_ex(req, "tools", &tools) &&
json_object_get_type(tools) == json_type_array &&
json_object_array_length(tools) > 0) {
const char *ts = json_object_to_json_string(tools);
if (ts) total += tok_count_str(ts);
}
json_object *messages = NULL; json_object *messages = NULL;
if (!json_object_object_get_ex(req, "messages", &messages) || if (!json_object_object_get_ex(req, "messages", &messages) ||
json_object_get_type(messages) != json_type_array) json_object_get_type(messages) != json_type_array)
return 0; return total;
int total = 0;
int n = json_object_array_length(messages); int n = json_object_array_length(messages);
int sys_idx = -1;
int last_nonsys_idx = -1;
bool sys_has_content = false;
for (int i = 0; i < n; i++) { for (int i = 0; i < n; i++) {
json_object *m = json_object_array_get_idx(messages, i); json_object *m = json_object_array_get_idx(messages, i);
if (!m) continue; if (!m) continue;
const char *roleStr = get_str(m, "role");
if (strcmp(roleStr, "system") == 0 && !sys_has_content) {
sys_idx = i;
if (count_content_tokens(json_object_object_get(m, "content")) > 0)
sys_has_content = true;
} else {
last_nonsys_idx = i;
}
}
for (int i = 0; i < n; i++) {
json_object *m = json_object_array_get_idx(messages, i);
if (!m) continue;
const char *roleStr = get_str(m, "role");
json_object *content = json_object_object_get(m, "content"); json_object *content = json_object_object_get(m, "content");
char *cs = content_to_string(content);
total += tok_count_str(cs); if (i == sys_idx) {
free(cs); total += count_content_tokens(content);
continue;
}
if (i == last_nonsys_idx &&
(strcmp(roleStr, "tool") == 0 || strcmp(roleStr, "function") == 0)) {
const char *toolName = get_str(m, "name");
if (toolName[0] == 0) toolName = get_str(m, "tool_call_id");
if (toolName[0] != 0) {
char *lastContent = content_to_string(content);
const char *lc = lastContent ? lastContent : "";
size_t need = strlen("Tool result for : ") + strlen(toolName) +
strlen(lc) + 1;
char *s = (char *)malloc(need);
if (s) {
snprintf(s, need, "Tool result for %s: %s", toolName, lc);
total += tok_count_str(s);
free(s);
} else {
total += tok_count_str(toolName) + count_content_tokens(content);
}
free(lastContent);
} else {
total += count_content_tokens(content);
}
continue;
}
total += count_content_tokens(content);
if (strcmp(roleStr, "tool") == 0 || strcmp(roleStr, "function") == 0) {
const char *toolID = get_str(m, "tool_call_id");
const char *nm = get_str(m, "name");
if (toolID[0] != 0) total += tok_count_str(toolID);
if (nm[0] != 0) total += tok_count_str(nm);
continue;
}
if (strcmp(roleStr, "assistant") == 0) {
json_object *rc = NULL; json_object *rc = NULL;
if (json_object_object_get_ex(m, "reasoning_content", &rc) && if (json_object_object_get_ex(m, "reasoning_content", &rc) &&
json_object_get_type(rc) == json_type_string) json_object_get_type(rc) == json_type_string)
total += tok_count_str(json_object_get_string(rc)); total += tok_count(json_object_get_string(rc),
json_object_get_string_len(rc));
json_object *tcs = NULL; json_object *tcs = NULL;
if (json_object_object_get_ex(m, "tool_calls", &tcs) && if (json_object_object_get_ex(m, "tool_calls", &tcs) &&
json_object_get_type(tcs) == json_type_array) { json_object_get_type(tcs) == json_type_array) {
@@ -42,13 +141,29 @@ int count_messages_tokens(void *req_obj) {
json_object *nm = NULL, *ar = NULL; json_object *nm = NULL, *ar = NULL;
if (json_object_object_get_ex(fn, "name", &nm) && if (json_object_object_get_ex(fn, "name", &nm) &&
json_object_get_type(nm) == json_type_string) json_object_get_type(nm) == json_type_string)
total += tok_count_str(json_object_get_string(nm)); total += tok_count(json_object_get_string(nm),
json_object_get_string_len(nm));
if (json_object_object_get_ex(fn, "arguments", &ar) && if (json_object_object_get_ex(fn, "arguments", &ar) &&
json_object_get_type(ar) == json_type_string) json_object_get_type(ar) == json_type_string)
total += tok_count_str(json_object_get_string(ar)); total += tok_count(json_object_get_string(ar),
json_object_get_string_len(ar));
} }
} }
} }
json_object *fc = NULL;
if (json_object_object_get_ex(m, "function_call", &fc) &&
json_object_get_type(fc) == json_type_object) {
json_object *nm = NULL, *ar = NULL;
if (json_object_object_get_ex(fc, "name", &nm) &&
json_object_get_type(nm) == json_type_string)
total += tok_count(json_object_get_string(nm),
json_object_get_string_len(nm));
if (json_object_object_get_ex(fc, "arguments", &ar) &&
json_object_get_type(ar) == json_type_string)
total += tok_count(json_object_get_string(ar),
json_object_get_string_len(ar));
}
}
} }
return total; return total;
} }
@@ -69,11 +184,13 @@ int count_completion_tokens(const char *content, const char *reasoning,
static char *content_to_string(json_object *content) { static char *content_to_string(json_object *content) {
if (!content) return strdup(""); if (!content) return strdup("");
enum json_type t = json_object_get_type(content); enum json_type t = json_object_get_type(content);
if (t == json_type_null) return strdup("");
if (t == json_type_string) if (t == json_type_string)
return strdup(json_object_get_string(content)); return strdup(json_object_get_string(content));
if (t == json_type_array) { if (t == json_type_array) {
strbuf sb; strbuf sb;
sb_init(&sb); sb_init(&sb);
if (!sb.buf) return strdup("");
int n = json_object_array_length(content); int n = json_object_array_length(content);
for (int i = 0; i < n; i++) { for (int i = 0; i < n; i++) {
json_object *part = json_object_array_get_idx(content, i); json_object *part = json_object_array_get_idx(content, i);
@@ -88,9 +205,7 @@ static char *content_to_string(json_object *content) {
sb_append_str(&sb, json_object_get_string(txt)); sb_append_str(&sb, json_object_get_string(txt));
} }
} }
char *r = strdup(sb.buf); return sb.buf;
sb_free(&sb);
return r;
} }
const char *s = json_object_to_json_string(content); const char *s = json_object_to_json_string(content);
return s ? strdup(s) : strdup(""); return s ? strdup(s) : strdup("");
@@ -124,8 +239,7 @@ static double get_double(json_object *req, const char *key) {
/* ------------------------------------------------------------------ */ /* ------------------------------------------------------------------ */
/* Build Gradio payload */ /* Build Gradio payload */
/* ------------------------------------------------------------------ */ /* ------------------------------------------------------------------ */
char *build_gradio_payload(void *req_obj, const char *model_default) { char *build_gradio_payload(void *req_obj) {
(void)model_default;
json_object *req = (json_object *)req_obj; json_object *req = (json_object *)req_obj;
json_object *messages = NULL; json_object *messages = NULL;
if (!json_object_object_get_ex(req, "messages", &messages) || if (!json_object_object_get_ex(req, "messages", &messages) ||
@@ -133,19 +247,25 @@ char *build_gradio_payload(void *req_obj, const char *model_default) {
return NULL; return NULL;
int n = json_object_array_length(messages); int n = json_object_array_length(messages);
char *system_str = strdup(""); int *nonSys = (int *)malloc(sizeof(int) * (size_t)(n > 0 ? n : 1));
char *message_str = strdup(""); if (!nonSys) return NULL;
json_object *history_arr = json_object_new_array(); json_object *history_arr = json_object_new_array();
if (!history_arr) {
free(nonSys);
return NULL;
}
char *system_str = NULL;
char *message_str = NULL;
/* First pass: pick first system message as system prompt, collect the /* First pass: pick first system message as system prompt, collect the
* rest as non-system messages. */ * rest as non-system messages. */
int *nonSys = (int *)malloc(sizeof(int) * (size_t)(n > 0 ? n : 1));
int nonSysN = 0; int nonSysN = 0;
for (int i = 0; i < n; i++) { for (int i = 0; i < n; i++) {
json_object *m = json_object_array_get_idx(messages, i); json_object *m = json_object_array_get_idx(messages, i);
if (!m) continue; if (!m) continue;
const char *roleStr = get_str(m, "role"); const char *roleStr = get_str(m, "role");
if (strcmp(roleStr, "system") == 0 && system_str[0] == 0) { if (strcmp(roleStr, "system") == 0 && (!system_str || system_str[0] == 0)) {
char *cs = content_to_string(json_object_object_get(m, "content")); char *cs = content_to_string(json_object_object_get(m, "content"));
free(system_str); free(system_str);
system_str = cs; system_str = cs;
@@ -160,17 +280,19 @@ char *build_gradio_payload(void *req_obj, const char *model_default) {
json_object *m = json_object_array_get_idx(messages, i); json_object *m = json_object_array_get_idx(messages, i);
const char *roleStr = get_str(m, "role"); const char *roleStr = get_str(m, "role");
json_object *item = json_object_new_object(); json_object *item = json_object_new_object();
if (!item) continue;
json_object_object_add(item, "role", json_object_new_string(roleStr)); json_object_object_add(item, "role", json_object_new_string(roleStr));
json_object *content = json_object_object_get(m, "content"); json_object *content = json_object_object_get(m, "content");
char *cs = content_to_string(content); char *cs = content_to_string(content);
const char *cstr = cs ? cs : "";
if (strcmp(roleStr, "assistant") == 0) { if (strcmp(roleStr, "assistant") == 0) {
if (cs[0] != 0) if (cstr[0] != 0)
json_object_object_add(item, "content", json_object_new_string(cs)); json_object_object_add(item, "content", json_object_new_string(cstr));
else else
json_object_object_add(item, "content", json_object_new_null()); json_object_object_add(item, "content", json_object_new_null());
} else { } else {
json_object_object_add(item, "content", json_object_new_string(cs)); json_object_object_add(item, "content", json_object_new_string(cstr));
} }
free(cs); free(cs);
@@ -214,25 +336,25 @@ char *build_gradio_payload(void *req_obj, const char *model_default) {
json_object *m = json_object_array_get_idx(messages, i); json_object *m = json_object_array_get_idx(messages, i);
const char *roleStr = get_str(m, "role"); const char *roleStr = get_str(m, "role");
char *lastContent = content_to_string(json_object_object_get(m, "content")); char *lastContent = content_to_string(json_object_object_get(m, "content"));
const char *lc = lastContent ? lastContent : "";
if (strcmp(roleStr, "tool") == 0 || strcmp(roleStr, "function") == 0) { if (strcmp(roleStr, "tool") == 0 || strcmp(roleStr, "function") == 0) {
const char *toolName = get_str(m, "name"); const char *toolName = get_str(m, "name");
if (toolName[0] == 0) toolName = get_str(m, "tool_call_id"); if (toolName[0] == 0) toolName = get_str(m, "tool_call_id");
if (toolName[0] != 0) { if (toolName[0] != 0) {
size_t need = strlen("Tool result for : ") + strlen(toolName) + size_t need = strlen("Tool result for : ") + strlen(toolName) +
strlen(lastContent) + 1; strlen(lc) + 1;
char *s = (char *)malloc(need); char *s = (char *)malloc(need);
snprintf(s, need, "Tool result for %s: %s", toolName, lastContent); if (s) {
free(message_str); snprintf(s, need, "Tool result for %s: %s", toolName, lc);
message_str = s; message_str = s;
} else {
free(message_str);
message_str = strdup(lastContent);
}
} else {
free(message_str);
message_str = strdup(lastContent);
} }
free(lastContent); free(lastContent);
} else {
message_str = lastContent;
}
} else {
message_str = lastContent;
}
} }
free(nonSys); free(nonSys);
@@ -250,20 +372,27 @@ char *build_gradio_payload(void *req_obj, const char *model_default) {
bool topp_present = has_nonnull(req, "top_p"); bool topp_present = has_nonnull(req, "top_p");
double topp = topp_present ? get_double(req, "top_p") : 0.0; double topp = topp_present ? get_double(req, "top_p") : 0.0;
char *functions_json = NULL; const char *functions_json = "";
json_object *tools = NULL; json_object *tools = NULL;
if (json_object_object_get_ex(req, "tools", &tools) && if (json_object_object_get_ex(req, "tools", &tools) &&
json_object_get_type(tools) == json_type_array) { json_object_get_type(tools) == json_type_array) {
const char *s = json_object_to_json_string(tools); const char *s = json_object_to_json_string(tools);
if (s) functions_json = strdup(s); if (s) functions_json = s;
} }
if (!functions_json) functions_json = strdup("");
/* Assemble the Gradio data array. */ /* Assemble the Gradio data array. */
json_object *root = json_object_new_object(); json_object *root = json_object_new_object();
json_object *data = json_object_new_array(); json_object *data = json_object_new_array();
json_object_array_add(data, json_object_new_string(message_str)); if (!root || !data) {
json_object_array_add(data, json_object_new_string(system_str)); if (root) json_object_put(root);
if (data) json_object_put(data);
json_object_put(history_arr);
free(system_str);
free(message_str);
return NULL;
}
json_object_array_add(data, json_object_new_string(message_str ? message_str : ""));
json_object_array_add(data, json_object_new_string(system_str ? system_str : ""));
json_object_array_add(data, history_arr); json_object_array_add(data, history_arr);
json_object_array_add(data, json_object_new_string("high")); json_object_array_add(data, json_object_new_string("high"));
if (temp_present) if (temp_present)
@@ -284,7 +413,6 @@ char *build_gradio_payload(void *req_obj, const char *model_default) {
json_object_put(root); json_object_put(root);
free(system_str); free(system_str);
free(message_str); free(message_str);
free(functions_json);
return payload; return payload;
} }
@@ -336,6 +464,14 @@ int parse_gradio_stream(const char *data_json,
*out_reasoning = (r && json_object_get_type(r) == json_type_string) *out_reasoning = (r && json_object_get_type(r) == json_type_string)
? strdup(json_object_get_string(r)) ? strdup(json_object_get_string(r))
: strdup(""); : strdup("");
if (!*out_content || !*out_reasoning) {
free(*out_content);
free(*out_reasoning);
*out_content = NULL;
*out_reasoning = NULL;
json_object_put(root);
return 0;
}
if (m >= 3) { if (m >= 3) {
json_object *tcobj = json_object_array_get_idx(inner, 2); json_object *tcobj = json_object_array_get_idx(inner, 2);
@@ -343,12 +479,14 @@ int parse_gradio_stream(const char *data_json,
int len = json_object_array_length(tcobj); int len = json_object_array_length(tcobj);
if (len > 0) { if (len > 0) {
tool_call_t *arr = (tool_call_t *)calloc((size_t)len, sizeof(tool_call_t)); tool_call_t *arr = (tool_call_t *)calloc((size_t)len, sizeof(tool_call_t));
if (arr) {
int cnt = 0; int cnt = 0;
for (int i = 0; i < len; i++) { for (int i = 0; i < len; i++) {
json_object *e = json_object_array_get_idx(tcobj, i); json_object *e = json_object_array_get_idx(tcobj, i);
if (json_object_get_type(e) != json_type_object) continue; if (json_object_get_type(e) != json_type_object) continue;
json_object *fid = NULL, *ffn = NULL, *fname = NULL, *fargs = NULL; json_object *fid = NULL, *ftype = NULL, *ffn = NULL, *fname = NULL, *fargs = NULL;
json_object_object_get_ex(e, "id", &fid); json_object_object_get_ex(e, "id", &fid);
json_object_object_get_ex(e, "type", &ftype);
json_object_object_get_ex(e, "function", &ffn); json_object_object_get_ex(e, "function", &ffn);
if (ffn && json_object_get_type(ffn) == json_type_object) { if (ffn && json_object_get_type(ffn) == json_type_object) {
json_object_object_get_ex(ffn, "name", &fname); json_object_object_get_ex(ffn, "name", &fname);
@@ -357,9 +495,8 @@ int parse_gradio_stream(const char *data_json,
arr[cnt].id = (fid && json_object_get_type(fid) == json_type_string) arr[cnt].id = (fid && json_object_get_type(fid) == json_type_string)
? strdup(json_object_get_string(fid)) ? strdup(json_object_get_string(fid))
: strdup(""); : strdup("");
arr[cnt].type = (json_object_object_get_ex(e, "type", &fid) && arr[cnt].type = (ftype && json_object_get_type(ftype) == json_type_string)
json_object_get_type(fid) == json_type_string) ? strdup(json_object_get_string(ftype))
? strdup(json_object_get_string(fid))
: strdup(""); : strdup("");
arr[cnt].name = (fname && json_object_get_type(fname) == json_type_string) arr[cnt].name = (fname && json_object_get_type(fname) == json_type_string)
? strdup(json_object_get_string(fname)) ? strdup(json_object_get_string(fname))
@@ -368,6 +505,13 @@ int parse_gradio_stream(const char *data_json,
(fargs && json_object_get_type(fargs) == json_type_string) (fargs && json_object_get_type(fargs) == json_type_string)
? strdup(json_object_get_string(fargs)) ? strdup(json_object_get_string(fargs))
: strdup(""); : strdup("");
if (!arr[cnt].id || !arr[cnt].type || !arr[cnt].name || !arr[cnt].arguments) {
free(arr[cnt].id);
free(arr[cnt].type);
free(arr[cnt].name);
free(arr[cnt].arguments);
continue;
}
cnt++; cnt++;
} }
if (cnt > 0) { if (cnt > 0) {
@@ -379,6 +523,7 @@ int parse_gradio_stream(const char *data_json,
} }
} }
} }
}
json_object_put(root); json_object_put(root);
return 1; return 1;
@@ -436,11 +581,15 @@ char *build_completion_response(const char *id, long created, const char *model,
json_object_object_add(root, "choices", choices); json_object_object_add(root, "choices", choices);
int completion_tokens = count_completion_tokens(content, reasoning, tc, n_tc); int completion_tokens = count_completion_tokens(content, reasoning, tc, n_tc);
int reasoning_tokens = (reasoning && reasoning[0]) ? tok_count_str(reasoning) : 0;
int total_tokens = prompt_tokens + completion_tokens; int total_tokens = prompt_tokens + completion_tokens;
json_object *usage = json_object_new_object(); json_object *usage = json_object_new_object();
json_object_object_add(usage, "prompt_tokens", json_object_new_int(prompt_tokens)); json_object_object_add(usage, "prompt_tokens", json_object_new_int(prompt_tokens));
json_object_object_add(usage, "completion_tokens", json_object_new_int(completion_tokens)); json_object_object_add(usage, "completion_tokens", json_object_new_int(completion_tokens));
json_object_object_add(usage, "total_tokens", json_object_new_int(total_tokens)); json_object_object_add(usage, "total_tokens", json_object_new_int(total_tokens));
json_object *details = json_object_new_object();
json_object_object_add(details, "reasoning_tokens", json_object_new_int(reasoning_tokens));
json_object_object_add(usage, "completion_tokens_details", details);
json_object_object_add(root, "usage", usage); json_object_object_add(root, "usage", usage);
const char *s = json_object_to_json_string(root); const char *s = json_object_to_json_string(root);
main.c
+6 -6
View File
@@ -46,12 +46,12 @@ int main(int argc, char **argv) {
val = argv[i + 1]; val = argv[i + 1];
i++; i++;
} }
if (strcmp(a, "-port") == 0) cfg.port = val; if (strcmp(a, "-port") == 0) { if (val) cfg.port = val; }
else if (strcmp(a, "-endpoint") == 0) cfg.endpoint = val; else if (strcmp(a, "-endpoint") == 0) { if (val) cfg.endpoint = val; }
else if (strcmp(a, "-model") == 0) cfg.model = val; else if (strcmp(a, "-model") == 0) { if (val) cfg.model = val; }
else if (strcmp(a, "-user-agent") == 0) cfg.user_agent = val; else if (strcmp(a, "-user-agent") == 0 || strcmp(a, "-ua") == 0) {
else if (strcmp(a, "-ua") == 0) cfg.user_agent = val; if (val) cfg.user_agent = val;
else if (strcmp(a, "-max-body") == 0) { } else if (strcmp(a, "-max-body") == 0) {
if (val) { if (val) {
long v = (long)strtoul(val, NULL, 10); long v = (long)strtoul(val, NULL, 10);
if (v > 0) cfg.max_body_bytes = v; if (v > 0) cfg.max_body_bytes = v;
server.c
+96 -49
View File
@@ -4,7 +4,6 @@
#include <string.h> #include <string.h>
#include <unistd.h> #include <unistd.h>
#include <time.h> #include <time.h>
#include <errno.h>
#include <json-c/json.h> #include <json-c/json.h>
#include <curl/curl.h> #include <curl/curl.h>
@@ -13,6 +12,7 @@
/* ------------------------------------------------------------------ */ /* ------------------------------------------------------------------ */
struct tcst { struct tcst {
char *key; char *key;
char *name;
char *prev_args; char *prev_args;
bool emitted; bool emitted;
}; };
@@ -126,14 +126,17 @@ static void emit_chunk(sse_ctx_t *ctx, const char *role, const char *content,
if (s) { if (s) {
size_t need = strlen(s) + 32; size_t need = strlen(s) + 32;
char *buf = (char *)malloc(need); char *buf = (char *)malloc(need);
if (buf) {
int bl = snprintf(buf, need, "data: %s\n\n", s); int bl = snprintf(buf, need, "data: %s\n\n", s);
write_all(ctx->client_fd, buf, (size_t)bl); write_all(ctx->client_fd, buf, (size_t)bl);
free(buf); free(buf);
} }
}
json_object_put(root); json_object_put(root);
} }
static void emit_usage_chunk(sse_ctx_t *ctx, int prompt_tokens, int completion_tokens) { static void emit_usage_chunk(sse_ctx_t *ctx, int prompt_tokens,
int completion_tokens, int reasoning_tokens) {
json_object *root = json_object_new_object(); json_object *root = json_object_new_object();
json_object_object_add(root, "id", json_object_new_string(ctx->completion_id)); json_object_object_add(root, "id", json_object_new_string(ctx->completion_id));
json_object_object_add(root, "object", json_object_new_string("chat.completion.chunk")); json_object_object_add(root, "object", json_object_new_string("chat.completion.chunk"));
@@ -146,15 +149,20 @@ static void emit_usage_chunk(sse_ctx_t *ctx, int prompt_tokens, int completion_t
json_object_object_add(usage, "completion_tokens", json_object_new_int(completion_tokens)); json_object_object_add(usage, "completion_tokens", json_object_new_int(completion_tokens));
json_object_object_add(usage, "total_tokens", json_object_object_add(usage, "total_tokens",
json_object_new_int(prompt_tokens + completion_tokens)); json_object_new_int(prompt_tokens + completion_tokens));
json_object *details = json_object_new_object();
json_object_object_add(details, "reasoning_tokens", json_object_new_int(reasoning_tokens));
json_object_object_add(usage, "completion_tokens_details", details);
json_object_object_add(root, "usage", usage); json_object_object_add(root, "usage", usage);
const char *j = json_object_to_json_string(root); const char *j = json_object_to_json_string(root);
if (j) { if (j) {
size_t need = strlen(j) + 32; size_t need = strlen(j) + 32;
char *buf = (char *)malloc(need); char *buf = (char *)malloc(need);
if (buf) {
int bl = snprintf(buf, need, "data: %s\n\n", j); int bl = snprintf(buf, need, "data: %s\n\n", j);
write_all(ctx->client_fd, buf, (size_t)bl); write_all(ctx->client_fd, buf, (size_t)bl);
free(buf); free(buf);
} }
}
json_object_put(root); json_object_put(root);
} }
@@ -175,12 +183,7 @@ static void process_one_line(sse_ctx_t *ctx, char *line) {
char *c = NULL, *r = NULL; char *c = NULL, *r = NULL;
tool_call_t *tc = NULL; tool_call_t *tc = NULL;
int ntc = 0; int ntc = 0;
if (!parse_gradio_stream(payload, &c, &r, &tc, &ntc)) { if (!parse_gradio_stream(payload, &c, &r, &tc, &ntc)) return;
free(c);
free(r);
free_toolcalls(tc, ntc);
return;
}
if (ctx->streaming) { if (ctx->streaming) {
handle_stream_delta(ctx, c, r, tc, ntc); handle_stream_delta(ctx, c, r, tc, ntc);
@@ -188,32 +191,36 @@ static void process_one_line(sse_ctx_t *ctx, char *line) {
free(r); free(r);
free_toolcalls(tc, ntc); free_toolcalls(tc, ntc);
} else { } else {
if (!ctx->final_content || c[0] != 0) {
free(ctx->final_content); free(ctx->final_content);
ctx->final_content = c ? c : strdup(""); ctx->final_content = c;
} else {
free(c);
}
if (!ctx->final_reasoning || r[0] != 0) {
free(ctx->final_reasoning); free(ctx->final_reasoning);
ctx->final_reasoning = r ? r : strdup(""); ctx->final_reasoning = r;
free_toolcalls(ctx->final_tc, ctx->final_n_tc); } else {
free(r);
}
if (ntc > 0) { if (ntc > 0) {
free_toolcalls(ctx->final_tc, ctx->final_n_tc);
ctx->final_tc = tc; ctx->final_tc = tc;
ctx->final_n_tc = ntc; ctx->final_n_tc = ntc;
} else { } else {
ctx->final_tc = NULL; free_toolcalls(tc, ntc);
ctx->final_n_tc = 0;
free(tc);
} }
} }
} }
static void process_complete_lines(sse_ctx_t *ctx) { static void process_complete_lines(sse_ctx_t *ctx) {
char *buf = ctx->line.buf; char *buf = ctx->line.buf;
if (!buf) return;
while (1) { while (1) {
char *nl = strchr(buf, '\n'); char *nl = strchr(buf, '\n');
if (!nl) break; if (!nl) break;
*nl = 0; *nl = 0;
char *line = buf; process_one_line(ctx, buf);
size_t ll = strlen(line);
if (ll > 0 && line[ll - 1] == '\r') line[ll - 1] = 0;
process_one_line(ctx, line);
size_t consumed = (size_t)(nl - buf) + 1; size_t consumed = (size_t)(nl - buf) + 1;
size_t rest = ctx->line.len - consumed; size_t rest = ctx->line.len - consumed;
memmove(buf, nl + 1, rest + 1); memmove(buf, nl + 1, rest + 1);
@@ -222,12 +229,8 @@ static void process_complete_lines(sse_ctx_t *ctx) {
} }
static void flush_remaining(sse_ctx_t *ctx) { static void flush_remaining(sse_ctx_t *ctx) {
if (ctx->line.len == 0) return; if (ctx->line.len == 0 || !ctx->line.buf) return;
char *line = ctx->line.buf; process_one_line(ctx, ctx->line.buf);
size_t ll = ctx->line.len;
if (ll > 0 && line[ll - 1] == '\r') line[ll - 1] = 0;
if (ll > 0 && line[ll - 1] == '\n') line[ll - 1] = 0;
process_one_line(ctx, line);
sb_reset(&ctx->line); sb_reset(&ctx->line);
} }
@@ -253,37 +256,64 @@ static void handle_stream_delta(sse_ctx_t *ctx, char *c, char *r,
if (strcmp(ctx->tc_state[j].key, key) == 0) { st = &ctx->tc_state[j]; break; } if (strcmp(ctx->tc_state[j].key, key) == 0) { st = &ctx->tc_state[j]; break; }
if (!st) { if (!st) {
if (ctx->tc_state_n >= ctx->tc_state_cap) { if (ctx->tc_state_n >= ctx->tc_state_cap) {
ctx->tc_state_cap = ctx->tc_state_cap ? ctx->tc_state_cap * 2 : 8; int ncap = ctx->tc_state_cap ? ctx->tc_state_cap * 2 : 8;
ctx->tc_state = (struct tcst *)realloc(ctx->tc_state, struct tcst *nt = (struct tcst *)realloc(ctx->tc_state,
(size_t)ctx->tc_state_cap * sizeof(*ctx->tc_state)); (size_t)ncap * sizeof(*ctx->tc_state));
if (!nt) continue;
ctx->tc_state = nt;
ctx->tc_state_cap = ncap;
}
char *kcopy = strdup(key);
char *ncopy = strdup(tc[i].name ? tc[i].name : "");
char *acopy = strdup("");
if (!kcopy || !ncopy || !acopy) {
free(kcopy);
free(ncopy);
free(acopy);
continue;
} }
st = &ctx->tc_state[ctx->tc_state_n++]; st = &ctx->tc_state[ctx->tc_state_n++];
st->key = strdup(key); st->key = kcopy;
st->prev_args = strdup(""); st->name = ncopy;
st->prev_args = acopy;
st->emitted = false; st->emitted = false;
} else if (tc[i].name && tc[i].name[0] && st->name[0] == 0) {
char *ncopy = strdup(tc[i].name);
if (ncopy) {
free(st->name);
st->name = ncopy;
}
} }
const char *cur = tc[i].arguments ? tc[i].arguments : ""; const char *cur = tc[i].arguments ? tc[i].arguments : "";
if (!st->emitted) { if (!st->emitted) {
char *ncur = strdup(cur);
if (!ncur) continue;
st->emitted = true; st->emitted = true;
free(st->prev_args); free(st->prev_args);
st->prev_args = strdup(cur); st->prev_args = ncur;
ctx->any_toolcall = true; ctx->any_toolcall = true;
char *js = toolcall_delta_json(&tc[i], i); char *js = toolcall_delta_json(&tc[i], i);
if (js) {
emit_chunk(ctx, NULL, NULL, NULL, js, NULL); emit_chunk(ctx, NULL, NULL, NULL, js, NULL);
free(js); free(js);
}
} else { } else {
size_t prev = strlen(st->prev_args); size_t prev = strlen(st->prev_args);
size_t curlen = strlen(cur); size_t curlen = strlen(cur);
if (curlen > prev) { if (curlen > prev) {
const char *delta = cur + prev; char *ncur = strdup(cur);
if (!ncur) continue;
const char *delta = ncur + prev;
free(st->prev_args); free(st->prev_args);
st->prev_args = strdup(cur); st->prev_args = ncur;
char *js = toolcall_delta_argdelta(i, delta); char *js = toolcall_delta_argdelta(i, delta);
if (js) {
emit_chunk(ctx, NULL, NULL, NULL, js, NULL); emit_chunk(ctx, NULL, NULL, NULL, js, NULL);
free(js); free(js);
} }
} }
} }
}
} }
/* ------------------------------------------------------------------ */ /* ------------------------------------------------------------------ */
@@ -322,8 +352,9 @@ static size_t write_sse_cb(void *ptr, size_t sz, size_t nmemb, void *data) {
/* Upstream calls (with Fibonacci backoff retry) */ /* Upstream calls (with Fibonacci backoff retry) */
/* ------------------------------------------------------------------ */ /* ------------------------------------------------------------------ */
static int upstream_post(const char *url, const char *body, const char *ua, static int upstream_post(const char *url, const char *body, const char *ua,
strbuf *resp, long *http_code) { strbuf *resp) {
for (int attempt = 1; attempt <= MAX_RETRIES; attempt++) { for (int attempt = 1; attempt <= MAX_RETRIES; attempt++) {
sb_reset(resp);
CURL *c = curl_easy_init(); CURL *c = curl_easy_init();
if (!c) return -1; if (!c) return -1;
struct curl_slist *hdrs = NULL; struct curl_slist *hdrs = NULL;
@@ -344,10 +375,7 @@ static int upstream_post(const char *url, const char *body, const char *ua,
curl_easy_getinfo(c, CURLINFO_RESPONSE_CODE, &code); curl_easy_getinfo(c, CURLINFO_RESPONSE_CODE, &code);
curl_slist_free_all(hdrs); curl_slist_free_all(hdrs);
curl_easy_cleanup(c); curl_easy_cleanup(c);
if (rc == CURLE_OK && code == 200) { if (rc == CURLE_OK && code == 200) return 0;
if (http_code) *http_code = code;
return 0;
}
if (attempt < MAX_RETRIES) sleep((unsigned int)fib_delay(attempt)); if (attempt < MAX_RETRIES) sleep((unsigned int)fib_delay(attempt));
} }
return -1; return -1;
@@ -355,6 +383,15 @@ static int upstream_post(const char *url, const char *body, const char *ua,
static int upstream_get_stream(const char *url, const char *ua, sse_ctx_t *ctx) { static int upstream_get_stream(const char *url, const char *ua, sse_ctx_t *ctx) {
for (int attempt = 1; attempt <= MAX_RETRIES; attempt++) { for (int attempt = 1; attempt <= MAX_RETRIES; attempt++) {
sb_reset(&ctx->line);
ctx->status = 0;
free(ctx->final_content);
ctx->final_content = NULL;
free(ctx->final_reasoning);
ctx->final_reasoning = NULL;
free_toolcalls(ctx->final_tc, ctx->final_n_tc);
ctx->final_tc = NULL;
ctx->final_n_tc = 0;
CURL *c = curl_easy_init(); CURL *c = curl_easy_init();
if (!c) return -1; if (!c) return -1;
struct curl_slist *hdrs = NULL; struct curl_slist *hdrs = NULL;
@@ -376,6 +413,7 @@ static int upstream_get_stream(const char *url, const char *ua, sse_ctx_t *ctx)
curl_slist_free_all(hdrs); curl_slist_free_all(hdrs);
curl_easy_cleanup(c); curl_easy_cleanup(c);
if (rc == CURLE_OK && code == 200) return 0; if (rc == CURLE_OK && code == 200) return 0;
if (ctx->headers_sent) break;
if (attempt < MAX_RETRIES) sleep((unsigned int)fib_delay(attempt)); if (attempt < MAX_RETRIES) sleep((unsigned int)fib_delay(attempt));
} }
return -1; return -1;
@@ -421,7 +459,7 @@ static void handle_chat(int fd, http_req_t *req, const config_t *cfg) {
model = model_buf; model = model_buf;
} }
char *payload = build_gradio_payload(jreq, model); char *payload = build_gradio_payload(jreq);
int prompt_tokens = count_messages_tokens(jreq); int prompt_tokens = count_messages_tokens(jreq);
bool include_usage = false; bool include_usage = false;
@@ -448,8 +486,9 @@ static void handle_chat(int fd, http_req_t *req, const config_t *cfg) {
size_t eplen = strlen(ep); size_t eplen = strlen(ep);
char epclean[2048]; char epclean[2048];
if (eplen > 0 && ep[eplen - 1] == '/') { if (eplen > 0 && ep[eplen - 1] == '/') {
memcpy(epclean, ep, eplen - 1); size_t clen = eplen - 1 < sizeof epclean - 1 ? eplen - 1 : sizeof epclean - 1;
epclean[eplen - 1] = 0; memcpy(epclean, ep, clen);
epclean[clen] = 0;
} else { } else {
strncpy(epclean, ep, sizeof epclean - 1); strncpy(epclean, ep, sizeof epclean - 1);
epclean[sizeof epclean - 1] = 0; epclean[sizeof epclean - 1] = 0;
@@ -460,8 +499,7 @@ static void handle_chat(int fd, http_req_t *req, const config_t *cfg) {
strbuf post_resp; strbuf post_resp;
sb_init(&post_resp); sb_init(&post_resp);
long code = 0; if (!post_resp.buf || upstream_post(call_url, payload, ua, &post_resp) != 0) {
if (upstream_post(call_url, payload, ua, &post_resp, &code) != 0) {
send_simple(fd, 502, "Bad Gateway", send_simple(fd, 502, "Bad Gateway",
"Upstream error: failed to reach Gradio call endpoint"); "Upstream error: failed to reach Gradio call endpoint");
sb_free(&post_resp); sb_free(&post_resp);
@@ -475,8 +513,10 @@ static void handle_chat(int fd, http_req_t *req, const config_t *cfg) {
if (jr) { if (jr) {
json_object *ev = NULL; json_object *ev = NULL;
if (json_object_object_get_ex(jr, "event_id", &ev) && if (json_object_object_get_ex(jr, "event_id", &ev) &&
json_object_get_type(ev) == json_type_string) json_object_get_type(ev) == json_type_string) {
strncpy(event_id, json_object_get_string(ev), sizeof event_id - 1); strncpy(event_id, json_object_get_string(ev), sizeof event_id - 1);
event_id[sizeof event_id - 1] = 0;
}
json_object_put(jr); json_object_put(jr);
} }
sb_free(&post_resp); sb_free(&post_resp);
@@ -501,10 +541,10 @@ static void handle_chat(int fd, http_req_t *req, const config_t *cfg) {
generate_uuid(uuid); generate_uuid(uuid);
snprintf(ctx.completion_id, sizeof ctx.completion_id, "chatcmpl-%s", uuid); snprintf(ctx.completion_id, sizeof ctx.completion_id, "chatcmpl-%s", uuid);
sb_init(&ctx.line); sb_init(&ctx.line);
if (stream) {
sb_init(&ctx.emitted_content); sb_init(&ctx.emitted_content);
sb_init(&ctx.emitted_reasoning); sb_init(&ctx.emitted_reasoning);
ctx.final_content = strdup(""); }
ctx.final_reasoning = strdup("");
int rc = upstream_get_stream(stream_url, ua, &ctx); int rc = upstream_get_stream(stream_url, ua, &ctx);
@@ -519,11 +559,13 @@ static void handle_chat(int fd, http_req_t *req, const config_t *cfg) {
if (stream) { if (stream) {
emit_chunk(&ctx, NULL, NULL, NULL, NULL, ctx.any_toolcall ? "tool_calls" : "stop"); emit_chunk(&ctx, NULL, NULL, NULL, NULL, ctx.any_toolcall ? "tool_calls" : "stop");
if (ctx.include_usage) { if (ctx.include_usage) {
int completion_tokens = tok_count_str(ctx.emitted_content.buf); int reasoning_tokens = tok_count_str(ctx.emitted_reasoning.buf);
completion_tokens += tok_count_str(ctx.emitted_reasoning.buf); int completion_tokens = tok_count_str(ctx.emitted_content.buf) + reasoning_tokens;
for (int i = 0; i < ctx.tc_state_n; i++) for (int i = 0; i < ctx.tc_state_n; i++) {
completion_tokens += tok_count_str(ctx.tc_state[i].name);
completion_tokens += tok_count_str(ctx.tc_state[i].prev_args); completion_tokens += tok_count_str(ctx.tc_state[i].prev_args);
emit_usage_chunk(&ctx, prompt_tokens, completion_tokens); }
emit_usage_chunk(&ctx, prompt_tokens, completion_tokens, reasoning_tokens);
} }
const char *done = "data: [DONE]\n\n"; const char *done = "data: [DONE]\n\n";
write_all(fd, done, strlen(done)); write_all(fd, done, strlen(done));
@@ -545,6 +587,7 @@ cleanup:
sb_free(&ctx.emitted_reasoning); sb_free(&ctx.emitted_reasoning);
for (int i = 0; i < ctx.tc_state_n; i++) { for (int i = 0; i < ctx.tc_state_n; i++) {
free(ctx.tc_state[i].key); free(ctx.tc_state[i].key);
free(ctx.tc_state[i].name);
free(ctx.tc_state[i].prev_args); free(ctx.tc_state[i].prev_args);
} }
free(ctx.tc_state); free(ctx.tc_state);
@@ -558,6 +601,7 @@ cleanup:
/* Request parsing / connection dispatch */ /* Request parsing / connection dispatch */
/* ------------------------------------------------------------------ */ /* ------------------------------------------------------------------ */
int read_headers(int fd, strbuf *buf) { int read_headers(int fd, strbuf *buf) {
if (!buf->buf) return -1;
char tmp[4096]; char tmp[4096];
while (1) { while (1) {
if (strstr(buf->buf, "\r\n\r\n")) break; if (strstr(buf->buf, "\r\n\r\n")) break;
@@ -570,6 +614,7 @@ int read_headers(int fd, strbuf *buf) {
} }
int parse_http_request(strbuf *buf, int fd, http_req_t *req, long max_body_bytes) { int parse_http_request(strbuf *buf, int fd, http_req_t *req, long max_body_bytes) {
if (!buf->buf) return -1;
char *p = buf->buf; char *p = buf->buf;
char *line_end = strstr(p, "\r\n"); char *line_end = strstr(p, "\r\n");
if (!line_end) return -1; if (!line_end) return -1;
@@ -617,6 +662,7 @@ int parse_http_request(strbuf *buf, int fd, http_req_t *req, long max_body_bytes
char *body_start = hlimit + 4; char *body_start = hlimit + 4;
size_t body_avail = buf->len - (size_t)(body_start - buf->buf); size_t body_avail = buf->len - (size_t)(body_start - buf->buf);
if (req->content_length < 0) return -1;
if (max_body_bytes > 0 && req->content_length > max_body_bytes) return -2; if (max_body_bytes > 0 && req->content_length > max_body_bytes) return -2;
if (req->content_length > 0) { if (req->content_length > 0) {
req->body = (char *)malloc((size_t)req->content_length + 1); req->body = (char *)malloc((size_t)req->content_length + 1);
@@ -634,6 +680,7 @@ int parse_http_request(strbuf *buf, int fd, http_req_t *req, long max_body_bytes
req->body_len = got; req->body_len = got;
} else { } else {
req->body = strdup(""); req->body = strdup("");
if (!req->body) return -1;
req->body_len = 0; req->body_len = 0;
} }
return 0; return 0;
tokenizer.c
+169 -74
View File
@@ -19,6 +19,7 @@
#include <string.h> #include <string.h>
#include <stdint.h> #include <stdint.h>
#include <stdlib.h> #include <stdlib.h>
#include <pthread.h>
extern const int tok_ntok; extern const int tok_ntok;
extern const unsigned char tok_blob[]; extern const unsigned char tok_blob[];
@@ -26,10 +27,11 @@ extern const int tok_rank[];
extern const uint16_t tok_len[]; extern const uint16_t tok_len[];
/* ----------------------------- vocab hash ----------------------------- */ /* ----------------------------- vocab hash ----------------------------- */
static int tok_inited = 0; static pthread_once_t tok_once = PTHREAD_ONCE_INIT;
static const char **tok_keys = NULL; static const char **tok_keys = NULL;
static int *tok_ranks = NULL; static int *tok_ranks = NULL;
static int *tok_klen = NULL;static unsigned int tok_mask = 0; static int *tok_klen = NULL;
static unsigned int tok_mask = 0;
static unsigned long tok_fnv1a(const unsigned char *s, int len) { static unsigned long tok_fnv1a(const unsigned char *s, int len) {
unsigned long h = 1469598103934665603UL; unsigned long h = 1469598103934665603UL;
@@ -41,34 +43,39 @@ static unsigned long tok_fnv1a(const unsigned char *s, int len) {
} }
static void tok_init(void) { static void tok_init(void) {
if (tok_inited) return;
tok_inited = 1;
unsigned int size = 1; unsigned int size = 1;
while (size < (unsigned int)tok_ntok * 2u) size <<= 1; while (size < (unsigned int)tok_ntok * 2u) size <<= 1;
tok_mask = size - 1; const char **keys = (const char **)calloc(size, sizeof(char *));
tok_keys = (const char **)calloc(size, sizeof(char *)); int *ranks = (int *)calloc(size, sizeof(int));
tok_ranks = (int *)calloc(size, sizeof(int)); int *klen = (int *)calloc(size, sizeof(int));
tok_klen = (int *)calloc(size, sizeof(int)); if (!keys || !ranks || !klen) {
if (!tok_keys || !tok_ranks || !tok_klen) return; free(keys);
free(ranks);
free(klen);
return;
}
unsigned int mask = size - 1;
int offset = 0; int offset = 0;
for (int i = 0; i < tok_ntok; i++) { for (int i = 0; i < tok_ntok; i++) {
int klen = tok_len[i]; int kl = tok_len[i];
const char *key = (const char *)(tok_blob + offset); const char *key = (const char *)(tok_blob + offset);
unsigned long h = tok_fnv1a((const unsigned char *)key, klen) & tok_mask; unsigned long h = tok_fnv1a((const unsigned char *)key, kl) & mask;
while (tok_keys[h]) { while (keys[h]) {
const char *k = tok_keys[h]; h = (h + 1) & mask;
if (tok_klen[h] == klen && memcmp(k, key, klen) == 0) break; /* dup (none expected) */
h = (h + 1) & tok_mask;
} }
tok_keys[h] = key; keys[h] = key;
tok_ranks[h] = tok_rank[i]; ranks[h] = tok_rank[i];
tok_klen[h] = klen; klen[h] = kl;
offset += klen; offset += kl;
} }
tok_mask = mask;
tok_ranks = ranks;
tok_klen = klen;
tok_keys = keys;
} }
static int tok_lookup(const unsigned char *s, int len) { static int tok_lookup(const unsigned char *s, int len) {
if (!tok_inited) tok_init(); pthread_once(&tok_once, tok_init);
if (!tok_keys) return -1; if (!tok_keys) return -1;
unsigned long h = tok_fnv1a(s, len) & tok_mask; unsigned long h = tok_fnv1a(s, len) & tok_mask;
while (tok_keys[h]) { while (tok_keys[h]) {
@@ -100,15 +107,39 @@ static int u_is_number(int cp) {
} }
static int u_is_letter(int cp) { static int u_is_letter(int cp) {
if ((cp >= 'a' && cp <= 'z') || (cp >= 'A' && cp <= 'Z')) return 1; if ((cp >= 'a' && cp <= 'z') || (cp >= 'A' && cp <= 'Z')) return 1;
if (cp >= 0x00c0 && cp <= 0x024f) return 1; /* Latin-1 + Latin Extended-A/B */ if (cp >= 0x00c0 && cp <= 0x02af) {
if (cp >= 0x0370 && cp <= 0x03ff) return 1; /* Greek */ return (cp != 0x00d7 && cp != 0x00f7); /* Latin-1 + Ext-A/B + IPA (excl. × ÷) */
if (cp >= 0x0400 && cp <= 0x04ff) return 1; /* Cyrillic */ }
if (cp >= 0x3040 && cp <= 0x30ff) return 1; /* Hiragana + Katakana */ if (cp >= 0x0370 && cp <= 0x03ff) {
return (cp != 0x037e && cp != 0x0387); /* Greek */
}
if (cp >= 0x0400 && cp <= 0x052f) return 1; /* Cyrillic + Supplement */
if (cp >= 0x0531 && cp <= 0x0587) return 1; /* Armenian */
if (cp >= 0x05d0 && cp <= 0x05ea) return 1; /* Hebrew */
if ((cp >= 0x0620 && cp <= 0x064a) ||
(cp >= 0x066e && cp <= 0x06d3) ||
(cp >= 0x06ee && cp <= 0x06ff) ||
(cp >= 0x0750 && cp <= 0x077f) ||
(cp >= 0x08a0 && cp <= 0x08ff)) return 1; /* Arabic */
if (cp >= 0x0904 && cp <= 0x0d7f) {
return !(cp >= 0x0964 && cp <= 0x096f); /* Devanagari + Indic scripts */
}
if ((cp >= 0x0e01 && cp <= 0x0e30) ||
(cp >= 0x0e32 && cp <= 0x0e33) ||
(cp >= 0x0e40 && cp <= 0x0e46)) return 1; /* Thai */
if (cp >= 0x10a0 && cp <= 0x10ff) return 1; /* Georgian */
if (cp >= 0x1100 && cp <= 0x11ff) return 1; /* Hangul Jamo */
if (cp >= 0x1e00 && cp <= 0x1fff) return 1; /* Latin Ext Additional + Greek Ext */
if (cp >= 0x3041 && cp <= 0x3096) return 1; /* Hiragana */
if (cp >= 0x30a1 && cp <= 0x30fc && cp != 0x30fb) return 1; /* Katakana (excl. ・) */
if (cp >= 0x3131 && cp <= 0x318e) return 1; /* Hangul Compatibility Jamo */
if (cp >= 0x3400 && cp <= 0x4dbf) return 1; /* CJK Ext A */ if (cp >= 0x3400 && cp <= 0x4dbf) return 1; /* CJK Ext A */
if (cp >= 0x4e00 && cp <= 0x9fff) return 1; /* CJK Unified Ideographs */ if (cp >= 0x4e00 && cp <= 0x9fff) return 1; /* CJK Unified Ideographs */
if (cp >= 0xac00 && cp <= 0xd7a3) return 1; /* Hangul Syllables */ if (cp >= 0xac00 && cp <= 0xd7a3) return 1; /* Hangul Syllables */
if (cp >= 0xff21 && cp <= 0xff5a) return 1; /* fullwidth A-Z */ if (cp >= 0xf900 && cp <= 0xfaff) return 1; /* CJK Compatibility Ideographs */
if (cp >= 0xff21 && cp <= 0xff3a) return 1; /* fullwidth A-Z */
if (cp >= 0xff41 && cp <= 0xff5a) return 1; /* fullwidth a-z */ if (cp >= 0xff41 && cp <= 0xff5a) return 1; /* fullwidth a-z */
if (cp >= 0x20000 && cp <= 0x2fa1f) return 1; /* CJK Ext B-F + Compat */
return 0; return 0;
} }
@@ -122,7 +153,7 @@ static void decode_cp(const unsigned char *t, int len, int pos, int *cp, int *cp
else if ((b >> 3) == 0x1e) { c = b & 0x07; l = 4; } else if ((b >> 3) == 0x1e) { c = b & 0x07; l = 4; }
else { c = b; l = 1; } /* invalid lead byte: treat as single byte */ else { c = b; l = 1; } /* invalid lead byte: treat as single byte */
for (int j = 1; j < l; j++) { for (int j = 1; j < l; j++) {
if (pos + j >= len) { c = b; l = 1; break; } if (pos + j >= len || (t[pos + j] & 0xc0) != 0x80) { c = b; l = 1; break; }
c = (c << 6) | (t[pos + j] & 0x3f); c = (c << 6) | (t[pos + j] & 0x3f);
} }
*cp = c; *cplen = l; *cp = c; *cplen = l;
@@ -183,7 +214,7 @@ static int tok_piece_end(const unsigned char *t, int len, int i) {
} }
/* 4) ?[^\s\p{L}\p{N}]++[\r\n]*+ (leading space optional) */ /* 4) ?[^\s\p{L}\p{N}]++[\r\n]*+ (leading space optional) */
int is_other = (!u_is_space(cp) && !u_is_letter(cp) && !u_is_number(cp)); int is_other = (!isS && !isL && !isN);
if (cp == 0x20 && ni < len && (!u_is_space(ncp) && !u_is_letter(ncp) && !u_is_number(ncp))) { if (cp == 0x20 && ni < len && (!u_is_space(ncp) && !u_is_letter(ncp) && !u_is_number(ncp))) {
int e = ni; int e = ni;
while (e < len) { while (e < len) {
@@ -213,80 +244,143 @@ static int tok_piece_end(const unsigned char *t, int len, int i) {
return e; return e;
} }
/* 5-8) whitespace runs */ /* 5-8) whitespace runs: \s++$ | \s*[\r\n] | \s+(?!\S) | \s */
if (isS) {
int e = i; int e = i;
int last_ws_len = cplen; int last_ws_len = cplen;
int has_nl = (cp == 0x0d || cp == 0x0a); int last_nl_end = 0;
while (e < len) { while (e < len) {
int c, cl; decode_cp(t, len, e, &c, &cl); int c, cl; decode_cp(t, len, e, &c, &cl);
if (!u_is_space(c)) break; if (!u_is_space(c)) break;
last_ws_len = cl; last_ws_len = cl;
if (c == 0x0d || c == 0x0a) has_nl = 1; if (c == 0x0d || c == 0x0a) last_nl_end = e + cl;
e += cl; e += cl;
} }
if (e == len) return e; /* 5) trailing whitespace */ if (e == len) return e; /* 5) \s++$ (trailing whitespace) */
if (has_nl) return e; /* 6) whitespace containing a newline */ if (last_nl_end > i) return last_nl_end; /* 6) \s*[\r\n] (up to last CR/LF) */
if ((e - i) > last_ws_len) return e - last_ws_len; /* 7) leave last whitespace */ if ((e - i) > last_ws_len) return e - last_ws_len; /* 7) \s+(?!\S) (leave last ws) */
return e; /* 8) single whitespace */ return e; /* 8) \s (single whitespace) */
}
return i + cplen; /* fallback */
} }
/* ----------------------------- BPE ------------------------------------ */ /* ----------------------------- BPE ------------------------------------ */
/* Count tokens in a single raw-byte piece via greedy lowest-rank merges. */ #define MAX_TOKEN_BYTES 128
typedef struct {
int prev;
int next;
int rank;
} bpe_node_t;
typedef struct {
int rank;
int pos;
} bpe_heap_item_t;
static int heap_less(bpe_heap_item_t a, bpe_heap_item_t b) {
return (a.rank < b.rank) || (a.rank == b.rank && a.pos < b.pos);
}
static void heap_push(bpe_heap_item_t *h, int *hn, bpe_heap_item_t v) {
int i = (*hn)++;
while (i > 0) {
int p = (i - 1) >> 1;
if (!heap_less(v, h[p])) break;
h[i] = h[p];
i = p;
}
h[i] = v;
}
static bpe_heap_item_t heap_pop(bpe_heap_item_t *h, int *hn) {
bpe_heap_item_t top = h[0];
bpe_heap_item_t last = h[--(*hn)];
int n = *hn, i = 0;
while ((i << 1) + 1 < n) {
int l = (i << 1) + 1, r = l + 1;
int c = (r < n && heap_less(h[r], h[l])) ? r : l;
if (!heap_less(h[c], last)) break;
h[i] = h[c];
i = c;
}
if (n > 0) h[i] = last;
return top;
}
/* Count tokens in a single raw-byte piece via O(N log N) lowest-rank merges. */
static int tok_bpe(const unsigned char *piece, int plen) { static int tok_bpe(const unsigned char *piece, int plen) {
if (plen <= 0) return 0; if (plen <= 0) return 0;
if (plen > 4096) return plen / 4; /* pathological guard; estimate */ if (plen == 1) return 1;
char **sym = (char **)malloc(sizeof(char *) * (size_t)plen); bpe_node_t *nodes = (bpe_node_t *)malloc(sizeof(bpe_node_t) * (size_t)plen);
if (!sym) return plen / 4; bpe_heap_item_t *heap = (bpe_heap_item_t *)malloc(sizeof(bpe_heap_item_t) * (size_t)plen * 2u);
int nsym = 0; if (!nodes || !heap) {
free(nodes);
free(heap);
return (plen + 3) / 4;
}
int hn = 0;
for (int k = 0; k < plen; k++) { for (int k = 0; k < plen; k++) {
char *s = (char *)malloc(2); nodes[k].prev = k - 1;
if (!s) { for (int j = 0; j < nsym; j++) free(sym[j]); free(sym); return plen / 4; } nodes[k].next = k + 1;
s[0] = (char)piece[k]; s[1] = 0; if (k + 1 < plen) {
sym[nsym++] = s; int r = tok_lookup(piece + k, 2);
nodes[k].rank = r;
if (r >= 0) {
bpe_heap_item_t item = { r, k };
heap_push(heap, &hn, item);
} }
char buf[256]; } else {
while (nsym > 1) { nodes[k].rank = -1;
int best = -1, bestr = 0x7fffffff;
for (int k = 0; k < nsym - 1; k++) {
int l1 = (int)strlen(sym[k]), l2 = (int)strlen(sym[k + 1]);
if (l1 + l2 >= (int)sizeof(buf)) continue;
memcpy(buf, sym[k], l1);
memcpy(buf + l1, sym[k + 1], l2);
buf[l1 + l2] = 0;
int r = tok_lookup((const unsigned char *)buf, l1 + l2);
if (r >= 0 && r < bestr) { bestr = r; best = k; }
} }
if (best < 0) break; }
int l1 = (int)strlen(sym[best]), l2 = (int)strlen(sym[best + 1]); int nsym = plen;
char *m = (char *)malloc((size_t)(l1 + l2 + 1)); while (nsym > 1 && hn > 0) {
memcpy(m, sym[best], l1); bpe_heap_item_t top = heap_pop(heap, &hn);
memcpy(m + l1, sym[best + 1], l2); int best = top.pos;
m[l1 + l2] = 0; int right = nodes[best].next;
free(sym[best]); if (right >= plen || nodes[best].rank != top.rank) continue;
free(sym[best + 1]);
sym[best] = m; int nr = nodes[right].next;
for (int k = best + 1; k < nsym - 1; k++) sym[k] = sym[k + 1]; nodes[best].next = nr;
if (nr < plen) nodes[nr].prev = best;
nodes[right].next = plen + 1;
nodes[right].rank = -1;
nsym--; nsym--;
if (nr < plen) {
int clen = nodes[nr].next - best;
int r = (clen <= MAX_TOKEN_BYTES) ? tok_lookup(piece + best, clen) : -1;
nodes[best].rank = r;
if (r >= 0) {
bpe_heap_item_t item = { r, best };
heap_push(heap, &hn, item);
} }
int n = nsym; } else {
for (int k = 0; k < nsym; k++) free(sym[k]); nodes[best].rank = -1;
free(sym); }
return n;
int left = nodes[best].prev;
if (left >= 0) {
int clen = nr - left;
int r = (clen <= MAX_TOKEN_BYTES) ? tok_lookup(piece + left, clen) : -1;
nodes[left].rank = r;
if (r >= 0) {
bpe_heap_item_t item = { r, left };
heap_push(heap, &hn, item);
}
}
}
free(nodes);
free(heap);
return nsym;
} }
/* ----------------------------- public --------------------------------- */ /* ----------------------------- public --------------------------------- */
int tok_count(const char *text, int len) { int tok_count(const char *text, int len) {
if (!text || len <= 0) return 0; if (!text || len <= 0) return 0;
if (!tok_inited) tok_init(); pthread_once(&tok_once, tok_init);
const unsigned char *t = (const unsigned char *)text; const unsigned char *t = (const unsigned char *)text;
int total = 0, i = 0; int total = 0, i = 0;
while (i < len) { while (i < len) {
int end = tok_piece_end(t, len, i); int end = tok_piece_end(t, len, i);
if (end <= i) end = i + 1;
total += tok_bpe(t + i, end - i); total += tok_bpe(t + i, end - i);
i = end; i = end;
} }
@@ -294,5 +388,6 @@ int tok_count(const char *text, int len) {
} }
int tok_count_str(const char *text) { int tok_count_str(const char *text) {
if (!text) return 0;
return tok_count(text, (int)strlen(text)); return tok_count(text, (int)strlen(text));
} }
util.c
+11 -5
View File
@@ -4,8 +4,7 @@
#include <string.h> #include <string.h>
#include <unistd.h> #include <unistd.h>
#include <errno.h> #include <errno.h>
#include <time.h> #include <stdint.h>
#include <fcntl.h>
/* ----------------------------- strbuf ------------------------------ */ /* ----------------------------- strbuf ------------------------------ */
@@ -25,9 +24,15 @@ void sb_free(strbuf *s) {
int sb_append(strbuf *s, const char *data, size_t n) { int sb_append(strbuf *s, const char *data, size_t n) {
if (n == 0) return 0; if (n == 0) return 0;
if (s->len + n + 1 > s->cap) { if (!data) return -1;
size_t ncap = s->cap ? s->cap * 2 : 64; if (n > SIZE_MAX - s->len - 1) return -1;
while (ncap < s->len + n + 1) ncap *= 2; size_t need = s->len + n + 1;
if (need > s->cap) {
size_t ncap = s->cap ? s->cap : 64;
while (ncap < need) {
if (ncap > SIZE_MAX / 2) { ncap = need; break; }
ncap *= 2;
}
char *nb = (char *)realloc(s->buf, ncap); char *nb = (char *)realloc(s->buf, ncap);
if (!nb) return -1; if (!nb) return -1;
s->buf = nb; s->buf = nb;
@@ -40,6 +45,7 @@ int sb_append(strbuf *s, const char *data, size_t n) {
} }
int sb_append_str(strbuf *s, const char *str) { int sb_append_str(strbuf *s, const char *str) {
if (!str) return 0;
return sb_append(s, str, strlen(str)); return sb_append(s, str, strlen(str));
} }