luxferre/hygate-c
1
0
Fork 0
Code Actions Activity

Tokenizer implemented

Browse Source
This commit is contained in:
Luxferre
2026-10-13 20:36:29 +03:00
parent 41cfd6f25e
commit 2d21b6f77e
5 changed files with 150 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hygate.h
+13 -1
View File
@@ -68,8 +68,20 @@ char *build_gradio_payload(void *req_obj, const char *model_default);
* frees the result. */
char *build_completion_response(const char *id, long created, const char *model,
const char *content, const char *reasoning,
tool_call_t *tc, int n_tc);
tool_call_t *tc, int n_tc, int prompt_tokens);
/* Token counting (embedded cl100k_base BPE tokenizer). Counts are
* deterministic subword counts; the served model (Hunyuan) uses a
* different tokenizer, so these are an approximation, not exact. */
int tok_count(const char *text, int len);
int tok_count_str(const char *text);
/* Count tokens across an OpenAI request's messages (prompt side). */
int count_messages_tokens(void *req_obj);
/* Count tokens of a generated completion (content + reasoning + tool calls). */
int count_completion_tokens(const char *content, const char *reasoning,
tool_call_t *tc, int n_tc);
/* Build a streaming tool-call delta JSON array (first emission: full
* tool call with index). Caller frees the result. */
char *toolcall_delta_json(tool_call_t *tc, int idx);