Switch embedded tokenizer to Tencent Hy3 120k ByteLevel BPE

This commit is contained in:
Luxferre
2026-10-14 14:15:28 +03:00
parent 80ad504d16
commit 561d90629f
5 changed files with 17110 additions and 19636 deletions
+1 -3
View File
@@ -71,9 +71,7 @@ char *build_completion_response(const char *id, long created, const char *model,
tool_call_t *tc, int n_tc, int prompt_tokens);
/* Token counting (embedded cl100k_base BPE tokenizer). Counts are
* deterministic subword counts; the served model (Hunyuan) uses a
* different tokenizer, so these are an approximation, not exact. */
/* Token counting (embedded Tencent Hy3 120k ByteLevel BPE tokenizer). */
int tok_count(const char *text, int len);
int tok_count_str(const char *text);