Switch embedded tokenizer to Tencent Hy3 120k ByteLevel BPE
This commit is contained in:
@@ -71,9 +71,7 @@ char *build_completion_response(const char *id, long created, const char *model,
|
||||
tool_call_t *tc, int n_tc, int prompt_tokens);
|
||||
|
||||
|
||||
/* Token counting (embedded cl100k_base BPE tokenizer). Counts are
|
||||
* deterministic subword counts; the served model (Hunyuan) uses a
|
||||
* different tokenizer, so these are an approximation, not exact. */
|
||||
/* Token counting (embedded Tencent Hy3 120k ByteLevel BPE tokenizer). */
|
||||
int tok_count(const char *text, int len);
|
||||
int tok_count_str(const char *text);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user