luxferre/hygate-c
1
0
Fork 0
Code Actions Activity

Switch embedded tokenizer to Tencent Hy3 120k ByteLevel BPE

Browse Source
This commit is contained in:
Luxferre
2026-10-14 14:15:28 +03:00
parent 80ad504d16
commit 561d90629f
5 changed files with 17110 additions and 19636 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+14 -9
View File
@@ -70,9 +70,11 @@ CORS is enabled for all origins (`Access-Control-Allow-Origin: *`).
Responses include an OpenAI-style `usage` object with `prompt_tokens`, Responses include an OpenAI-style `usage` object with `prompt_tokens`,
`completion_tokens`, `total_tokens`, and `completion_tokens_details` `completion_tokens`, `total_tokens`, and `completion_tokens_details`
(`reasoning_tokens`). These are computed locally by an **embedded cl100k_base (`reasoning_tokens`). These are computed locally by an **embedded Tencent Hy3
BPE tokenizer** (vocabulary data in `tokenizer_data.c`, logic in `tokenizer.c`) ByteLevel BPE tokenizer** (`tencent/Hy3`, 120,000 BPE tokens + 818 added
— no upstream token counts are available from the Gradio backend. special/control tokens; vocabulary data in `tokenizer_data.c`, 4-stage
pre-tokenizer and BPE logic in `tokenizer.c`) — no upstream token counts are
available from the Gradio backend.
- `prompt_tokens` sums the active `system` prompt, each message's `content` - `prompt_tokens` sums the active `system` prompt, each message's `content`
(or individual multi-part content parts), `reasoning_content`, `tool_calls` (or individual multi-part content parts), `reasoning_content`, `tool_calls`
@@ -83,11 +85,12 @@ BPE tokenizer** (vocabulary data in `tokenizer_data.c`, logic in `tokenizer.c`)
broken out under `completion_tokens_details`. broken out under `completion_tokens_details`.
- `total_tokens = prompt_tokens + completion_tokens`. - `total_tokens = prompt_tokens + completion_tokens`.
cl100k_base is OpenAI's canonical multilingual tokenizer and is used as the The embedded tokenizer implements `tencent/Hy3`'s exact 4-stage `Sequence`
best available approximation for Hunyuan; the resulting counts are pre-tokenizer (`\p{N}{1,3}` digit splitting, CJK/Hiragana/Katakana isolation,
deterministic subword counts and may differ slightly from the exact Hunyuan word/punctuation/whitespace regex splitting, and `ByteLevel` byte mapping),
vocabulary. No per-message formatting overhead (the `+3`/priming tokens added special-token matching (`<｜hy_...｜>`, `<think:opensource>`, tool tags,
OpenAI applies) is added. etc.), and rank-ordered BPE merges. No per-message chat-template wrapper
overhead is added.
## Security considerations ## Security considerations
@@ -104,7 +107,9 @@ OpenAI applies) is added.
|------|------| |------|------|
| `main.c` | Entry point, flag parsing, TCP listen, thread-per-connection with concurrency cap. | | `main.c` | Entry point, flag parsing, TCP listen, thread-per-connection with concurrency cap. |
| `server.c` | HTTP parsing, routing, SSE emitters, libcurl upstream calls, streaming state machine. | | `server.c` | HTTP parsing, routing, SSE emitters, libcurl upstream calls, streaming state machine. |
| `jsonapi.c` | OpenAI <-> Gradio payload translation and stream parsing. | | `jsonapi.c` | OpenAI <-> Gradio payload translation, stream parsing, and message token counting. |
| `tokenizer.c` | Embedded Tencent Hy3 4-stage pre-tokenizer, added-token matcher, and BPE engine. |
| `tokenizer_data.c` | Generated Tencent Hy3 120k BPE vocabulary tables. |
| `util.c` | Growable string buffer (`strbuf`), UUID, backoff, helpers. | | `util.c` | Growable string buffer (`strbuf`), UUID, backoff, helpers. |
| `hygate.h` | Shared types and declarations. | | `hygate.h` | Shared types and declarations. |
hygate.h
+1 -3
View File
@@ -71,9 +71,7 @@ char *build_completion_response(const char *id, long created, const char *model,
tool_call_t *tc, int n_tc, int prompt_tokens); tool_call_t *tc, int n_tc, int prompt_tokens);
/* Token counting (embedded cl100k_base BPE tokenizer). Counts are /* Token counting (embedded Tencent Hy3 120k ByteLevel BPE tokenizer). */
* deterministic subword counts; the served model (Hunyuan) uses a
* different tokenizer, so these are an approximation, not exact. */
int tok_count(const char *text, int len); int tok_count(const char *text, int len);
int tok_count_str(const char *text); int tok_count_str(const char *text);
jsonapi.c
+1 -1
View File
@@ -40,7 +40,7 @@ static int count_content_tokens(json_object *content) {
} }
/* ------------------------------------------------------------------ */ /* ------------------------------------------------------------------ */
/* Token counting (embedded cl100k_base BPE) */ /* Token counting (embedded Tencent Hy3 BPE) */
/* ------------------------------------------------------------------ */ /* ------------------------------------------------------------------ */
int count_messages_tokens(void *req_obj) { int count_messages_tokens(void *req_obj) {
json_object *req = (json_object *)req_obj; json_object *req = (json_object *)req_obj;
tokenizer.c
+549 -169
View File
@@ -1,19 +1,17 @@
/* tokenizer.c - embedded cl100k_base BPE tokenizer (pure C, no deps). /* tokenizer.c - embedded Tencent Hy3 ByteLevel BPE tokenizer (pure C, no deps).
* *
* Counts tokens for arbitrary UTF-8 text using the OpenAI cl100k_base * Counts tokens for arbitrary UTF-8 text using the official Tencent Hy3
* vocabulary (embedded in tokenizer_data.c). The model served by this * (tencent/Hy3, HYV3ForCausalLM) 120k ByteLevel BPE vocabulary (embedded
* gateway (Hunyuan) uses a different tokenizer, but cl100k_base is the * in tokenizer_data.c) and its 818 added special/control tokens.
* canonical multilingual OpenAI tokenizer and gives deterministic,
* real subword counts that are far closer to the truth than a naive
* bytes/4 heuristic.
* *
* The pre-tokenizer mirrors tiktoken's cl100k regex: * The pre-tokenizer implements Hy3's 4-stage Sequence pre-tokenizer:
* '(?i:[sdmt]|ll|ve|re)|[^\r\n\p{L}\p{N}]?+\p{L}++|\p{N}{1,3}+| * 1) Split(Isolated): \p{N}{1,3}
* ?[^\s\p{L}\p{N}]++[\r\n]*+|\s++$|\s*[\r\n]|\s+(?!\S)|\s * 2) Split(Isolated): [\u4e00-\u9fa5\u3040-\u309f\u30a0-\u30ff]+
* Unicode letter/number classification is approximated with compact * 3) Split(Isolated): [!"#$%&'()*+,\-./:;<=>?@\[\\\]^_`{|}~][A-Za-z]+
* codepoint-range tables (covers Latin, CJK, Kana, Hangul, Cyrillic, * |[^\r\n\p{L}\p{P}\p{S}]?[\p{L}\p{M}]+
* Greek and common digits); this only affects script boundaries, not * | ?[\p{P}\p{S}]+[\r\n]*
* the BPE merge counts for in-script text. * |\s*[\r\n]+|\s+(?!\S)|\s+
* 4) ByteLevel(add_prefix_space=false, use_regex=false)
*/ */
#include "hygate.h" #include "hygate.h"
#include <string.h> #include <string.h>
@@ -23,14 +21,13 @@
extern const int tok_ntok; extern const int tok_ntok;
extern const unsigned char tok_blob[]; extern const unsigned char tok_blob[];
extern const int tok_rank[];
extern const uint16_t tok_len[]; extern const uint16_t tok_len[];
/* ----------------------------- vocab hash ----------------------------- */ /* ----------------------------- vocab hash ----------------------------- */
static pthread_once_t tok_once = PTHREAD_ONCE_INIT; static pthread_once_t tok_once = PTHREAD_ONCE_INIT;
static const char **tok_keys = NULL; static const char **tok_keys = NULL;
static int *tok_ranks = NULL; static int *tok_ranks = NULL;
static int *tok_klen = NULL; static uint16_t *tok_klen = NULL;
static unsigned int tok_mask = 0; static unsigned int tok_mask = 0;
static unsigned long tok_fnv1a(const unsigned char *s, int len) { static unsigned long tok_fnv1a(const unsigned char *s, int len) {
@@ -47,7 +44,7 @@ static void tok_init(void) {
while (size < (unsigned int)tok_ntok * 2u) size <<= 1; while (size < (unsigned int)tok_ntok * 2u) size <<= 1;
const char **keys = (const char **)calloc(size, sizeof(char *)); const char **keys = (const char **)calloc(size, sizeof(char *));
int *ranks = (int *)calloc(size, sizeof(int)); int *ranks = (int *)calloc(size, sizeof(int));
int *klen = (int *)calloc(size, sizeof(int)); uint16_t *klen = (uint16_t *)calloc(size, sizeof(uint16_t));
if (!keys || !ranks || !klen) { if (!keys || !ranks || !klen) {
free(keys); free(keys);
free(ranks); free(ranks);
@@ -57,14 +54,14 @@ static void tok_init(void) {
unsigned int mask = size - 1; unsigned int mask = size - 1;
int offset = 0; int offset = 0;
for (int i = 0; i < tok_ntok; i++) { for (int i = 0; i < tok_ntok; i++) {
int kl = tok_len[i]; uint16_t kl = tok_len[i];
const char *key = (const char *)(tok_blob + offset); const char *key = (const char *)(tok_blob + offset);
unsigned long h = tok_fnv1a((const unsigned char *)key, kl) & mask; unsigned long h = tok_fnv1a((const unsigned char *)key, kl) & mask;
while (keys[h]) { while (keys[h]) {
h = (h + 1) & mask; h = (h + 1) & mask;
} }
keys[h] = key; keys[h] = key;
ranks[h] = tok_rank[i]; ranks[h] = i;
klen[h] = kl; klen[h] = kl;
offset += kl; offset += kl;
} }
@@ -75,194 +72,524 @@ static void tok_init(void) {
} }
static int tok_lookup(const unsigned char *s, int len) { static int tok_lookup(const unsigned char *s, int len) {
pthread_once(&tok_once, tok_init);
if (!tok_keys) return -1; if (!tok_keys) return -1;
unsigned long h = tok_fnv1a(s, len) & tok_mask; unsigned long h = tok_fnv1a(s, len) & tok_mask;
while (tok_keys[h]) { while (tok_keys[h]) {
const char *k = tok_keys[h]; if ((int)tok_klen[h] == len && memcmp(tok_keys[h], s, (size_t)len) == 0)
if (tok_klen[h] == len && memcmp(k, s, len) == 0) return tok_ranks[h]; return tok_ranks[h];
h = (h + 1) & tok_mask; h = (h + 1) & tok_mask;
} }
return -1; return -1;
} }
/* ----------------------------- unicode -------------------------------- */ /* ----------------------------- unicode -------------------------------- */
static int u_is_space(int cp) { enum {
switch (cp) { UCAT_OTHER = 0,
case 0x20: case 0x09: case 0x0a: case 0x0d: case 0x0b: case 0x0c: UCAT_WS = 1, /* \s (Unicode White_Space) */
case 0xa0: case 0x1680: case 0x2028: case 0x2029: case 0x202f: UCAT_NUM = 2, /* \p{N} */
case 0x205f: case 0x3000: UCAT_LM = 3, /* [\p{L}\p{M}] */
return 1; UCAT_PS = 4 /* [\p{P}\p{S}] */
};
static const unsigned char u_ascii_cat[128] = {
0,0,0,0,0,0,0,0,0,1,1,1,1,1,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
1,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,
2,2,2,2,2,2,2,2,2,2,4,4,4,4,4,4,
4,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,
3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,
4,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,
3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,0,
};
/* Packed (start_codepoint << 3) | category for U+0080..U+10FFFF (1976 transitions). */
static const uint32_t u_cat_trans[1976] = {
0x400u,0x429u,0x430u,0x501u,0x50cu,0x553u,0x55cu,0x568u,0x574u,0x592u,0x5a4u,0x5abu,
0x5b4u,0x5cau,0x5d3u,0x5dcu,0x5e2u,0x5fcu,0x603u,0x6bcu,0x6c3u,0x7bcu,0x7c3u,0x1614u,
0x1633u,0x1694u,0x1703u,0x172cu,0x1763u,0x176cu,0x1773u,0x177cu,0x1803u,0x1bacu,0x1bb3u,0x1bc0u,
0x1bd3u,0x1bf4u,0x1bfbu,0x1c00u,0x1c24u,0x1c33u,0x1c3cu,0x1c43u,0x1c58u,0x1c63u,0x1c68u,0x1c73u,
0x1d10u,0x1d1bu,0x1fb4u,0x1fbbu,0x2414u,0x241bu,0x2980u,0x298bu,0x2ab8u,0x2acbu,0x2ad4u,0x2b03u,
0x2c4cu,0x2c58u,0x2c6cu,0x2c80u,0x2c8bu,0x2df4u,0x2dfbu,0x2e04u,0x2e0bu,0x2e1cu,0x2e23u,0x2e34u,
0x2e3bu,0x2e40u,0x2e83u,0x2f58u,0x2f7bu,0x2f9cu,0x2fa8u,0x3034u,0x3083u,0x30dcu,0x30e0u,0x30ecu,
0x3103u,0x3302u,0x3354u,0x3373u,0x36a4u,0x36abu,0x36e8u,0x36f4u,0x36fbu,0x374cu,0x3753u,0x3782u,
0x37d3u,0x37ecu,0x37fbu,0x3804u,0x3870u,0x3883u,0x3a58u,0x3a6bu,0x3d90u,0x3e02u,0x3e53u,0x3fb4u,
0x3fd3u,0x3fd8u,0x3febu,0x3ff4u,0x4003u,0x4170u,0x4184u,0x41f8u,0x4203u,0x42e0u,0x42f4u,0x42f8u,
0x4303u,0x4358u,0x4383u,0x4444u,0x444bu,0x4478u,0x44bbu,0x4710u,0x471bu,0x4b24u,0x4b32u,0x4b84u,
0x4b8bu,0x4c20u,0x4c2bu,0x4c68u,0x4c7bu,0x4c88u,0x4c9bu,0x4d48u,0x4d53u,0x4d88u,0x4d93u,0x4d98u,
0x4db3u,0x4dd0u,0x4de3u,0x4e28u,0x4e3bu,0x4e48u,0x4e5bu,0x4e78u,0x4ebbu,0x4ec0u,0x4ee3u,0x4ef0u,
0x4efbu,0x4f20u,0x4f32u,0x4f83u,0x4f94u,0x4fa2u,0x4fd4u,0x4fe3u,0x4fecu,0x4ff3u,0x4ff8u,0x500bu,
0x5020u,0x502bu,0x5058u,0x507bu,0x5088u,0x509bu,0x5148u,0x5153u,0x5188u,0x5193u,0x51a0u,0x51abu,
0x51b8u,0x51c3u,0x51d0u,0x51e3u,0x51e8u,0x51f3u,0x5218u,0x523bu,0x5248u,0x525bu,0x5270u,0x528bu,
0x5290u,0x52cbu,0x52e8u,0x52f3u,0x52f8u,0x5332u,0x5383u,0x53b4u,0x53b8u,0x540bu,0x5420u,0x542bu,
0x5470u,0x547bu,0x5490u,0x549bu,0x5548u,0x5553u,0x5588u,0x5593u,0x55a0u,0x55abu,0x55d0u,0x55e3u,
0x5630u,0x563bu,0x5650u,0x565bu,0x5670u,0x5683u,0x5688u,0x5703u,0x5720u,0x5732u,0x5784u,0x5790u,
0x57cbu,0x5800u,0x580bu,0x5820u,0x582bu,0x5868u,0x587bu,0x5888u,0x589bu,0x5948u,0x5953u,0x5988u,
0x5993u,0x59a0u,0x59abu,0x59d0u,0x59e3u,0x5a28u,0x5a3bu,0x5a48u,0x5a5bu,0x5a70u,0x5aabu,0x5ac0u,
0x5ae3u,0x5af0u,0x5afbu,0x5b20u,0x5b32u,0x5b84u,0x5b8bu,0x5b92u,0x5bc0u,0x5c13u,0x5c20u,0x5c2bu,
0x5c58u,0x5c73u,0x5c88u,0x5c93u,0x5cb0u,0x5ccbu,0x5cd8u,0x5ce3u,0x5ce8u,0x5cf3u,0x5d00u,0x5d1bu,
0x5d28u,0x5d43u,0x5d58u,0x5d73u,0x5dd0u,0x5df3u,0x5e18u,0x5e33u,0x5e48u,0x5e53u,0x5e70u,0x5e83u,
0x5e88u,0x5ebbu,0x5ec0u,0x5f32u,0x5f9cu,0x5fd8u,0x6003u,0x6068u,0x6073u,0x6088u,0x6093u,0x6148u,
0x6153u,0x61d0u,0x61e3u,0x6228u,0x6233u,0x6248u,0x6253u,0x6270u,0x62abu,0x62b8u,0x62c3u,0x62d8u,
0x62ebu,0x62f0u,0x6303u,0x6320u,0x6332u,0x6380u,0x63bcu,0x63c2u,0x63fcu,0x6403u,0x6424u,0x642bu,
0x6468u,0x6473u,0x6488u,0x6493u,0x6548u,0x6553u,0x65a0u,0x65abu,0x65d0u,0x65e3u,0x6628u,0x6633u,
0x6648u,0x6653u,0x6670u,0x66abu,0x66b8u,0x66ebu,0x66f8u,0x6703u,0x6720u,0x6732u,0x6780u,0x678bu,
0x67a0u,0x6803u,0x6868u,0x6873u,0x6888u,0x6893u,0x6a28u,0x6a33u,0x6a48u,0x6a53u,0x6a7cu,0x6a80u,
0x6aa3u,0x6ac2u,0x6afbu,0x6b20u,0x6b32u,0x6bccu,0x6bd3u,0x6c00u,0x6c0bu,0x6c20u,0x6c2bu,0x6cb8u,
0x6cd3u,0x6d90u,0x6d9bu,0x6de0u,0x6debu,0x6df0u,0x6e03u,0x6e38u,0x6e53u,0x6e58u,0x6e7bu,0x6ea8u,
0x6eb3u,0x6eb8u,0x6ec3u,0x6f00u,0x6f32u,0x6f80u,0x6f93u,0x6fa4u,0x6fa8u,0x700bu,0x71d8u,0x71fcu,
0x7203u,0x727cu,0x7282u,0x72d4u,0x72e0u,0x740bu,0x7418u,0x7423u,0x7428u,0x7433u,0x7458u,0x7463u,
0x7520u,0x752bu,0x7530u,0x753bu,0x75f0u,0x7603u,0x7628u,0x7633u,0x7638u,0x7643u,0x7678u,0x7682u,
0x76d0u,0x76e3u,0x7700u,0x7803u,0x780cu,0x78c3u,0x78d4u,0x7902u,0x79a4u,0x79abu,0x79b4u,0x79bbu,
0x79c4u,0x79cbu,0x79d4u,0x79f3u,0x7a40u,0x7a4bu,0x7b68u,0x7b8bu,0x7c2cu,0x7c33u,0x7cc0u,0x7ccbu,
0x7de8u,0x7df4u,0x7e33u,0x7e3cu,0x7e68u,0x7e74u,0x7ed8u,0x8003u,0x8202u,0x8254u,0x8283u,0x8482u,
0x84d3u,0x84f4u,0x8503u,0x8630u,0x863bu,0x8640u,0x866bu,0x8670u,0x8683u,0x87dcu,0x87e3u,0x9248u,
0x9253u,0x9270u,0x9283u,0x92b8u,0x92c3u,0x92c8u,0x92d3u,0x92f0u,0x9303u,0x9448u,0x9453u,0x9470u,
0x9483u,0x9588u,0x9593u,0x95b0u,0x95c3u,0x95f8u,0x9603u,0x9608u,0x9613u,0x9630u,0x9643u,0x96b8u,
0x96c3u,0x9888u,0x9893u,0x98b0u,0x98c3u,0x9ad8u,0x9aebu,0x9b04u,0x9b4au,0x9be8u,0x9c03u,0x9c84u,
0x9cd0u,0x9d03u,0x9fb0u,0x9fc3u,0x9ff0u,0xa004u,0xa00bu,0xb36cu,0xb37bu,0xb401u,0xb40bu,0xb4dcu,
0xb4e8u,0xb503u,0xb75cu,0xb772u,0xb78bu,0xb7c8u,0xb803u,0xb8b0u,0xb8fbu,0xb9acu,0xb9b8u,0xba03u,
0xbaa0u,0xbb03u,0xbb68u,0xbb73u,0xbb88u,0xbb93u,0xbba0u,0xbc03u,0xbea4u,0xbebbu,0xbec4u,0xbee3u,
0xbef0u,0xbf02u,0xbf50u,0xbf82u,0xbfd0u,0xc004u,0xc05bu,0xc070u,0xc07bu,0xc082u,0xc0d0u,0xc103u,
0xc3c8u,0xc403u,0xc558u,0xc583u,0xc7b0u,0xc803u,0xc8f8u,0xc903u,0xc960u,0xc983u,0xc9e0u,0xca04u,
0xca08u,0xca24u,0xca32u,0xca83u,0xcb70u,0xcb83u,0xcba8u,0xcc03u,0xcd60u,0xcd83u,0xce50u,0xce82u,
0xced8u,0xcef4u,0xd003u,0xd0e0u,0xd0f4u,0xd103u,0xd2f8u,0xd303u,0xd3e8u,0xd3fbu,0xd402u,0xd450u,
0xd482u,0xd4d0u,0xd504u,0xd53bu,0xd544u,0xd570u,0xd583u,0xd678u,0xd803u,0xda68u,0xda74u,0xda82u,
0xdad4u,0xdb5bu,0xdba4u,0xdc03u,0xdd82u,0xddd3u,0xdfa0u,0xdfe4u,0xe003u,0xe1c0u,0xe1dcu,0xe202u,
0xe250u,0xe26bu,0xe282u,0xe2d3u,0xe3f4u,0xe403u,0xe458u,0xe483u,0xe5d8u,0xe5ebu,0xe604u,0xe640u,
0xe683u,0xe69cu,0xe6a3u,0xe7d8u,0xe803u,0xf8b0u,0xf8c3u,0xf8f0u,0xf903u,0xfa30u,0xfa43u,0xfa70u,
0xfa83u,0xfac0u,0xfacbu,0xfad0u,0xfadbu,0xfae0u,0xfaebu,0xfaf0u,0xfafbu,0xfbf0u,0xfc03u,0xfda8u,
0xfdb3u,0xfdecu,0xfdf3u,0xfdfcu,0xfe13u,0xfe28u,0xfe33u,0xfe6cu,0xfe83u,0xfea0u,0xfeb3u,0xfee0u,
0xfeecu,0xff03u,0xff6cu,0xff80u,0xff93u,0xffa8u,0xffb3u,0xffecu,0xfff8u,0x10001u,0x10058u,0x10084u,
0x10141u,0x10150u,0x10179u,0x10184u,0x102f9u,0x10300u,0x10382u,0x1038bu,0x10390u,0x103a2u,0x103d4u,0x103fbu,
0x10402u,0x10454u,0x10478u,0x10483u,0x104e8u,0x10504u,0x10608u,0x10683u,0x10788u,0x10804u,0x10813u,0x1081cu,
0x1083bu,0x10844u,0x10853u,0x108a4u,0x108abu,0x108b4u,0x108cbu,0x108f4u,0x10923u,0x1092cu,0x10933u,0x1093cu,
0x10943u,0x1094cu,0x10953u,0x10974u,0x1097bu,0x109d4u,0x109e3u,0x10a04u,0x10a2bu,0x10a54u,0x10a73u,0x10a7cu,
0x10a82u,0x10c1bu,0x10c2au,0x10c54u,0x10c60u,0x10c84u,0x12150u,0x12204u,0x12258u,0x12302u,0x124e4u,0x12752u,
0x12804u,0x13bb2u,0x13ca4u,0x15ba0u,0x15bb4u,0x15cb0u,0x15cbcu,0x16003u,0x1672cu,0x1675bu,0x167a0u,0x167ccu,
0x167eau,0x167f4u,0x16803u,0x16930u,0x1693bu,0x16940u,0x1696bu,0x16970u,0x16983u,0x16b40u,0x16b7bu,0x16b84u,
0x16b88u,0x16bfbu,0x16cb8u,0x16d03u,0x16d38u,0x16d43u,0x16d78u,0x16d83u,0x16db8u,0x16dc3u,0x16df8u,0x16e03u,
0x16e38u,0x16e43u,0x16e78u,0x16e83u,0x16eb8u,0x16ec3u,0x16ef8u,0x16f03u,0x17004u,0x1717bu,0x17184u,0x172f0u,
0x17404u,0x174d0u,0x174dcu,0x177a0u,0x17804u,0x17eb0u,0x17f84u,0x18001u,0x1800cu,0x1802bu,0x1803au,0x18044u,
0x1810au,0x18153u,0x18184u,0x1818bu,0x181b4u,0x181c2u,0x181dbu,0x181ecu,0x18200u,0x1820bu,0x184b8u,0x184cbu,
0x184dcu,0x184ebu,0x18504u,0x1850bu,0x187dcu,0x187e3u,0x18800u,0x1882bu,0x18980u,0x1898bu,0x18c78u,0x18c84u,
0x18c92u,0x18cb4u,0x18d03u,0x18e04u,0x18f30u,0x18f7cu,0x18f83u,0x19004u,0x190f8u,0x19102u,0x19154u,0x19242u,
0x19284u,0x1928au,0x19304u,0x19402u,0x19454u,0x1958au,0x19604u,0x1a003u,0x26e04u,0x27003u,0x52468u,0x52484u,
0x52638u,0x52683u,0x527f4u,0x52803u,0x5306cu,0x53083u,0x53102u,0x53153u,0x53160u,0x53203u,0x5339cu,0x533a3u,
0x533f4u,0x533fbu,0x53732u,0x53783u,0x53794u,0x537c0u,0x53804u,0x538bbu,0x53904u,0x53913u,0x53c4cu,0x53c5bu,
0x53e70u,0x53e83u,0x53e90u,0x53e9bu,0x53ea0u,0x53eabu,0x53ee8u,0x53f93u,0x54144u,0x54163u,0x54168u,0x54182u,
0x541b4u,0x541d0u,0x54203u,0x543a4u,0x543c0u,0x54403u,0x54630u,0x54674u,0x54682u,0x546d0u,0x54703u,0x547c4u,
0x547dbu,0x547e4u,0x547ebu,0x54802u,0x54853u,0x54974u,0x54983u,0x54aa0u,0x54afcu,0x54b03u,0x54be8u,0x54c03u,
0x54e0cu,0x54e70u,0x54e7bu,0x54e82u,0x54ed0u,0x54ef4u,0x54f03u,0x54f82u,0x54fd3u,0x54ff8u,0x55003u,0x551b8u,
0x55203u,0x55270u,0x55282u,0x552d0u,0x552e4u,0x55303u,0x553bcu,0x553d3u,0x55618u,0x556dbu,0x556f4u,0x55703u,
0x55784u,0x55793u,0x557b8u,0x5580bu,0x55838u,0x5584bu,0x55878u,0x5588bu,0x558b8u,0x55903u,0x55938u,0x55943u,
0x55978u,0x55983u,0x55adcu,0x55ae3u,0x55b54u,0x55b60u,0x55b83u,0x55f5cu,0x55f63u,0x55f70u,0x55f82u,0x55fd0u,
0x56003u,0x6bd20u,0x6bd83u,0x6be38u,0x6be5bu,0x6bfe0u,0x7c803u,0x7d370u,0x7d383u,0x7d6d0u,0x7d803u,0x7d838u,
0x7d89bu,0x7d8c0u,0x7d8ebu,0x7d94cu,0x7d953u,0x7d9b8u,0x7d9c3u,0x7d9e8u,0x7d9f3u,0x7d9f8u,0x7da03u,0x7da10u,
0x7da1bu,0x7da28u,0x7da33u,0x7dd94u,0x7de18u,0x7de9bu,0x7e9f4u,0x7ea83u,0x7ec80u,0x7ec93u,0x7ee40u,0x7ee7cu,
0x7ee80u,0x7ef83u,0x7efe4u,0x7f003u,0x7f084u,0x7f0d0u,0x7f103u,0x7f184u,0x7f298u,0x7f2a4u,0x7f338u,0x7f344u,
0x7f360u,0x7f383u,0x7f3a8u,0x7f3b3u,0x7f7e8u,0x7f80cu,0x7f882u,0x7f8d4u,0x7f90bu,0x7f9dcu,0x7fa0bu,0x7fadcu,
0x7fb33u,0x7fdf8u,0x7fe13u,0x7fe40u,0x7fe53u,0x7fe80u,0x7fe93u,0x7fec0u,0x7fed3u,0x7fee8u,0x7ff04u,0x7ff38u,
0x7ff44u,0x7ff78u,0x7ffe4u,0x7fff0u,0x80003u,0x80060u,0x8006bu,0x80138u,0x80143u,0x801d8u,0x801e3u,0x801f0u,
0x801fbu,0x80270u,0x80283u,0x802f0u,0x80403u,0x807d8u,0x80804u,0x80818u,0x8083au,0x809a0u,0x809bcu,0x80a02u,
0x80bccu,0x80c52u,0x80c64u,0x80c78u,0x80c84u,0x80ce8u,0x80d04u,0x80d08u,0x80e84u,0x80febu,0x80ff0u,0x81403u,
0x814e8u,0x81503u,0x81688u,0x81703u,0x8170au,0x817e0u,0x81803u,0x81902u,0x81920u,0x8196bu,0x81a0au,0x81a13u,
0x81a52u,0x81a58u,0x81a83u,0x81bd8u,0x81c03u,0x81cf0u,0x81cfcu,0x81d03u,0x81e20u,0x81e43u,0x81e84u,0x81e8au,
0x81eb0u,0x82003u,0x824f0u,0x82502u,0x82550u,0x82583u,0x826a0u,0x826c3u,0x827e0u,0x82803u,0x82940u,0x82983u,
0x82b20u,0x82b7cu,0x82b83u,0x82bd8u,0x82be3u,0x82c58u,0x82c63u,0x82c98u,0x82ca3u,0x82cb0u,0x82cbbu,0x82d10u,
0x82d1bu,0x82d90u,0x82d9bu,0x82dd0u,0x82ddbu,0x82de8u,0x82e03u,0x82fa0u,0x83003u,0x839b8u,0x83a03u,0x83ab0u,
0x83b03u,0x83b40u,0x83c03u,0x83c30u,0x83c3bu,0x83d88u,0x83d93u,0x83dd8u,0x84003u,0x84030u,0x84043u,0x84048u,
0x84053u,0x841b0u,0x841bbu,0x841c8u,0x841e3u,0x841e8u,0x841fbu,0x842b0u,0x842bcu,0x842c2u,0x84303u,0x843bcu,
0x843cau,0x84403u,0x844f8u,0x8453au,0x84580u,0x84703u,0x84798u,0x847a3u,0x847b0u,0x847dau,0x84803u,0x848b2u,
0x848e0u,0x848fcu,0x84903u,0x849d0u,0x849fcu,0x84a00u,0x84c03u,0x84dc0u,0x84de2u,0x84df3u,0x84e02u,0x84e80u,
0x84e92u,0x85003u,0x85020u,0x8502bu,0x85038u,0x85063u,0x850a0u,0x850abu,0x850c0u,0x850cbu,0x851b0u,0x851c3u,
0x851d8u,0x851fbu,0x85202u,0x85248u,0x85284u,0x852c8u,0x85303u,0x853eau,0x853fcu,0x85403u,0x854eau,0x85500u,
0x85603u,0x85644u,0x8564bu,0x85738u,0x8575au,0x85784u,0x857b8u,0x85803u,0x859b0u,0x859ccu,0x85a03u,0x85ab0u,
0x85ac2u,0x85b03u,0x85b98u,0x85bc2u,0x85c03u,0x85c90u,0x85cccu,0x85ce8u,0x85d4au,0x85d80u,0x86003u,0x86248u,
0x86403u,0x86598u,0x86603u,0x86798u,0x867d2u,0x86803u,0x86940u,0x86982u,0x869d0u,0x86a02u,0x86a53u,0x86b30u,
0x86b4bu,0x86b74u,0x86b7bu,0x86c30u,0x86c74u,0x86c80u,0x87302u,0x873f8u,0x87403u,0x87550u,0x8755bu,0x8756cu,
0x87570u,0x87583u,0x87590u,0x87613u,0x87628u,0x877e3u,0x878eau,0x8793bu,0x87940u,0x87983u,0x87a8au,0x87aacu,
0x87ad0u,0x87b83u,0x87c34u,0x87c50u,0x87d83u,0x87e2au,0x87e60u,0x87f03u,0x87fb8u,0x88003u,0x8823cu,0x88270u,
0x88292u,0x88383u,0x883b0u,0x883fbu,0x885dcu,0x885e8u,0x885f4u,0x88613u,0x88618u,0x88683u,0x88748u,0x88782u,
0x887d0u,0x88803u,0x889a8u,0x889b2u,0x88a04u,0x88a23u,0x88a40u,0x88a83u,0x88ba4u,0x88bb3u,0x88bb8u,0x88c03u,
0x88e2cu,0x88e4bu,0x88e6cu,0x88e73u,0x88e82u,0x88ed3u,0x88edcu,0x88ee3u,0x88eecu,0x88f00u,0x88f0au,0x88fa8u,
0x89003u,0x89090u,0x8909bu,0x891c4u,0x891f3u,0x89210u,0x89403u,0x89438u,0x89443u,0x89448u,0x89453u,0x89470u,
0x8947bu,0x894f0u,0x894fbu,0x8954cu,0x89550u,0x89583u,0x89758u,0x89782u,0x897d0u,0x89803u,0x89820u,0x8982bu,
0x89868u,0x8987bu,0x89888u,0x8989bu,0x89948u,0x89953u,0x89988u,0x89993u,0x899a0u,0x899abu,0x899d0u,0x899dbu,
0x89a28u,0x89a3bu,0x89a48u,0x89a5bu,0x89a70u,0x89a83u,0x89a88u,0x89abbu,0x89ac0u,0x89aebu,0x89b20u,0x89b33u,
0x89b68u,0x89b83u,0x89ba8u,0x89c03u,0x89c50u,0x89c5bu,0x89c60u,0x89c73u,0x89c78u,0x89c83u,0x89db0u,0x89dbbu,
0x89e08u,0x89e13u,0x89e18u,0x89e2bu,0x89e30u,0x89e3bu,0x89e58u,0x89e63u,0x89ea4u,0x89eb0u,0x89ebcu,0x89ec8u,
0x89f0bu,0x89f18u,0x8a003u,0x8a25cu,0x8a282u,0x8a2d4u,0x8a2e0u,0x8a2ecu,0x8a2f3u,0x8a310u,0x8a403u,0x8a634u,
0x8a63bu,0x8a640u,0x8a682u,0x8a6d0u,0x8ac03u,0x8adb0u,0x8adc3u,0x8ae0cu,0x8aec3u,0x8aef0u,0x8b003u,0x8b20cu,
0x8b223u,0x8b228u,0x8b282u,0x8b2d0u,0x8b304u,0x8b368u,0x8b403u,0x8b5ccu,0x8b5d0u,0x8b602u,0x8b650u,0x8b682u,
0x8b720u,0x8b803u,0x8b8d8u,0x8b8ebu,0x8b960u,0x8b982u,0x8b9e4u,0x8ba03u,0x8ba38u,0x8c003u,0x8c1dcu,0x8c1e0u,
0x8c503u,0x8c702u,0x8c798u,0x8c7fbu,0x8c838u,0x8c84bu,0x8c850u,0x8c863u,0x8c8a0u,0x8c8abu,0x8c8b8u,0x8c8c3u,
0x8c9b0u,0x8c9bbu,0x8c9c8u,0x8c9dbu,0x8ca24u,0x8ca38u,0x8ca82u,0x8cad0u,0x8cd03u,0x8cd40u,0x8cd53u,0x8cec0u,
0x8ced3u,0x8cf14u,0x8cf1bu,0x8cf28u,0x8d003u,0x8d1fcu,0x8d23bu,0x8d240u,0x8d283u,0x8d4d4u,0x8d4ebu,0x8d4f4u,
0x8d518u,0x8d583u,0x8d7c8u,0x8d804u,0x8d850u,0x8de03u,0x8df0cu,0x8df10u,0x8df82u,0x8dfd0u,0x8e003u,0x8e048u,
0x8e053u,0x8e1b8u,0x8e1c3u,0x8e20cu,0x8e230u,0x8e282u,0x8e368u,0x8e384u,0x8e393u,0x8e480u,0x8e493u,0x8e540u,
0x8e54bu,0x8e5b8u,0x8e803u,0x8e838u,0x8e843u,0x8e850u,0x8e85bu,0x8e9b8u,0x8e9d3u,0x8e9d8u,0x8e9e3u,0x8e9f0u,
0x8e9fbu,0x8ea40u,0x8ea82u,0x8ead0u,0x8eb03u,0x8eb30u,0x8eb3bu,0x8eb48u,0x8eb53u,0x8ec78u,0x8ec83u,0x8ec90u,
0x8ec9bu,0x8ecc8u,0x8ed02u,0x8ed50u,0x8f703u,0x8f7bcu,0x8f7c8u,0x8f803u,0x8f888u,0x8f893u,0x8f9d8u,0x8f9f3u,
0x8fa1cu,0x8fa82u,0x8fad3u,0x8fad8u,0x8fd83u,0x8fd88u,0x8fe02u,0x8feacu,0x8ff90u,0x8fffcu,0x90003u,0x91cd0u,
0x92002u,0x92378u,0x92384u,0x923a8u,0x92403u,0x92a20u,0x97c83u,0x97f8cu,0x97f98u,0x98003u,0x9a180u,0x9a203u,
0x9a2b0u,0x9a303u,0xa1fd8u,0xa2003u,0xa3238u,0xb0803u,0xb0982u,0xb09d0u,0xb4003u,0xb51c8u,0xb5203u,0xb52f8u,
0xb5302u,0xb5350u,0xb5374u,0xb5383u,0xb55f8u,0xb5602u,0xb5650u,0xb5683u,0xb5770u,0xb5783u,0xb57acu,0xb57b0u,
0xb5803u,0xb59bcu,0xb5a03u,0xb5a24u,0xb5a30u,0xb5a82u,0xb5ad0u,0xb5adau,0xb5b10u,0xb5b1bu,0xb5bc0u,0xb5bebu,
0xb5c80u,0xb6a03u,0xb6b6cu,0xb6b82u,0xb6bd0u,0xb7203u,0xb7402u,0xb74bcu,0xb74d8u,0xb7803u,0xb7a58u,0xb7a7bu,
0xb7c40u,0xb7c7bu,0xb7d00u,0xb7f03u,0xb7f14u,0xb7f1bu,0xb7f28u,0xb7f83u,0xb7f90u,0xb8003u,0xc3fc0u,0xc4003u,
0xc66b0u,0xc67fbu,0xc6848u,0xd7f83u,0xd7fa0u,0xd7fabu,0xd7fe0u,0xd7febu,0xd7ff8u,0xd8003u,0xd8918u,0xd8993u,
0xd8998u,0xd8a83u,0xd8a98u,0xd8aabu,0xd8ab0u,0xd8b23u,0xd8b40u,0xd8b83u,0xd97e0u,0xde003u,0xde358u,0xde383u,
0xde3e8u,0xde403u,0xde448u,0xde483u,0xde4d0u,0xde4e4u,0xde4ebu,0xde4fcu,0xde500u,0xe6004u,0xe6782u,0xe67d0u,
0xe6804u,0xe75a0u,0xe7803u,0xe7970u,0xe7983u,0xe7a38u,0xe7a84u,0xe7e20u,0xe8004u,0xe87b0u,0xe8804u,0xe8938u,
0xe894cu,0xe8b2bu,0xe8b54u,0xe8b6bu,0xe8b98u,0xe8bdbu,0xe8c1cu,0xe8c2bu,0xe8c64u,0xe8d53u,0xe8d74u,0xe8f58u,
0xe9004u,0xe9213u,0xe922cu,0xe9230u,0xe9602u,0xe96a0u,0xe9702u,0xe97a0u,0xe9804u,0xe9ab8u,0xe9b02u,0xe9bc8u,
0xea003u,0xea2a8u,0xea2b3u,0xea4e8u,0xea4f3u,0xea500u,0xea513u,0xea518u,0xea52bu,0xea538u,0xea54bu,0xea568u,
0xea573u,0xea5d0u,0xea5dbu,0xea5e0u,0xea5ebu,0xea620u,0xea62bu,0xea830u,0xea83bu,0xea858u,0xea86bu,0xea8a8u,
0xea8b3u,0xea8e8u,0xea8f3u,0xea9d0u,0xea9dbu,0xea9f8u,0xeaa03u,0xeaa28u,0xeaa33u,0xeaa38u,0xeaa53u,0xeaa88u,
0xeaa93u,0xeb530u,0xeb543u,0xeb60cu,0xeb613u,0xeb6dcu,0xeb6e3u,0xeb7dcu,0xeb7e3u,0xeb8acu,0xeb8b3u,0xeb9acu,
0xeb9b3u,0xeba7cu,0xeba83u,0xebb7cu,0xebb83u,0xebc4cu,0xebc53u,0xebd4cu,0xebd53u,0xebe1cu,0xebe23u,0xebe60u,
0xebe72u,0xec004u,0xed003u,0xed1bcu,0xed1dbu,0xed36cu,0xed3abu,0xed3b4u,0xed423u,0xed42cu,0xed460u,0xed4dbu,
0xed500u,0xed50bu,0xed580u,0xef803u,0xef8f8u,0xef92bu,0xef958u,0xf0003u,0xf0038u,0xf0043u,0xf00c8u,0xf00dbu,
0xf0110u,0xf011bu,0xf0128u,0xf0133u,0xf0158u,0xf0183u,0xf0370u,0xf047bu,0xf0480u,0xf0803u,0xf0968u,0xf0983u,
0xf09f0u,0xf0a02u,0xf0a50u,0xf0a73u,0xf0a7cu,0xf0a80u,0xf1483u,0xf1578u,0xf1603u,0xf1782u,0xf17d0u,0xf17fcu,
0xf1800u,0xf2683u,0xf2782u,0xf27d0u,0xf2e83u,0xf2f8au,0xf2fd8u,0xf2ffcu,0xf3000u,0xf3f03u,0xf3f38u,0xf3f43u,
0xf3f60u,0xf3f6bu,0xf3f78u,0xf3f83u,0xf3ff8u,0xf4003u,0xf4628u,0xf463au,0xf4683u,0xf46b8u,0xf4803u,0xf4a60u,
0xf4a82u,0xf4ad0u,0xf4af4u,0xf4b00u,0xf638au,0xf6564u,0xf656au,0xf6584u,0xf658au,0xf65a8u,0xf680au,0xf6974u,
0xf697au,0xf69f0u,0xf7003u,0xf7020u,0xf702bu,0xf7100u,0xf710bu,0xf7118u,0xf7123u,0xf7128u,0xf713bu,0xf7140u,
0xf714bu,0xf7198u,0xf71a3u,0xf71c0u,0xf71cbu,0xf71d0u,0xf71dbu,0xf71e0u,0xf7213u,0xf7218u,0xf723bu,0xf7240u,
0xf724bu,0xf7250u,0xf725bu,0xf7260u,0xf726bu,0xf7280u,0xf728bu,0xf7298u,0xf72a3u,0xf72a8u,0xf72bbu,0xf72c0u,
0xf72cbu,0xf72d0u,0xf72dbu,0xf72e0u,0xf72ebu,0xf72f0u,0xf72fbu,0xf7300u,0xf730bu,0xf7318u,0xf7323u,0xf7328u,
0xf733bu,0xf7358u,0xf7363u,0xf7398u,0xf73a3u,0xf73c0u,0xf73cbu,0xf73e8u,0xf73f3u,0xf73f8u,0xf7403u,0xf7450u,
0xf745bu,0xf74e0u,0xf750bu,0xf7520u,0xf752bu,0xf7550u,0xf755bu,0xf75e0u,0xf7784u,0xf7790u,0xf8004u,0xf8160u,
0xf8184u,0xf84a0u,0xf8504u,0xf8578u,0xf858cu,0xf8600u,0xf860cu,0xf8680u,0xf868cu,0xf87b0u,0xf8802u,0xf886cu,
0xf8d70u,0xf8f34u,0xf9018u,0xf9084u,0xf91e0u,0xf9204u,0xf9248u,0xf9284u,0xf9290u,0xf9304u,0xf9330u,0xf9804u,
0xfb6c0u,0xfb6e4u,0xfb768u,0xfb784u,0xfb7e8u,0xfb804u,0xfbbb8u,0xfbbdcu,0xfbed0u,0xfbf04u,0xfbf60u,0xfbf84u,
0xfbf88u,0xfc004u,0xfc060u,0xfc084u,0xfc240u,0xfc284u,0xfc2d0u,0xfc304u,0xfc440u,0xfc484u,0xfc570u,0xfc584u,
0xfc5e0u,0xfc604u,0xfc610u,0xfc804u,0xfd2a0u,0xfd304u,0xfd370u,0xfd384u,0xfd3e8u,0xfd404u,0xfd450u,0xfd47cu,
0xfd638u,0xfd674u,0xfd6e8u,0xfd6fcu,0xfd750u,0xfd784u,0xfd7c8u,0xfd804u,0xfdc98u,0xfdca4u,0xfdf82u,0xfdfd0u,
0x100003u,0x153700u,0x153803u,0x15b9d0u,0x15ba03u,0x15c0f0u,0x15c103u,0x167510u,0x167583u,0x175f08u,0x175f83u,0x1772f0u,
0x17c003u,0x17d0f0u,0x180003u,0x189a58u,0x189a83u,0x191d80u,0x700803u,0x700f80u,
};
static int u_cat(int cp) {
if (cp < 0) return UCAT_OTHER;
if (cp < 0x80) return (int)u_ascii_cat[cp];
if (cp > 0x10ffff) return UCAT_OTHER;
uint32_t target = ((uint32_t)cp << 3) | 7u;
int lo = 0, hi = (int)(sizeof(u_cat_trans) / sizeof(u_cat_trans[0])) - 1;
while (lo < hi) {
int mid = (lo + hi + 1) >> 1;
if (u_cat_trans[mid] <= target) lo = mid;
else hi = mid - 1;
} }
if (cp >= 0x2000 && cp <= 0x200a) return 1; return (int)(u_cat_trans[lo] & 7u);
return 0;
}
static int u_is_number(int cp) {
if (cp >= 0x30 && cp <= 0x39) return 1; /* ASCII digits */
if (cp >= 0xff10 && cp <= 0xff19) return 1; /* fullwidth digits */
if (cp >= 0x0660 && cp <= 0x0669) return 1; /* Arabic-Indic */
if (cp >= 0x06f0 && cp <= 0x06f9) return 1; /* Extended Arabic-Indic */
if (cp >= 0x0966 && cp <= 0x096f) return 1; /* Devanagari */
return 0;
}
static int u_is_letter(int cp) {
if ((cp >= 'a' && cp <= 'z') || (cp >= 'A' && cp <= 'Z')) return 1;
if (cp >= 0x00c0 && cp <= 0x02af) {
return (cp != 0x00d7 && cp != 0x00f7); /* Latin-1 + Ext-A/B + IPA (excl. × ÷) */
}
if (cp >= 0x0370 && cp <= 0x03ff) {
return (cp != 0x037e && cp != 0x0387); /* Greek */
}
if (cp >= 0x0400 && cp <= 0x052f) return 1; /* Cyrillic + Supplement */
if (cp >= 0x0531 && cp <= 0x0587) return 1; /* Armenian */
if (cp >= 0x05d0 && cp <= 0x05ea) return 1; /* Hebrew */
if ((cp >= 0x0620 && cp <= 0x064a) ||
(cp >= 0x066e && cp <= 0x06d3) ||
(cp >= 0x06ee && cp <= 0x06ff) ||
(cp >= 0x0750 && cp <= 0x077f) ||
(cp >= 0x08a0 && cp <= 0x08ff)) return 1; /* Arabic */
if (cp >= 0x0904 && cp <= 0x0d7f) {
return !(cp >= 0x0964 && cp <= 0x096f); /* Devanagari + Indic scripts */
}
if ((cp >= 0x0e01 && cp <= 0x0e30) ||
(cp >= 0x0e32 && cp <= 0x0e33) ||
(cp >= 0x0e40 && cp <= 0x0e46)) return 1; /* Thai */
if (cp >= 0x10a0 && cp <= 0x10ff) return 1; /* Georgian */
if (cp >= 0x1100 && cp <= 0x11ff) return 1; /* Hangul Jamo */
if (cp >= 0x1e00 && cp <= 0x1fff) return 1; /* Latin Ext Additional + Greek Ext */
if (cp >= 0x3041 && cp <= 0x3096) return 1; /* Hiragana */
if (cp >= 0x30a1 && cp <= 0x30fc && cp != 0x30fb) return 1; /* Katakana (excl. ・) */
if (cp >= 0x3131 && cp <= 0x318e) return 1; /* Hangul Compatibility Jamo */
if (cp >= 0x3400 && cp <= 0x4dbf) return 1; /* CJK Ext A */
if (cp >= 0x4e00 && cp <= 0x9fff) return 1; /* CJK Unified Ideographs */
if (cp >= 0xac00 && cp <= 0xd7a3) return 1; /* Hangul Syllables */
if (cp >= 0xf900 && cp <= 0xfaff) return 1; /* CJK Compatibility Ideographs */
if (cp >= 0xff21 && cp <= 0xff3a) return 1; /* fullwidth A-Z */
if (cp >= 0xff41 && cp <= 0xff5a) return 1; /* fullwidth a-z */
if (cp >= 0x20000 && cp <= 0x2fa1f) return 1; /* CJK Ext B-F + Compat */
return 0;
} }
/* Decode one UTF-8 codepoint at t[pos]; sets *cp and *cplen. */ static int u_is_cjk_kana(int cp) {
return (cp >= 0x4e00 && cp <= 0x9fa5) || (cp >= 0x3040 && cp <= 0x30ff);
}
static int u_is_ascii_punct(int cp) {
return (cp >= 0x21 && cp <= 0x2f) || (cp >= 0x3a && cp <= 0x40) ||
(cp >= 0x5b && cp <= 0x60) || (cp >= 0x7b && cp <= 0x7e);
}
static int u_is_ascii_alpha(int cp) {
return (cp >= 'A' && cp <= 'Z') || (cp >= 'a' && cp <= 'z');
}
/* Decode one UTF-8 codepoint at t[pos]; sets *cp (-1 if invalid byte) and *cplen. */
static void decode_cp(const unsigned char *t, int len, int pos, int *cp, int *cplen) { static void decode_cp(const unsigned char *t, int len, int pos, int *cp, int *cplen) {
unsigned char b = t[pos]; unsigned char b = t[pos];
int c, l; if (b < 0x80) { *cp = b; *cplen = 1; return; }
if (b < 0x80) { c = b; l = 1; } int c, l, min_cp;
else if ((b >> 5) == 0x6) { c = b & 0x1f; l = 2; } if ((b >> 5) == 0x6) { c = b & 0x1f; l = 2; min_cp = 0x80; }
else if ((b >> 4) == 0xe) { c = b & 0x0f; l = 3; } else if ((b >> 4) == 0xe) { c = b & 0x0f; l = 3; min_cp = 0x800; }
else if ((b >> 3) == 0x1e) { c = b & 0x07; l = 4; } else if ((b >> 3) == 0x1e) { c = b & 0x07; l = 4; min_cp = 0x10000; }
else { c = b; l = 1; } /* invalid lead byte: treat as single byte */ else { *cp = -1; *cplen = 1; return; }
if (pos + l > len) { *cp = -1; *cplen = 1; return; }
for (int j = 1; j < l; j++) { for (int j = 1; j < l; j++) {
if (pos + j >= len || (t[pos + j] & 0xc0) != 0x80) { c = b; l = 1; break; } if ((t[pos + j] & 0xc0) != 0x80) { *cp = -1; *cplen = 1; return; }
c = (c << 6) | (t[pos + j] & 0x3f); c = (c << 6) | (t[pos + j] & 0x3f);
} }
if (c < min_cp || c > 0x10ffff || (c >= 0xd800 && c <= 0xdfff)) {
*cp = -1; *cplen = 1; return;
}
*cp = c; *cplen = l; *cp = c; *cplen = l;
} }
/* ----------------------------- added tokens --------------------------- */
typedef struct {
const char *str;
uint8_t len;
} added_tok_t;
static const added_tok_t tok_added_named[] = {
{ "<\357\275\234tool\342\226\201output\342\226\201begin:opensource\357\275\234>", 40 },
{ "<\357\275\234hy_begin_of_sentence:opensource\357\275\234>", 39 },
{ "<\357\275\234tool\342\226\201calls\342\226\201begin:opensource\357\275\234>", 39 },
{ "<\357\275\234tool\342\226\201call\342\226\201begin:opensource\357\275\234>", 38 },
{ "<\357\275\234tool\342\226\201output\342\226\201end:opensource\357\275\234>", 38 },
{ "<\357\275\234tool\342\226\201calls\342\226\201end:opensource\357\275\234>", 37 },
{ "<\357\275\234tool\342\226\201call\342\226\201end:opensource\357\275\234>", 36 },
{ "<\357\275\234reasoning_mode:opensource\357\275\234>", 33 },
{ "<\357\275\234hy_Assistant:opensource\357\275\234>", 31 },
{ "<\357\275\234hy_end\342\226\201of\342\226\201sentence\357\275\234>", 30 },
{ "<\357\275\234tool\342\226\201sep:opensource\357\275\234>", 29 },
{ "</tool_responses:opensource>", 28 },
{ "</tool_response:opensource>", 27 },
{ "<tool_responses:opensource>", 27 },
{ "<\357\275\234hy_User:opensource\357\275\234>", 26 },
{ "<tool_response:opensource>", 26 },
{ "<\357\275\234hy_pad:opensource\357\275\234>", 25 },
{ "<\357\275\234hy_eos:opensource\357\275\234>", 25 },
{ "</tool_calls:opensource>", 24 },
{ "</tool_call:opensource>", 23 },
{ "<tool_calls:opensource>", 23 },
{ "</arg_value:opensource>", 23 },
{ "<\357\275\234hy_fim\342\226\201begin\357\275\234>", 22 },
{ "<tool_call:opensource>", 22 },
{ "<arg_value:opensource>", 22 },
{ "<\357\275\234hy_fim\342\226\201hole\357\275\234>", 21 },
{ "<tool_sep:opensource>", 21 },
{ "</arg_key:opensource>", 21 },
{ "<\357\275\234hy_fim\342\226\201end\357\275\234>", 20 },
{ "<arg_key:opensource>", 20 },
{ "</think:opensource>", 19 },
{ "<think:opensource>", 18 },
{ "<\357\275\234hy_EOT\357\275\234>", 14 },
{ "</answer>", 9 },
{ "/no_think", 9 },
{ "<answer>", 8 },
{ "/think", 6 },
};
/* Match an added special/control token at t[pos..len). Returns byte length or 0. */
static int tok_match_added(const unsigned char *t, int len, int pos) {
unsigned char b = t[pos];
if (b != '<' && b != '/') return 0;
int rem = len - pos;
const char *p = (const char *)(t + pos);
/* Check <｜hy_place▁holder▁no▁N｜> (11 <= N <= 799, excl. 20..33 and 48..51) */
static const char ph_prefix[] = "<\357\275\234hy_place\342\226\201holder\342\226\201no\342\226\201";
static const char ph_suffix[] = "\357\275\234>";
const int ph_plen = (int)(sizeof(ph_prefix) - 1);
const int ph_slen = (int)(sizeof(ph_suffix) - 1);
if (b == '<' && rem >= ph_plen + 2 + ph_slen &&
memcmp(p, ph_prefix, (size_t)ph_plen) == 0) {
int k = ph_plen, num = 0, ndig = 0;
while (k < rem && ndig < 3 && p[k] >= '0' && p[k] <= '9') {
num = num * 10 + (p[k] - '0');
k++;
ndig++;
}
if (ndig >= 2 && p[ph_plen] != '0' && num >= 11 && num <= 799 &&
!(num >= 20 && num <= 33) && !(num >= 48 && num <= 51) &&
rem - k >= ph_slen && memcmp(p + k, ph_suffix, (size_t)ph_slen) == 0) {
return k + ph_slen;
}
}
int n_added = (int)(sizeof(tok_added_named) / sizeof(tok_added_named[0]));
for (int i = 0; i < n_added; i++) {
int alen = (int)tok_added_named[i].len;
if (rem >= alen && p[1] == tok_added_named[i].str[1] &&
memcmp(p, tok_added_named[i].str, (size_t)alen) == 0) {
return alen;
}
}
return 0;
}
/* ----------------------------- pre-tokenizer -------------------------- */ /* ----------------------------- pre-tokenizer -------------------------- */
/* Returns the exclusive end byte offset of the token piece that starts at static int in_stage3_seg(int cp, int cat) {
* position i, implementing the cl100k regex alternatives in order. */ return cat != UCAT_NUM && !u_is_cjk_kana(cp);
}
/* Returns the exclusive end byte offset of the pre-tokenized piece starting
* at position i within slice t[0..len), implementing Hy3's 3-stage Split. */
static int tok_piece_end(const unsigned char *t, int len, int i) { static int tok_piece_end(const unsigned char *t, int len, int i) {
int cp, cplen; int cp, cplen;
decode_cp(t, len, i, &cp, &cplen); decode_cp(t, len, i, &cp, &cplen);
int cat = u_cat(cp);
int ni = i + cplen; int ni = i + cplen;
int ncp = 0, ncl = 0;
if (ni < len) decode_cp(t, len, ni, &ncp, &ncl);
/* 1) '(?i:[sdmt]|ll|ve|re) */ /* Stage 1: \p{N}{1,3} */
if (cp == 0x27) { if (cat == UCAT_NUM) {
if (ni < len) { int e = ni, cnt = 1;
int j2 = ni + ncl;
if (j2 < len) {
int c2, l2; decode_cp(t, len, j2, &c2, &l2);
int a = ncp, b = c2;
if (a >= 'A' && a <= 'Z') a += 32;
if (b >= 'A' && b <= 'Z') b += 32;
if ((a == 'l' && b == 'l') || (a == 'v' && b == 'e') || (a == 'r' && b == 'e'))
return j2 + l2;
}
int a = ncp; if (a >= 'A' && a <= 'Z') a += 32;
if (a == 's' || a == 'd' || a == 'm' || a == 't') return ni + ncl;
}
/* apostrophe alone falls through to whitespace/other handling */
}
int isL = u_is_letter(cp), isN = u_is_number(cp), isS = u_is_space(cp);
int is_not_cln = (!isL && !isN && cp != 0x0d && cp != 0x0a);
/* 2) [^\r\n\p{L}\p{N}]?+\p{L}++ */
if (isL) {
int e = i;
while (e < len) { int c, cl; decode_cp(t, len, e, &c, &cl); if (!u_is_letter(c)) break; e += cl; }
return e;
}
if (is_not_cln && ni < len && u_is_letter(ncp)) {
int e = ni;
while (e < len) { int c, cl; decode_cp(t, len, e, &c, &cl); if (!u_is_letter(c)) break; e += cl; }
return e;
}
/* 3) \p{N}{1,3}+ */
if (isN) {
int e = i, cnt = 0;
while (e < len && cnt < 3) { while (e < len && cnt < 3) {
int c, cl; decode_cp(t, len, e, &c, &cl); int c, cl;
if (!u_is_number(c)) break; decode_cp(t, len, e, &c, &cl);
e += cl; cnt++; if (u_cat(c) != UCAT_NUM) break;
e += cl;
cnt++;
} }
return e; return e;
} }
/* 4) ?[^\s\p{L}\p{N}]++[\r\n]*+ (leading space optional) */ /* Stage 2: [\u4e00-\u9fa5\u3040-\u309f\u30a0-\u30ff]+ sub-slice with Stage 3 applied */
int is_other = (!isS && !isL && !isN); if (u_is_cjk_kana(cp)) {
if (cp == 0x20 && ni < len && (!u_is_space(ncp) && !u_is_letter(ncp) && !u_is_number(ncp))) { int ncp = -1, ncl = 0, ncat = UCAT_OTHER;
if (ni < len) {
decode_cp(t, len, ni, &ncp, &ncl);
if (u_is_cjk_kana(ncp)) ncat = u_cat(ncp);
else ncp = -1;
}
if (cat == UCAT_LM) {
int e = ni;
while (e < len) {
int c, cl;
decode_cp(t, len, e, &c, &cl);
if (!u_is_cjk_kana(c) || u_cat(c) != UCAT_LM) break;
e += cl;
}
return e;
}
if (cat != UCAT_PS && ncp >= 0 && ncat == UCAT_LM) {
int e = ni + ncl;
while (e < len) {
int c, cl;
decode_cp(t, len, e, &c, &cl);
if (!u_is_cjk_kana(c) || u_cat(c) != UCAT_LM) break;
e += cl;
}
return e;
}
if (cat == UCAT_PS) {
int e = ni;
while (e < len) {
int c, cl;
decode_cp(t, len, e, &c, &cl);
if (!u_is_cjk_kana(c) || u_cat(c) != UCAT_PS) break;
e += cl;
}
return e;
}
int e = ni; int e = ni;
while (e < len) { while (e < len) {
int c, cl; decode_cp(t, len, e, &c, &cl); int c, cl;
if (u_is_space(c) || u_is_letter(c) || u_is_number(c)) break; decode_cp(t, len, e, &c, &cl);
e += cl; if (!u_is_cjk_kana(c)) break;
} int cc = u_cat(c);
while (e < len) { if (cc == UCAT_LM || cc == UCAT_PS) break;
int c, cl; decode_cp(t, len, e, &c, &cl); if (e + cl < len) {
if (c != 0x0d && c != 0x0a) break; int nc, nl;
e += cl; decode_cp(t, len, e + cl, &nc, &nl);
} if (u_is_cjk_kana(nc) && u_cat(nc) == UCAT_LM) break;
return e; }
}
if (is_other) {
int e = i;
while (e < len) {
int c, cl; decode_cp(t, len, e, &c, &cl);
if (u_is_space(c) || u_is_letter(c) || u_is_number(c)) break;
e += cl;
}
while (e < len) {
int c, cl; decode_cp(t, len, e, &c, &cl);
if (c != 0x0d && c != 0x0a) break;
e += cl; e += cl;
} }
return e; return e;
} }
/* 5-8) whitespace runs: \s++$ | \s*[\r\n] | \s+(?!\S) | \s */ /* Stage 3 on non-number, non-CJK/Kana segment */
int e = i; int has_ni = 0, ncp = -1, ncl = 0, ncat = UCAT_OTHER;
int last_ws_len = cplen; if (ni < len) {
int last_nl_end = 0; decode_cp(t, len, ni, &ncp, &ncl);
ncat = u_cat(ncp);
has_ni = in_stage3_seg(ncp, ncat);
}
/* 1) [!"#$%&'()*+,\-./:;<=>?@\[\\\]^_`{|}~][A-Za-z]+ */
if (u_is_ascii_punct(cp) && has_ni && u_is_ascii_alpha(ncp)) {
int e = ni + ncl;
while (e < len && u_is_ascii_alpha((int)t[e])) e++;
return e;
}
/* 2) [^\r\n\p{L}\p{P}\p{S}]?[\p{L}\p{M}]+ */
if (cat == UCAT_LM) {
int e = ni;
while (e < len) {
int c, cl;
decode_cp(t, len, e, &c, &cl);
int cc = u_cat(c);
if (!in_stage3_seg(c, cc) || cc != UCAT_LM) break;
e += cl;
}
return e;
}
if (cp != 0x0d && cp != 0x0a && cat != UCAT_PS && has_ni && ncat == UCAT_LM) {
int e = ni + ncl;
while (e < len) {
int c, cl;
decode_cp(t, len, e, &c, &cl);
int cc = u_cat(c);
if (!in_stage3_seg(c, cc) || cc != UCAT_LM) break;
e += cl;
}
return e;
}
/* 3) ?[\p{P}\p{S}]+[\r\n]* */
if (cp == 0x20 && has_ni && ncat == UCAT_PS) {
int e = ni + ncl;
while (e < len) {
int c, cl;
decode_cp(t, len, e, &c, &cl);
int cc = u_cat(c);
if (!in_stage3_seg(c, cc) || cc != UCAT_PS) break;
e += cl;
}
while (e < len && (t[e] == 0x0d || t[e] == 0x0a)) e++;
return e;
}
if (cat == UCAT_PS) {
int e = ni;
while (e < len) {
int c, cl;
decode_cp(t, len, e, &c, &cl);
int cc = u_cat(c);
if (!in_stage3_seg(c, cc) || cc != UCAT_PS) break;
e += cl;
}
while (e < len && (t[e] == 0x0d || t[e] == 0x0a)) e++;
return e;
}
/* 4-6) \s*[\r\n]+ | \s+(?!\S) | \s+ */
if (cat == UCAT_WS) {
int e = i;
int last_ws_len = cplen;
int last_nl_end = 0;
while (e < len) {
int c, cl;
decode_cp(t, len, e, &c, &cl);
if (u_cat(c) != UCAT_WS) break;
last_ws_len = cl;
if (c == 0x0d || c == 0x0a) last_nl_end = e + cl;
e += cl;
}
if (last_nl_end > i) return last_nl_end;
if (e == len) return e;
int ac, acl;
decode_cp(t, len, e, &ac, &acl);
if (!in_stage3_seg(ac, u_cat(ac))) return e;
if ((e - i) > last_ws_len) return e - last_ws_len;
return e;
}
/* 7) Non-matching gap of UCAT_OTHER characters not followed by in-segment UCAT_LM */
int e = ni;
while (e < len) { while (e < len) {
int c, cl; decode_cp(t, len, e, &c, &cl); int c, cl;
if (!u_is_space(c)) break; decode_cp(t, len, e, &c, &cl);
last_ws_len = cl; int cc = u_cat(c);
if (c == 0x0d || c == 0x0a) last_nl_end = e + cl; if (!in_stage3_seg(c, cc) || cc != UCAT_OTHER) break;
if (e + cl < len) {
int nc, nl;
decode_cp(t, len, e + cl, &nc, &nl);
int ncc = u_cat(nc);
if (in_stage3_seg(nc, ncc) && ncc == UCAT_LM) break;
}
e += cl; e += cl;
} }
if (e == len) return e; /* 5) \s++$ (trailing whitespace) */ return e;
if (last_nl_end > i) return last_nl_end; /* 6) \s*[\r\n] (up to last CR/LF) */
if ((e - i) > last_ws_len) return e - last_ws_len; /* 7) \s+(?!\S) (leave last ws) */
return e; /* 8) \s (single whitespace) */
} }
/* ----------------------------- BPE ------------------------------------ */ /* ----------------------------- BPE ------------------------------------ */
#define MAX_TOKEN_BYTES 128 #define MAX_TOKEN_BYTES 512
typedef struct { typedef struct {
int prev; int prev;
@@ -305,8 +632,15 @@ static bpe_heap_item_t heap_pop(bpe_heap_item_t *h, int *hn) {
return top; return top;
} }
/* Hy3's BPE vocab omits 14 single-byte values (0x0d CR and 13 invalid UTF-8
* bytes 0xc0, 0xc1, 0xf5..0xff) with unk_token=null, so HF tokenizers drops
* them when constructing a BPE Word. */
static int is_unmapped_byte(unsigned char b) {
return b == 13 || b == 192 || b == 193 || b >= 245;
}
/* Count tokens in a single raw-byte piece via O(N log N) lowest-rank merges. */ /* Count tokens in a single raw-byte piece via O(N log N) lowest-rank merges. */
static int tok_bpe(const unsigned char *piece, int plen) { static int tok_bpe_raw(const unsigned char *piece, int plen) {
if (plen <= 0) return 0; if (plen <= 0) return 0;
if (plen == 1) return 1; if (plen == 1) return 1;
bpe_node_t *nodes = (bpe_node_t *)malloc(sizeof(bpe_node_t) * (size_t)plen); bpe_node_t *nodes = (bpe_node_t *)malloc(sizeof(bpe_node_t) * (size_t)plen);
@@ -373,11 +707,31 @@ static int tok_bpe(const unsigned char *piece, int plen) {
return nsym; return nsym;
} }
/* ----------------------------- public --------------------------------- */ static int tok_bpe(const unsigned char *piece, int plen) {
int tok_count(const char *text, int len) { int has_unmapped = 0;
if (!text || len <= 0) return 0; for (int i = 0; i < plen; i++) {
pthread_once(&tok_once, tok_init); if (is_unmapped_byte(piece[i])) {
const unsigned char *t = (const unsigned char *)text; has_unmapped = 1;
break;
}
}
if (!has_unmapped) return tok_bpe_raw(piece, plen);
unsigned char stack_buf[256];
unsigned char *buf = (plen <= (int)sizeof(stack_buf))
? stack_buf
: (unsigned char *)malloc((size_t)plen);
if (!buf) return (plen + 3) / 4;
int flen = 0;
for (int i = 0; i < plen; i++) {
if (!is_unmapped_byte(piece[i])) buf[flen++] = piece[i];
}
int res = tok_bpe_raw(buf, flen);
if (buf != stack_buf) free(buf);
return res;
}
static int tok_count_slice(const unsigned char *t, int len) {
int total = 0, i = 0; int total = 0, i = 0;
while (i < len) { while (i < len) {
int end = tok_piece_end(t, len, i); int end = tok_piece_end(t, len, i);
@@ -387,6 +741,32 @@ int tok_count(const char *text, int len) {
return total; return total;
} }
/* ----------------------------- public --------------------------------- */
int tok_count(const char *text, int len) {
if (!text || len <= 0) return 0;
pthread_once(&tok_once, tok_init);
const unsigned char *t = (const unsigned char *)text;
int total = 0, i = 0;
while (i < len) {
int m = i, alen = 0;
while (m < len) {
if (t[m] == '<' || t[m] == '/') {
alen = tok_match_added(t, len, m);
if (alen > 0) break;
}
m++;
}
if (m > i) total += tok_count_slice(t + i, m - i);
if (alen > 0) {
total += 1;
i = m + alen;
} else {
break;
}
}
return total;
}
int tok_count_str(const char *text) { int tok_count_str(const char *text) {
if (!text) return 0; if (!text) return 0;
return tok_count(text, (int)strlen(text)); return tok_count(text, (int)strlen(text));
tokenizer_data.c
+16545 -19454
View File
File diff suppressed because it is too large Load Diff