774 lines
35 KiB
C
774 lines
35 KiB
C
/* tokenizer.c - embedded Tencent Hy3 ByteLevel BPE tokenizer (pure C, no deps).
|
||
*
|
||
* Counts tokens for arbitrary UTF-8 text using the official Tencent Hy3
|
||
* (tencent/Hy3, HYV3ForCausalLM) 120k ByteLevel BPE vocabulary (embedded
|
||
* in tokenizer_data.c) and its 818 added special/control tokens.
|
||
*
|
||
* The pre-tokenizer implements Hy3's 4-stage Sequence pre-tokenizer:
|
||
* 1) Split(Isolated): \p{N}{1,3}
|
||
* 2) Split(Isolated): [\u4e00-\u9fa5\u3040-\u309f\u30a0-\u30ff]+
|
||
* 3) Split(Isolated): [!"#$%&'()*+,\-./:;<=>?@\[\\\]^_`{|}~][A-Za-z]+
|
||
* |[^\r\n\p{L}\p{P}\p{S}]?[\p{L}\p{M}]+
|
||
* | ?[\p{P}\p{S}]+[\r\n]*
|
||
* |\s*[\r\n]+|\s+(?!\S)|\s+
|
||
* 4) ByteLevel(add_prefix_space=false, use_regex=false)
|
||
*/
|
||
#include "hygate.h"
|
||
#include <string.h>
|
||
#include <stdint.h>
|
||
#include <stdlib.h>
|
||
#include <pthread.h>
|
||
|
||
extern const int tok_ntok;
|
||
extern const unsigned char tok_blob[];
|
||
extern const uint16_t tok_len[];
|
||
|
||
/* ----------------------------- vocab hash ----------------------------- */
|
||
static pthread_once_t tok_once = PTHREAD_ONCE_INIT;
|
||
static const char **tok_keys = NULL;
|
||
static int *tok_ranks = NULL;
|
||
static uint16_t *tok_klen = NULL;
|
||
static unsigned int tok_mask = 0;
|
||
|
||
static uint64_t tok_fnv1a(const unsigned char *s, int len) {
|
||
uint64_t h = 1469598103934665603ULL;
|
||
for (int i = 0; i < len; i++) {
|
||
h ^= (uint64_t)s[i];
|
||
h *= 1099511628211ULL;
|
||
}
|
||
return h;
|
||
}
|
||
|
||
static void tok_init(void) {
|
||
unsigned int size = 1;
|
||
while (size < (unsigned int)tok_ntok * 2u) size <<= 1;
|
||
const char **keys = (const char **)calloc(size, sizeof(char *));
|
||
int *ranks = (int *)calloc(size, sizeof(int));
|
||
uint16_t *klen = (uint16_t *)calloc(size, sizeof(uint16_t));
|
||
if (!keys || !ranks || !klen) {
|
||
free(keys);
|
||
free(ranks);
|
||
free(klen);
|
||
return;
|
||
}
|
||
unsigned int mask = size - 1;
|
||
int offset = 0;
|
||
for (int i = 0; i < tok_ntok; i++) {
|
||
uint16_t kl = tok_len[i];
|
||
const char *key = (const char *)(tok_blob + offset);
|
||
unsigned int h = (unsigned int)tok_fnv1a((const unsigned char *)key, kl) & mask;
|
||
while (keys[h]) {
|
||
h = (h + 1) & mask;
|
||
}
|
||
keys[h] = key;
|
||
ranks[h] = i;
|
||
klen[h] = kl;
|
||
offset += kl;
|
||
}
|
||
tok_mask = mask;
|
||
tok_ranks = ranks;
|
||
tok_klen = klen;
|
||
tok_keys = keys;
|
||
}
|
||
|
||
static int tok_lookup(const unsigned char *s, int len) {
|
||
if (!tok_keys) return -1;
|
||
unsigned int h = (unsigned int)tok_fnv1a(s, len) & tok_mask;
|
||
while (tok_keys[h]) {
|
||
if ((int)tok_klen[h] == len && memcmp(tok_keys[h], s, (size_t)len) == 0)
|
||
return tok_ranks[h];
|
||
h = (h + 1) & tok_mask;
|
||
}
|
||
return -1;
|
||
}
|
||
|
||
/* ----------------------------- unicode -------------------------------- */
|
||
enum {
|
||
UCAT_OTHER = 0,
|
||
UCAT_WS = 1, /* \s (Unicode White_Space) */
|
||
UCAT_NUM = 2, /* \p{N} */
|
||
UCAT_LM = 3, /* [\p{L}\p{M}] */
|
||
UCAT_PS = 4 /* [\p{P}\p{S}] */
|
||
};
|
||
|
||
static const unsigned char u_ascii_cat[128] = {
|
||
0,0,0,0,0,0,0,0,0,1,1,1,1,1,0,0,
|
||
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
|
||
1,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,
|
||
2,2,2,2,2,2,2,2,2,2,4,4,4,4,4,4,
|
||
4,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,
|
||
3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,
|
||
4,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,
|
||
3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,0,
|
||
};
|
||
|
||
/* Packed (start_codepoint << 3) | category for U+0080..U+10FFFF (1976 transitions). */
|
||
static const uint32_t u_cat_trans[1976] = {
|
||
0x400u,0x429u,0x430u,0x501u,0x50cu,0x553u,0x55cu,0x568u,0x574u,0x592u,0x5a4u,0x5abu,
|
||
0x5b4u,0x5cau,0x5d3u,0x5dcu,0x5e2u,0x5fcu,0x603u,0x6bcu,0x6c3u,0x7bcu,0x7c3u,0x1614u,
|
||
0x1633u,0x1694u,0x1703u,0x172cu,0x1763u,0x176cu,0x1773u,0x177cu,0x1803u,0x1bacu,0x1bb3u,0x1bc0u,
|
||
0x1bd3u,0x1bf4u,0x1bfbu,0x1c00u,0x1c24u,0x1c33u,0x1c3cu,0x1c43u,0x1c58u,0x1c63u,0x1c68u,0x1c73u,
|
||
0x1d10u,0x1d1bu,0x1fb4u,0x1fbbu,0x2414u,0x241bu,0x2980u,0x298bu,0x2ab8u,0x2acbu,0x2ad4u,0x2b03u,
|
||
0x2c4cu,0x2c58u,0x2c6cu,0x2c80u,0x2c8bu,0x2df4u,0x2dfbu,0x2e04u,0x2e0bu,0x2e1cu,0x2e23u,0x2e34u,
|
||
0x2e3bu,0x2e40u,0x2e83u,0x2f58u,0x2f7bu,0x2f9cu,0x2fa8u,0x3034u,0x3083u,0x30dcu,0x30e0u,0x30ecu,
|
||
0x3103u,0x3302u,0x3354u,0x3373u,0x36a4u,0x36abu,0x36e8u,0x36f4u,0x36fbu,0x374cu,0x3753u,0x3782u,
|
||
0x37d3u,0x37ecu,0x37fbu,0x3804u,0x3870u,0x3883u,0x3a58u,0x3a6bu,0x3d90u,0x3e02u,0x3e53u,0x3fb4u,
|
||
0x3fd3u,0x3fd8u,0x3febu,0x3ff4u,0x4003u,0x4170u,0x4184u,0x41f8u,0x4203u,0x42e0u,0x42f4u,0x42f8u,
|
||
0x4303u,0x4358u,0x4383u,0x4444u,0x444bu,0x4478u,0x44bbu,0x4710u,0x471bu,0x4b24u,0x4b32u,0x4b84u,
|
||
0x4b8bu,0x4c20u,0x4c2bu,0x4c68u,0x4c7bu,0x4c88u,0x4c9bu,0x4d48u,0x4d53u,0x4d88u,0x4d93u,0x4d98u,
|
||
0x4db3u,0x4dd0u,0x4de3u,0x4e28u,0x4e3bu,0x4e48u,0x4e5bu,0x4e78u,0x4ebbu,0x4ec0u,0x4ee3u,0x4ef0u,
|
||
0x4efbu,0x4f20u,0x4f32u,0x4f83u,0x4f94u,0x4fa2u,0x4fd4u,0x4fe3u,0x4fecu,0x4ff3u,0x4ff8u,0x500bu,
|
||
0x5020u,0x502bu,0x5058u,0x507bu,0x5088u,0x509bu,0x5148u,0x5153u,0x5188u,0x5193u,0x51a0u,0x51abu,
|
||
0x51b8u,0x51c3u,0x51d0u,0x51e3u,0x51e8u,0x51f3u,0x5218u,0x523bu,0x5248u,0x525bu,0x5270u,0x528bu,
|
||
0x5290u,0x52cbu,0x52e8u,0x52f3u,0x52f8u,0x5332u,0x5383u,0x53b4u,0x53b8u,0x540bu,0x5420u,0x542bu,
|
||
0x5470u,0x547bu,0x5490u,0x549bu,0x5548u,0x5553u,0x5588u,0x5593u,0x55a0u,0x55abu,0x55d0u,0x55e3u,
|
||
0x5630u,0x563bu,0x5650u,0x565bu,0x5670u,0x5683u,0x5688u,0x5703u,0x5720u,0x5732u,0x5784u,0x5790u,
|
||
0x57cbu,0x5800u,0x580bu,0x5820u,0x582bu,0x5868u,0x587bu,0x5888u,0x589bu,0x5948u,0x5953u,0x5988u,
|
||
0x5993u,0x59a0u,0x59abu,0x59d0u,0x59e3u,0x5a28u,0x5a3bu,0x5a48u,0x5a5bu,0x5a70u,0x5aabu,0x5ac0u,
|
||
0x5ae3u,0x5af0u,0x5afbu,0x5b20u,0x5b32u,0x5b84u,0x5b8bu,0x5b92u,0x5bc0u,0x5c13u,0x5c20u,0x5c2bu,
|
||
0x5c58u,0x5c73u,0x5c88u,0x5c93u,0x5cb0u,0x5ccbu,0x5cd8u,0x5ce3u,0x5ce8u,0x5cf3u,0x5d00u,0x5d1bu,
|
||
0x5d28u,0x5d43u,0x5d58u,0x5d73u,0x5dd0u,0x5df3u,0x5e18u,0x5e33u,0x5e48u,0x5e53u,0x5e70u,0x5e83u,
|
||
0x5e88u,0x5ebbu,0x5ec0u,0x5f32u,0x5f9cu,0x5fd8u,0x6003u,0x6068u,0x6073u,0x6088u,0x6093u,0x6148u,
|
||
0x6153u,0x61d0u,0x61e3u,0x6228u,0x6233u,0x6248u,0x6253u,0x6270u,0x62abu,0x62b8u,0x62c3u,0x62d8u,
|
||
0x62ebu,0x62f0u,0x6303u,0x6320u,0x6332u,0x6380u,0x63bcu,0x63c2u,0x63fcu,0x6403u,0x6424u,0x642bu,
|
||
0x6468u,0x6473u,0x6488u,0x6493u,0x6548u,0x6553u,0x65a0u,0x65abu,0x65d0u,0x65e3u,0x6628u,0x6633u,
|
||
0x6648u,0x6653u,0x6670u,0x66abu,0x66b8u,0x66ebu,0x66f8u,0x6703u,0x6720u,0x6732u,0x6780u,0x678bu,
|
||
0x67a0u,0x6803u,0x6868u,0x6873u,0x6888u,0x6893u,0x6a28u,0x6a33u,0x6a48u,0x6a53u,0x6a7cu,0x6a80u,
|
||
0x6aa3u,0x6ac2u,0x6afbu,0x6b20u,0x6b32u,0x6bccu,0x6bd3u,0x6c00u,0x6c0bu,0x6c20u,0x6c2bu,0x6cb8u,
|
||
0x6cd3u,0x6d90u,0x6d9bu,0x6de0u,0x6debu,0x6df0u,0x6e03u,0x6e38u,0x6e53u,0x6e58u,0x6e7bu,0x6ea8u,
|
||
0x6eb3u,0x6eb8u,0x6ec3u,0x6f00u,0x6f32u,0x6f80u,0x6f93u,0x6fa4u,0x6fa8u,0x700bu,0x71d8u,0x71fcu,
|
||
0x7203u,0x727cu,0x7282u,0x72d4u,0x72e0u,0x740bu,0x7418u,0x7423u,0x7428u,0x7433u,0x7458u,0x7463u,
|
||
0x7520u,0x752bu,0x7530u,0x753bu,0x75f0u,0x7603u,0x7628u,0x7633u,0x7638u,0x7643u,0x7678u,0x7682u,
|
||
0x76d0u,0x76e3u,0x7700u,0x7803u,0x780cu,0x78c3u,0x78d4u,0x7902u,0x79a4u,0x79abu,0x79b4u,0x79bbu,
|
||
0x79c4u,0x79cbu,0x79d4u,0x79f3u,0x7a40u,0x7a4bu,0x7b68u,0x7b8bu,0x7c2cu,0x7c33u,0x7cc0u,0x7ccbu,
|
||
0x7de8u,0x7df4u,0x7e33u,0x7e3cu,0x7e68u,0x7e74u,0x7ed8u,0x8003u,0x8202u,0x8254u,0x8283u,0x8482u,
|
||
0x84d3u,0x84f4u,0x8503u,0x8630u,0x863bu,0x8640u,0x866bu,0x8670u,0x8683u,0x87dcu,0x87e3u,0x9248u,
|
||
0x9253u,0x9270u,0x9283u,0x92b8u,0x92c3u,0x92c8u,0x92d3u,0x92f0u,0x9303u,0x9448u,0x9453u,0x9470u,
|
||
0x9483u,0x9588u,0x9593u,0x95b0u,0x95c3u,0x95f8u,0x9603u,0x9608u,0x9613u,0x9630u,0x9643u,0x96b8u,
|
||
0x96c3u,0x9888u,0x9893u,0x98b0u,0x98c3u,0x9ad8u,0x9aebu,0x9b04u,0x9b4au,0x9be8u,0x9c03u,0x9c84u,
|
||
0x9cd0u,0x9d03u,0x9fb0u,0x9fc3u,0x9ff0u,0xa004u,0xa00bu,0xb36cu,0xb37bu,0xb401u,0xb40bu,0xb4dcu,
|
||
0xb4e8u,0xb503u,0xb75cu,0xb772u,0xb78bu,0xb7c8u,0xb803u,0xb8b0u,0xb8fbu,0xb9acu,0xb9b8u,0xba03u,
|
||
0xbaa0u,0xbb03u,0xbb68u,0xbb73u,0xbb88u,0xbb93u,0xbba0u,0xbc03u,0xbea4u,0xbebbu,0xbec4u,0xbee3u,
|
||
0xbef0u,0xbf02u,0xbf50u,0xbf82u,0xbfd0u,0xc004u,0xc05bu,0xc070u,0xc07bu,0xc082u,0xc0d0u,0xc103u,
|
||
0xc3c8u,0xc403u,0xc558u,0xc583u,0xc7b0u,0xc803u,0xc8f8u,0xc903u,0xc960u,0xc983u,0xc9e0u,0xca04u,
|
||
0xca08u,0xca24u,0xca32u,0xca83u,0xcb70u,0xcb83u,0xcba8u,0xcc03u,0xcd60u,0xcd83u,0xce50u,0xce82u,
|
||
0xced8u,0xcef4u,0xd003u,0xd0e0u,0xd0f4u,0xd103u,0xd2f8u,0xd303u,0xd3e8u,0xd3fbu,0xd402u,0xd450u,
|
||
0xd482u,0xd4d0u,0xd504u,0xd53bu,0xd544u,0xd570u,0xd583u,0xd678u,0xd803u,0xda68u,0xda74u,0xda82u,
|
||
0xdad4u,0xdb5bu,0xdba4u,0xdc03u,0xdd82u,0xddd3u,0xdfa0u,0xdfe4u,0xe003u,0xe1c0u,0xe1dcu,0xe202u,
|
||
0xe250u,0xe26bu,0xe282u,0xe2d3u,0xe3f4u,0xe403u,0xe458u,0xe483u,0xe5d8u,0xe5ebu,0xe604u,0xe640u,
|
||
0xe683u,0xe69cu,0xe6a3u,0xe7d8u,0xe803u,0xf8b0u,0xf8c3u,0xf8f0u,0xf903u,0xfa30u,0xfa43u,0xfa70u,
|
||
0xfa83u,0xfac0u,0xfacbu,0xfad0u,0xfadbu,0xfae0u,0xfaebu,0xfaf0u,0xfafbu,0xfbf0u,0xfc03u,0xfda8u,
|
||
0xfdb3u,0xfdecu,0xfdf3u,0xfdfcu,0xfe13u,0xfe28u,0xfe33u,0xfe6cu,0xfe83u,0xfea0u,0xfeb3u,0xfee0u,
|
||
0xfeecu,0xff03u,0xff6cu,0xff80u,0xff93u,0xffa8u,0xffb3u,0xffecu,0xfff8u,0x10001u,0x10058u,0x10084u,
|
||
0x10141u,0x10150u,0x10179u,0x10184u,0x102f9u,0x10300u,0x10382u,0x1038bu,0x10390u,0x103a2u,0x103d4u,0x103fbu,
|
||
0x10402u,0x10454u,0x10478u,0x10483u,0x104e8u,0x10504u,0x10608u,0x10683u,0x10788u,0x10804u,0x10813u,0x1081cu,
|
||
0x1083bu,0x10844u,0x10853u,0x108a4u,0x108abu,0x108b4u,0x108cbu,0x108f4u,0x10923u,0x1092cu,0x10933u,0x1093cu,
|
||
0x10943u,0x1094cu,0x10953u,0x10974u,0x1097bu,0x109d4u,0x109e3u,0x10a04u,0x10a2bu,0x10a54u,0x10a73u,0x10a7cu,
|
||
0x10a82u,0x10c1bu,0x10c2au,0x10c54u,0x10c60u,0x10c84u,0x12150u,0x12204u,0x12258u,0x12302u,0x124e4u,0x12752u,
|
||
0x12804u,0x13bb2u,0x13ca4u,0x15ba0u,0x15bb4u,0x15cb0u,0x15cbcu,0x16003u,0x1672cu,0x1675bu,0x167a0u,0x167ccu,
|
||
0x167eau,0x167f4u,0x16803u,0x16930u,0x1693bu,0x16940u,0x1696bu,0x16970u,0x16983u,0x16b40u,0x16b7bu,0x16b84u,
|
||
0x16b88u,0x16bfbu,0x16cb8u,0x16d03u,0x16d38u,0x16d43u,0x16d78u,0x16d83u,0x16db8u,0x16dc3u,0x16df8u,0x16e03u,
|
||
0x16e38u,0x16e43u,0x16e78u,0x16e83u,0x16eb8u,0x16ec3u,0x16ef8u,0x16f03u,0x17004u,0x1717bu,0x17184u,0x172f0u,
|
||
0x17404u,0x174d0u,0x174dcu,0x177a0u,0x17804u,0x17eb0u,0x17f84u,0x18001u,0x1800cu,0x1802bu,0x1803au,0x18044u,
|
||
0x1810au,0x18153u,0x18184u,0x1818bu,0x181b4u,0x181c2u,0x181dbu,0x181ecu,0x18200u,0x1820bu,0x184b8u,0x184cbu,
|
||
0x184dcu,0x184ebu,0x18504u,0x1850bu,0x187dcu,0x187e3u,0x18800u,0x1882bu,0x18980u,0x1898bu,0x18c78u,0x18c84u,
|
||
0x18c92u,0x18cb4u,0x18d03u,0x18e04u,0x18f30u,0x18f7cu,0x18f83u,0x19004u,0x190f8u,0x19102u,0x19154u,0x19242u,
|
||
0x19284u,0x1928au,0x19304u,0x19402u,0x19454u,0x1958au,0x19604u,0x1a003u,0x26e04u,0x27003u,0x52468u,0x52484u,
|
||
0x52638u,0x52683u,0x527f4u,0x52803u,0x5306cu,0x53083u,0x53102u,0x53153u,0x53160u,0x53203u,0x5339cu,0x533a3u,
|
||
0x533f4u,0x533fbu,0x53732u,0x53783u,0x53794u,0x537c0u,0x53804u,0x538bbu,0x53904u,0x53913u,0x53c4cu,0x53c5bu,
|
||
0x53e70u,0x53e83u,0x53e90u,0x53e9bu,0x53ea0u,0x53eabu,0x53ee8u,0x53f93u,0x54144u,0x54163u,0x54168u,0x54182u,
|
||
0x541b4u,0x541d0u,0x54203u,0x543a4u,0x543c0u,0x54403u,0x54630u,0x54674u,0x54682u,0x546d0u,0x54703u,0x547c4u,
|
||
0x547dbu,0x547e4u,0x547ebu,0x54802u,0x54853u,0x54974u,0x54983u,0x54aa0u,0x54afcu,0x54b03u,0x54be8u,0x54c03u,
|
||
0x54e0cu,0x54e70u,0x54e7bu,0x54e82u,0x54ed0u,0x54ef4u,0x54f03u,0x54f82u,0x54fd3u,0x54ff8u,0x55003u,0x551b8u,
|
||
0x55203u,0x55270u,0x55282u,0x552d0u,0x552e4u,0x55303u,0x553bcu,0x553d3u,0x55618u,0x556dbu,0x556f4u,0x55703u,
|
||
0x55784u,0x55793u,0x557b8u,0x5580bu,0x55838u,0x5584bu,0x55878u,0x5588bu,0x558b8u,0x55903u,0x55938u,0x55943u,
|
||
0x55978u,0x55983u,0x55adcu,0x55ae3u,0x55b54u,0x55b60u,0x55b83u,0x55f5cu,0x55f63u,0x55f70u,0x55f82u,0x55fd0u,
|
||
0x56003u,0x6bd20u,0x6bd83u,0x6be38u,0x6be5bu,0x6bfe0u,0x7c803u,0x7d370u,0x7d383u,0x7d6d0u,0x7d803u,0x7d838u,
|
||
0x7d89bu,0x7d8c0u,0x7d8ebu,0x7d94cu,0x7d953u,0x7d9b8u,0x7d9c3u,0x7d9e8u,0x7d9f3u,0x7d9f8u,0x7da03u,0x7da10u,
|
||
0x7da1bu,0x7da28u,0x7da33u,0x7dd94u,0x7de18u,0x7de9bu,0x7e9f4u,0x7ea83u,0x7ec80u,0x7ec93u,0x7ee40u,0x7ee7cu,
|
||
0x7ee80u,0x7ef83u,0x7efe4u,0x7f003u,0x7f084u,0x7f0d0u,0x7f103u,0x7f184u,0x7f298u,0x7f2a4u,0x7f338u,0x7f344u,
|
||
0x7f360u,0x7f383u,0x7f3a8u,0x7f3b3u,0x7f7e8u,0x7f80cu,0x7f882u,0x7f8d4u,0x7f90bu,0x7f9dcu,0x7fa0bu,0x7fadcu,
|
||
0x7fb33u,0x7fdf8u,0x7fe13u,0x7fe40u,0x7fe53u,0x7fe80u,0x7fe93u,0x7fec0u,0x7fed3u,0x7fee8u,0x7ff04u,0x7ff38u,
|
||
0x7ff44u,0x7ff78u,0x7ffe4u,0x7fff0u,0x80003u,0x80060u,0x8006bu,0x80138u,0x80143u,0x801d8u,0x801e3u,0x801f0u,
|
||
0x801fbu,0x80270u,0x80283u,0x802f0u,0x80403u,0x807d8u,0x80804u,0x80818u,0x8083au,0x809a0u,0x809bcu,0x80a02u,
|
||
0x80bccu,0x80c52u,0x80c64u,0x80c78u,0x80c84u,0x80ce8u,0x80d04u,0x80d08u,0x80e84u,0x80febu,0x80ff0u,0x81403u,
|
||
0x814e8u,0x81503u,0x81688u,0x81703u,0x8170au,0x817e0u,0x81803u,0x81902u,0x81920u,0x8196bu,0x81a0au,0x81a13u,
|
||
0x81a52u,0x81a58u,0x81a83u,0x81bd8u,0x81c03u,0x81cf0u,0x81cfcu,0x81d03u,0x81e20u,0x81e43u,0x81e84u,0x81e8au,
|
||
0x81eb0u,0x82003u,0x824f0u,0x82502u,0x82550u,0x82583u,0x826a0u,0x826c3u,0x827e0u,0x82803u,0x82940u,0x82983u,
|
||
0x82b20u,0x82b7cu,0x82b83u,0x82bd8u,0x82be3u,0x82c58u,0x82c63u,0x82c98u,0x82ca3u,0x82cb0u,0x82cbbu,0x82d10u,
|
||
0x82d1bu,0x82d90u,0x82d9bu,0x82dd0u,0x82ddbu,0x82de8u,0x82e03u,0x82fa0u,0x83003u,0x839b8u,0x83a03u,0x83ab0u,
|
||
0x83b03u,0x83b40u,0x83c03u,0x83c30u,0x83c3bu,0x83d88u,0x83d93u,0x83dd8u,0x84003u,0x84030u,0x84043u,0x84048u,
|
||
0x84053u,0x841b0u,0x841bbu,0x841c8u,0x841e3u,0x841e8u,0x841fbu,0x842b0u,0x842bcu,0x842c2u,0x84303u,0x843bcu,
|
||
0x843cau,0x84403u,0x844f8u,0x8453au,0x84580u,0x84703u,0x84798u,0x847a3u,0x847b0u,0x847dau,0x84803u,0x848b2u,
|
||
0x848e0u,0x848fcu,0x84903u,0x849d0u,0x849fcu,0x84a00u,0x84c03u,0x84dc0u,0x84de2u,0x84df3u,0x84e02u,0x84e80u,
|
||
0x84e92u,0x85003u,0x85020u,0x8502bu,0x85038u,0x85063u,0x850a0u,0x850abu,0x850c0u,0x850cbu,0x851b0u,0x851c3u,
|
||
0x851d8u,0x851fbu,0x85202u,0x85248u,0x85284u,0x852c8u,0x85303u,0x853eau,0x853fcu,0x85403u,0x854eau,0x85500u,
|
||
0x85603u,0x85644u,0x8564bu,0x85738u,0x8575au,0x85784u,0x857b8u,0x85803u,0x859b0u,0x859ccu,0x85a03u,0x85ab0u,
|
||
0x85ac2u,0x85b03u,0x85b98u,0x85bc2u,0x85c03u,0x85c90u,0x85cccu,0x85ce8u,0x85d4au,0x85d80u,0x86003u,0x86248u,
|
||
0x86403u,0x86598u,0x86603u,0x86798u,0x867d2u,0x86803u,0x86940u,0x86982u,0x869d0u,0x86a02u,0x86a53u,0x86b30u,
|
||
0x86b4bu,0x86b74u,0x86b7bu,0x86c30u,0x86c74u,0x86c80u,0x87302u,0x873f8u,0x87403u,0x87550u,0x8755bu,0x8756cu,
|
||
0x87570u,0x87583u,0x87590u,0x87613u,0x87628u,0x877e3u,0x878eau,0x8793bu,0x87940u,0x87983u,0x87a8au,0x87aacu,
|
||
0x87ad0u,0x87b83u,0x87c34u,0x87c50u,0x87d83u,0x87e2au,0x87e60u,0x87f03u,0x87fb8u,0x88003u,0x8823cu,0x88270u,
|
||
0x88292u,0x88383u,0x883b0u,0x883fbu,0x885dcu,0x885e8u,0x885f4u,0x88613u,0x88618u,0x88683u,0x88748u,0x88782u,
|
||
0x887d0u,0x88803u,0x889a8u,0x889b2u,0x88a04u,0x88a23u,0x88a40u,0x88a83u,0x88ba4u,0x88bb3u,0x88bb8u,0x88c03u,
|
||
0x88e2cu,0x88e4bu,0x88e6cu,0x88e73u,0x88e82u,0x88ed3u,0x88edcu,0x88ee3u,0x88eecu,0x88f00u,0x88f0au,0x88fa8u,
|
||
0x89003u,0x89090u,0x8909bu,0x891c4u,0x891f3u,0x89210u,0x89403u,0x89438u,0x89443u,0x89448u,0x89453u,0x89470u,
|
||
0x8947bu,0x894f0u,0x894fbu,0x8954cu,0x89550u,0x89583u,0x89758u,0x89782u,0x897d0u,0x89803u,0x89820u,0x8982bu,
|
||
0x89868u,0x8987bu,0x89888u,0x8989bu,0x89948u,0x89953u,0x89988u,0x89993u,0x899a0u,0x899abu,0x899d0u,0x899dbu,
|
||
0x89a28u,0x89a3bu,0x89a48u,0x89a5bu,0x89a70u,0x89a83u,0x89a88u,0x89abbu,0x89ac0u,0x89aebu,0x89b20u,0x89b33u,
|
||
0x89b68u,0x89b83u,0x89ba8u,0x89c03u,0x89c50u,0x89c5bu,0x89c60u,0x89c73u,0x89c78u,0x89c83u,0x89db0u,0x89dbbu,
|
||
0x89e08u,0x89e13u,0x89e18u,0x89e2bu,0x89e30u,0x89e3bu,0x89e58u,0x89e63u,0x89ea4u,0x89eb0u,0x89ebcu,0x89ec8u,
|
||
0x89f0bu,0x89f18u,0x8a003u,0x8a25cu,0x8a282u,0x8a2d4u,0x8a2e0u,0x8a2ecu,0x8a2f3u,0x8a310u,0x8a403u,0x8a634u,
|
||
0x8a63bu,0x8a640u,0x8a682u,0x8a6d0u,0x8ac03u,0x8adb0u,0x8adc3u,0x8ae0cu,0x8aec3u,0x8aef0u,0x8b003u,0x8b20cu,
|
||
0x8b223u,0x8b228u,0x8b282u,0x8b2d0u,0x8b304u,0x8b368u,0x8b403u,0x8b5ccu,0x8b5d0u,0x8b602u,0x8b650u,0x8b682u,
|
||
0x8b720u,0x8b803u,0x8b8d8u,0x8b8ebu,0x8b960u,0x8b982u,0x8b9e4u,0x8ba03u,0x8ba38u,0x8c003u,0x8c1dcu,0x8c1e0u,
|
||
0x8c503u,0x8c702u,0x8c798u,0x8c7fbu,0x8c838u,0x8c84bu,0x8c850u,0x8c863u,0x8c8a0u,0x8c8abu,0x8c8b8u,0x8c8c3u,
|
||
0x8c9b0u,0x8c9bbu,0x8c9c8u,0x8c9dbu,0x8ca24u,0x8ca38u,0x8ca82u,0x8cad0u,0x8cd03u,0x8cd40u,0x8cd53u,0x8cec0u,
|
||
0x8ced3u,0x8cf14u,0x8cf1bu,0x8cf28u,0x8d003u,0x8d1fcu,0x8d23bu,0x8d240u,0x8d283u,0x8d4d4u,0x8d4ebu,0x8d4f4u,
|
||
0x8d518u,0x8d583u,0x8d7c8u,0x8d804u,0x8d850u,0x8de03u,0x8df0cu,0x8df10u,0x8df82u,0x8dfd0u,0x8e003u,0x8e048u,
|
||
0x8e053u,0x8e1b8u,0x8e1c3u,0x8e20cu,0x8e230u,0x8e282u,0x8e368u,0x8e384u,0x8e393u,0x8e480u,0x8e493u,0x8e540u,
|
||
0x8e54bu,0x8e5b8u,0x8e803u,0x8e838u,0x8e843u,0x8e850u,0x8e85bu,0x8e9b8u,0x8e9d3u,0x8e9d8u,0x8e9e3u,0x8e9f0u,
|
||
0x8e9fbu,0x8ea40u,0x8ea82u,0x8ead0u,0x8eb03u,0x8eb30u,0x8eb3bu,0x8eb48u,0x8eb53u,0x8ec78u,0x8ec83u,0x8ec90u,
|
||
0x8ec9bu,0x8ecc8u,0x8ed02u,0x8ed50u,0x8f703u,0x8f7bcu,0x8f7c8u,0x8f803u,0x8f888u,0x8f893u,0x8f9d8u,0x8f9f3u,
|
||
0x8fa1cu,0x8fa82u,0x8fad3u,0x8fad8u,0x8fd83u,0x8fd88u,0x8fe02u,0x8feacu,0x8ff90u,0x8fffcu,0x90003u,0x91cd0u,
|
||
0x92002u,0x92378u,0x92384u,0x923a8u,0x92403u,0x92a20u,0x97c83u,0x97f8cu,0x97f98u,0x98003u,0x9a180u,0x9a203u,
|
||
0x9a2b0u,0x9a303u,0xa1fd8u,0xa2003u,0xa3238u,0xb0803u,0xb0982u,0xb09d0u,0xb4003u,0xb51c8u,0xb5203u,0xb52f8u,
|
||
0xb5302u,0xb5350u,0xb5374u,0xb5383u,0xb55f8u,0xb5602u,0xb5650u,0xb5683u,0xb5770u,0xb5783u,0xb57acu,0xb57b0u,
|
||
0xb5803u,0xb59bcu,0xb5a03u,0xb5a24u,0xb5a30u,0xb5a82u,0xb5ad0u,0xb5adau,0xb5b10u,0xb5b1bu,0xb5bc0u,0xb5bebu,
|
||
0xb5c80u,0xb6a03u,0xb6b6cu,0xb6b82u,0xb6bd0u,0xb7203u,0xb7402u,0xb74bcu,0xb74d8u,0xb7803u,0xb7a58u,0xb7a7bu,
|
||
0xb7c40u,0xb7c7bu,0xb7d00u,0xb7f03u,0xb7f14u,0xb7f1bu,0xb7f28u,0xb7f83u,0xb7f90u,0xb8003u,0xc3fc0u,0xc4003u,
|
||
0xc66b0u,0xc67fbu,0xc6848u,0xd7f83u,0xd7fa0u,0xd7fabu,0xd7fe0u,0xd7febu,0xd7ff8u,0xd8003u,0xd8918u,0xd8993u,
|
||
0xd8998u,0xd8a83u,0xd8a98u,0xd8aabu,0xd8ab0u,0xd8b23u,0xd8b40u,0xd8b83u,0xd97e0u,0xde003u,0xde358u,0xde383u,
|
||
0xde3e8u,0xde403u,0xde448u,0xde483u,0xde4d0u,0xde4e4u,0xde4ebu,0xde4fcu,0xde500u,0xe6004u,0xe6782u,0xe67d0u,
|
||
0xe6804u,0xe75a0u,0xe7803u,0xe7970u,0xe7983u,0xe7a38u,0xe7a84u,0xe7e20u,0xe8004u,0xe87b0u,0xe8804u,0xe8938u,
|
||
0xe894cu,0xe8b2bu,0xe8b54u,0xe8b6bu,0xe8b98u,0xe8bdbu,0xe8c1cu,0xe8c2bu,0xe8c64u,0xe8d53u,0xe8d74u,0xe8f58u,
|
||
0xe9004u,0xe9213u,0xe922cu,0xe9230u,0xe9602u,0xe96a0u,0xe9702u,0xe97a0u,0xe9804u,0xe9ab8u,0xe9b02u,0xe9bc8u,
|
||
0xea003u,0xea2a8u,0xea2b3u,0xea4e8u,0xea4f3u,0xea500u,0xea513u,0xea518u,0xea52bu,0xea538u,0xea54bu,0xea568u,
|
||
0xea573u,0xea5d0u,0xea5dbu,0xea5e0u,0xea5ebu,0xea620u,0xea62bu,0xea830u,0xea83bu,0xea858u,0xea86bu,0xea8a8u,
|
||
0xea8b3u,0xea8e8u,0xea8f3u,0xea9d0u,0xea9dbu,0xea9f8u,0xeaa03u,0xeaa28u,0xeaa33u,0xeaa38u,0xeaa53u,0xeaa88u,
|
||
0xeaa93u,0xeb530u,0xeb543u,0xeb60cu,0xeb613u,0xeb6dcu,0xeb6e3u,0xeb7dcu,0xeb7e3u,0xeb8acu,0xeb8b3u,0xeb9acu,
|
||
0xeb9b3u,0xeba7cu,0xeba83u,0xebb7cu,0xebb83u,0xebc4cu,0xebc53u,0xebd4cu,0xebd53u,0xebe1cu,0xebe23u,0xebe60u,
|
||
0xebe72u,0xec004u,0xed003u,0xed1bcu,0xed1dbu,0xed36cu,0xed3abu,0xed3b4u,0xed423u,0xed42cu,0xed460u,0xed4dbu,
|
||
0xed500u,0xed50bu,0xed580u,0xef803u,0xef8f8u,0xef92bu,0xef958u,0xf0003u,0xf0038u,0xf0043u,0xf00c8u,0xf00dbu,
|
||
0xf0110u,0xf011bu,0xf0128u,0xf0133u,0xf0158u,0xf0183u,0xf0370u,0xf047bu,0xf0480u,0xf0803u,0xf0968u,0xf0983u,
|
||
0xf09f0u,0xf0a02u,0xf0a50u,0xf0a73u,0xf0a7cu,0xf0a80u,0xf1483u,0xf1578u,0xf1603u,0xf1782u,0xf17d0u,0xf17fcu,
|
||
0xf1800u,0xf2683u,0xf2782u,0xf27d0u,0xf2e83u,0xf2f8au,0xf2fd8u,0xf2ffcu,0xf3000u,0xf3f03u,0xf3f38u,0xf3f43u,
|
||
0xf3f60u,0xf3f6bu,0xf3f78u,0xf3f83u,0xf3ff8u,0xf4003u,0xf4628u,0xf463au,0xf4683u,0xf46b8u,0xf4803u,0xf4a60u,
|
||
0xf4a82u,0xf4ad0u,0xf4af4u,0xf4b00u,0xf638au,0xf6564u,0xf656au,0xf6584u,0xf658au,0xf65a8u,0xf680au,0xf6974u,
|
||
0xf697au,0xf69f0u,0xf7003u,0xf7020u,0xf702bu,0xf7100u,0xf710bu,0xf7118u,0xf7123u,0xf7128u,0xf713bu,0xf7140u,
|
||
0xf714bu,0xf7198u,0xf71a3u,0xf71c0u,0xf71cbu,0xf71d0u,0xf71dbu,0xf71e0u,0xf7213u,0xf7218u,0xf723bu,0xf7240u,
|
||
0xf724bu,0xf7250u,0xf725bu,0xf7260u,0xf726bu,0xf7280u,0xf728bu,0xf7298u,0xf72a3u,0xf72a8u,0xf72bbu,0xf72c0u,
|
||
0xf72cbu,0xf72d0u,0xf72dbu,0xf72e0u,0xf72ebu,0xf72f0u,0xf72fbu,0xf7300u,0xf730bu,0xf7318u,0xf7323u,0xf7328u,
|
||
0xf733bu,0xf7358u,0xf7363u,0xf7398u,0xf73a3u,0xf73c0u,0xf73cbu,0xf73e8u,0xf73f3u,0xf73f8u,0xf7403u,0xf7450u,
|
||
0xf745bu,0xf74e0u,0xf750bu,0xf7520u,0xf752bu,0xf7550u,0xf755bu,0xf75e0u,0xf7784u,0xf7790u,0xf8004u,0xf8160u,
|
||
0xf8184u,0xf84a0u,0xf8504u,0xf8578u,0xf858cu,0xf8600u,0xf860cu,0xf8680u,0xf868cu,0xf87b0u,0xf8802u,0xf886cu,
|
||
0xf8d70u,0xf8f34u,0xf9018u,0xf9084u,0xf91e0u,0xf9204u,0xf9248u,0xf9284u,0xf9290u,0xf9304u,0xf9330u,0xf9804u,
|
||
0xfb6c0u,0xfb6e4u,0xfb768u,0xfb784u,0xfb7e8u,0xfb804u,0xfbbb8u,0xfbbdcu,0xfbed0u,0xfbf04u,0xfbf60u,0xfbf84u,
|
||
0xfbf88u,0xfc004u,0xfc060u,0xfc084u,0xfc240u,0xfc284u,0xfc2d0u,0xfc304u,0xfc440u,0xfc484u,0xfc570u,0xfc584u,
|
||
0xfc5e0u,0xfc604u,0xfc610u,0xfc804u,0xfd2a0u,0xfd304u,0xfd370u,0xfd384u,0xfd3e8u,0xfd404u,0xfd450u,0xfd47cu,
|
||
0xfd638u,0xfd674u,0xfd6e8u,0xfd6fcu,0xfd750u,0xfd784u,0xfd7c8u,0xfd804u,0xfdc98u,0xfdca4u,0xfdf82u,0xfdfd0u,
|
||
0x100003u,0x153700u,0x153803u,0x15b9d0u,0x15ba03u,0x15c0f0u,0x15c103u,0x167510u,0x167583u,0x175f08u,0x175f83u,0x1772f0u,
|
||
0x17c003u,0x17d0f0u,0x180003u,0x189a58u,0x189a83u,0x191d80u,0x700803u,0x700f80u,
|
||
};
|
||
|
||
static int u_cat(int cp) {
|
||
if (cp < 0) return UCAT_OTHER;
|
||
if (cp < 0x80) return (int)u_ascii_cat[cp];
|
||
if (cp > 0x10ffff) return UCAT_OTHER;
|
||
uint32_t target = ((uint32_t)cp << 3) | 7u;
|
||
int lo = 0, hi = (int)(sizeof(u_cat_trans) / sizeof(u_cat_trans[0])) - 1;
|
||
while (lo < hi) {
|
||
int mid = (lo + hi + 1) >> 1;
|
||
if (u_cat_trans[mid] <= target) lo = mid;
|
||
else hi = mid - 1;
|
||
}
|
||
return (int)(u_cat_trans[lo] & 7u);
|
||
}
|
||
|
||
static int u_is_cjk_kana(int cp) {
|
||
return (cp >= 0x4e00 && cp <= 0x9fa5) || (cp >= 0x3040 && cp <= 0x30ff);
|
||
}
|
||
|
||
static int u_is_ascii_punct(int cp) {
|
||
return (cp >= 0x21 && cp <= 0x2f) || (cp >= 0x3a && cp <= 0x40) ||
|
||
(cp >= 0x5b && cp <= 0x60) || (cp >= 0x7b && cp <= 0x7e);
|
||
}
|
||
|
||
static int u_is_ascii_alpha(int cp) {
|
||
return (cp >= 'A' && cp <= 'Z') || (cp >= 'a' && cp <= 'z');
|
||
}
|
||
|
||
/* Decode one UTF-8 codepoint at t[pos]; sets *cp (-1 if invalid byte) and *cplen. */
|
||
static void decode_cp(const unsigned char *t, int len, int pos, int *cp, int *cplen) {
|
||
unsigned char b = t[pos];
|
||
if (b < 0x80) { *cp = b; *cplen = 1; return; }
|
||
int c, l, min_cp;
|
||
if ((b >> 5) == 0x6) { c = b & 0x1f; l = 2; min_cp = 0x80; }
|
||
else if ((b >> 4) == 0xe) { c = b & 0x0f; l = 3; min_cp = 0x800; }
|
||
else if ((b >> 3) == 0x1e) { c = b & 0x07; l = 4; min_cp = 0x10000; }
|
||
else { *cp = -1; *cplen = 1; return; }
|
||
if (pos + l > len) { *cp = -1; *cplen = 1; return; }
|
||
for (int j = 1; j < l; j++) {
|
||
if ((t[pos + j] & 0xc0) != 0x80) { *cp = -1; *cplen = 1; return; }
|
||
c = (c << 6) | (t[pos + j] & 0x3f);
|
||
}
|
||
if (c < min_cp || c > 0x10ffff || (c >= 0xd800 && c <= 0xdfff)) {
|
||
*cp = -1; *cplen = 1; return;
|
||
}
|
||
*cp = c; *cplen = l;
|
||
}
|
||
|
||
/* ----------------------------- added tokens --------------------------- */
|
||
typedef struct {
|
||
const char *str;
|
||
uint8_t len;
|
||
} added_tok_t;
|
||
|
||
static const added_tok_t tok_added_named[] = {
|
||
{ "<\357\275\234tool\342\226\201output\342\226\201begin:opensource\357\275\234>", 40 },
|
||
{ "<\357\275\234hy_begin_of_sentence:opensource\357\275\234>", 39 },
|
||
{ "<\357\275\234tool\342\226\201calls\342\226\201begin:opensource\357\275\234>", 39 },
|
||
{ "<\357\275\234tool\342\226\201call\342\226\201begin:opensource\357\275\234>", 38 },
|
||
{ "<\357\275\234tool\342\226\201output\342\226\201end:opensource\357\275\234>", 38 },
|
||
{ "<\357\275\234tool\342\226\201calls\342\226\201end:opensource\357\275\234>", 37 },
|
||
{ "<\357\275\234tool\342\226\201call\342\226\201end:opensource\357\275\234>", 36 },
|
||
{ "<\357\275\234reasoning_mode:opensource\357\275\234>", 33 },
|
||
{ "<\357\275\234hy_Assistant:opensource\357\275\234>", 31 },
|
||
{ "<\357\275\234hy_end\342\226\201of\342\226\201sentence\357\275\234>", 30 },
|
||
{ "<\357\275\234tool\342\226\201sep:opensource\357\275\234>", 29 },
|
||
{ "</tool_responses:opensource>", 28 },
|
||
{ "</tool_response:opensource>", 27 },
|
||
{ "<tool_responses:opensource>", 27 },
|
||
{ "<\357\275\234hy_User:opensource\357\275\234>", 26 },
|
||
{ "<tool_response:opensource>", 26 },
|
||
{ "<\357\275\234hy_pad:opensource\357\275\234>", 25 },
|
||
{ "<\357\275\234hy_eos:opensource\357\275\234>", 25 },
|
||
{ "</tool_calls:opensource>", 24 },
|
||
{ "</tool_call:opensource>", 23 },
|
||
{ "<tool_calls:opensource>", 23 },
|
||
{ "</arg_value:opensource>", 23 },
|
||
{ "<\357\275\234hy_fim\342\226\201begin\357\275\234>", 22 },
|
||
{ "<tool_call:opensource>", 22 },
|
||
{ "<arg_value:opensource>", 22 },
|
||
{ "<\357\275\234hy_fim\342\226\201hole\357\275\234>", 21 },
|
||
{ "<tool_sep:opensource>", 21 },
|
||
{ "</arg_key:opensource>", 21 },
|
||
{ "<\357\275\234hy_fim\342\226\201end\357\275\234>", 20 },
|
||
{ "<arg_key:opensource>", 20 },
|
||
{ "</think:opensource>", 19 },
|
||
{ "<think:opensource>", 18 },
|
||
{ "<\357\275\234hy_EOT\357\275\234>", 14 },
|
||
{ "</answer>", 9 },
|
||
{ "/no_think", 9 },
|
||
{ "<answer>", 8 },
|
||
{ "/think", 6 },
|
||
};
|
||
|
||
/* Match an added special/control token at t[pos..len). Returns byte length or 0. */
|
||
static int tok_match_added(const unsigned char *t, int len, int pos) {
|
||
unsigned char b = t[pos];
|
||
if (b != '<' && b != '/') return 0;
|
||
int rem = len - pos;
|
||
const char *p = (const char *)(t + pos);
|
||
|
||
/* Check <｜hy_place▁holder▁no▁N｜> (11 <= N <= 799, excl. 20..33 and 48..51) */
|
||
static const char ph_prefix[] = "<\357\275\234hy_place\342\226\201holder\342\226\201no\342\226\201";
|
||
static const char ph_suffix[] = "\357\275\234>";
|
||
const int ph_plen = (int)(sizeof(ph_prefix) - 1);
|
||
const int ph_slen = (int)(sizeof(ph_suffix) - 1);
|
||
if (b == '<' && rem >= ph_plen + 2 + ph_slen &&
|
||
memcmp(p, ph_prefix, (size_t)ph_plen) == 0) {
|
||
int k = ph_plen, num = 0, ndig = 0;
|
||
while (k < rem && ndig < 3 && p[k] >= '0' && p[k] <= '9') {
|
||
num = num * 10 + (p[k] - '0');
|
||
k++;
|
||
ndig++;
|
||
}
|
||
if (ndig >= 2 && p[ph_plen] != '0' && num >= 11 && num <= 799 &&
|
||
!(num >= 20 && num <= 33) && !(num >= 48 && num <= 51) &&
|
||
rem - k >= ph_slen && memcmp(p + k, ph_suffix, (size_t)ph_slen) == 0) {
|
||
return k + ph_slen;
|
||
}
|
||
}
|
||
|
||
int n_added = (int)(sizeof(tok_added_named) / sizeof(tok_added_named[0]));
|
||
for (int i = 0; i < n_added; i++) {
|
||
int alen = (int)tok_added_named[i].len;
|
||
if (rem >= alen && p[1] == tok_added_named[i].str[1] &&
|
||
memcmp(p, tok_added_named[i].str, (size_t)alen) == 0) {
|
||
return alen;
|
||
}
|
||
}
|
||
return 0;
|
||
}
|
||
|
||
/* ----------------------------- pre-tokenizer -------------------------- */
|
||
static int in_stage3_seg(int cp, int cat) {
|
||
return cat != UCAT_NUM && !u_is_cjk_kana(cp);
|
||
}
|
||
|
||
/* Returns the exclusive end byte offset of the pre-tokenized piece starting
|
||
* at position i within slice t[0..len), implementing Hy3's 3-stage Split. */
|
||
static int tok_piece_end(const unsigned char *t, int len, int i) {
|
||
int cp, cplen;
|
||
decode_cp(t, len, i, &cp, &cplen);
|
||
int cat = u_cat(cp);
|
||
int ni = i + cplen;
|
||
|
||
/* Stage 1: \p{N}{1,3} */
|
||
if (cat == UCAT_NUM) {
|
||
int e = ni, cnt = 1;
|
||
while (e < len && cnt < 3) {
|
||
int c, cl;
|
||
decode_cp(t, len, e, &c, &cl);
|
||
if (u_cat(c) != UCAT_NUM) break;
|
||
e += cl;
|
||
cnt++;
|
||
}
|
||
return e;
|
||
}
|
||
|
||
/* Stage 2: [\u4e00-\u9fa5\u3040-\u309f\u30a0-\u30ff]+ sub-slice with Stage 3 applied */
|
||
if (u_is_cjk_kana(cp)) {
|
||
int ncp = -1, ncl = 0, ncat = UCAT_OTHER;
|
||
if (ni < len) {
|
||
decode_cp(t, len, ni, &ncp, &ncl);
|
||
if (u_is_cjk_kana(ncp)) ncat = u_cat(ncp);
|
||
else ncp = -1;
|
||
}
|
||
if (cat == UCAT_LM) {
|
||
int e = ni;
|
||
while (e < len) {
|
||
int c, cl;
|
||
decode_cp(t, len, e, &c, &cl);
|
||
if (!u_is_cjk_kana(c) || u_cat(c) != UCAT_LM) break;
|
||
e += cl;
|
||
}
|
||
return e;
|
||
}
|
||
if (cat != UCAT_PS && ncp >= 0 && ncat == UCAT_LM) {
|
||
int e = ni + ncl;
|
||
while (e < len) {
|
||
int c, cl;
|
||
decode_cp(t, len, e, &c, &cl);
|
||
if (!u_is_cjk_kana(c) || u_cat(c) != UCAT_LM) break;
|
||
e += cl;
|
||
}
|
||
return e;
|
||
}
|
||
if (cat == UCAT_PS) {
|
||
int e = ni;
|
||
while (e < len) {
|
||
int c, cl;
|
||
decode_cp(t, len, e, &c, &cl);
|
||
if (!u_is_cjk_kana(c) || u_cat(c) != UCAT_PS) break;
|
||
e += cl;
|
||
}
|
||
return e;
|
||
}
|
||
int e = ni;
|
||
while (e < len) {
|
||
int c, cl;
|
||
decode_cp(t, len, e, &c, &cl);
|
||
if (!u_is_cjk_kana(c)) break;
|
||
int cc = u_cat(c);
|
||
if (cc == UCAT_LM || cc == UCAT_PS) break;
|
||
if (e + cl < len) {
|
||
int nc, nl;
|
||
decode_cp(t, len, e + cl, &nc, &nl);
|
||
if (u_is_cjk_kana(nc) && u_cat(nc) == UCAT_LM) break;
|
||
}
|
||
e += cl;
|
||
}
|
||
return e;
|
||
}
|
||
|
||
/* Stage 3 on non-number, non-CJK/Kana segment */
|
||
int has_ni = 0, ncp = -1, ncl = 0, ncat = UCAT_OTHER;
|
||
if (ni < len) {
|
||
decode_cp(t, len, ni, &ncp, &ncl);
|
||
ncat = u_cat(ncp);
|
||
has_ni = in_stage3_seg(ncp, ncat);
|
||
}
|
||
|
||
/* 1) [!"#$%&'()*+,\-./:;<=>?@\[\\\]^_`{|}~][A-Za-z]+ */
|
||
if (u_is_ascii_punct(cp) && has_ni && u_is_ascii_alpha(ncp)) {
|
||
int e = ni + ncl;
|
||
while (e < len && u_is_ascii_alpha((int)t[e])) e++;
|
||
return e;
|
||
}
|
||
|
||
/* 2) [^\r\n\p{L}\p{P}\p{S}]?[\p{L}\p{M}]+ */
|
||
if (cat == UCAT_LM) {
|
||
int e = ni;
|
||
while (e < len) {
|
||
int c, cl;
|
||
decode_cp(t, len, e, &c, &cl);
|
||
int cc = u_cat(c);
|
||
if (!in_stage3_seg(c, cc) || cc != UCAT_LM) break;
|
||
e += cl;
|
||
}
|
||
return e;
|
||
}
|
||
if (cp != 0x0d && cp != 0x0a && cat != UCAT_PS && has_ni && ncat == UCAT_LM) {
|
||
int e = ni + ncl;
|
||
while (e < len) {
|
||
int c, cl;
|
||
decode_cp(t, len, e, &c, &cl);
|
||
int cc = u_cat(c);
|
||
if (!in_stage3_seg(c, cc) || cc != UCAT_LM) break;
|
||
e += cl;
|
||
}
|
||
return e;
|
||
}
|
||
|
||
/* 3) ?[\p{P}\p{S}]+[\r\n]* */
|
||
if (cp == 0x20 && has_ni && ncat == UCAT_PS) {
|
||
int e = ni + ncl;
|
||
while (e < len) {
|
||
int c, cl;
|
||
decode_cp(t, len, e, &c, &cl);
|
||
int cc = u_cat(c);
|
||
if (!in_stage3_seg(c, cc) || cc != UCAT_PS) break;
|
||
e += cl;
|
||
}
|
||
while (e < len && (t[e] == 0x0d || t[e] == 0x0a)) e++;
|
||
return e;
|
||
}
|
||
if (cat == UCAT_PS) {
|
||
int e = ni;
|
||
while (e < len) {
|
||
int c, cl;
|
||
decode_cp(t, len, e, &c, &cl);
|
||
int cc = u_cat(c);
|
||
if (!in_stage3_seg(c, cc) || cc != UCAT_PS) break;
|
||
e += cl;
|
||
}
|
||
while (e < len && (t[e] == 0x0d || t[e] == 0x0a)) e++;
|
||
return e;
|
||
}
|
||
|
||
/* 4-6) \s*[\r\n]+ | \s+(?!\S) | \s+ */
|
||
if (cat == UCAT_WS) {
|
||
int e = i;
|
||
int last_ws_len = cplen;
|
||
int last_nl_end = 0;
|
||
while (e < len) {
|
||
int c, cl;
|
||
decode_cp(t, len, e, &c, &cl);
|
||
if (u_cat(c) != UCAT_WS) break;
|
||
last_ws_len = cl;
|
||
if (c == 0x0d || c == 0x0a) last_nl_end = e + cl;
|
||
e += cl;
|
||
}
|
||
if (last_nl_end > i) return last_nl_end;
|
||
if (e == len) return e;
|
||
int ac, acl;
|
||
decode_cp(t, len, e, &ac, &acl);
|
||
if (!in_stage3_seg(ac, u_cat(ac))) return e;
|
||
if ((e - i) > last_ws_len) return e - last_ws_len;
|
||
return e;
|
||
}
|
||
|
||
/* 7) Non-matching gap of UCAT_OTHER characters not followed by in-segment UCAT_LM */
|
||
int e = ni;
|
||
while (e < len) {
|
||
int c, cl;
|
||
decode_cp(t, len, e, &c, &cl);
|
||
int cc = u_cat(c);
|
||
if (!in_stage3_seg(c, cc) || cc != UCAT_OTHER) break;
|
||
if (e + cl < len) {
|
||
int nc, nl;
|
||
decode_cp(t, len, e + cl, &nc, &nl);
|
||
int ncc = u_cat(nc);
|
||
if (in_stage3_seg(nc, ncc) && ncc == UCAT_LM) break;
|
||
}
|
||
e += cl;
|
||
}
|
||
return e;
|
||
}
|
||
|
||
/* ----------------------------- BPE ------------------------------------ */
|
||
#define MAX_TOKEN_BYTES 512
|
||
|
||
typedef struct {
|
||
int prev;
|
||
int next;
|
||
int rank;
|
||
} bpe_node_t;
|
||
|
||
typedef struct {
|
||
int rank;
|
||
int pos;
|
||
} bpe_heap_item_t;
|
||
|
||
static int heap_less(bpe_heap_item_t a, bpe_heap_item_t b) {
|
||
return (a.rank < b.rank) || (a.rank == b.rank && a.pos < b.pos);
|
||
}
|
||
|
||
static void heap_push(bpe_heap_item_t *h, int *hn, bpe_heap_item_t v) {
|
||
int i = (*hn)++;
|
||
while (i > 0) {
|
||
int p = (i - 1) >> 1;
|
||
if (!heap_less(v, h[p])) break;
|
||
h[i] = h[p];
|
||
i = p;
|
||
}
|
||
h[i] = v;
|
||
}
|
||
|
||
static bpe_heap_item_t heap_pop(bpe_heap_item_t *h, int *hn) {
|
||
bpe_heap_item_t top = h[0];
|
||
bpe_heap_item_t last = h[--(*hn)];
|
||
int n = *hn, i = 0;
|
||
while ((i << 1) + 1 < n) {
|
||
int l = (i << 1) + 1, r = l + 1;
|
||
int c = (r < n && heap_less(h[r], h[l])) ? r : l;
|
||
if (!heap_less(h[c], last)) break;
|
||
h[i] = h[c];
|
||
i = c;
|
||
}
|
||
if (n > 0) h[i] = last;
|
||
return top;
|
||
}
|
||
|
||
/* Hy3's BPE vocab omits 14 single-byte values (0x0d CR and 13 invalid UTF-8
|
||
* bytes 0xc0, 0xc1, 0xf5..0xff) with unk_token=null, so HF tokenizers drops
|
||
* them when constructing a BPE Word. */
|
||
static int is_unmapped_byte(unsigned char b) {
|
||
return b == 13 || b == 192 || b == 193 || b >= 245;
|
||
}
|
||
|
||
/* Count tokens in a single raw-byte piece via O(N log N) lowest-rank merges. */
|
||
static int tok_bpe_raw(const unsigned char *piece, int plen) {
|
||
if (plen <= 0) return 0;
|
||
if (plen == 1) return 1;
|
||
bpe_node_t *nodes = (bpe_node_t *)malloc(sizeof(bpe_node_t) * (size_t)plen);
|
||
bpe_heap_item_t *heap = (bpe_heap_item_t *)malloc(sizeof(bpe_heap_item_t) * (size_t)plen * 2u);
|
||
if (!nodes || !heap) {
|
||
free(nodes);
|
||
free(heap);
|
||
return (plen + 3) / 4;
|
||
}
|
||
int hn = 0;
|
||
for (int k = 0; k < plen; k++) {
|
||
nodes[k].prev = k - 1;
|
||
nodes[k].next = k + 1;
|
||
if (k + 1 < plen) {
|
||
int r = tok_lookup(piece + k, 2);
|
||
nodes[k].rank = r;
|
||
if (r >= 0) {
|
||
bpe_heap_item_t item = { r, k };
|
||
heap_push(heap, &hn, item);
|
||
}
|
||
} else {
|
||
nodes[k].rank = -1;
|
||
}
|
||
}
|
||
int nsym = plen;
|
||
while (nsym > 1 && hn > 0) {
|
||
bpe_heap_item_t top = heap_pop(heap, &hn);
|
||
int best = top.pos;
|
||
int right = nodes[best].next;
|
||
if (right >= plen || nodes[best].rank != top.rank) continue;
|
||
|
||
int nr = nodes[right].next;
|
||
nodes[best].next = nr;
|
||
if (nr < plen) nodes[nr].prev = best;
|
||
nodes[right].next = plen + 1;
|
||
nodes[right].rank = -1;
|
||
nsym--;
|
||
|
||
if (nr < plen) {
|
||
int clen = nodes[nr].next - best;
|
||
int r = (clen <= MAX_TOKEN_BYTES) ? tok_lookup(piece + best, clen) : -1;
|
||
nodes[best].rank = r;
|
||
if (r >= 0) {
|
||
bpe_heap_item_t item = { r, best };
|
||
heap_push(heap, &hn, item);
|
||
}
|
||
} else {
|
||
nodes[best].rank = -1;
|
||
}
|
||
|
||
int left = nodes[best].prev;
|
||
if (left >= 0) {
|
||
int clen = nr - left;
|
||
int r = (clen <= MAX_TOKEN_BYTES) ? tok_lookup(piece + left, clen) : -1;
|
||
nodes[left].rank = r;
|
||
if (r >= 0) {
|
||
bpe_heap_item_t item = { r, left };
|
||
heap_push(heap, &hn, item);
|
||
}
|
||
}
|
||
}
|
||
free(nodes);
|
||
free(heap);
|
||
return nsym;
|
||
}
|
||
|
||
static int tok_bpe(const unsigned char *piece, int plen) {
|
||
int has_unmapped = 0;
|
||
for (int i = 0; i < plen; i++) {
|
||
if (is_unmapped_byte(piece[i])) {
|
||
has_unmapped = 1;
|
||
break;
|
||
}
|
||
}
|
||
if (!has_unmapped) return tok_bpe_raw(piece, plen);
|
||
|
||
unsigned char stack_buf[256];
|
||
unsigned char *buf = (plen <= (int)sizeof(stack_buf))
|
||
? stack_buf
|
||
: (unsigned char *)malloc((size_t)plen);
|
||
if (!buf) return (plen + 3) / 4;
|
||
int flen = 0;
|
||
for (int i = 0; i < plen; i++) {
|
||
if (!is_unmapped_byte(piece[i])) buf[flen++] = piece[i];
|
||
}
|
||
int res = tok_bpe_raw(buf, flen);
|
||
if (buf != stack_buf) free(buf);
|
||
return res;
|
||
}
|
||
|
||
static int tok_count_slice(const unsigned char *t, int len) {
|
||
int total = 0, i = 0;
|
||
while (i < len) {
|
||
int end = tok_piece_end(t, len, i);
|
||
total += tok_bpe(t + i, end - i);
|
||
i = end;
|
||
}
|
||
return total;
|
||
}
|
||
|
||
/* ----------------------------- public --------------------------------- */
|
||
int tok_count(const char *text, int len) {
|
||
if (!text || len <= 0) return 0;
|
||
pthread_once(&tok_once, tok_init);
|
||
const unsigned char *t = (const unsigned char *)text;
|
||
int total = 0, i = 0;
|
||
while (i < len) {
|
||
int m = i, alen = 0;
|
||
while (m < len) {
|
||
if (t[m] == '<' || t[m] == '/') {
|
||
alen = tok_match_added(t, len, m);
|
||
if (alen > 0) break;
|
||
}
|
||
m++;
|
||
}
|
||
if (m > i) total += tok_count_slice(t + i, m - i);
|
||
if (alen > 0) {
|
||
total += 1;
|
||
i = m + alen;
|
||
} else {
|
||
break;
|
||
}
|
||
}
|
||
return total;
|
||
}
|
||
|
||
int tok_count_str(const char *text) {
|
||
if (!text) return 0;
|
||
return tok_count(text, (int)strlen(text));
|
||
}
|