hygate-c/tokenizer.c

/* tokenizer.c - embedded Tencent Hy3 ByteLevel BPE tokenizer (pure C, no deps).
 *
 * Counts tokens for arbitrary UTF-8 text using the official Tencent Hy3
 * (tencent/Hy3, HYV3ForCausalLM) 120k ByteLevel BPE vocabulary (embedded
 * in tokenizer_data.c) and its 818 added special/control tokens.
 *
 * The pre-tokenizer implements Hy3's 4-stage Sequence pre-tokenizer:
 *   1) Split(Isolated): \p{N}{1,3}
 *   2) Split(Isolated): [\u4e00-\u9fa5\u3040-\u309f\u30a0-\u30ff]+
 *   3) Split(Isolated): [!"#$%&'()*+,\-./:;<=>?@\[\\\]^_`{|}~][A-Za-z]+
 *                       |[^\r\n\p{L}\p{P}\p{S}]?[\p{L}\p{M}]+
 *                       | ?[\p{P}\p{S}]+[\r\n]*
 *                       |\s*[\r\n]+|\s+(?!\S)|\s+
 *   4) ByteLevel(add_prefix_space=false, use_regex=false)
 */
#include "hygate.h"
#include <string.h>
#include <stdint.h>
#include <stdlib.h>
#include <pthread.h>

extern const int tok_ntok;
extern const unsigned char tok_blob[];
extern const uint16_t tok_len[];

/* ----------------------------- vocab hash ----------------------------- */
static pthread_once_t tok_once = PTHREAD_ONCE_INIT;
static const char **tok_keys = NULL;
static int *tok_ranks = NULL;
static uint16_t *tok_klen = NULL;
static unsigned int tok_mask = 0;

static unsigned long tok_fnv1a(const unsigned char *s, int len) {
  unsigned long h = 1469598103934665603UL;
  for (int i = 0; i < len; i++) {
    h ^= (unsigned long)s[i];
    h *= 1099511628211UL;
  }
  return h;
}

static void tok_init(void) {
  unsigned int size = 1;
  while (size < (unsigned int)tok_ntok * 2u) size <<= 1;
  const char **keys = (const char **)calloc(size, sizeof(char *));
  int *ranks = (int *)calloc(size, sizeof(int));
  uint16_t *klen = (uint16_t *)calloc(size, sizeof(uint16_t));
  if (!keys || !ranks || !klen) {
    free(keys);
    free(ranks);
    free(klen);
    return;
  }
  unsigned int mask = size - 1;
  int offset = 0;
  for (int i = 0; i < tok_ntok; i++) {
    uint16_t kl = tok_len[i];
    const char *key = (const char *)(tok_blob + offset);
    unsigned long h = tok_fnv1a((const unsigned char *)key, kl) & mask;
    while (keys[h]) {
      h = (h + 1) & mask;
    }
    keys[h] = key;
    ranks[h] = i;
    klen[h] = kl;
    offset += kl;
  }
  tok_mask = mask;
  tok_ranks = ranks;
  tok_klen = klen;
  tok_keys = keys;
}

static int tok_lookup(const unsigned char *s, int len) {
  if (!tok_keys) return -1;
  unsigned long h = tok_fnv1a(s, len) & tok_mask;
  while (tok_keys[h]) {
    if ((int)tok_klen[h] == len && memcmp(tok_keys[h], s, (size_t)len) == 0)
      return tok_ranks[h];
    h = (h + 1) & tok_mask;
  }
  return -1;
}

/* ----------------------------- unicode -------------------------------- */
enum {
  UCAT_OTHER = 0,
  UCAT_WS    = 1, /* \s (Unicode White_Space) */
  UCAT_NUM   = 2, /* \p{N} */
  UCAT_LM    = 3, /* [\p{L}\p{M}] */
  UCAT_PS    = 4  /* [\p{P}\p{S}] */
};

static const unsigned char u_ascii_cat[128] = {
  0,0,0,0,0,0,0,0,0,1,1,1,1,1,0,0,
  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
  1,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,
  2,2,2,2,2,2,2,2,2,2,4,4,4,4,4,4,
  4,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,
  3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,
  4,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,
  3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,0,
};

/* Packed (start_codepoint << 3) | category for U+0080..U+10FFFF (1976 transitions). */
static const uint32_t u_cat_trans[1976] = {
  0x400u,0x429u,0x430u,0x501u,0x50cu,0x553u,0x55cu,0x568u,0x574u,0x592u,0x5a4u,0x5abu,
  0x5b4u,0x5cau,0x5d3u,0x5dcu,0x5e2u,0x5fcu,0x603u,0x6bcu,0x6c3u,0x7bcu,0x7c3u,0x1614u,
  0x1633u,0x1694u,0x1703u,0x172cu,0x1763u,0x176cu,0x1773u,0x177cu,0x1803u,0x1bacu,0x1bb3u,0x1bc0u,
  0x1bd3u,0x1bf4u,0x1bfbu,0x1c00u,0x1c24u,0x1c33u,0x1c3cu,0x1c43u,0x1c58u,0x1c63u,0x1c68u,0x1c73u,
  0x1d10u,0x1d1bu,0x1fb4u,0x1fbbu,0x2414u,0x241bu,0x2980u,0x298bu,0x2ab8u,0x2acbu,0x2ad4u,0x2b03u,
  0x2c4cu,0x2c58u,0x2c6cu,0x2c80u,0x2c8bu,0x2df4u,0x2dfbu,0x2e04u,0x2e0bu,0x2e1cu,0x2e23u,0x2e34u,
  0x2e3bu,0x2e40u,0x2e83u,0x2f58u,0x2f7bu,0x2f9cu,0x2fa8u,0x3034u,0x3083u,0x30dcu,0x30e0u,0x30ecu,
  0x3103u,0x3302u,0x3354u,0x3373u,0x36a4u,0x36abu,0x36e8u,0x36f4u,0x36fbu,0x374cu,0x3753u,0x3782u,
  0x37d3u,0x37ecu,0x37fbu,0x3804u,0x3870u,0x3883u,0x3a58u,0x3a6bu,0x3d90u,0x3e02u,0x3e53u,0x3fb4u,
  0x3fd3u,0x3fd8u,0x3febu,0x3ff4u,0x4003u,0x4170u,0x4184u,0x41f8u,0x4203u,0x42e0u,0x42f4u,0x42f8u,
  0x4303u,0x4358u,0x4383u,0x4444u,0x444bu,0x4478u,0x44bbu,0x4710u,0x471bu,0x4b24u,0x4b32u,0x4b84u,
  0x4b8bu,0x4c20u,0x4c2bu,0x4c68u,0x4c7bu,0x4c88u,0x4c9bu,0x4d48u,0x4d53u,0x4d88u,0x4d93u,0x4d98u,
  0x4db3u,0x4dd0u,0x4de3u,0x4e28u,0x4e3bu,0x4e48u,0x4e5bu,0x4e78u,0x4ebbu,0x4ec0u,0x4ee3u,0x4ef0u,
  0x4efbu,0x4f20u,0x4f32u,0x4f83u,0x4f94u,0x4fa2u,0x4fd4u,0x4fe3u,0x4fecu,0x4ff3u,0x4ff8u,0x500bu,
  0x5020u,0x502bu,0x5058u,0x507bu,0x5088u,0x509bu,0x5148u,0x5153u,0x5188u,0x5193u,0x51a0u,0x51abu,
  0x51b8u,0x51c3u,0x51d0u,0x51e3u,0x51e8u,0x51f3u,0x5218u,0x523bu,0x5248u,0x525bu,0x5270u,0x528bu,
  0x5290u,0x52cbu,0x52e8u,0x52f3u,0x52f8u,0x5332u,0x5383u,0x53b4u,0x53b8u,0x540bu,0x5420u,0x542bu,
  0x5470u,0x547bu,0x5490u,0x549bu,0x5548u,0x5553u,0x5588u,0x5593u,0x55a0u,0x55abu,0x55d0u,0x55e3u,
  0x5630u,0x563bu,0x5650u,0x565bu,0x5670u,0x5683u,0x5688u,0x5703u,0x5720u,0x5732u,0x5784u,0x5790u,
  0x57cbu,0x5800u,0x580bu,0x5820u,0x582bu,0x5868u,0x587bu,0x5888u,0x589bu,0x5948u,0x5953u,0x5988u,
  0x5993u,0x59a0u,0x59abu,0x59d0u,0x59e3u,0x5a28u,0x5a3bu,0x5a48u,0x5a5bu,0x5a70u,0x5aabu,0x5ac0u,
  0x5ae3u,0x5af0u,0x5afbu,0x5b20u,0x5b32u,0x5b84u,0x5b8bu,0x5b92u,0x5bc0u,0x5c13u,0x5c20u,0x5c2bu,
  0x5c58u,0x5c73u,0x5c88u,0x5c93u,0x5cb0u,0x5ccbu,0x5cd8u,0x5ce3u,0x5ce8u,0x5cf3u,0x5d00u,0x5d1bu,
  0x5d28u,0x5d43u,0x5d58u,0x5d73u,0x5dd0u,0x5df3u,0x5e18u,0x5e33u,0x5e48u,0x5e53u,0x5e70u,0x5e83u,
  0x5e88u,0x5ebbu,0x5ec0u,0x5f32u,0x5f9cu,0x5fd8u,0x6003u,0x6068u,0x6073u,0x6088u,0x6093u,0x6148u,
  0x6153u,0x61d0u,0x61e3u,0x6228u,0x6233u,0x6248u,0x6253u,0x6270u,0x62abu,0x62b8u,0x62c3u,0x62d8u,
  0x62ebu,0x62f0u,0x6303u,0x6320u,0x6332u,0x6380u,0x63bcu,0x63c2u,0x63fcu,0x6403u,0x6424u,0x642bu,
  0x6468u,0x6473u,0x6488u,0x6493u,0x6548u,0x6553u,0x65a0u,0x65abu,0x65d0u,0x65e3u,0x6628u,0x6633u,
  0x6648u,0x6653u,0x6670u,0x66abu,0x66b8u,0x66ebu,0x66f8u,0x6703u,0x6720u,0x6732u,0x6780u,0x678bu,
  0x67a0u,0x6803u,0x6868u,0x6873u,0x6888u,0x6893u,0x6a28u,0x6a33u,0x6a48u,0x6a53u,0x6a7cu,0x6a80u,
  0x6aa3u,0x6ac2u,0x6afbu,0x6b20u,0x6b32u,0x6bccu,0x6bd3u,0x6c00u,0x6c0bu,0x6c20u,0x6c2bu,0x6cb8u,
  0x6cd3u,0x6d90u,0x6d9bu,0x6de0u,0x6debu,0x6df0u,0x6e03u,0x6e38u,0x6e53u,0x6e58u,0x6e7bu,0x6ea8u,
  0x6eb3u,0x6eb8u,0x6ec3u,0x6f00u,0x6f32u,0x6f80u,0x6f93u,0x6fa4u,0x6fa8u,0x700bu,0x71d8u,0x71fcu,
  0x7203u,0x727cu,0x7282u,0x72d4u,0x72e0u,0x740bu,0x7418u,0x7423u,0x7428u,0x7433u,0x7458u,0x7463u,
  0x7520u,0x752bu,0x7530u,0x753bu,0x75f0u,0x7603u,0x7628u,0x7633u,0x7638u,0x7643u,0x7678u,0x7682u,
  0x76d0u,0x76e3u,0x7700u,0x7803u,0x780cu,0x78c3u,0x78d4u,0x7902u,0x79a4u,0x79abu,0x79b4u,0x79bbu,
  0x79c4u,0x79cbu,0x79d4u,0x79f3u,0x7a40u,0x7a4bu,0x7b68u,0x7b8bu,0x7c2cu,0x7c33u,0x7cc0u,0x7ccbu,
  0x7de8u,0x7df4u,0x7e33u,0x7e3cu,0x7e68u,0x7e74u,0x7ed8u,0x8003u,0x8202u,0x8254u,0x8283u,0x8482u,
  0x84d3u,0x84f4u,0x8503u,0x8630u,0x863bu,0x8640u,0x866bu,0x8670u,0x8683u,0x87dcu,0x87e3u,0x9248u,
  0x9253u,0x9270u,0x9283u,0x92b8u,0x92c3u,0x92c8u,0x92d3u,0x92f0u,0x9303u,0x9448u,0x9453u,0x9470u,
  0x9483u,0x9588u,0x9593u,0x95b0u,0x95c3u,0x95f8u,0x9603u,0x9608u,0x9613u,0x9630u,0x9643u,0x96b8u,
  0x96c3u,0x9888u,0x9893u,0x98b0u,0x98c3u,0x9ad8u,0x9aebu,0x9b04u,0x9b4au,0x9be8u,0x9c03u,0x9c84u,
  0x9cd0u,0x9d03u,0x9fb0u,0x9fc3u,0x9ff0u,0xa004u,0xa00bu,0xb36cu,0xb37bu,0xb401u,0xb40bu,0xb4dcu,
  0xb4e8u,0xb503u,0xb75cu,0xb772u,0xb78bu,0xb7c8u,0xb803u,0xb8b0u,0xb8fbu,0xb9acu,0xb9b8u,0xba03u,
  0xbaa0u,0xbb03u,0xbb68u,0xbb73u,0xbb88u,0xbb93u,0xbba0u,0xbc03u,0xbea4u,0xbebbu,0xbec4u,0xbee3u,
  0xbef0u,0xbf02u,0xbf50u,0xbf82u,0xbfd0u,0xc004u,0xc05bu,0xc070u,0xc07bu,0xc082u,0xc0d0u,0xc103u,
  0xc3c8u,0xc403u,0xc558u,0xc583u,0xc7b0u,0xc803u,0xc8f8u,0xc903u,0xc960u,0xc983u,0xc9e0u,0xca04u,
  0xca08u,0xca24u,0xca32u,0xca83u,0xcb70u,0xcb83u,0xcba8u,0xcc03u,0xcd60u,0xcd83u,0xce50u,0xce82u,
  0xced8u,0xcef4u,0xd003u,0xd0e0u,0xd0f4u,0xd103u,0xd2f8u,0xd303u,0xd3e8u,0xd3fbu,0xd402u,0xd450u,
  0xd482u,0xd4d0u,0xd504u,0xd53bu,0xd544u,0xd570u,0xd583u,0xd678u,0xd803u,0xda68u,0xda74u,0xda82u,
  0xdad4u,0xdb5bu,0xdba4u,0xdc03u,0xdd82u,0xddd3u,0xdfa0u,0xdfe4u,0xe003u,0xe1c0u,0xe1dcu,0xe202u,
  0xe250u,0xe26bu,0xe282u,0xe2d3u,0xe3f4u,0xe403u,0xe458u,0xe483u,0xe5d8u,0xe5ebu,0xe604u,0xe640u,
  0xe683u,0xe69cu,0xe6a3u,0xe7d8u,0xe803u,0xf8b0u,0xf8c3u,0xf8f0u,0xf903u,0xfa30u,0xfa43u,0xfa70u,
  0xfa83u,0xfac0u,0xfacbu,0xfad0u,0xfadbu,0xfae0u,0xfaebu,0xfaf0u,0xfafbu,0xfbf0u,0xfc03u,0xfda8u,
  0xfdb3u,0xfdecu,0xfdf3u,0xfdfcu,0xfe13u,0xfe28u,0xfe33u,0xfe6cu,0xfe83u,0xfea0u,0xfeb3u,0xfee0u,
  0xfeecu,0xff03u,0xff6cu,0xff80u,0xff93u,0xffa8u,0xffb3u,0xffecu,0xfff8u,0x10001u,0x10058u,0x10084u,
  0x10141u,0x10150u,0x10179u,0x10184u,0x102f9u,0x10300u,0x10382u,0x1038bu,0x10390u,0x103a2u,0x103d4u,0x103fbu,
  0x10402u,0x10454u,0x10478u,0x10483u,0x104e8u,0x10504u,0x10608u,0x10683u,0x10788u,0x10804u,0x10813u,0x1081cu,
  0x1083bu,0x10844u,0x10853u,0x108a4u,0x108abu,0x108b4u,0x108cbu,0x108f4u,0x10923u,0x1092cu,0x10933u,0x1093cu,
  0x10943u,0x1094cu,0x10953u,0x10974u,0x1097bu,0x109d4u,0x109e3u,0x10a04u,0x10a2bu,0x10a54u,0x10a73u,0x10a7cu,
  0x10a82u,0x10c1bu,0x10c2au,0x10c54u,0x10c60u,0x10c84u,0x12150u,0x12204u,0x12258u,0x12302u,0x124e4u,0x12752u,
  0x12804u,0x13bb2u,0x13ca4u,0x15ba0u,0x15bb4u,0x15cb0u,0x15cbcu,0x16003u,0x1672cu,0x1675bu,0x167a0u,0x167ccu,
  0x167eau,0x167f4u,0x16803u,0x16930u,0x1693bu,0x16940u,0x1696bu,0x16970u,0x16983u,0x16b40u,0x16b7bu,0x16b84u,
  0x16b88u,0x16bfbu,0x16cb8u,0x16d03u,0x16d38u,0x16d43u,0x16d78u,0x16d83u,0x16db8u,0x16dc3u,0x16df8u,0x16e03u,
  0x16e38u,0x16e43u,0x16e78u,0x16e83u,0x16eb8u,0x16ec3u,0x16ef8u,0x16f03u,0x17004u,0x1717bu,0x17184u,0x172f0u,
  0x17404u,0x174d0u,0x174dcu,0x177a0u,0x17804u,0x17eb0u,0x17f84u,0x18001u,0x1800cu,0x1802bu,0x1803au,0x18044u,
  0x1810au,0x18153u,0x18184u,0x1818bu,0x181b4u,0x181c2u,0x181dbu,0x181ecu,0x18200u,0x1820bu,0x184b8u,0x184cbu,
  0x184dcu,0x184ebu,0x18504u,0x1850bu,0x187dcu,0x187e3u,0x18800u,0x1882bu,0x18980u,0x1898bu,0x18c78u,0x18c84u,
  0x18c92u,0x18cb4u,0x18d03u,0x18e04u,0x18f30u,0x18f7cu,0x18f83u,0x19004u,0x190f8u,0x19102u,0x19154u,0x19242u,
  0x19284u,0x1928au,0x19304u,0x19402u,0x19454u,0x1958au,0x19604u,0x1a003u,0x26e04u,0x27003u,0x52468u,0x52484u,
  0x52638u,0x52683u,0x527f4u,0x52803u,0x5306cu,0x53083u,0x53102u,0x53153u,0x53160u,0x53203u,0x5339cu,0x533a3u,
  0x533f4u,0x533fbu,0x53732u,0x53783u,0x53794u,0x537c0u,0x53804u,0x538bbu,0x53904u,0x53913u,0x53c4cu,0x53c5bu,
  0x53e70u,0x53e83u,0x53e90u,0x53e9bu,0x53ea0u,0x53eabu,0x53ee8u,0x53f93u,0x54144u,0x54163u,0x54168u,0x54182u,
  0x541b4u,0x541d0u,0x54203u,0x543a4u,0x543c0u,0x54403u,0x54630u,0x54674u,0x54682u,0x546d0u,0x54703u,0x547c4u,
  0x547dbu,0x547e4u,0x547ebu,0x54802u,0x54853u,0x54974u,0x54983u,0x54aa0u,0x54afcu,0x54b03u,0x54be8u,0x54c03u,
  0x54e0cu,0x54e70u,0x54e7bu,0x54e82u,0x54ed0u,0x54ef4u,0x54f03u,0x54f82u,0x54fd3u,0x54ff8u,0x55003u,0x551b8u,
  0x55203u,0x55270u,0x55282u,0x552d0u,0x552e4u,0x55303u,0x553bcu,0x553d3u,0x55618u,0x556dbu,0x556f4u,0x55703u,
  0x55784u,0x55793u,0x557b8u,0x5580bu,0x55838u,0x5584bu,0x55878u,0x5588bu,0x558b8u,0x55903u,0x55938u,0x55943u,
  0x55978u,0x55983u,0x55adcu,0x55ae3u,0x55b54u,0x55b60u,0x55b83u,0x55f5cu,0x55f63u,0x55f70u,0x55f82u,0x55fd0u,
  0x56003u,0x6bd20u,0x6bd83u,0x6be38u,0x6be5bu,0x6bfe0u,0x7c803u,0x7d370u,0x7d383u,0x7d6d0u,0x7d803u,0x7d838u,
  0x7d89bu,0x7d8c0u,0x7d8ebu,0x7d94cu,0x7d953u,0x7d9b8u,0x7d9c3u,0x7d9e8u,0x7d9f3u,0x7d9f8u,0x7da03u,0x7da10u,
  0x7da1bu,0x7da28u,0x7da33u,0x7dd94u,0x7de18u,0x7de9bu,0x7e9f4u,0x7ea83u,0x7ec80u,0x7ec93u,0x7ee40u,0x7ee7cu,
  0x7ee80u,0x7ef83u,0x7efe4u,0x7f003u,0x7f084u,0x7f0d0u,0x7f103u,0x7f184u,0x7f298u,0x7f2a4u,0x7f338u,0x7f344u,
  0x7f360u,0x7f383u,0x7f3a8u,0x7f3b3u,0x7f7e8u,0x7f80cu,0x7f882u,0x7f8d4u,0x7f90bu,0x7f9dcu,0x7fa0bu,0x7fadcu,
  0x7fb33u,0x7fdf8u,0x7fe13u,0x7fe40u,0x7fe53u,0x7fe80u,0x7fe93u,0x7fec0u,0x7fed3u,0x7fee8u,0x7ff04u,0x7ff38u,
  0x7ff44u,0x7ff78u,0x7ffe4u,0x7fff0u,0x80003u,0x80060u,0x8006bu,0x80138u,0x80143u,0x801d8u,0x801e3u,0x801f0u,
  0x801fbu,0x80270u,0x80283u,0x802f0u,0x80403u,0x807d8u,0x80804u,0x80818u,0x8083au,0x809a0u,0x809bcu,0x80a02u,
  0x80bccu,0x80c52u,0x80c64u,0x80c78u,0x80c84u,0x80ce8u,0x80d04u,0x80d08u,0x80e84u,0x80febu,0x80ff0u,0x81403u,
  0x814e8u,0x81503u,0x81688u,0x81703u,0x8170au,0x817e0u,0x81803u,0x81902u,0x81920u,0x8196bu,0x81a0au,0x81a13u,
  0x81a52u,0x81a58u,0x81a83u,0x81bd8u,0x81c03u,0x81cf0u,0x81cfcu,0x81d03u,0x81e20u,0x81e43u,0x81e84u,0x81e8au,
  0x81eb0u,0x82003u,0x824f0u,0x82502u,0x82550u,0x82583u,0x826a0u,0x826c3u,0x827e0u,0x82803u,0x82940u,0x82983u,
  0x82b20u,0x82b7cu,0x82b83u,0x82bd8u,0x82be3u,0x82c58u,0x82c63u,0x82c98u,0x82ca3u,0x82cb0u,0x82cbbu,0x82d10u,
  0x82d1bu,0x82d90u,0x82d9bu,0x82dd0u,0x82ddbu,0x82de8u,0x82e03u,0x82fa0u,0x83003u,0x839b8u,0x83a03u,0x83ab0u,
  0x83b03u,0x83b40u,0x83c03u,0x83c30u,0x83c3bu,0x83d88u,0x83d93u,0x83dd8u,0x84003u,0x84030u,0x84043u,0x84048u,
  0x84053u,0x841b0u,0x841bbu,0x841c8u,0x841e3u,0x841e8u,0x841fbu,0x842b0u,0x842bcu,0x842c2u,0x84303u,0x843bcu,
  0x843cau,0x84403u,0x844f8u,0x8453au,0x84580u,0x84703u,0x84798u,0x847a3u,0x847b0u,0x847dau,0x84803u,0x848b2u,
  0x848e0u,0x848fcu,0x84903u,0x849d0u,0x849fcu,0x84a00u,0x84c03u,0x84dc0u,0x84de2u,0x84df3u,0x84e02u,0x84e80u,
  0x84e92u,0x85003u,0x85020u,0x8502bu,0x85038u,0x85063u,0x850a0u,0x850abu,0x850c0u,0x850cbu,0x851b0u,0x851c3u,
  0x851d8u,0x851fbu,0x85202u,0x85248u,0x85284u,0x852c8u,0x85303u,0x853eau,0x853fcu,0x85403u,0x854eau,0x85500u,
  0x85603u,0x85644u,0x8564bu,0x85738u,0x8575au,0x85784u,0x857b8u,0x85803u,0x859b0u,0x859ccu,0x85a03u,0x85ab0u,
  0x85ac2u,0x85b03u,0x85b98u,0x85bc2u,0x85c03u,0x85c90u,0x85cccu,0x85ce8u,0x85d4au,0x85d80u,0x86003u,0x86248u,
  0x86403u,0x86598u,0x86603u,0x86798u,0x867d2u,0x86803u,0x86940u,0x86982u,0x869d0u,0x86a02u,0x86a53u,0x86b30u,
  0x86b4bu,0x86b74u,0x86b7bu,0x86c30u,0x86c74u,0x86c80u,0x87302u,0x873f8u,0x87403u,0x87550u,0x8755bu,0x8756cu,
  0x87570u,0x87583u,0x87590u,0x87613u,0x87628u,0x877e3u,0x878eau,0x8793bu,0x87940u,0x87983u,0x87a8au,0x87aacu,
  0x87ad0u,0x87b83u,0x87c34u,0x87c50u,0x87d83u,0x87e2au,0x87e60u,0x87f03u,0x87fb8u,0x88003u,0x8823cu,0x88270u,
  0x88292u,0x88383u,0x883b0u,0x883fbu,0x885dcu,0x885e8u,0x885f4u,0x88613u,0x88618u,0x88683u,0x88748u,0x88782u,
  0x887d0u,0x88803u,0x889a8u,0x889b2u,0x88a04u,0x88a23u,0x88a40u,0x88a83u,0x88ba4u,0x88bb3u,0x88bb8u,0x88c03u,
  0x88e2cu,0x88e4bu,0x88e6cu,0x88e73u,0x88e82u,0x88ed3u,0x88edcu,0x88ee3u,0x88eecu,0x88f00u,0x88f0au,0x88fa8u,
  0x89003u,0x89090u,0x8909bu,0x891c4u,0x891f3u,0x89210u,0x89403u,0x89438u,0x89443u,0x89448u,0x89453u,0x89470u,
  0x8947bu,0x894f0u,0x894fbu,0x8954cu,0x89550u,0x89583u,0x89758u,0x89782u,0x897d0u,0x89803u,0x89820u,0x8982bu,
  0x89868u,0x8987bu,0x89888u,0x8989bu,0x89948u,0x89953u,0x89988u,0x89993u,0x899a0u,0x899abu,0x899d0u,0x899dbu,
  0x89a28u,0x89a3bu,0x89a48u,0x89a5bu,0x89a70u,0x89a83u,0x89a88u,0x89abbu,0x89ac0u,0x89aebu,0x89b20u,0x89b33u,
  0x89b68u,0x89b83u,0x89ba8u,0x89c03u,0x89c50u,0x89c5bu,0x89c60u,0x89c73u,0x89c78u,0x89c83u,0x89db0u,0x89dbbu,
  0x89e08u,0x89e13u,0x89e18u,0x89e2bu,0x89e30u,0x89e3bu,0x89e58u,0x89e63u,0x89ea4u,0x89eb0u,0x89ebcu,0x89ec8u,
  0x89f0bu,0x89f18u,0x8a003u,0x8a25cu,0x8a282u,0x8a2d4u,0x8a2e0u,0x8a2ecu,0x8a2f3u,0x8a310u,0x8a403u,0x8a634u,
  0x8a63bu,0x8a640u,0x8a682u,0x8a6d0u,0x8ac03u,0x8adb0u,0x8adc3u,0x8ae0cu,0x8aec3u,0x8aef0u,0x8b003u,0x8b20cu,
  0x8b223u,0x8b228u,0x8b282u,0x8b2d0u,0x8b304u,0x8b368u,0x8b403u,0x8b5ccu,0x8b5d0u,0x8b602u,0x8b650u,0x8b682u,
  0x8b720u,0x8b803u,0x8b8d8u,0x8b8ebu,0x8b960u,0x8b982u,0x8b9e4u,0x8ba03u,0x8ba38u,0x8c003u,0x8c1dcu,0x8c1e0u,
  0x8c503u,0x8c702u,0x8c798u,0x8c7fbu,0x8c838u,0x8c84bu,0x8c850u,0x8c863u,0x8c8a0u,0x8c8abu,0x8c8b8u,0x8c8c3u,
  0x8c9b0u,0x8c9bbu,0x8c9c8u,0x8c9dbu,0x8ca24u,0x8ca38u,0x8ca82u,0x8cad0u,0x8cd03u,0x8cd40u,0x8cd53u,0x8cec0u,
  0x8ced3u,0x8cf14u,0x8cf1bu,0x8cf28u,0x8d003u,0x8d1fcu,0x8d23bu,0x8d240u,0x8d283u,0x8d4d4u,0x8d4ebu,0x8d4f4u,
  0x8d518u,0x8d583u,0x8d7c8u,0x8d804u,0x8d850u,0x8de03u,0x8df0cu,0x8df10u,0x8df82u,0x8dfd0u,0x8e003u,0x8e048u,
  0x8e053u,0x8e1b8u,0x8e1c3u,0x8e20cu,0x8e230u,0x8e282u,0x8e368u,0x8e384u,0x8e393u,0x8e480u,0x8e493u,0x8e540u,
  0x8e54bu,0x8e5b8u,0x8e803u,0x8e838u,0x8e843u,0x8e850u,0x8e85bu,0x8e9b8u,0x8e9d3u,0x8e9d8u,0x8e9e3u,0x8e9f0u,
  0x8e9fbu,0x8ea40u,0x8ea82u,0x8ead0u,0x8eb03u,0x8eb30u,0x8eb3bu,0x8eb48u,0x8eb53u,0x8ec78u,0x8ec83u,0x8ec90u,
  0x8ec9bu,0x8ecc8u,0x8ed02u,0x8ed50u,0x8f703u,0x8f7bcu,0x8f7c8u,0x8f803u,0x8f888u,0x8f893u,0x8f9d8u,0x8f9f3u,
  0x8fa1cu,0x8fa82u,0x8fad3u,0x8fad8u,0x8fd83u,0x8fd88u,0x8fe02u,0x8feacu,0x8ff90u,0x8fffcu,0x90003u,0x91cd0u,
  0x92002u,0x92378u,0x92384u,0x923a8u,0x92403u,0x92a20u,0x97c83u,0x97f8cu,0x97f98u,0x98003u,0x9a180u,0x9a203u,
  0x9a2b0u,0x9a303u,0xa1fd8u,0xa2003u,0xa3238u,0xb0803u,0xb0982u,0xb09d0u,0xb4003u,0xb51c8u,0xb5203u,0xb52f8u,
  0xb5302u,0xb5350u,0xb5374u,0xb5383u,0xb55f8u,0xb5602u,0xb5650u,0xb5683u,0xb5770u,0xb5783u,0xb57acu,0xb57b0u,
  0xb5803u,0xb59bcu,0xb5a03u,0xb5a24u,0xb5a30u,0xb5a82u,0xb5ad0u,0xb5adau,0xb5b10u,0xb5b1bu,0xb5bc0u,0xb5bebu,
  0xb5c80u,0xb6a03u,0xb6b6cu,0xb6b82u,0xb6bd0u,0xb7203u,0xb7402u,0xb74bcu,0xb74d8u,0xb7803u,0xb7a58u,0xb7a7bu,
  0xb7c40u,0xb7c7bu,0xb7d00u,0xb7f03u,0xb7f14u,0xb7f1bu,0xb7f28u,0xb7f83u,0xb7f90u,0xb8003u,0xc3fc0u,0xc4003u,
  0xc66b0u,0xc67fbu,0xc6848u,0xd7f83u,0xd7fa0u,0xd7fabu,0xd7fe0u,0xd7febu,0xd7ff8u,0xd8003u,0xd8918u,0xd8993u,
  0xd8998u,0xd8a83u,0xd8a98u,0xd8aabu,0xd8ab0u,0xd8b23u,0xd8b40u,0xd8b83u,0xd97e0u,0xde003u,0xde358u,0xde383u,
  0xde3e8u,0xde403u,0xde448u,0xde483u,0xde4d0u,0xde4e4u,0xde4ebu,0xde4fcu,0xde500u,0xe6004u,0xe6782u,0xe67d0u,
  0xe6804u,0xe75a0u,0xe7803u,0xe7970u,0xe7983u,0xe7a38u,0xe7a84u,0xe7e20u,0xe8004u,0xe87b0u,0xe8804u,0xe8938u,
  0xe894cu,0xe8b2bu,0xe8b54u,0xe8b6bu,0xe8b98u,0xe8bdbu,0xe8c1cu,0xe8c2bu,0xe8c64u,0xe8d53u,0xe8d74u,0xe8f58u,
  0xe9004u,0xe9213u,0xe922cu,0xe9230u,0xe9602u,0xe96a0u,0xe9702u,0xe97a0u,0xe9804u,0xe9ab8u,0xe9b02u,0xe9bc8u,
  0xea003u,0xea2a8u,0xea2b3u,0xea4e8u,0xea4f3u,0xea500u,0xea513u,0xea518u,0xea52bu,0xea538u,0xea54bu,0xea568u,
  0xea573u,0xea5d0u,0xea5dbu,0xea5e0u,0xea5ebu,0xea620u,0xea62bu,0xea830u,0xea83bu,0xea858u,0xea86bu,0xea8a8u,
  0xea8b3u,0xea8e8u,0xea8f3u,0xea9d0u,0xea9dbu,0xea9f8u,0xeaa03u,0xeaa28u,0xeaa33u,0xeaa38u,0xeaa53u,0xeaa88u,
  0xeaa93u,0xeb530u,0xeb543u,0xeb60cu,0xeb613u,0xeb6dcu,0xeb6e3u,0xeb7dcu,0xeb7e3u,0xeb8acu,0xeb8b3u,0xeb9acu,
  0xeb9b3u,0xeba7cu,0xeba83u,0xebb7cu,0xebb83u,0xebc4cu,0xebc53u,0xebd4cu,0xebd53u,0xebe1cu,0xebe23u,0xebe60u,
  0xebe72u,0xec004u,0xed003u,0xed1bcu,0xed1dbu,0xed36cu,0xed3abu,0xed3b4u,0xed423u,0xed42cu,0xed460u,0xed4dbu,
  0xed500u,0xed50bu,0xed580u,0xef803u,0xef8f8u,0xef92bu,0xef958u,0xf0003u,0xf0038u,0xf0043u,0xf00c8u,0xf00dbu,
  0xf0110u,0xf011bu,0xf0128u,0xf0133u,0xf0158u,0xf0183u,0xf0370u,0xf047bu,0xf0480u,0xf0803u,0xf0968u,0xf0983u,
  0xf09f0u,0xf0a02u,0xf0a50u,0xf0a73u,0xf0a7cu,0xf0a80u,0xf1483u,0xf1578u,0xf1603u,0xf1782u,0xf17d0u,0xf17fcu,
  0xf1800u,0xf2683u,0xf2782u,0xf27d0u,0xf2e83u,0xf2f8au,0xf2fd8u,0xf2ffcu,0xf3000u,0xf3f03u,0xf3f38u,0xf3f43u,
  0xf3f60u,0xf3f6bu,0xf3f78u,0xf3f83u,0xf3ff8u,0xf4003u,0xf4628u,0xf463au,0xf4683u,0xf46b8u,0xf4803u,0xf4a60u,
  0xf4a82u,0xf4ad0u,0xf4af4u,0xf4b00u,0xf638au,0xf6564u,0xf656au,0xf6584u,0xf658au,0xf65a8u,0xf680au,0xf6974u,
  0xf697au,0xf69f0u,0xf7003u,0xf7020u,0xf702bu,0xf7100u,0xf710bu,0xf7118u,0xf7123u,0xf7128u,0xf713bu,0xf7140u,
  0xf714bu,0xf7198u,0xf71a3u,0xf71c0u,0xf71cbu,0xf71d0u,0xf71dbu,0xf71e0u,0xf7213u,0xf7218u,0xf723bu,0xf7240u,
  0xf724bu,0xf7250u,0xf725bu,0xf7260u,0xf726bu,0xf7280u,0xf728bu,0xf7298u,0xf72a3u,0xf72a8u,0xf72bbu,0xf72c0u,
  0xf72cbu,0xf72d0u,0xf72dbu,0xf72e0u,0xf72ebu,0xf72f0u,0xf72fbu,0xf7300u,0xf730bu,0xf7318u,0xf7323u,0xf7328u,
  0xf733bu,0xf7358u,0xf7363u,0xf7398u,0xf73a3u,0xf73c0u,0xf73cbu,0xf73e8u,0xf73f3u,0xf73f8u,0xf7403u,0xf7450u,
  0xf745bu,0xf74e0u,0xf750bu,0xf7520u,0xf752bu,0xf7550u,0xf755bu,0xf75e0u,0xf7784u,0xf7790u,0xf8004u,0xf8160u,
  0xf8184u,0xf84a0u,0xf8504u,0xf8578u,0xf858cu,0xf8600u,0xf860cu,0xf8680u,0xf868cu,0xf87b0u,0xf8802u,0xf886cu,
  0xf8d70u,0xf8f34u,0xf9018u,0xf9084u,0xf91e0u,0xf9204u,0xf9248u,0xf9284u,0xf9290u,0xf9304u,0xf9330u,0xf9804u,
  0xfb6c0u,0xfb6e4u,0xfb768u,0xfb784u,0xfb7e8u,0xfb804u,0xfbbb8u,0xfbbdcu,0xfbed0u,0xfbf04u,0xfbf60u,0xfbf84u,
  0xfbf88u,0xfc004u,0xfc060u,0xfc084u,0xfc240u,0xfc284u,0xfc2d0u,0xfc304u,0xfc440u,0xfc484u,0xfc570u,0xfc584u,
  0xfc5e0u,0xfc604u,0xfc610u,0xfc804u,0xfd2a0u,0xfd304u,0xfd370u,0xfd384u,0xfd3e8u,0xfd404u,0xfd450u,0xfd47cu,
  0xfd638u,0xfd674u,0xfd6e8u,0xfd6fcu,0xfd750u,0xfd784u,0xfd7c8u,0xfd804u,0xfdc98u,0xfdca4u,0xfdf82u,0xfdfd0u,
  0x100003u,0x153700u,0x153803u,0x15b9d0u,0x15ba03u,0x15c0f0u,0x15c103u,0x167510u,0x167583u,0x175f08u,0x175f83u,0x1772f0u,
  0x17c003u,0x17d0f0u,0x180003u,0x189a58u,0x189a83u,0x191d80u,0x700803u,0x700f80u,
};

static int u_cat(int cp) {
  if (cp < 0) return UCAT_OTHER;
  if (cp < 0x80) return (int)u_ascii_cat[cp];
  if (cp > 0x10ffff) return UCAT_OTHER;
  uint32_t target = ((uint32_t)cp << 3) | 7u;
  int lo = 0, hi = (int)(sizeof(u_cat_trans) / sizeof(u_cat_trans[0])) - 1;
  while (lo < hi) {
    int mid = (lo + hi + 1) >> 1;
    if (u_cat_trans[mid] <= target) lo = mid;
    else hi = mid - 1;
  }
  return (int)(u_cat_trans[lo] & 7u);
}

static int u_is_cjk_kana(int cp) {
  return (cp >= 0x4e00 && cp <= 0x9fa5) || (cp >= 0x3040 && cp <= 0x30ff);
}

static int u_is_ascii_punct(int cp) {
  return (cp >= 0x21 && cp <= 0x2f) || (cp >= 0x3a && cp <= 0x40) ||
         (cp >= 0x5b && cp <= 0x60) || (cp >= 0x7b && cp <= 0x7e);
}

static int u_is_ascii_alpha(int cp) {
  return (cp >= 'A' && cp <= 'Z') || (cp >= 'a' && cp <= 'z');
}

/* Decode one UTF-8 codepoint at t[pos]; sets *cp (-1 if invalid byte) and *cplen. */
static void decode_cp(const unsigned char *t, int len, int pos, int *cp, int *cplen) {
  unsigned char b = t[pos];
  if (b < 0x80) { *cp = b; *cplen = 1; return; }
  int c, l, min_cp;
  if ((b >> 5) == 0x6) { c = b & 0x1f; l = 2; min_cp = 0x80; }
  else if ((b >> 4) == 0xe) { c = b & 0x0f; l = 3; min_cp = 0x800; }
  else if ((b >> 3) == 0x1e) { c = b & 0x07; l = 4; min_cp = 0x10000; }
  else { *cp = -1; *cplen = 1; return; }
  if (pos + l > len) { *cp = -1; *cplen = 1; return; }
  for (int j = 1; j < l; j++) {
    if ((t[pos + j] & 0xc0) != 0x80) { *cp = -1; *cplen = 1; return; }
    c = (c << 6) | (t[pos + j] & 0x3f);
  }
  if (c < min_cp || c > 0x10ffff || (c >= 0xd800 && c <= 0xdfff)) {
    *cp = -1; *cplen = 1; return;
  }
  *cp = c; *cplen = l;
}

/* ----------------------------- added tokens --------------------------- */
typedef struct {
  const char *str;
  uint8_t len;
} added_tok_t;

static const added_tok_t tok_added_named[] = {
  { "<\357\275\234tool\342\226\201output\342\226\201begin:opensource\357\275\234>", 40 },
  { "<\357\275\234hy_begin_of_sentence:opensource\357\275\234>", 39 },
  { "<\357\275\234tool\342\226\201calls\342\226\201begin:opensource\357\275\234>", 39 },
  { "<\357\275\234tool\342\226\201call\342\226\201begin:opensource\357\275\234>", 38 },
  { "<\357\275\234tool\342\226\201output\342\226\201end:opensource\357\275\234>", 38 },
  { "<\357\275\234tool\342\226\201calls\342\226\201end:opensource\357\275\234>", 37 },
  { "<\357\275\234tool\342\226\201call\342\226\201end:opensource\357\275\234>", 36 },
  { "<\357\275\234reasoning_mode:opensource\357\275\234>", 33 },
  { "<\357\275\234hy_Assistant:opensource\357\275\234>", 31 },
  { "<\357\275\234hy_end\342\226\201of\342\226\201sentence\357\275\234>", 30 },
  { "<\357\275\234tool\342\226\201sep:opensource\357\275\234>", 29 },
  { "</tool_responses:opensource>", 28 },
  { "</tool_response:opensource>", 27 },
  { "<tool_responses:opensource>", 27 },
  { "<\357\275\234hy_User:opensource\357\275\234>", 26 },
  { "<tool_response:opensource>", 26 },
  { "<\357\275\234hy_pad:opensource\357\275\234>", 25 },
  { "<\357\275\234hy_eos:opensource\357\275\234>", 25 },
  { "</tool_calls:opensource>", 24 },
  { "</tool_call:opensource>", 23 },
  { "<tool_calls:opensource>", 23 },
  { "</arg_value:opensource>", 23 },
  { "<\357\275\234hy_fim\342\226\201begin\357\275\234>", 22 },
  { "<tool_call:opensource>", 22 },
  { "<arg_value:opensource>", 22 },
  { "<\357\275\234hy_fim\342\226\201hole\357\275\234>", 21 },
  { "<tool_sep:opensource>", 21 },
  { "</arg_key:opensource>", 21 },
  { "<\357\275\234hy_fim\342\226\201end\357\275\234>", 20 },
  { "<arg_key:opensource>", 20 },
  { "</think:opensource>", 19 },
  { "<think:opensource>", 18 },
  { "<\357\275\234hy_EOT\357\275\234>", 14 },
  { "</answer>", 9 },
  { "/no_think", 9 },
  { "<answer>", 8 },
  { "/think", 6 },
};

/* Match an added special/control token at t[pos..len). Returns byte length or 0. */
static int tok_match_added(const unsigned char *t, int len, int pos) {
  unsigned char b = t[pos];
  if (b != '<' && b != '/') return 0;
  int rem = len - pos;
  const char *p = (const char *)(t + pos);

  /* Check <｜hy_place▁holder▁no▁N｜> (11 <= N <= 799, excl. 20..33 and 48..51) */
  static const char ph_prefix[] = "<\357\275\234hy_place\342\226\201holder\342\226\201no\342\226\201";
  static const char ph_suffix[] = "\357\275\234>";
  const int ph_plen = (int)(sizeof(ph_prefix) - 1);
  const int ph_slen = (int)(sizeof(ph_suffix) - 1);
  if (b == '<' && rem >= ph_plen + 2 + ph_slen &&
      memcmp(p, ph_prefix, (size_t)ph_plen) == 0) {
    int k = ph_plen, num = 0, ndig = 0;
    while (k < rem && ndig < 3 && p[k] >= '0' && p[k] <= '9') {
      num = num * 10 + (p[k] - '0');
      k++;
      ndig++;
    }
    if (ndig >= 2 && p[ph_plen] != '0' && num >= 11 && num <= 799 &&
        !(num >= 20 && num <= 33) && !(num >= 48 && num <= 51) &&
        rem - k >= ph_slen && memcmp(p + k, ph_suffix, (size_t)ph_slen) == 0) {
      return k + ph_slen;
    }
  }

  int n_added = (int)(sizeof(tok_added_named) / sizeof(tok_added_named[0]));
  for (int i = 0; i < n_added; i++) {
    int alen = (int)tok_added_named[i].len;
    if (rem >= alen && p[1] == tok_added_named[i].str[1] &&
        memcmp(p, tok_added_named[i].str, (size_t)alen) == 0) {
      return alen;
    }
  }
  return 0;
}

/* ----------------------------- pre-tokenizer -------------------------- */
static int in_stage3_seg(int cp, int cat) {
  return cat != UCAT_NUM && !u_is_cjk_kana(cp);
}

/* Returns the exclusive end byte offset of the pre-tokenized piece starting
 * at position i within slice t[0..len), implementing Hy3's 3-stage Split. */
static int tok_piece_end(const unsigned char *t, int len, int i) {
  int cp, cplen;
  decode_cp(t, len, i, &cp, &cplen);
  int cat = u_cat(cp);
  int ni = i + cplen;

  /* Stage 1: \p{N}{1,3} */
  if (cat == UCAT_NUM) {
    int e = ni, cnt = 1;
    while (e < len && cnt < 3) {
      int c, cl;
      decode_cp(t, len, e, &c, &cl);
      if (u_cat(c) != UCAT_NUM) break;
      e += cl;
      cnt++;
    }
    return e;
  }

  /* Stage 2: [\u4e00-\u9fa5\u3040-\u309f\u30a0-\u30ff]+ sub-slice with Stage 3 applied */
  if (u_is_cjk_kana(cp)) {
    int ncp = -1, ncl = 0, ncat = UCAT_OTHER;
    if (ni < len) {
      decode_cp(t, len, ni, &ncp, &ncl);
      if (u_is_cjk_kana(ncp)) ncat = u_cat(ncp);
      else ncp = -1;
    }
    if (cat == UCAT_LM) {
      int e = ni;
      while (e < len) {
        int c, cl;
        decode_cp(t, len, e, &c, &cl);
        if (!u_is_cjk_kana(c) || u_cat(c) != UCAT_LM) break;
        e += cl;
      }
      return e;
    }
    if (cat != UCAT_PS && ncp >= 0 && ncat == UCAT_LM) {
      int e = ni + ncl;
      while (e < len) {
        int c, cl;
        decode_cp(t, len, e, &c, &cl);
        if (!u_is_cjk_kana(c) || u_cat(c) != UCAT_LM) break;
        e += cl;
      }
      return e;
    }
    if (cat == UCAT_PS) {
      int e = ni;
      while (e < len) {
        int c, cl;
        decode_cp(t, len, e, &c, &cl);
        if (!u_is_cjk_kana(c) || u_cat(c) != UCAT_PS) break;
        e += cl;
      }
      return e;
    }
    int e = ni;
    while (e < len) {
      int c, cl;
      decode_cp(t, len, e, &c, &cl);
      if (!u_is_cjk_kana(c)) break;
      int cc = u_cat(c);
      if (cc == UCAT_LM || cc == UCAT_PS) break;
      if (e + cl < len) {
        int nc, nl;
        decode_cp(t, len, e + cl, &nc, &nl);
        if (u_is_cjk_kana(nc) && u_cat(nc) == UCAT_LM) break;
      }
      e += cl;
    }
    return e;
  }

  /* Stage 3 on non-number, non-CJK/Kana segment */
  int has_ni = 0, ncp = -1, ncl = 0, ncat = UCAT_OTHER;
  if (ni < len) {
    decode_cp(t, len, ni, &ncp, &ncl);
    ncat = u_cat(ncp);
    has_ni = in_stage3_seg(ncp, ncat);
  }

  /* 1) [!"#$%&'()*+,\-./:;<=>?@\[\\\]^_`{|}~][A-Za-z]+ */
  if (u_is_ascii_punct(cp) && has_ni && u_is_ascii_alpha(ncp)) {
    int e = ni + ncl;
    while (e < len && u_is_ascii_alpha((int)t[e])) e++;
    return e;
  }

  /* 2) [^\r\n\p{L}\p{P}\p{S}]?[\p{L}\p{M}]+ */
  if (cat == UCAT_LM) {
    int e = ni;
    while (e < len) {
      int c, cl;
      decode_cp(t, len, e, &c, &cl);
      int cc = u_cat(c);
      if (!in_stage3_seg(c, cc) || cc != UCAT_LM) break;
      e += cl;
    }
    return e;
  }
  if (cp != 0x0d && cp != 0x0a && cat != UCAT_PS && has_ni && ncat == UCAT_LM) {
    int e = ni + ncl;
    while (e < len) {
      int c, cl;
      decode_cp(t, len, e, &c, &cl);
      int cc = u_cat(c);
      if (!in_stage3_seg(c, cc) || cc != UCAT_LM) break;
      e += cl;
    }
    return e;
  }

  /* 3)  ?[\p{P}\p{S}]+[\r\n]* */
  if (cp == 0x20 && has_ni && ncat == UCAT_PS) {
    int e = ni + ncl;
    while (e < len) {
      int c, cl;
      decode_cp(t, len, e, &c, &cl);
      int cc = u_cat(c);
      if (!in_stage3_seg(c, cc) || cc != UCAT_PS) break;
      e += cl;
    }
    while (e < len && (t[e] == 0x0d || t[e] == 0x0a)) e++;
    return e;
  }
  if (cat == UCAT_PS) {
    int e = ni;
    while (e < len) {
      int c, cl;
      decode_cp(t, len, e, &c, &cl);
      int cc = u_cat(c);
      if (!in_stage3_seg(c, cc) || cc != UCAT_PS) break;
      e += cl;
    }
    while (e < len && (t[e] == 0x0d || t[e] == 0x0a)) e++;
    return e;
  }

  /* 4-6) \s*[\r\n]+ | \s+(?!\S) | \s+ */
  if (cat == UCAT_WS) {
    int e = i;
    int last_ws_len = cplen;
    int last_nl_end = 0;
    while (e < len) {
      int c, cl;
      decode_cp(t, len, e, &c, &cl);
      if (u_cat(c) != UCAT_WS) break;
      last_ws_len = cl;
      if (c == 0x0d || c == 0x0a) last_nl_end = e + cl;
      e += cl;
    }
    if (last_nl_end > i) return last_nl_end;
    if (e == len) return e;
    int ac, acl;
    decode_cp(t, len, e, &ac, &acl);
    if (!in_stage3_seg(ac, u_cat(ac))) return e;
    if ((e - i) > last_ws_len) return e - last_ws_len;
    return e;
  }

  /* 7) Non-matching gap of UCAT_OTHER characters not followed by in-segment UCAT_LM */
  int e = ni;
  while (e < len) {
    int c, cl;
    decode_cp(t, len, e, &c, &cl);
    int cc = u_cat(c);
    if (!in_stage3_seg(c, cc) || cc != UCAT_OTHER) break;
    if (e + cl < len) {
      int nc, nl;
      decode_cp(t, len, e + cl, &nc, &nl);
      int ncc = u_cat(nc);
      if (in_stage3_seg(nc, ncc) && ncc == UCAT_LM) break;
    }
    e += cl;
  }
  return e;
}

/* ----------------------------- BPE ------------------------------------ */
#define MAX_TOKEN_BYTES 512

typedef struct {
  int prev;
  int next;
  int rank;
} bpe_node_t;

typedef struct {
  int rank;
  int pos;
} bpe_heap_item_t;

static int heap_less(bpe_heap_item_t a, bpe_heap_item_t b) {
  return (a.rank < b.rank) || (a.rank == b.rank && a.pos < b.pos);
}

static void heap_push(bpe_heap_item_t *h, int *hn, bpe_heap_item_t v) {
  int i = (*hn)++;
  while (i > 0) {
    int p = (i - 1) >> 1;
    if (!heap_less(v, h[p])) break;
    h[i] = h[p];
    i = p;
  }
  h[i] = v;
}

static bpe_heap_item_t heap_pop(bpe_heap_item_t *h, int *hn) {
  bpe_heap_item_t top = h[0];
  bpe_heap_item_t last = h[--(*hn)];
  int n = *hn, i = 0;
  while ((i << 1) + 1 < n) {
    int l = (i << 1) + 1, r = l + 1;
    int c = (r < n && heap_less(h[r], h[l])) ? r : l;
    if (!heap_less(h[c], last)) break;
    h[i] = h[c];
    i = c;
  }
  if (n > 0) h[i] = last;
  return top;
}

/* Hy3's BPE vocab omits 14 single-byte values (0x0d CR and 13 invalid UTF-8
 * bytes 0xc0, 0xc1, 0xf5..0xff) with unk_token=null, so HF tokenizers drops
 * them when constructing a BPE Word. */
static int is_unmapped_byte(unsigned char b) {
  return b == 13 || b == 192 || b == 193 || b >= 245;
}

/* Count tokens in a single raw-byte piece via O(N log N) lowest-rank merges. */
static int tok_bpe_raw(const unsigned char *piece, int plen) {
  if (plen <= 0) return 0;
  if (plen == 1) return 1;
  bpe_node_t *nodes = (bpe_node_t *)malloc(sizeof(bpe_node_t) * (size_t)plen);
  bpe_heap_item_t *heap = (bpe_heap_item_t *)malloc(sizeof(bpe_heap_item_t) * (size_t)plen * 2u);
  if (!nodes || !heap) {
    free(nodes);
    free(heap);
    return (plen + 3) / 4;
  }
  int hn = 0;
  for (int k = 0; k < plen; k++) {
    nodes[k].prev = k - 1;
    nodes[k].next = k + 1;
    if (k + 1 < plen) {
      int r = tok_lookup(piece + k, 2);
      nodes[k].rank = r;
      if (r >= 0) {
        bpe_heap_item_t item = { r, k };
        heap_push(heap, &hn, item);
      }
    } else {
      nodes[k].rank = -1;
    }
  }
  int nsym = plen;
  while (nsym > 1 && hn > 0) {
    bpe_heap_item_t top = heap_pop(heap, &hn);
    int best = top.pos;
    int right = nodes[best].next;
    if (right >= plen || nodes[best].rank != top.rank) continue;

    int nr = nodes[right].next;
    nodes[best].next = nr;
    if (nr < plen) nodes[nr].prev = best;
    nodes[right].next = plen + 1;
    nodes[right].rank = -1;
    nsym--;

    if (nr < plen) {
      int clen = nodes[nr].next - best;
      int r = (clen <= MAX_TOKEN_BYTES) ? tok_lookup(piece + best, clen) : -1;
      nodes[best].rank = r;
      if (r >= 0) {
        bpe_heap_item_t item = { r, best };
        heap_push(heap, &hn, item);
      }
    } else {
      nodes[best].rank = -1;
    }

    int left = nodes[best].prev;
    if (left >= 0) {
      int clen = nr - left;
      int r = (clen <= MAX_TOKEN_BYTES) ? tok_lookup(piece + left, clen) : -1;
      nodes[left].rank = r;
      if (r >= 0) {
        bpe_heap_item_t item = { r, left };
        heap_push(heap, &hn, item);
      }
    }
  }
  free(nodes);
  free(heap);
  return nsym;
}

static int tok_bpe(const unsigned char *piece, int plen) {
  int has_unmapped = 0;
  for (int i = 0; i < plen; i++) {
    if (is_unmapped_byte(piece[i])) {
      has_unmapped = 1;
      break;
    }
  }
  if (!has_unmapped) return tok_bpe_raw(piece, plen);

  unsigned char stack_buf[256];
  unsigned char *buf = (plen <= (int)sizeof(stack_buf))
                           ? stack_buf
                           : (unsigned char *)malloc((size_t)plen);
  if (!buf) return (plen + 3) / 4;
  int flen = 0;
  for (int i = 0; i < plen; i++) {
    if (!is_unmapped_byte(piece[i])) buf[flen++] = piece[i];
  }
  int res = tok_bpe_raw(buf, flen);
  if (buf != stack_buf) free(buf);
  return res;
}

static int tok_count_slice(const unsigned char *t, int len) {
  int total = 0, i = 0;
  while (i < len) {
    int end = tok_piece_end(t, len, i);
    total += tok_bpe(t + i, end - i);
    i = end;
  }
  return total;
}

/* ----------------------------- public --------------------------------- */
int tok_count(const char *text, int len) {
  if (!text || len <= 0) return 0;
  pthread_once(&tok_once, tok_init);
  const unsigned char *t = (const unsigned char *)text;
  int total = 0, i = 0;
  while (i < len) {
    int m = i, alen = 0;
    while (m < len) {
      if (t[m] == '<' || t[m] == '/') {
        alen = tok_match_added(t, len, m);
        if (alen > 0) break;
      }
      m++;
    }
    if (m > i) total += tok_count_slice(t + i, m - i);
    if (alen > 0) {
      total += 1;
      i = m + alen;
    } else {
      break;
    }
  }
  return total;
}

int tok_count_str(const char *text) {
  if (!text) return 0;
  return tok_count(text, (int)strlen(text));
}