luxferre/hygate-c
1
0
Fork 0
Code Actions Activity

recent changes

Browse Source
This commit is contained in:
Luxferre
2026-10-14 00:00:02 +03:00
parent 8a0c1d0ec0
commit 80ad504d16
7 changed files with 550 additions and 251 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tokenizer.c
+178 -83
View File
@@ -19,6 +19,7 @@
#include <string.h>
#include <stdint.h>
#include <stdlib.h>
#include <pthread.h>
extern const int tok_ntok;
extern const unsigned char tok_blob[];
@@ -26,10 +27,11 @@ extern const int tok_rank[];
extern const uint16_t tok_len[];
/* ----------------------------- vocab hash ----------------------------- */
static int tok_inited = 0;
static pthread_once_t tok_once = PTHREAD_ONCE_INIT;
static const char **tok_keys = NULL;
static int *tok_ranks = NULL;
static int *tok_klen = NULL;static unsigned int tok_mask = 0;
static int *tok_klen = NULL;
static unsigned int tok_mask = 0;
static unsigned long tok_fnv1a(const unsigned char *s, int len) {
unsigned long h = 1469598103934665603UL;
@@ -41,34 +43,39 @@ static unsigned long tok_fnv1a(const unsigned char *s, int len) {
}
static void tok_init(void) {
if (tok_inited) return;
tok_inited = 1;
unsigned int size = 1;
while (size < (unsigned int)tok_ntok * 2u) size <<= 1;
tok_mask = size - 1;
tok_keys = (const char **)calloc(size, sizeof(char *));
tok_ranks = (int *)calloc(size, sizeof(int));
tok_klen = (int *)calloc(size, sizeof(int));
if (!tok_keys || !tok_ranks || !tok_klen) return;
const char **keys = (const char **)calloc(size, sizeof(char *));
int *ranks = (int *)calloc(size, sizeof(int));
int *klen = (int *)calloc(size, sizeof(int));
if (!keys || !ranks || !klen) {
free(keys);
free(ranks);
free(klen);
return;
}
unsigned int mask = size - 1;
int offset = 0;
for (int i = 0; i < tok_ntok; i++) {
int klen = tok_len[i];
int kl = tok_len[i];
const char *key = (const char *)(tok_blob + offset);
unsigned long h = tok_fnv1a((const unsigned char *)key, klen) & tok_mask;
while (tok_keys[h]) {
const char *k = tok_keys[h];
if (tok_klen[h] == klen && memcmp(k, key, klen) == 0) break; /* dup (none expected) */
h = (h + 1) & tok_mask;
unsigned long h = tok_fnv1a((const unsigned char *)key, kl) & mask;
while (keys[h]) {
h = (h + 1) & mask;
}
tok_keys[h] = key;
tok_ranks[h] = tok_rank[i];
tok_klen[h] = klen;
offset += klen;
keys[h] = key;
ranks[h] = tok_rank[i];
klen[h] = kl;
offset += kl;
}
tok_mask = mask;
tok_ranks = ranks;
tok_klen = klen;
tok_keys = keys;
}
static int tok_lookup(const unsigned char *s, int len) {
if (!tok_inited) tok_init();
pthread_once(&tok_once, tok_init);
if (!tok_keys) return -1;
unsigned long h = tok_fnv1a(s, len) & tok_mask;
while (tok_keys[h]) {
@@ -100,15 +107,39 @@ static int u_is_number(int cp) {
}
static int u_is_letter(int cp) {
if ((cp >= 'a' && cp <= 'z') || (cp >= 'A' && cp <= 'Z')) return 1;
if (cp >= 0x00c0 && cp <= 0x024f) return 1; /* Latin-1 + Latin Extended-A/B */
if (cp >= 0x0370 && cp <= 0x03ff) return 1; /* Greek */
if (cp >= 0x0400 && cp <= 0x04ff) return 1; /* Cyrillic */
if (cp >= 0x3040 && cp <= 0x30ff) return 1; /* Hiragana + Katakana */
if (cp >= 0x00c0 && cp <= 0x02af) {
return (cp != 0x00d7 && cp != 0x00f7); /* Latin-1 + Ext-A/B + IPA (excl. × ÷) */
}
if (cp >= 0x0370 && cp <= 0x03ff) {
return (cp != 0x037e && cp != 0x0387); /* Greek */
}
if (cp >= 0x0400 && cp <= 0x052f) return 1; /* Cyrillic + Supplement */
if (cp >= 0x0531 && cp <= 0x0587) return 1; /* Armenian */
if (cp >= 0x05d0 && cp <= 0x05ea) return 1; /* Hebrew */
if ((cp >= 0x0620 && cp <= 0x064a) ||
(cp >= 0x066e && cp <= 0x06d3) ||
(cp >= 0x06ee && cp <= 0x06ff) ||
(cp >= 0x0750 && cp <= 0x077f) ||
(cp >= 0x08a0 && cp <= 0x08ff)) return 1; /* Arabic */
if (cp >= 0x0904 && cp <= 0x0d7f) {
return !(cp >= 0x0964 && cp <= 0x096f); /* Devanagari + Indic scripts */
}
if ((cp >= 0x0e01 && cp <= 0x0e30) ||
(cp >= 0x0e32 && cp <= 0x0e33) ||
(cp >= 0x0e40 && cp <= 0x0e46)) return 1; /* Thai */
if (cp >= 0x10a0 && cp <= 0x10ff) return 1; /* Georgian */
if (cp >= 0x1100 && cp <= 0x11ff) return 1; /* Hangul Jamo */
if (cp >= 0x1e00 && cp <= 0x1fff) return 1; /* Latin Ext Additional + Greek Ext */
if (cp >= 0x3041 && cp <= 0x3096) return 1; /* Hiragana */
if (cp >= 0x30a1 && cp <= 0x30fc && cp != 0x30fb) return 1; /* Katakana (excl. ・) */
if (cp >= 0x3131 && cp <= 0x318e) return 1; /* Hangul Compatibility Jamo */
if (cp >= 0x3400 && cp <= 0x4dbf) return 1; /* CJK Ext A */
if (cp >= 0x4e00 && cp <= 0x9fff) return 1; /* CJK Unified Ideographs */
if (cp >= 0xac00 && cp <= 0xd7a3) return 1; /* Hangul Syllables */
if (cp >= 0xff21 && cp <= 0xff5a) return 1; /* fullwidth A-Z */
if (cp >= 0xf900 && cp <= 0xfaff) return 1; /* CJK Compatibility Ideographs */
if (cp >= 0xff21 && cp <= 0xff3a) return 1; /* fullwidth A-Z */
if (cp >= 0xff41 && cp <= 0xff5a) return 1; /* fullwidth a-z */
if (cp >= 0x20000 && cp <= 0x2fa1f) return 1; /* CJK Ext B-F + Compat */
return 0;
}
@@ -122,7 +153,7 @@ static void decode_cp(const unsigned char *t, int len, int pos, int *cp, int *cp
else if ((b >> 3) == 0x1e) { c = b & 0x07; l = 4; }
else { c = b; l = 1; } /* invalid lead byte: treat as single byte */
for (int j = 1; j < l; j++) {
if (pos + j >= len) { c = b; l = 1; break; }
if (pos + j >= len || (t[pos + j] & 0xc0) != 0x80) { c = b; l = 1; break; }
c = (c << 6) | (t[pos + j] & 0x3f);
}
*cp = c; *cplen = l;
@@ -183,7 +214,7 @@ static int tok_piece_end(const unsigned char *t, int len, int i) {
}
/* 4) ?[^\s\p{L}\p{N}]++[\r\n]*+ (leading space optional) */
int is_other = (!u_is_space(cp) && !u_is_letter(cp) && !u_is_number(cp));
int is_other = (!isS && !isL && !isN);
if (cp == 0x20 && ni < len && (!u_is_space(ncp) && !u_is_letter(ncp) && !u_is_number(ncp))) {
int e = ni;
while (e < len) {
@@ -213,80 +244,143 @@ static int tok_piece_end(const unsigned char *t, int len, int i) {
return e;
}
/* 5-8) whitespace runs */
if (isS) {
int e = i;
int last_ws_len = cplen;
int has_nl = (cp == 0x0d || cp == 0x0a);
while (e < len) {
int c, cl; decode_cp(t, len, e, &c, &cl);
if (!u_is_space(c)) break;
last_ws_len = cl;
if (c == 0x0d || c == 0x0a) has_nl = 1;
e += cl;
}
if (e == len) return e; /* 5) trailing whitespace */
if (has_nl) return e; /* 6) whitespace containing a newline */
if ((e - i) > last_ws_len) return e - last_ws_len; /* 7) leave last whitespace */
return e; /* 8) single whitespace */
/* 5-8) whitespace runs: \s++$ | \s*[\r\n] | \s+(?!\S) | \s */
int e = i;
int last_ws_len = cplen;
int last_nl_end = 0;
while (e < len) {
int c, cl; decode_cp(t, len, e, &c, &cl);
if (!u_is_space(c)) break;
last_ws_len = cl;
if (c == 0x0d || c == 0x0a) last_nl_end = e + cl;
e += cl;
}
return i + cplen; /* fallback */
if (e == len) return e; /* 5) \s++$ (trailing whitespace) */
if (last_nl_end > i) return last_nl_end; /* 6) \s*[\r\n] (up to last CR/LF) */
if ((e - i) > last_ws_len) return e - last_ws_len; /* 7) \s+(?!\S) (leave last ws) */
return e; /* 8) \s (single whitespace) */
}
/* ----------------------------- BPE ------------------------------------ */
/* Count tokens in a single raw-byte piece via greedy lowest-rank merges. */
#define MAX_TOKEN_BYTES 128
typedef struct {
int prev;
int next;
int rank;
} bpe_node_t;
typedef struct {
int rank;
int pos;
} bpe_heap_item_t;
static int heap_less(bpe_heap_item_t a, bpe_heap_item_t b) {
return (a.rank < b.rank) || (a.rank == b.rank && a.pos < b.pos);
}
static void heap_push(bpe_heap_item_t *h, int *hn, bpe_heap_item_t v) {
int i = (*hn)++;
while (i > 0) {
int p = (i - 1) >> 1;
if (!heap_less(v, h[p])) break;
h[i] = h[p];
i = p;
}
h[i] = v;
}
static bpe_heap_item_t heap_pop(bpe_heap_item_t *h, int *hn) {
bpe_heap_item_t top = h[0];
bpe_heap_item_t last = h[--(*hn)];
int n = *hn, i = 0;
while ((i << 1) + 1 < n) {
int l = (i << 1) + 1, r = l + 1;
int c = (r < n && heap_less(h[r], h[l])) ? r : l;
if (!heap_less(h[c], last)) break;
h[i] = h[c];
i = c;
}
if (n > 0) h[i] = last;
return top;
}
/* Count tokens in a single raw-byte piece via O(N log N) lowest-rank merges. */
static int tok_bpe(const unsigned char *piece, int plen) {
if (plen <= 0) return 0;
if (plen > 4096) return plen / 4; /* pathological guard; estimate */
char **sym = (char **)malloc(sizeof(char *) * (size_t)plen);
if (!sym) return plen / 4;
int nsym = 0;
if (plen == 1) return 1;
bpe_node_t *nodes = (bpe_node_t *)malloc(sizeof(bpe_node_t) * (size_t)plen);
bpe_heap_item_t *heap = (bpe_heap_item_t *)malloc(sizeof(bpe_heap_item_t) * (size_t)plen * 2u);
if (!nodes || !heap) {
free(nodes);
free(heap);
return (plen + 3) / 4;
}
int hn = 0;
for (int k = 0; k < plen; k++) {
char *s = (char *)malloc(2);
if (!s) { for (int j = 0; j < nsym; j++) free(sym[j]); free(sym); return plen / 4; }
s[0] = (char)piece[k]; s[1] = 0;
sym[nsym++] = s;
}
char buf[256];
while (nsym > 1) {
int best = -1, bestr = 0x7fffffff;
for (int k = 0; k < nsym - 1; k++) {
int l1 = (int)strlen(sym[k]), l2 = (int)strlen(sym[k + 1]);
if (l1 + l2 >= (int)sizeof(buf)) continue;
memcpy(buf, sym[k], l1);
memcpy(buf + l1, sym[k + 1], l2);
buf[l1 + l2] = 0;
int r = tok_lookup((const unsigned char *)buf, l1 + l2);
if (r >= 0 && r < bestr) { bestr = r; best = k; }
nodes[k].prev = k - 1;
nodes[k].next = k + 1;
if (k + 1 < plen) {
int r = tok_lookup(piece + k, 2);
nodes[k].rank = r;
if (r >= 0) {
bpe_heap_item_t item = { r, k };
heap_push(heap, &hn, item);
}
} else {
nodes[k].rank = -1;
}
if (best < 0) break;
int l1 = (int)strlen(sym[best]), l2 = (int)strlen(sym[best + 1]);
char *m = (char *)malloc((size_t)(l1 + l2 + 1));
memcpy(m, sym[best], l1);
memcpy(m + l1, sym[best + 1], l2);
m[l1 + l2] = 0;
free(sym[best]);
free(sym[best + 1]);
sym[best] = m;
for (int k = best + 1; k < nsym - 1; k++) sym[k] = sym[k + 1];
nsym--;
}
int n = nsym;
for (int k = 0; k < nsym; k++) free(sym[k]);
free(sym);
return n;
int nsym = plen;
while (nsym > 1 && hn > 0) {
bpe_heap_item_t top = heap_pop(heap, &hn);
int best = top.pos;
int right = nodes[best].next;
if (right >= plen || nodes[best].rank != top.rank) continue;
int nr = nodes[right].next;
nodes[best].next = nr;
if (nr < plen) nodes[nr].prev = best;
nodes[right].next = plen + 1;
nodes[right].rank = -1;
nsym--;
if (nr < plen) {
int clen = nodes[nr].next - best;
int r = (clen <= MAX_TOKEN_BYTES) ? tok_lookup(piece + best, clen) : -1;
nodes[best].rank = r;
if (r >= 0) {
bpe_heap_item_t item = { r, best };
heap_push(heap, &hn, item);
}
} else {
nodes[best].rank = -1;
}
int left = nodes[best].prev;
if (left >= 0) {
int clen = nr - left;
int r = (clen <= MAX_TOKEN_BYTES) ? tok_lookup(piece + left, clen) : -1;
nodes[left].rank = r;
if (r >= 0) {
bpe_heap_item_t item = { r, left };
heap_push(heap, &hn, item);
}
}
}
free(nodes);
free(heap);
return nsym;
}
/* ----------------------------- public --------------------------------- */
int tok_count(const char *text, int len) {
if (!text || len <= 0) return 0;
if (!tok_inited) tok_init();
pthread_once(&tok_once, tok_init);
const unsigned char *t = (const unsigned char *)text;
int total = 0, i = 0;
while (i < len) {
int end = tok_piece_end(t, len, i);
if (end <= i) end = i + 1;
total += tok_bpe(t + i, end - i);
i = end;
}
@@ -294,5 +388,6 @@ int tok_count(const char *text, int len) {
}
int tok_count_str(const char *text) {
if (!text) return 0;
return tok_count(text, (int)strlen(text));
}