luxferre/hygate-c
1
0
Fork 0
Code Actions Activity

tok/mkf upd

Browse Source
This commit is contained in:
Luxferre
2026-10-15 23:13:11 +03:00
parent 561d90629f
commit 18ebe25d34
4 changed files with 92 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tokenizer.c
+6 -6
View File
@@ -30,11 +30,11 @@ static int *tok_ranks = NULL;
static uint16_t *tok_klen = NULL;
static unsigned int tok_mask = 0;
static unsigned long tok_fnv1a(const unsigned char *s, int len) {
unsigned long h = 1469598103934665603UL;
static uint64_t tok_fnv1a(const unsigned char *s, int len) {
uint64_t h = 1469598103934665603ULL;
for (int i = 0; i < len; i++) {
h ^= (unsigned long)s[i];
h *= 1099511628211UL;
h ^= (uint64_t)s[i];
h *= 1099511628211ULL;
}
return h;
}
@@ -56,7 +56,7 @@ static void tok_init(void) {
for (int i = 0; i < tok_ntok; i++) {
uint16_t kl = tok_len[i];
const char *key = (const char *)(tok_blob + offset);
unsigned long h = tok_fnv1a((const unsigned char *)key, kl) & mask;
unsigned int h = (unsigned int)tok_fnv1a((const unsigned char *)key, kl) & mask;
while (keys[h]) {
h = (h + 1) & mask;
}
@@ -73,7 +73,7 @@ static void tok_init(void) {
static int tok_lookup(const unsigned char *s, int len) {
if (!tok_keys) return -1;
unsigned long h = tok_fnv1a(s, len) & tok_mask;
unsigned int h = (unsigned int)tok_fnv1a(s, len) & tok_mask;
while (tok_keys[h]) {
if ((int)tok_klen[h] == len && memcmp(tok_keys[h], s, (size_t)len) == 0)
return tok_ranks[h];