diff --git a/README.md b/README.md
index 69e4798..b2bb308 100644
--- a/README.md
+++ b/README.md
@@ -69,16 +69,18 @@ CORS is enabled for all origins (`Access-Control-Allow-Origin: *`).
 ## Token usage counting
 
 Responses include an OpenAI-style `usage` object with `prompt_tokens`,
-`completion_tokens`, and `total_tokens`. These are computed locally by an
-**embedded cl100k_base BPE tokenizer** (vocabulary data in
-`tokenizer_data.c`, logic in `tokenizer.c`) — no upstream token counts are
-available from the Gradio backend.
+`completion_tokens`, `total_tokens`, and `completion_tokens_details`
+(`reasoning_tokens`). These are computed locally by an **embedded cl100k_base
+BPE tokenizer** (vocabulary data in `tokenizer_data.c`, logic in `tokenizer.c`)
+— no upstream token counts are available from the Gradio backend.
 
-- `prompt_tokens` sums each message's `content` (or multi-part content
-  parts), `reasoning_content`, and any `tool_calls` function `name` /
-  `arguments`.
+- `prompt_tokens` sums the active `system` prompt, each message's `content`
+  (or individual multi-part content parts), `reasoning_content`, `tool_calls`
+  / `function_call` (`name` and `arguments`), `tool` result wrappers, and any
+  request `tools` function definitions (`name`, `description`, `parameters`).
 - `completion_tokens` sums the assistant `content`, `reasoning_content`, and
-  any emitted `tool_calls`.
+  any emitted `tool_calls` (`name` and `arguments`), with `reasoning_tokens`
+  broken out under `completion_tokens_details`.
 - `total_tokens = prompt_tokens + completion_tokens`.
 
 cl100k_base is OpenAI's canonical multilingual tokenizer and is used as the
diff --git a/hygate.h b/hygate.h
index f93e6c2..1ec96c8 100644
--- a/hygate.h
+++ b/hygate.h
@@ -62,7 +62,7 @@ int parse_gradio_stream(const char *data_json,
 /* Build the Gradio call payload {"data":[...]} from an OpenAI chat
  * completion request object. Caller frees the result. Returns NULL on
  * malformed input. */
-char *build_gradio_payload(void *req_obj, const char *model_default);
+char *build_gradio_payload(void *req_obj);
 
 /* Build a non-streaming OpenAI completion response JSON string. Caller
  * frees the result. */
diff --git a/jsonapi.c b/jsonapi.c
index efc7655..2943975 100644
--- a/jsonapi.c
+++ b/jsonapi.c
@@ -4,49 +4,164 @@
 #include <string.h>
 #include <json-c/json.h>
 
-/* forward declaration (defined later in this file) */
+/* forward declarations (defined later in this file) */
 static char *content_to_string(json_object *content);
+static const char *get_str(json_object *obj, const char *key);
+
+static int count_content_tokens(json_object *content) {
+  if (!content) return 0;
+  enum json_type t = json_object_get_type(content);
+  if (t == json_type_null) return 0;
+  if (t == json_type_string)
+    return tok_count(json_object_get_string(content),
+                     json_object_get_string_len(content));
+  if (t == json_type_array) {
+    int total = 0;
+    int n = json_object_array_length(content);
+    for (int i = 0; i < n; i++) {
+      json_object *part = json_object_array_get_idx(content, i);
+      if (!part) continue;
+      enum json_type pt = json_object_get_type(part);
+      if (pt == json_type_string) {
+        total += tok_count(json_object_get_string(part),
+                           json_object_get_string_len(part));
+      } else if (pt == json_type_object) {
+        json_object *txt = NULL;
+        if (json_object_object_get_ex(part, "text", &txt) &&
+            json_object_get_type(txt) == json_type_string)
+          total += tok_count(json_object_get_string(txt),
+                             json_object_get_string_len(txt));
+      }
+    }
+    return total;
+  }
+  const char *s = json_object_to_json_string(content);
+  return s ? tok_count_str(s) : 0;
+}
 
 /* ------------------------------------------------------------------ */
 /* Token counting (embedded cl100k_base BPE)                            */
 /* ------------------------------------------------------------------ */
 int count_messages_tokens(void *req_obj) {
   json_object *req = (json_object *)req_obj;
+  if (!req) return 0;
+  int total = 0;
+
+  json_object *tools = NULL;
+  if (json_object_object_get_ex(req, "tools", &tools) &&
+      json_object_get_type(tools) == json_type_array &&
+      json_object_array_length(tools) > 0) {
+    const char *ts = json_object_to_json_string(tools);
+    if (ts) total += tok_count_str(ts);
+  }
+
   json_object *messages = NULL;
   if (!json_object_object_get_ex(req, "messages", &messages) ||
       json_object_get_type(messages) != json_type_array)
-    return 0;
-  int total = 0;
+    return total;
+
   int n = json_object_array_length(messages);
+  int sys_idx = -1;
+  int last_nonsys_idx = -1;
+  bool sys_has_content = false;
   for (int i = 0; i < n; i++) {
     json_object *m = json_object_array_get_idx(messages, i);
     if (!m) continue;
+    const char *roleStr = get_str(m, "role");
+    if (strcmp(roleStr, "system") == 0 && !sys_has_content) {
+      sys_idx = i;
+      if (count_content_tokens(json_object_object_get(m, "content")) > 0)
+        sys_has_content = true;
+    } else {
+      last_nonsys_idx = i;
+    }
+  }
+
+  for (int i = 0; i < n; i++) {
+    json_object *m = json_object_array_get_idx(messages, i);
+    if (!m) continue;
+    const char *roleStr = get_str(m, "role");
     json_object *content = json_object_object_get(m, "content");
-    char *cs = content_to_string(content);
-    total += tok_count_str(cs);
-    free(cs);
-    json_object *rc = NULL;
-    if (json_object_object_get_ex(m, "reasoning_content", &rc) &&
-        json_object_get_type(rc) == json_type_string)
-      total += tok_count_str(json_object_get_string(rc));
-    json_object *tcs = NULL;
-    if (json_object_object_get_ex(m, "tool_calls", &tcs) &&
-        json_object_get_type(tcs) == json_type_array) {
-      int tn = json_object_array_length(tcs);
-      for (int j = 0; j < tn; j++) {
-        json_object *tc = json_object_array_get_idx(tcs, j);
-        if (!tc) continue;
-        json_object *fn = NULL;
-        if (json_object_object_get_ex(tc, "function", &fn) &&
-            json_object_get_type(fn) == json_type_object) {
-          json_object *nm = NULL, *ar = NULL;
-          if (json_object_object_get_ex(fn, "name", &nm) &&
-              json_object_get_type(nm) == json_type_string)
-            total += tok_count_str(json_object_get_string(nm));
-          if (json_object_object_get_ex(fn, "arguments", &ar) &&
-              json_object_get_type(ar) == json_type_string)
-            total += tok_count_str(json_object_get_string(ar));
+
+    if (i == sys_idx) {
+      total += count_content_tokens(content);
+      continue;
+    }
+
+    if (i == last_nonsys_idx &&
+        (strcmp(roleStr, "tool") == 0 || strcmp(roleStr, "function") == 0)) {
+      const char *toolName = get_str(m, "name");
+      if (toolName[0] == 0) toolName = get_str(m, "tool_call_id");
+      if (toolName[0] != 0) {
+        char *lastContent = content_to_string(content);
+        const char *lc = lastContent ? lastContent : "";
+        size_t need = strlen("Tool result for : ") + strlen(toolName) +
+                      strlen(lc) + 1;
+        char *s = (char *)malloc(need);
+        if (s) {
+          snprintf(s, need, "Tool result for %s: %s", toolName, lc);
+          total += tok_count_str(s);
+          free(s);
+        } else {
+          total += tok_count_str(toolName) + count_content_tokens(content);
         }
+        free(lastContent);
+      } else {
+        total += count_content_tokens(content);
+      }
+      continue;
+    }
+
+    total += count_content_tokens(content);
+
+    if (strcmp(roleStr, "tool") == 0 || strcmp(roleStr, "function") == 0) {
+      const char *toolID = get_str(m, "tool_call_id");
+      const char *nm = get_str(m, "name");
+      if (toolID[0] != 0) total += tok_count_str(toolID);
+      if (nm[0] != 0) total += tok_count_str(nm);
+      continue;
+    }
+
+    if (strcmp(roleStr, "assistant") == 0) {
+      json_object *rc = NULL;
+      if (json_object_object_get_ex(m, "reasoning_content", &rc) &&
+          json_object_get_type(rc) == json_type_string)
+        total += tok_count(json_object_get_string(rc),
+                           json_object_get_string_len(rc));
+      json_object *tcs = NULL;
+      if (json_object_object_get_ex(m, "tool_calls", &tcs) &&
+          json_object_get_type(tcs) == json_type_array) {
+        int tn = json_object_array_length(tcs);
+        for (int j = 0; j < tn; j++) {
+          json_object *tc = json_object_array_get_idx(tcs, j);
+          if (!tc) continue;
+          json_object *fn = NULL;
+          if (json_object_object_get_ex(tc, "function", &fn) &&
+              json_object_get_type(fn) == json_type_object) {
+            json_object *nm = NULL, *ar = NULL;
+            if (json_object_object_get_ex(fn, "name", &nm) &&
+                json_object_get_type(nm) == json_type_string)
+              total += tok_count(json_object_get_string(nm),
+                                 json_object_get_string_len(nm));
+            if (json_object_object_get_ex(fn, "arguments", &ar) &&
+                json_object_get_type(ar) == json_type_string)
+              total += tok_count(json_object_get_string(ar),
+                                 json_object_get_string_len(ar));
+          }
+        }
+      }
+      json_object *fc = NULL;
+      if (json_object_object_get_ex(m, "function_call", &fc) &&
+          json_object_get_type(fc) == json_type_object) {
+        json_object *nm = NULL, *ar = NULL;
+        if (json_object_object_get_ex(fc, "name", &nm) &&
+            json_object_get_type(nm) == json_type_string)
+          total += tok_count(json_object_get_string(nm),
+                             json_object_get_string_len(nm));
+        if (json_object_object_get_ex(fc, "arguments", &ar) &&
+            json_object_get_type(ar) == json_type_string)
+          total += tok_count(json_object_get_string(ar),
+                             json_object_get_string_len(ar));
       }
     }
   }
@@ -69,11 +184,13 @@ int count_completion_tokens(const char *content, const char *reasoning,
 static char *content_to_string(json_object *content) {
   if (!content) return strdup("");
   enum json_type t = json_object_get_type(content);
+  if (t == json_type_null) return strdup("");
   if (t == json_type_string)
     return strdup(json_object_get_string(content));
   if (t == json_type_array) {
     strbuf sb;
     sb_init(&sb);
+    if (!sb.buf) return strdup("");
     int n = json_object_array_length(content);
     for (int i = 0; i < n; i++) {
       json_object *part = json_object_array_get_idx(content, i);
@@ -88,9 +205,7 @@ static char *content_to_string(json_object *content) {
           sb_append_str(&sb, json_object_get_string(txt));
       }
     }
-    char *r = strdup(sb.buf);
-    sb_free(&sb);
-    return r;
+    return sb.buf;
   }
   const char *s = json_object_to_json_string(content);
   return s ? strdup(s) : strdup("");
@@ -124,8 +239,7 @@ static double get_double(json_object *req, const char *key) {
 /* ------------------------------------------------------------------ */
 /* Build Gradio payload                                                */
 /* ------------------------------------------------------------------ */
-char *build_gradio_payload(void *req_obj, const char *model_default) {
-  (void)model_default;
+char *build_gradio_payload(void *req_obj) {
   json_object *req = (json_object *)req_obj;
   json_object *messages = NULL;
   if (!json_object_object_get_ex(req, "messages", &messages) ||
@@ -133,19 +247,25 @@ char *build_gradio_payload(void *req_obj, const char *model_default) {
     return NULL;
   int n = json_object_array_length(messages);
 
-  char *system_str = strdup("");
-  char *message_str = strdup("");
+  int *nonSys = (int *)malloc(sizeof(int) * (size_t)(n > 0 ? n : 1));
+  if (!nonSys) return NULL;
   json_object *history_arr = json_object_new_array();
+  if (!history_arr) {
+    free(nonSys);
+    return NULL;
+  }
+
+  char *system_str = NULL;
+  char *message_str = NULL;
 
   /* First pass: pick first system message as system prompt, collect the
    * rest as non-system messages. */
-  int *nonSys = (int *)malloc(sizeof(int) * (size_t)(n > 0 ? n : 1));
   int nonSysN = 0;
   for (int i = 0; i < n; i++) {
     json_object *m = json_object_array_get_idx(messages, i);
     if (!m) continue;
     const char *roleStr = get_str(m, "role");
-    if (strcmp(roleStr, "system") == 0 && system_str[0] == 0) {
+    if (strcmp(roleStr, "system") == 0 && (!system_str || system_str[0] == 0)) {
       char *cs = content_to_string(json_object_object_get(m, "content"));
       free(system_str);
       system_str = cs;
@@ -160,17 +280,19 @@ char *build_gradio_payload(void *req_obj, const char *model_default) {
     json_object *m = json_object_array_get_idx(messages, i);
     const char *roleStr = get_str(m, "role");
     json_object *item = json_object_new_object();
+    if (!item) continue;
     json_object_object_add(item, "role", json_object_new_string(roleStr));
 
     json_object *content = json_object_object_get(m, "content");
     char *cs = content_to_string(content);
+    const char *cstr = cs ? cs : "";
     if (strcmp(roleStr, "assistant") == 0) {
-      if (cs[0] != 0)
-        json_object_object_add(item, "content", json_object_new_string(cs));
+      if (cstr[0] != 0)
+        json_object_object_add(item, "content", json_object_new_string(cstr));
       else
         json_object_object_add(item, "content", json_object_new_null());
     } else {
-      json_object_object_add(item, "content", json_object_new_string(cs));
+      json_object_object_add(item, "content", json_object_new_string(cstr));
     }
     free(cs);
 
@@ -214,25 +336,25 @@ char *build_gradio_payload(void *req_obj, const char *model_default) {
     json_object *m = json_object_array_get_idx(messages, i);
     const char *roleStr = get_str(m, "role");
     char *lastContent = content_to_string(json_object_object_get(m, "content"));
+    const char *lc = lastContent ? lastContent : "";
     if (strcmp(roleStr, "tool") == 0 || strcmp(roleStr, "function") == 0) {
       const char *toolName = get_str(m, "name");
       if (toolName[0] == 0) toolName = get_str(m, "tool_call_id");
       if (toolName[0] != 0) {
         size_t need = strlen("Tool result for : ") + strlen(toolName) +
-                      strlen(lastContent) + 1;
+                      strlen(lc) + 1;
         char *s = (char *)malloc(need);
-        snprintf(s, need, "Tool result for %s: %s", toolName, lastContent);
-        free(message_str);
-        message_str = s;
+        if (s) {
+          snprintf(s, need, "Tool result for %s: %s", toolName, lc);
+          message_str = s;
+        }
+        free(lastContent);
       } else {
-        free(message_str);
-        message_str = strdup(lastContent);
+        message_str = lastContent;
       }
     } else {
-      free(message_str);
-      message_str = strdup(lastContent);
+      message_str = lastContent;
     }
-    free(lastContent);
   }
 
   free(nonSys);
@@ -250,20 +372,27 @@ char *build_gradio_payload(void *req_obj, const char *model_default) {
   bool topp_present = has_nonnull(req, "top_p");
   double topp = topp_present ? get_double(req, "top_p") : 0.0;
 
-  char *functions_json = NULL;
+  const char *functions_json = "";
   json_object *tools = NULL;
   if (json_object_object_get_ex(req, "tools", &tools) &&
       json_object_get_type(tools) == json_type_array) {
     const char *s = json_object_to_json_string(tools);
-    if (s) functions_json = strdup(s);
+    if (s) functions_json = s;
   }
-  if (!functions_json) functions_json = strdup("");
 
   /* Assemble the Gradio data array. */
   json_object *root = json_object_new_object();
   json_object *data = json_object_new_array();
-  json_object_array_add(data, json_object_new_string(message_str));
-  json_object_array_add(data, json_object_new_string(system_str));
+  if (!root || !data) {
+    if (root) json_object_put(root);
+    if (data) json_object_put(data);
+    json_object_put(history_arr);
+    free(system_str);
+    free(message_str);
+    return NULL;
+  }
+  json_object_array_add(data, json_object_new_string(message_str ? message_str : ""));
+  json_object_array_add(data, json_object_new_string(system_str ? system_str : ""));
   json_object_array_add(data, history_arr);
   json_object_array_add(data, json_object_new_string("high"));
   if (temp_present)
@@ -284,7 +413,6 @@ char *build_gradio_payload(void *req_obj, const char *model_default) {
   json_object_put(root);
   free(system_str);
   free(message_str);
-  free(functions_json);
   return payload;
 }
 
@@ -336,6 +464,14 @@ int parse_gradio_stream(const char *data_json,
   *out_reasoning = (r && json_object_get_type(r) == json_type_string)
                      ? strdup(json_object_get_string(r))
                      : strdup("");
+  if (!*out_content || !*out_reasoning) {
+    free(*out_content);
+    free(*out_reasoning);
+    *out_content = NULL;
+    *out_reasoning = NULL;
+    json_object_put(root);
+    return 0;
+  }
 
   if (m >= 3) {
     json_object *tcobj = json_object_array_get_idx(inner, 2);
@@ -343,38 +479,47 @@ int parse_gradio_stream(const char *data_json,
       int len = json_object_array_length(tcobj);
       if (len > 0) {
         tool_call_t *arr = (tool_call_t *)calloc((size_t)len, sizeof(tool_call_t));
-        int cnt = 0;
-        for (int i = 0; i < len; i++) {
-          json_object *e = json_object_array_get_idx(tcobj, i);
-          if (json_object_get_type(e) != json_type_object) continue;
-          json_object *fid = NULL, *ffn = NULL, *fname = NULL, *fargs = NULL;
-          json_object_object_get_ex(e, "id", &fid);
-          json_object_object_get_ex(e, "function", &ffn);
-          if (ffn && json_object_get_type(ffn) == json_type_object) {
-            json_object_object_get_ex(ffn, "name", &fname);
-            json_object_object_get_ex(ffn, "arguments", &fargs);
+        if (arr) {
+          int cnt = 0;
+          for (int i = 0; i < len; i++) {
+            json_object *e = json_object_array_get_idx(tcobj, i);
+            if (json_object_get_type(e) != json_type_object) continue;
+            json_object *fid = NULL, *ftype = NULL, *ffn = NULL, *fname = NULL, *fargs = NULL;
+            json_object_object_get_ex(e, "id", &fid);
+            json_object_object_get_ex(e, "type", &ftype);
+            json_object_object_get_ex(e, "function", &ffn);
+            if (ffn && json_object_get_type(ffn) == json_type_object) {
+              json_object_object_get_ex(ffn, "name", &fname);
+              json_object_object_get_ex(ffn, "arguments", &fargs);
+            }
+            arr[cnt].id = (fid && json_object_get_type(fid) == json_type_string)
+                              ? strdup(json_object_get_string(fid))
+                              : strdup("");
+            arr[cnt].type = (ftype && json_object_get_type(ftype) == json_type_string)
+                               ? strdup(json_object_get_string(ftype))
+                               : strdup("");
+            arr[cnt].name = (fname && json_object_get_type(fname) == json_type_string)
+                               ? strdup(json_object_get_string(fname))
+                               : strdup("");
+            arr[cnt].arguments =
+                (fargs && json_object_get_type(fargs) == json_type_string)
+                    ? strdup(json_object_get_string(fargs))
+                    : strdup("");
+            if (!arr[cnt].id || !arr[cnt].type || !arr[cnt].name || !arr[cnt].arguments) {
+              free(arr[cnt].id);
+              free(arr[cnt].type);
+              free(arr[cnt].name);
+              free(arr[cnt].arguments);
+              continue;
+            }
+            cnt++;
+          }
+          if (cnt > 0) {
+            *out_tc = arr;
+            *out_n_tc = cnt;
+          } else {
+            free(arr);
           }
-          arr[cnt].id = (fid && json_object_get_type(fid) == json_type_string)
-                            ? strdup(json_object_get_string(fid))
-                            : strdup("");
-          arr[cnt].type = (json_object_object_get_ex(e, "type", &fid) &&
-                           json_object_get_type(fid) == json_type_string)
-                             ? strdup(json_object_get_string(fid))
-                             : strdup("");
-          arr[cnt].name = (fname && json_object_get_type(fname) == json_type_string)
-                             ? strdup(json_object_get_string(fname))
-                             : strdup("");
-          arr[cnt].arguments =
-              (fargs && json_object_get_type(fargs) == json_type_string)
-                  ? strdup(json_object_get_string(fargs))
-                  : strdup("");
-          cnt++;
-        }
-        if (cnt > 0) {
-          *out_tc = arr;
-          *out_n_tc = cnt;
-        } else {
-          free(arr);
         }
       }
     }
@@ -436,11 +581,15 @@ char *build_completion_response(const char *id, long created, const char *model,
   json_object_object_add(root, "choices", choices);
 
   int completion_tokens = count_completion_tokens(content, reasoning, tc, n_tc);
+  int reasoning_tokens = (reasoning && reasoning[0]) ? tok_count_str(reasoning) : 0;
   int total_tokens = prompt_tokens + completion_tokens;
   json_object *usage = json_object_new_object();
   json_object_object_add(usage, "prompt_tokens", json_object_new_int(prompt_tokens));
   json_object_object_add(usage, "completion_tokens", json_object_new_int(completion_tokens));
   json_object_object_add(usage, "total_tokens", json_object_new_int(total_tokens));
+  json_object *details = json_object_new_object();
+  json_object_object_add(details, "reasoning_tokens", json_object_new_int(reasoning_tokens));
+  json_object_object_add(usage, "completion_tokens_details", details);
   json_object_object_add(root, "usage", usage);
 
   const char *s = json_object_to_json_string(root);
diff --git a/main.c b/main.c
index 410990f..57abe96 100644
--- a/main.c
+++ b/main.c
@@ -46,12 +46,12 @@ int main(int argc, char **argv) {
         val = argv[i + 1];
         i++;
       }
-      if (strcmp(a, "-port") == 0) cfg.port = val;
-      else if (strcmp(a, "-endpoint") == 0) cfg.endpoint = val;
-      else if (strcmp(a, "-model") == 0) cfg.model = val;
-      else if (strcmp(a, "-user-agent") == 0) cfg.user_agent = val;
-      else if (strcmp(a, "-ua") == 0) cfg.user_agent = val;
-      else if (strcmp(a, "-max-body") == 0) {
+      if (strcmp(a, "-port") == 0) { if (val) cfg.port = val; }
+      else if (strcmp(a, "-endpoint") == 0) { if (val) cfg.endpoint = val; }
+      else if (strcmp(a, "-model") == 0) { if (val) cfg.model = val; }
+      else if (strcmp(a, "-user-agent") == 0 || strcmp(a, "-ua") == 0) {
+        if (val) cfg.user_agent = val;
+      } else if (strcmp(a, "-max-body") == 0) {
         if (val) {
           long v = (long)strtoul(val, NULL, 10);
           if (v > 0) cfg.max_body_bytes = v;
diff --git a/server.c b/server.c
index e93616c..49f912f 100644
--- a/server.c
+++ b/server.c
@@ -4,7 +4,6 @@
 #include <string.h>
 #include <unistd.h>
 #include <time.h>
-#include <errno.h>
 #include <json-c/json.h>
 #include <curl/curl.h>
 
@@ -13,6 +12,7 @@
 /* ------------------------------------------------------------------ */
 struct tcst {
   char *key;
+  char *name;
   char *prev_args;
   bool emitted;
 };
@@ -126,14 +126,17 @@ static void emit_chunk(sse_ctx_t *ctx, const char *role, const char *content,
   if (s) {
     size_t need = strlen(s) + 32;
     char *buf = (char *)malloc(need);
-    int bl = snprintf(buf, need, "data: %s\n\n", s);
-    write_all(ctx->client_fd, buf, (size_t)bl);
-    free(buf);
+    if (buf) {
+      int bl = snprintf(buf, need, "data: %s\n\n", s);
+      write_all(ctx->client_fd, buf, (size_t)bl);
+      free(buf);
+    }
   }
   json_object_put(root);
 }
 
-static void emit_usage_chunk(sse_ctx_t *ctx, int prompt_tokens, int completion_tokens) {
+static void emit_usage_chunk(sse_ctx_t *ctx, int prompt_tokens,
+                             int completion_tokens, int reasoning_tokens) {
   json_object *root = json_object_new_object();
   json_object_object_add(root, "id", json_object_new_string(ctx->completion_id));
   json_object_object_add(root, "object", json_object_new_string("chat.completion.chunk"));
@@ -146,14 +149,19 @@ static void emit_usage_chunk(sse_ctx_t *ctx, int prompt_tokens, int completion_t
   json_object_object_add(usage, "completion_tokens", json_object_new_int(completion_tokens));
   json_object_object_add(usage, "total_tokens",
                          json_object_new_int(prompt_tokens + completion_tokens));
+  json_object *details = json_object_new_object();
+  json_object_object_add(details, "reasoning_tokens", json_object_new_int(reasoning_tokens));
+  json_object_object_add(usage, "completion_tokens_details", details);
   json_object_object_add(root, "usage", usage);
   const char *j = json_object_to_json_string(root);
   if (j) {
     size_t need = strlen(j) + 32;
     char *buf = (char *)malloc(need);
-    int bl = snprintf(buf, need, "data: %s\n\n", j);
-    write_all(ctx->client_fd, buf, (size_t)bl);
-    free(buf);
+    if (buf) {
+      int bl = snprintf(buf, need, "data: %s\n\n", j);
+      write_all(ctx->client_fd, buf, (size_t)bl);
+      free(buf);
+    }
   }
   json_object_put(root);
 }
@@ -175,12 +183,7 @@ static void process_one_line(sse_ctx_t *ctx, char *line) {
   char *c = NULL, *r = NULL;
   tool_call_t *tc = NULL;
   int ntc = 0;
-  if (!parse_gradio_stream(payload, &c, &r, &tc, &ntc)) {
-    free(c);
-    free(r);
-    free_toolcalls(tc, ntc);
-    return;
-  }
+  if (!parse_gradio_stream(payload, &c, &r, &tc, &ntc)) return;
 
   if (ctx->streaming) {
     handle_stream_delta(ctx, c, r, tc, ntc);
@@ -188,32 +191,36 @@ static void process_one_line(sse_ctx_t *ctx, char *line) {
     free(r);
     free_toolcalls(tc, ntc);
   } else {
-    free(ctx->final_content);
-    ctx->final_content = c ? c : strdup("");
-    free(ctx->final_reasoning);
-    ctx->final_reasoning = r ? r : strdup("");
-    free_toolcalls(ctx->final_tc, ctx->final_n_tc);
+    if (!ctx->final_content || c[0] != 0) {
+      free(ctx->final_content);
+      ctx->final_content = c;
+    } else {
+      free(c);
+    }
+    if (!ctx->final_reasoning || r[0] != 0) {
+      free(ctx->final_reasoning);
+      ctx->final_reasoning = r;
+    } else {
+      free(r);
+    }
     if (ntc > 0) {
+      free_toolcalls(ctx->final_tc, ctx->final_n_tc);
       ctx->final_tc = tc;
       ctx->final_n_tc = ntc;
     } else {
-      ctx->final_tc = NULL;
-      ctx->final_n_tc = 0;
-      free(tc);
+      free_toolcalls(tc, ntc);
     }
   }
 }
 
 static void process_complete_lines(sse_ctx_t *ctx) {
   char *buf = ctx->line.buf;
+  if (!buf) return;
   while (1) {
     char *nl = strchr(buf, '\n');
     if (!nl) break;
     *nl = 0;
-    char *line = buf;
-    size_t ll = strlen(line);
-    if (ll > 0 && line[ll - 1] == '\r') line[ll - 1] = 0;
-    process_one_line(ctx, line);
+    process_one_line(ctx, buf);
     size_t consumed = (size_t)(nl - buf) + 1;
     size_t rest = ctx->line.len - consumed;
     memmove(buf, nl + 1, rest + 1);
@@ -222,12 +229,8 @@ static void process_complete_lines(sse_ctx_t *ctx) {
 }
 
 static void flush_remaining(sse_ctx_t *ctx) {
-  if (ctx->line.len == 0) return;
-  char *line = ctx->line.buf;
-  size_t ll = ctx->line.len;
-  if (ll > 0 && line[ll - 1] == '\r') line[ll - 1] = 0;
-  if (ll > 0 && line[ll - 1] == '\n') line[ll - 1] = 0;
-  process_one_line(ctx, line);
+  if (ctx->line.len == 0 || !ctx->line.buf) return;
+  process_one_line(ctx, ctx->line.buf);
   sb_reset(&ctx->line);
 }
 
@@ -253,34 +256,61 @@ static void handle_stream_delta(sse_ctx_t *ctx, char *c, char *r,
       if (strcmp(ctx->tc_state[j].key, key) == 0) { st = &ctx->tc_state[j]; break; }
     if (!st) {
       if (ctx->tc_state_n >= ctx->tc_state_cap) {
-        ctx->tc_state_cap = ctx->tc_state_cap ? ctx->tc_state_cap * 2 : 8;
-        ctx->tc_state = (struct tcst *)realloc(ctx->tc_state,
-                              (size_t)ctx->tc_state_cap * sizeof(*ctx->tc_state));
+        int ncap = ctx->tc_state_cap ? ctx->tc_state_cap * 2 : 8;
+        struct tcst *nt = (struct tcst *)realloc(ctx->tc_state,
+                            (size_t)ncap * sizeof(*ctx->tc_state));
+        if (!nt) continue;
+        ctx->tc_state = nt;
+        ctx->tc_state_cap = ncap;
+      }
+      char *kcopy = strdup(key);
+      char *ncopy = strdup(tc[i].name ? tc[i].name : "");
+      char *acopy = strdup("");
+      if (!kcopy || !ncopy || !acopy) {
+        free(kcopy);
+        free(ncopy);
+        free(acopy);
+        continue;
       }
       st = &ctx->tc_state[ctx->tc_state_n++];
-      st->key = strdup(key);
-      st->prev_args = strdup("");
+      st->key = kcopy;
+      st->name = ncopy;
+      st->prev_args = acopy;
       st->emitted = false;
+    } else if (tc[i].name && tc[i].name[0] && st->name[0] == 0) {
+      char *ncopy = strdup(tc[i].name);
+      if (ncopy) {
+        free(st->name);
+        st->name = ncopy;
+      }
     }
     const char *cur = tc[i].arguments ? tc[i].arguments : "";
     if (!st->emitted) {
+      char *ncur = strdup(cur);
+      if (!ncur) continue;
       st->emitted = true;
       free(st->prev_args);
-      st->prev_args = strdup(cur);
+      st->prev_args = ncur;
       ctx->any_toolcall = true;
       char *js = toolcall_delta_json(&tc[i], i);
-      emit_chunk(ctx, NULL, NULL, NULL, js, NULL);
-      free(js);
+      if (js) {
+        emit_chunk(ctx, NULL, NULL, NULL, js, NULL);
+        free(js);
+      }
     } else {
       size_t prev = strlen(st->prev_args);
       size_t curlen = strlen(cur);
       if (curlen > prev) {
-        const char *delta = cur + prev;
+        char *ncur = strdup(cur);
+        if (!ncur) continue;
+        const char *delta = ncur + prev;
         free(st->prev_args);
-        st->prev_args = strdup(cur);
+        st->prev_args = ncur;
         char *js = toolcall_delta_argdelta(i, delta);
-        emit_chunk(ctx, NULL, NULL, NULL, js, NULL);
-        free(js);
+        if (js) {
+          emit_chunk(ctx, NULL, NULL, NULL, js, NULL);
+          free(js);
+        }
       }
     }
   }
@@ -322,8 +352,9 @@ static size_t write_sse_cb(void *ptr, size_t sz, size_t nmemb, void *data) {
 /* Upstream calls (with Fibonacci backoff retry)                        */
 /* ------------------------------------------------------------------ */
 static int upstream_post(const char *url, const char *body, const char *ua,
-                         strbuf *resp, long *http_code) {
+                         strbuf *resp) {
   for (int attempt = 1; attempt <= MAX_RETRIES; attempt++) {
+    sb_reset(resp);
     CURL *c = curl_easy_init();
     if (!c) return -1;
     struct curl_slist *hdrs = NULL;
@@ -344,10 +375,7 @@ static int upstream_post(const char *url, const char *body, const char *ua,
     curl_easy_getinfo(c, CURLINFO_RESPONSE_CODE, &code);
     curl_slist_free_all(hdrs);
     curl_easy_cleanup(c);
-    if (rc == CURLE_OK && code == 200) {
-      if (http_code) *http_code = code;
-      return 0;
-    }
+    if (rc == CURLE_OK && code == 200) return 0;
     if (attempt < MAX_RETRIES) sleep((unsigned int)fib_delay(attempt));
   }
   return -1;
@@ -355,6 +383,15 @@ static int upstream_post(const char *url, const char *body, const char *ua,
 
 static int upstream_get_stream(const char *url, const char *ua, sse_ctx_t *ctx) {
   for (int attempt = 1; attempt <= MAX_RETRIES; attempt++) {
+    sb_reset(&ctx->line);
+    ctx->status = 0;
+    free(ctx->final_content);
+    ctx->final_content = NULL;
+    free(ctx->final_reasoning);
+    ctx->final_reasoning = NULL;
+    free_toolcalls(ctx->final_tc, ctx->final_n_tc);
+    ctx->final_tc = NULL;
+    ctx->final_n_tc = 0;
     CURL *c = curl_easy_init();
     if (!c) return -1;
     struct curl_slist *hdrs = NULL;
@@ -376,6 +413,7 @@ static int upstream_get_stream(const char *url, const char *ua, sse_ctx_t *ctx)
     curl_slist_free_all(hdrs);
     curl_easy_cleanup(c);
     if (rc == CURLE_OK && code == 200) return 0;
+    if (ctx->headers_sent) break;
     if (attempt < MAX_RETRIES) sleep((unsigned int)fib_delay(attempt));
   }
   return -1;
@@ -421,7 +459,7 @@ static void handle_chat(int fd, http_req_t *req, const config_t *cfg) {
     model = model_buf;
   }
 
-  char *payload = build_gradio_payload(jreq, model);
+  char *payload = build_gradio_payload(jreq);
 
   int prompt_tokens = count_messages_tokens(jreq);
   bool include_usage = false;
@@ -448,8 +486,9 @@ static void handle_chat(int fd, http_req_t *req, const config_t *cfg) {
   size_t eplen = strlen(ep);
   char epclean[2048];
   if (eplen > 0 && ep[eplen - 1] == '/') {
-    memcpy(epclean, ep, eplen - 1);
-    epclean[eplen - 1] = 0;
+    size_t clen = eplen - 1 < sizeof epclean - 1 ? eplen - 1 : sizeof epclean - 1;
+    memcpy(epclean, ep, clen);
+    epclean[clen] = 0;
   } else {
     strncpy(epclean, ep, sizeof epclean - 1);
     epclean[sizeof epclean - 1] = 0;
@@ -460,8 +499,7 @@ static void handle_chat(int fd, http_req_t *req, const config_t *cfg) {
 
   strbuf post_resp;
   sb_init(&post_resp);
-  long code = 0;
-  if (upstream_post(call_url, payload, ua, &post_resp, &code) != 0) {
+  if (!post_resp.buf || upstream_post(call_url, payload, ua, &post_resp) != 0) {
     send_simple(fd, 502, "Bad Gateway",
                 "Upstream error: failed to reach Gradio call endpoint");
     sb_free(&post_resp);
@@ -475,8 +513,10 @@ static void handle_chat(int fd, http_req_t *req, const config_t *cfg) {
   if (jr) {
     json_object *ev = NULL;
     if (json_object_object_get_ex(jr, "event_id", &ev) &&
-        json_object_get_type(ev) == json_type_string)
+        json_object_get_type(ev) == json_type_string) {
       strncpy(event_id, json_object_get_string(ev), sizeof event_id - 1);
+      event_id[sizeof event_id - 1] = 0;
+    }
     json_object_put(jr);
   }
   sb_free(&post_resp);
@@ -501,10 +541,10 @@ static void handle_chat(int fd, http_req_t *req, const config_t *cfg) {
   generate_uuid(uuid);
   snprintf(ctx.completion_id, sizeof ctx.completion_id, "chatcmpl-%s", uuid);
   sb_init(&ctx.line);
-  sb_init(&ctx.emitted_content);
-  sb_init(&ctx.emitted_reasoning);
-  ctx.final_content = strdup("");
-  ctx.final_reasoning = strdup("");
+  if (stream) {
+    sb_init(&ctx.emitted_content);
+    sb_init(&ctx.emitted_reasoning);
+  }
 
   int rc = upstream_get_stream(stream_url, ua, &ctx);
 
@@ -519,11 +559,13 @@ static void handle_chat(int fd, http_req_t *req, const config_t *cfg) {
   if (stream) {
     emit_chunk(&ctx, NULL, NULL, NULL, NULL, ctx.any_toolcall ? "tool_calls" : "stop");
     if (ctx.include_usage) {
-      int completion_tokens = tok_count_str(ctx.emitted_content.buf);
-      completion_tokens += tok_count_str(ctx.emitted_reasoning.buf);
-      for (int i = 0; i < ctx.tc_state_n; i++)
+      int reasoning_tokens = tok_count_str(ctx.emitted_reasoning.buf);
+      int completion_tokens = tok_count_str(ctx.emitted_content.buf) + reasoning_tokens;
+      for (int i = 0; i < ctx.tc_state_n; i++) {
+        completion_tokens += tok_count_str(ctx.tc_state[i].name);
         completion_tokens += tok_count_str(ctx.tc_state[i].prev_args);
-      emit_usage_chunk(&ctx, prompt_tokens, completion_tokens);
+      }
+      emit_usage_chunk(&ctx, prompt_tokens, completion_tokens, reasoning_tokens);
     }
     const char *done = "data: [DONE]\n\n";
     write_all(fd, done, strlen(done));
@@ -545,6 +587,7 @@ cleanup:
   sb_free(&ctx.emitted_reasoning);
   for (int i = 0; i < ctx.tc_state_n; i++) {
     free(ctx.tc_state[i].key);
+    free(ctx.tc_state[i].name);
     free(ctx.tc_state[i].prev_args);
   }
   free(ctx.tc_state);
@@ -558,6 +601,7 @@ cleanup:
 /* Request parsing / connection dispatch                               */
 /* ------------------------------------------------------------------ */
 int read_headers(int fd, strbuf *buf) {
+  if (!buf->buf) return -1;
   char tmp[4096];
   while (1) {
     if (strstr(buf->buf, "\r\n\r\n")) break;
@@ -570,6 +614,7 @@ int read_headers(int fd, strbuf *buf) {
 }
 
 int parse_http_request(strbuf *buf, int fd, http_req_t *req, long max_body_bytes) {
+  if (!buf->buf) return -1;
   char *p = buf->buf;
   char *line_end = strstr(p, "\r\n");
   if (!line_end) return -1;
@@ -617,6 +662,7 @@ int parse_http_request(strbuf *buf, int fd, http_req_t *req, long max_body_bytes
 
   char *body_start = hlimit + 4;
   size_t body_avail = buf->len - (size_t)(body_start - buf->buf);
+  if (req->content_length < 0) return -1;
   if (max_body_bytes > 0 && req->content_length > max_body_bytes) return -2;
   if (req->content_length > 0) {
     req->body = (char *)malloc((size_t)req->content_length + 1);
@@ -634,6 +680,7 @@ int parse_http_request(strbuf *buf, int fd, http_req_t *req, long max_body_bytes
     req->body_len = got;
   } else {
     req->body = strdup("");
+    if (!req->body) return -1;
     req->body_len = 0;
   }
   return 0;
diff --git a/tokenizer.c b/tokenizer.c
index e3a6c4f..e3db6b3 100644
--- a/tokenizer.c
+++ b/tokenizer.c
@@ -19,6 +19,7 @@
 #include <string.h>
 #include <stdint.h>
 #include <stdlib.h>
+#include <pthread.h>
 
 extern const int tok_ntok;
 extern const unsigned char tok_blob[];
@@ -26,10 +27,11 @@ extern const int tok_rank[];
 extern const uint16_t tok_len[];
 
 /* ----------------------------- vocab hash ----------------------------- */
-static int tok_inited = 0;
+static pthread_once_t tok_once = PTHREAD_ONCE_INIT;
 static const char **tok_keys = NULL;
 static int *tok_ranks = NULL;
-static int *tok_klen = NULL;static unsigned int tok_mask = 0;
+static int *tok_klen = NULL;
+static unsigned int tok_mask = 0;
 
 static unsigned long tok_fnv1a(const unsigned char *s, int len) {
   unsigned long h = 1469598103934665603UL;
@@ -41,34 +43,39 @@ static unsigned long tok_fnv1a(const unsigned char *s, int len) {
 }
 
 static void tok_init(void) {
-  if (tok_inited) return;
-  tok_inited = 1;
   unsigned int size = 1;
   while (size < (unsigned int)tok_ntok * 2u) size <<= 1;
-  tok_mask = size - 1;
-  tok_keys = (const char **)calloc(size, sizeof(char *));
-  tok_ranks = (int *)calloc(size, sizeof(int));
-  tok_klen = (int *)calloc(size, sizeof(int));
-  if (!tok_keys || !tok_ranks || !tok_klen) return;
+  const char **keys = (const char **)calloc(size, sizeof(char *));
+  int *ranks = (int *)calloc(size, sizeof(int));
+  int *klen = (int *)calloc(size, sizeof(int));
+  if (!keys || !ranks || !klen) {
+    free(keys);
+    free(ranks);
+    free(klen);
+    return;
+  }
+  unsigned int mask = size - 1;
   int offset = 0;
   for (int i = 0; i < tok_ntok; i++) {
-    int klen = tok_len[i];
+    int kl = tok_len[i];
     const char *key = (const char *)(tok_blob + offset);
-    unsigned long h = tok_fnv1a((const unsigned char *)key, klen) & tok_mask;
-    while (tok_keys[h]) {
-      const char *k = tok_keys[h];
-      if (tok_klen[h] == klen && memcmp(k, key, klen) == 0) break; /* dup (none expected) */
-      h = (h + 1) & tok_mask;
+    unsigned long h = tok_fnv1a((const unsigned char *)key, kl) & mask;
+    while (keys[h]) {
+      h = (h + 1) & mask;
     }
-    tok_keys[h] = key;
-    tok_ranks[h] = tok_rank[i];
-    tok_klen[h] = klen;
-    offset += klen;
+    keys[h] = key;
+    ranks[h] = tok_rank[i];
+    klen[h] = kl;
+    offset += kl;
   }
+  tok_mask = mask;
+  tok_ranks = ranks;
+  tok_klen = klen;
+  tok_keys = keys;
 }
 
 static int tok_lookup(const unsigned char *s, int len) {
-  if (!tok_inited) tok_init();
+  pthread_once(&tok_once, tok_init);
   if (!tok_keys) return -1;
   unsigned long h = tok_fnv1a(s, len) & tok_mask;
   while (tok_keys[h]) {
@@ -100,15 +107,39 @@ static int u_is_number(int cp) {
 }
 static int u_is_letter(int cp) {
   if ((cp >= 'a' && cp <= 'z') || (cp >= 'A' && cp <= 'Z')) return 1;
-  if (cp >= 0x00c0 && cp <= 0x024f) return 1;     /* Latin-1 + Latin Extended-A/B */
-  if (cp >= 0x0370 && cp <= 0x03ff) return 1;     /* Greek */
-  if (cp >= 0x0400 && cp <= 0x04ff) return 1;     /* Cyrillic */
-  if (cp >= 0x3040 && cp <= 0x30ff) return 1;     /* Hiragana + Katakana */
+  if (cp >= 0x00c0 && cp <= 0x02af) {
+    return (cp != 0x00d7 && cp != 0x00f7);        /* Latin-1 + Ext-A/B + IPA (excl. × ÷) */
+  }
+  if (cp >= 0x0370 && cp <= 0x03ff) {
+    return (cp != 0x037e && cp != 0x0387);        /* Greek */
+  }
+  if (cp >= 0x0400 && cp <= 0x052f) return 1;     /* Cyrillic + Supplement */
+  if (cp >= 0x0531 && cp <= 0x0587) return 1;     /* Armenian */
+  if (cp >= 0x05d0 && cp <= 0x05ea) return 1;     /* Hebrew */
+  if ((cp >= 0x0620 && cp <= 0x064a) ||
+      (cp >= 0x066e && cp <= 0x06d3) ||
+      (cp >= 0x06ee && cp <= 0x06ff) ||
+      (cp >= 0x0750 && cp <= 0x077f) ||
+      (cp >= 0x08a0 && cp <= 0x08ff)) return 1;   /* Arabic */
+  if (cp >= 0x0904 && cp <= 0x0d7f) {
+    return !(cp >= 0x0964 && cp <= 0x096f);       /* Devanagari + Indic scripts */
+  }
+  if ((cp >= 0x0e01 && cp <= 0x0e30) ||
+      (cp >= 0x0e32 && cp <= 0x0e33) ||
+      (cp >= 0x0e40 && cp <= 0x0e46)) return 1;   /* Thai */
+  if (cp >= 0x10a0 && cp <= 0x10ff) return 1;     /* Georgian */
+  if (cp >= 0x1100 && cp <= 0x11ff) return 1;     /* Hangul Jamo */
+  if (cp >= 0x1e00 && cp <= 0x1fff) return 1;     /* Latin Ext Additional + Greek Ext */
+  if (cp >= 0x3041 && cp <= 0x3096) return 1;     /* Hiragana */
+  if (cp >= 0x30a1 && cp <= 0x30fc && cp != 0x30fb) return 1; /* Katakana (excl. ・) */
+  if (cp >= 0x3131 && cp <= 0x318e) return 1;     /* Hangul Compatibility Jamo */
   if (cp >= 0x3400 && cp <= 0x4dbf) return 1;     /* CJK Ext A */
   if (cp >= 0x4e00 && cp <= 0x9fff) return 1;     /* CJK Unified Ideographs */
   if (cp >= 0xac00 && cp <= 0xd7a3) return 1;     /* Hangul Syllables */
-  if (cp >= 0xff21 && cp <= 0xff5a) return 1;     /* fullwidth A-Z */
+  if (cp >= 0xf900 && cp <= 0xfaff) return 1;     /* CJK Compatibility Ideographs */
+  if (cp >= 0xff21 && cp <= 0xff3a) return 1;     /* fullwidth A-Z */
   if (cp >= 0xff41 && cp <= 0xff5a) return 1;     /* fullwidth a-z */
+  if (cp >= 0x20000 && cp <= 0x2fa1f) return 1;   /* CJK Ext B-F + Compat */
   return 0;
 }
 
@@ -122,7 +153,7 @@ static void decode_cp(const unsigned char *t, int len, int pos, int *cp, int *cp
   else if ((b >> 3) == 0x1e) { c = b & 0x07; l = 4; }
   else { c = b; l = 1; } /* invalid lead byte: treat as single byte */
   for (int j = 1; j < l; j++) {
-    if (pos + j >= len) { c = b; l = 1; break; }
+    if (pos + j >= len || (t[pos + j] & 0xc0) != 0x80) { c = b; l = 1; break; }
     c = (c << 6) | (t[pos + j] & 0x3f);
   }
   *cp = c; *cplen = l;
@@ -183,7 +214,7 @@ static int tok_piece_end(const unsigned char *t, int len, int i) {
   }
 
   /* 4)  ?[^\s\p{L}\p{N}]++[\r\n]*+  (leading space optional) */
-  int is_other = (!u_is_space(cp) && !u_is_letter(cp) && !u_is_number(cp));
+  int is_other = (!isS && !isL && !isN);
   if (cp == 0x20 && ni < len && (!u_is_space(ncp) && !u_is_letter(ncp) && !u_is_number(ncp))) {
     int e = ni;
     while (e < len) {
@@ -213,80 +244,143 @@ static int tok_piece_end(const unsigned char *t, int len, int i) {
     return e;
   }
 
-  /* 5-8) whitespace runs */
-  if (isS) {
-    int e = i;
-    int last_ws_len = cplen;
-    int has_nl = (cp == 0x0d || cp == 0x0a);
-    while (e < len) {
-      int c, cl; decode_cp(t, len, e, &c, &cl);
-      if (!u_is_space(c)) break;
-      last_ws_len = cl;
-      if (c == 0x0d || c == 0x0a) has_nl = 1;
-      e += cl;
-    }
-    if (e == len) return e;                 /* 5) trailing whitespace */
-    if (has_nl) return e;                   /* 6) whitespace containing a newline */
-    if ((e - i) > last_ws_len) return e - last_ws_len; /* 7) leave last whitespace */
-    return e;                               /* 8) single whitespace */
+  /* 5-8) whitespace runs: \s++$ | \s*[\r\n] | \s+(?!\S) | \s */
+  int e = i;
+  int last_ws_len = cplen;
+  int last_nl_end = 0;
+  while (e < len) {
+    int c, cl; decode_cp(t, len, e, &c, &cl);
+    if (!u_is_space(c)) break;
+    last_ws_len = cl;
+    if (c == 0x0d || c == 0x0a) last_nl_end = e + cl;
+    e += cl;
   }
-
-  return i + cplen; /* fallback */
+  if (e == len) return e;                        /* 5) \s++$ (trailing whitespace) */
+  if (last_nl_end > i) return last_nl_end;       /* 6) \s*[\r\n] (up to last CR/LF) */
+  if ((e - i) > last_ws_len) return e - last_ws_len; /* 7) \s+(?!\S) (leave last ws) */
+  return e;                                      /* 8) \s (single whitespace) */
 }
 
 /* ----------------------------- BPE ------------------------------------ */
-/* Count tokens in a single raw-byte piece via greedy lowest-rank merges. */
+#define MAX_TOKEN_BYTES 128
+
+typedef struct {
+  int prev;
+  int next;
+  int rank;
+} bpe_node_t;
+
+typedef struct {
+  int rank;
+  int pos;
+} bpe_heap_item_t;
+
+static int heap_less(bpe_heap_item_t a, bpe_heap_item_t b) {
+  return (a.rank < b.rank) || (a.rank == b.rank && a.pos < b.pos);
+}
+
+static void heap_push(bpe_heap_item_t *h, int *hn, bpe_heap_item_t v) {
+  int i = (*hn)++;
+  while (i > 0) {
+    int p = (i - 1) >> 1;
+    if (!heap_less(v, h[p])) break;
+    h[i] = h[p];
+    i = p;
+  }
+  h[i] = v;
+}
+
+static bpe_heap_item_t heap_pop(bpe_heap_item_t *h, int *hn) {
+  bpe_heap_item_t top = h[0];
+  bpe_heap_item_t last = h[--(*hn)];
+  int n = *hn, i = 0;
+  while ((i << 1) + 1 < n) {
+    int l = (i << 1) + 1, r = l + 1;
+    int c = (r < n && heap_less(h[r], h[l])) ? r : l;
+    if (!heap_less(h[c], last)) break;
+    h[i] = h[c];
+    i = c;
+  }
+  if (n > 0) h[i] = last;
+  return top;
+}
+
+/* Count tokens in a single raw-byte piece via O(N log N) lowest-rank merges. */
 static int tok_bpe(const unsigned char *piece, int plen) {
   if (plen <= 0) return 0;
-  if (plen > 4096) return plen / 4; /* pathological guard; estimate */
-  char **sym = (char **)malloc(sizeof(char *) * (size_t)plen);
-  if (!sym) return plen / 4;
-  int nsym = 0;
+  if (plen == 1) return 1;
+  bpe_node_t *nodes = (bpe_node_t *)malloc(sizeof(bpe_node_t) * (size_t)plen);
+  bpe_heap_item_t *heap = (bpe_heap_item_t *)malloc(sizeof(bpe_heap_item_t) * (size_t)plen * 2u);
+  if (!nodes || !heap) {
+    free(nodes);
+    free(heap);
+    return (plen + 3) / 4;
+  }
+  int hn = 0;
   for (int k = 0; k < plen; k++) {
-    char *s = (char *)malloc(2);
-    if (!s) { for (int j = 0; j < nsym; j++) free(sym[j]); free(sym); return plen / 4; }
-    s[0] = (char)piece[k]; s[1] = 0;
-    sym[nsym++] = s;
-  }
-  char buf[256];
-  while (nsym > 1) {
-    int best = -1, bestr = 0x7fffffff;
-    for (int k = 0; k < nsym - 1; k++) {
-      int l1 = (int)strlen(sym[k]), l2 = (int)strlen(sym[k + 1]);
-      if (l1 + l2 >= (int)sizeof(buf)) continue;
-      memcpy(buf, sym[k], l1);
-      memcpy(buf + l1, sym[k + 1], l2);
-      buf[l1 + l2] = 0;
-      int r = tok_lookup((const unsigned char *)buf, l1 + l2);
-      if (r >= 0 && r < bestr) { bestr = r; best = k; }
+    nodes[k].prev = k - 1;
+    nodes[k].next = k + 1;
+    if (k + 1 < plen) {
+      int r = tok_lookup(piece + k, 2);
+      nodes[k].rank = r;
+      if (r >= 0) {
+        bpe_heap_item_t item = { r, k };
+        heap_push(heap, &hn, item);
+      }
+    } else {
+      nodes[k].rank = -1;
     }
-    if (best < 0) break;
-    int l1 = (int)strlen(sym[best]), l2 = (int)strlen(sym[best + 1]);
-    char *m = (char *)malloc((size_t)(l1 + l2 + 1));
-    memcpy(m, sym[best], l1);
-    memcpy(m + l1, sym[best + 1], l2);
-    m[l1 + l2] = 0;
-    free(sym[best]);
-    free(sym[best + 1]);
-    sym[best] = m;
-    for (int k = best + 1; k < nsym - 1; k++) sym[k] = sym[k + 1];
-    nsym--;
   }
-  int n = nsym;
-  for (int k = 0; k < nsym; k++) free(sym[k]);
-  free(sym);
-  return n;
+  int nsym = plen;
+  while (nsym > 1 && hn > 0) {
+    bpe_heap_item_t top = heap_pop(heap, &hn);
+    int best = top.pos;
+    int right = nodes[best].next;
+    if (right >= plen || nodes[best].rank != top.rank) continue;
+
+    int nr = nodes[right].next;
+    nodes[best].next = nr;
+    if (nr < plen) nodes[nr].prev = best;
+    nodes[right].next = plen + 1;
+    nodes[right].rank = -1;
+    nsym--;
+
+    if (nr < plen) {
+      int clen = nodes[nr].next - best;
+      int r = (clen <= MAX_TOKEN_BYTES) ? tok_lookup(piece + best, clen) : -1;
+      nodes[best].rank = r;
+      if (r >= 0) {
+        bpe_heap_item_t item = { r, best };
+        heap_push(heap, &hn, item);
+      }
+    } else {
+      nodes[best].rank = -1;
+    }
+
+    int left = nodes[best].prev;
+    if (left >= 0) {
+      int clen = nr - left;
+      int r = (clen <= MAX_TOKEN_BYTES) ? tok_lookup(piece + left, clen) : -1;
+      nodes[left].rank = r;
+      if (r >= 0) {
+        bpe_heap_item_t item = { r, left };
+        heap_push(heap, &hn, item);
+      }
+    }
+  }
+  free(nodes);
+  free(heap);
+  return nsym;
 }
 
 /* ----------------------------- public --------------------------------- */
 int tok_count(const char *text, int len) {
   if (!text || len <= 0) return 0;
-  if (!tok_inited) tok_init();
+  pthread_once(&tok_once, tok_init);
   const unsigned char *t = (const unsigned char *)text;
   int total = 0, i = 0;
   while (i < len) {
     int end = tok_piece_end(t, len, i);
-    if (end <= i) end = i + 1;
     total += tok_bpe(t + i, end - i);
     i = end;
   }
@@ -294,5 +388,6 @@ int tok_count(const char *text, int len) {
 }
 
 int tok_count_str(const char *text) {
+  if (!text) return 0;
   return tok_count(text, (int)strlen(text));
 }
diff --git a/util.c b/util.c
index ed81d97..f752956 100644
--- a/util.c
+++ b/util.c
@@ -4,8 +4,7 @@
 #include <string.h>
 #include <unistd.h>
 #include <errno.h>
-#include <time.h>
-#include <fcntl.h>
+#include <stdint.h>
 
 /* ----------------------------- strbuf ------------------------------ */
 
@@ -25,9 +24,15 @@ void sb_free(strbuf *s) {
 
 int sb_append(strbuf *s, const char *data, size_t n) {
   if (n == 0) return 0;
-  if (s->len + n + 1 > s->cap) {
-    size_t ncap = s->cap ? s->cap * 2 : 64;
-    while (ncap < s->len + n + 1) ncap *= 2;
+  if (!data) return -1;
+  if (n > SIZE_MAX - s->len - 1) return -1;
+  size_t need = s->len + n + 1;
+  if (need > s->cap) {
+    size_t ncap = s->cap ? s->cap : 64;
+    while (ncap < need) {
+      if (ncap > SIZE_MAX / 2) { ncap = need; break; }
+      ncap *= 2;
+    }
     char *nb = (char *)realloc(s->buf, ncap);
     if (!nb) return -1;
     s->buf = nb;
@@ -40,6 +45,7 @@ int sb_append(strbuf *s, const char *data, size_t n) {
 }
 
 int sb_append_str(strbuf *s, const char *str) {
+  if (!str) return 0;
   return sb_append(s, str, strlen(str));
 }