luxferre/bantam-cpp
1
0
Fork 0
Code Actions Activity

tkn cnt fix #3

Browse Source
This commit is contained in:
Luxferre
2026-10-16 08:37:38 +03:00
parent c970d96e91
commit d8af11fc47
2 changed files with 24 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
llm.cpp
+16 -7
View File
@@ -297,14 +297,23 @@ static void stream_handle_data(StreamCtx *sc, std::string_view data) {
sc->last_usage.prompt_tokens = pt;
sc->last_usage.completion_tokens = u["completion_tokens"].as_int();
sc->last_usage.total_tokens = tt;
/* Resolve this chunk's cached-token count only when the chunk actually
* reports it, so a later chunk that omits the field can never clobber a
* prior value (nor can a stale value spuriously inflate the total). */
const Json &ptd = u["prompt_tokens_details"];
if (ptd.is_object()) {
int ct_val = ptd["cached_tokens"].as_int();
if (ct_val > 0) sc->last_usage.cached_tokens = ct_val;
}
if (sc->last_usage.cached_tokens == 0) {
int ct_val = u["cached_tokens"].as_int();
if (ct_val > 0) sc->last_usage.cached_tokens = ct_val;
int chunk_cached = -1;
if (ptd.is_object() && ptd.contains("cached_tokens"))
chunk_cached = ptd["cached_tokens"].as_int();
else if (u.contains("cached_tokens"))
chunk_cached = u["cached_tokens"].as_int();
if (chunk_cached >= 0) {
sc->last_usage.cached_tokens = chunk_cached;
/* Some providers (e.g. Cohere on OpenRouter) report prompt_tokens as the
* *uncached* portion and cached_tokens as an *additional* amount rather
* than a subset. Fold cached into the true total input so the breakdown
* and the context percentage stay correct. */
if (chunk_cached > sc->last_usage.prompt_tokens)
sc->last_usage.prompt_tokens += chunk_cached;
}
}
}