diff --git a/json.cpp b/json.cpp index b7a6b74..7e676fb 100644 --- a/json.cpp +++ b/json.cpp @@ -680,6 +680,14 @@ bool parse_nonstream_response(std::string_view body, Message &out, Usage &usage) if (usage.cached_tokens == 0) { usage.cached_tokens = u["cached_tokens"].as_int(); } + if (usage.cached_tokens > usage.prompt_tokens) { + /* Some providers (e.g. Cohere models on OpenRouter) report + * prompt_tokens as the *uncached* portion and cached_tokens as an + * *additional* amount, instead of the OpenAI convention where + * cached_tokens is a subset of prompt_tokens. Normalize so + * prompt_tokens always holds the true total input. */ + usage.prompt_tokens += usage.cached_tokens; + } } usage.model = root["model"].str(); return true; diff --git a/llm.cpp b/llm.cpp index 93a8303..b1ec52f 100644 --- a/llm.cpp +++ b/llm.cpp @@ -297,14 +297,23 @@ static void stream_handle_data(StreamCtx *sc, std::string_view data) { sc->last_usage.prompt_tokens = pt; sc->last_usage.completion_tokens = u["completion_tokens"].as_int(); sc->last_usage.total_tokens = tt; + /* Resolve this chunk's cached-token count only when the chunk actually + * reports it, so a later chunk that omits the field can never clobber a + * prior value (nor can a stale value spuriously inflate the total). */ const Json &ptd = u["prompt_tokens_details"]; - if (ptd.is_object()) { - int ct_val = ptd["cached_tokens"].as_int(); - if (ct_val > 0) sc->last_usage.cached_tokens = ct_val; - } - if (sc->last_usage.cached_tokens == 0) { - int ct_val = u["cached_tokens"].as_int(); - if (ct_val > 0) sc->last_usage.cached_tokens = ct_val; + int chunk_cached = -1; + if (ptd.is_object() && ptd.contains("cached_tokens")) + chunk_cached = ptd["cached_tokens"].as_int(); + else if (u.contains("cached_tokens")) + chunk_cached = u["cached_tokens"].as_int(); + if (chunk_cached >= 0) { + sc->last_usage.cached_tokens = chunk_cached; + /* Some providers (e.g. Cohere on OpenRouter) report prompt_tokens as the + * *uncached* portion and cached_tokens as an *additional* amount rather + * than a subset. Fold cached into the true total input so the breakdown + * and the context percentage stay correct. */ + if (chunk_cached > sc->last_usage.prompt_tokens) + sc->last_usage.prompt_tokens += chunk_cached; } } }