luxferre/bantam-cpp
1
0
Fork 0
Code Actions Activity

Replace nlohmann/json with native C++20 JSON engine, cut build RAM/time, fix 32-bit build, enable -O2 -march=native

Browse Source
This commit is contained in:
Luxferre
2026-10-14 08:36:39 +03:00
parent 3e70adeb8b
commit 39d7640a5a
11 changed files with 910 additions and 25919 deletions
Download Patch File Download Diff File Expand all files Collapse all files
llm.cpp
+90 -100
View File
@@ -1,16 +1,14 @@
/* llm.cpp — libcurl HTTP client, Fibonacci backoff,
* streaming SSE parser and the llm() dispatch function. */
#include "bantam.hpp"
#include <unistd.h>
#include <array>
#include <chrono>
#include <thread>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <ctime>
#include <curl/curl.h>
#include <memory>
#include <nlohmann/json.hpp>
using json = nlohmann::json;
#include <unistd.h>
/* ----------------------------- memory callback ------------------------ */
static size_t mem_cb(char *ptr, size_t sz, size_t nm, void *ud) {
@@ -200,104 +198,95 @@ static void stream_maybe_erase(StreamCtx *sc) {
static void stream_handle_data(StreamCtx *sc, const std::string &data) {
if (data == "[DONE]") { sc->done = 1; return; }
try {
json d = json::parse(data);
if (!d.is_object()) return;
if (d.contains("usage") && d["usage"].is_object()) {
auto u = d["usage"];
int pt = (u.contains("prompt_tokens") && u["prompt_tokens"].is_number_integer())
? u["prompt_tokens"].get<int>() : 0;
int tt = (u.contains("total_tokens") && u["total_tokens"].is_number_integer())
? u["total_tokens"].get<int>() : 0;
if (pt > 0 || tt > 0) {
sc->last_usage.prompt_tokens = pt;
sc->last_usage.completion_tokens =
(u.contains("completion_tokens") && u["completion_tokens"].is_number_integer())
? u["completion_tokens"].get<int>() : 0;
sc->last_usage.total_tokens = tt;
if (u.contains("prompt_tokens_details") && u["prompt_tokens_details"].is_object()) {
auto ptd = u["prompt_tokens_details"];
if (ptd.contains("cached_tokens") && ptd["cached_tokens"].is_number_integer())
sc->last_usage.cached_tokens = ptd["cached_tokens"].get<int>();
}
if (sc->last_usage.cached_tokens == 0 &&
u.contains("cached_tokens") && u["cached_tokens"].is_number_integer())
sc->last_usage.cached_tokens = u["cached_tokens"].get<int>();
Json d_mut;
if (!Json::parse(data, d_mut) || !d_mut.is_object()) return;
const Json &d = d_mut;
const Json &u = d["usage"];
if (u.is_object()) {
int pt = u["prompt_tokens"].as_int();
int tt = u["total_tokens"].as_int();
if (pt > 0 || tt > 0) {
sc->last_usage.prompt_tokens = pt;
sc->last_usage.completion_tokens = u["completion_tokens"].as_int();
sc->last_usage.total_tokens = tt;
const Json &ptd = u["prompt_tokens_details"];
if (ptd.is_object()) {
int ct_val = ptd["cached_tokens"].as_int();
if (ct_val > 0) sc->last_usage.cached_tokens = ct_val;
}
if (sc->last_usage.cached_tokens == 0) {
int ct_val = u["cached_tokens"].as_int();
if (ct_val > 0) sc->last_usage.cached_tokens = ct_val;
}
}
if (d.contains("model") && d["model"].is_string())
sc->last_usage.model = d["model"].get<std::string>();
if (!d.contains("choices") || !d["choices"].is_array() || d["choices"].empty()) return;
auto ch = d["choices"][0];
if (!ch.is_object() || !ch.contains("delta") || !ch["delta"].is_object()) return;
auto delta = ch["delta"];
}
if (d["model"].is_string()) {
sc->last_usage.model = d["model"].str();
}
const Json &choices = d["choices"];
if (!choices.is_array() || choices.empty()) return;
const Json &ch = choices.at(0);
if (!ch.is_object()) return;
const Json &delta = ch["delta"];
if (!delta.is_object()) return;
std::string rc;
if (delta.contains("reasoning_content") && delta["reasoning_content"].is_string())
rc = delta["reasoning_content"].get<std::string>();
if (rc.empty() && delta.contains("reasoning") && delta["reasoning"].is_string())
rc = delta["reasoning"].get<std::string>();
if (rc.empty() && delta.contains("thought") && delta["thought"].is_string())
rc = delta["thought"].get<std::string>();
if (!rc.empty()) {
stream_maybe_erase(sc);
if (!sc->in_reasoning) {
if (!sc->content.empty()) {
flush_tbl(sc);
if (!sc->line_buf.empty()) { render_md_line_print(sc->line_buf, sc->md); sc->line_buf.clear(); }
printf("\n");
}
cprint("--- reasoning start ---\n", std::array{36});
sc->in_reasoning = true;
}
cprint(rc, std::array{2});
sc->reas += rc;
}
std::string ct;
if (delta.contains("content") && delta["content"].is_string())
ct = delta["content"].get<std::string>();
if (!ct.empty()) {
stream_maybe_erase(sc);
if (sc->in_reasoning) { cprint("\n--- reasoning end ---\n", std::array{36}); sc->in_reasoning = false; }
sc->content += ct;
sc->line_buf += ct;
while (true) {
auto nl = sc->line_buf.find('\n');
if (nl == std::string::npos) break;
std::string cur = sc->line_buf.substr(0, nl);
sc->line_buf.erase(0, nl + 1);
if (!sc->md.in_code && !is_table_line(cur)) {
flush_tbl(sc);
render_md_line_print(cur, sc->md);
} else {
sc->tbl.push_back(cur);
}
}
}
if (delta.contains("tool_calls") && delta["tool_calls"].is_array()) {
stream_maybe_erase(sc);
if (sc->in_reasoning) { cprint("\n--- reasoning end ---\n", std::array{36}); sc->in_reasoning = false; }
if (sc->live_idx == -1) {
std::string rc = delta["reasoning_content"].str();
if (rc.empty()) rc = delta["reasoning"].str();
if (rc.empty()) rc = delta["thought"].str();
if (!rc.empty()) {
stream_maybe_erase(sc);
if (!sc->in_reasoning) {
if (!sc->content.empty()) {
flush_tbl(sc);
if (!sc->line_buf.empty()) { render_md_line_print(sc->line_buf, sc->md); sc->line_buf.clear(); }
printf("\n");
}
for (auto &tc : delta["tool_calls"]) {
if (!tc.is_object()) continue;
int idx = (tc.contains("index") && tc["index"].is_number_integer())
? tc["index"].get<int>() : 0;
std::string id, name, args;
if (tc.contains("id") && tc["id"].is_string()) id = tc["id"].get<std::string>();
if (tc.contains("function") && tc["function"].is_object()) {
auto fn = tc["function"];
if (fn.contains("name") && fn["name"].is_string()) name = fn["name"].get<std::string>();
if (fn.contains("arguments") && fn["arguments"].is_string()) args = fn["arguments"].get<std::string>();
}
stream_upsert_tc(sc, idx, id, name, args);
}
step_live(sc);
cprint("--- reasoning start ---\n", std::array{36});
sc->in_reasoning = true;
}
} catch (...) {
return;
cprint(rc, std::array{2});
sc->reas += rc;
}
std::string ct = delta["content"].str();
if (!ct.empty()) {
stream_maybe_erase(sc);
if (sc->in_reasoning) { cprint("\n--- reasoning end ---\n", std::array{36}); sc->in_reasoning = false; }
sc->content += ct;
sc->line_buf += ct;
while (true) {
auto nl = sc->line_buf.find('\n');
if (nl == std::string::npos) break;
std::string cur = sc->line_buf.substr(0, nl);
sc->line_buf.erase(0, nl + 1);
if (!sc->md.in_code && !is_table_line(cur)) {
flush_tbl(sc);
render_md_line_print(cur, sc->md);
} else {
sc->tbl.push_back(cur);
}
}
}
const Json &tcs = delta["tool_calls"];
if (tcs.is_array()) {
stream_maybe_erase(sc);
if (sc->in_reasoning) { cprint("\n--- reasoning end ---\n", std::array{36}); sc->in_reasoning = false; }
if (sc->live_idx == -1) {
flush_tbl(sc);
if (!sc->line_buf.empty()) { render_md_line_print(sc->line_buf, sc->md); sc->line_buf.clear(); }
}
for (auto &tc : tcs.aval) {
if (!tc.is_object()) continue;
int idx = tc["index"].as_int();
std::string id = tc["id"].str();
std::string name, args;
const Json &fn = tc["function"];
if (fn.is_object()) {
name = fn["name"].str();
args = fn["arguments"].str();
}
stream_upsert_tc(sc, idx, id, name, args);
}
step_live(sc);
}
}
@@ -365,9 +354,10 @@ static size_t stream_write_cb(char *ptr, size_t sz, size_t nm, void *ud) {
/* ----------------------------- llm() ---------------------------------- */
static void sleep_interruptible(int secs, volatile int *interrupted) {
struct timespec ts{0, 100000000L};
for (int i = 0; i < secs * 10; i++) {
if (*interrupted) return;
std::this_thread::sleep_for(std::chrono::milliseconds(100));
nanosleep(&ts, nullptr);
}
}
@@ -407,8 +397,8 @@ int llm(Cfg &cfg, Messages &msgs, bool use_tools, Message &out, Usage &usage,
curl_easy_setopt(curl.c, CURLOPT_POSTFIELDSIZE, (long)payload.size());
curl_easy_setopt(curl.c, CURLOPT_HTTPHEADER, headers.list);
set_proxy(curl.c);
curl_easy_setopt(curl.c, CURLOPT_CONNECTTIMEOUT, cfg.timeout > 0 ? cfg.timeout : 30);
if (!stream) curl_easy_setopt(curl.c, CURLOPT_TIMEOUT, cfg.timeout > 0 ? cfg.timeout : 300);
curl_easy_setopt(curl.c, CURLOPT_CONNECTTIMEOUT, (long)(cfg.timeout > 0 ? cfg.timeout : 30));
if (!stream) curl_easy_setopt(curl.c, CURLOPT_TIMEOUT, (long)(cfg.timeout > 0 ? cfg.timeout : 300));
int attempt, rc = -1;
for (attempt = 0; attempt <= nf; attempt++) {