commit b8b03f3b419b59cab79ebe5a2a0f4eab7351aa15 Author: Luxferre Date: Wed Oct 14 09:18:19 2026 +0300 init commit diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..81febae --- /dev/null +++ b/Makefile @@ -0,0 +1,86 @@ +.POSIX: +# Dynagate LLM Gateway (C++20 port) - POSIX-compliant Makefile +# Targets: Linux, FreeBSD, macOS, Haiku (compatible with bmake, gmake, smake, etc.) +# Dependencies: libcurl (HTTP). JSON is handled natively in json.cpp +# (no extra or vendored dependency). +# Override compiler or flags if needed, e.g.: +# make CXX=clang++ CPPFLAGS="-I/opt/homebrew/include -I." LDLIBS="-L/opt/homebrew/lib -lcurl -pthread" +# +# 32-bit build: make 32bit +# Or: make CXXFLAGS="-std=c++20 -O2 -m32" LDFLAGS="-m32" +# Large file support on 32-bit: make CPPFLAGS="-D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE" + +CXX = c++ +STRIP = strip +CXXFLAGS = -std=c++20 -s -Wall -Wextra -Wpedantic -O2 -march=native -D_POSIX_C_SOURCE=200809L -D_DEFAULT_SOURCE -D_DARWIN_C_SOURCE +CPPFLAGS = -I/usr/local/include -I. -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE +CURL_CFLAGS = $$(pkg-config --cflags libcurl 2>/dev/null || curl-config --cflags 2>/dev/null) +CURL_LIBS = $$(pkg-config --libs libcurl 2>/dev/null || curl-config --libs 2>/dev/null || echo '-lcurl') +LDFLAGS = -s +LDLIBS = -L/usr/local/lib $(CURL_LIBS) -pthread + +# Location of the AddressSanitizer/UBSan runtime libraries plus a replacement +# libasan_preinit.o (so the runtime initializes before main()). On a normal +# system these ship with the compiler; override with +# make debug ASANDIR= (empty) — the gcc driver finds them automatically. +ASANDIR = /tmp/asanlib + +CORE_OBJS = json.o config.o handler.o server.o +OBJS = main.o $(CORE_OBJS) +TEST_OBJS = gateway_test.o $(CORE_OBJS) + +.PHONY: all build clean test run install debug 32bit + +all: build + +build: dynagate + +dynagate: $(OBJS) + $(CXX) $(CXXFLAGS) $(CPPFLAGS) $(LDFLAGS) -o dynagate $(OBJS) $(LDLIBS) + $(STRIP) dynagate + +main.o: main.cpp dynagate.hpp + $(CXX) $(CXXFLAGS) $(CPPFLAGS) $(CURL_CFLAGS) -c main.cpp + +json.o: json.cpp dynagate.hpp + $(CXX) $(CXXFLAGS) $(CPPFLAGS) $(CURL_CFLAGS) -c json.cpp + +config.o: config.cpp dynagate.hpp + $(CXX) $(CXXFLAGS) $(CPPFLAGS) $(CURL_CFLAGS) -c config.cpp + +handler.o: handler.cpp dynagate.hpp + $(CXX) $(CXXFLAGS) $(CPPFLAGS) $(CURL_CFLAGS) -c handler.cpp + +server.o: server.cpp dynagate.hpp + $(CXX) $(CXXFLAGS) $(CPPFLAGS) $(CURL_CFLAGS) -c server.cpp + +gateway_test.o: gateway_test.cpp dynagate.hpp + $(CXX) $(CXXFLAGS) $(CPPFLAGS) $(CURL_CFLAGS) -c gateway_test.cpp + +gateway_test: $(TEST_OBJS) + $(CXX) $(CXXFLAGS) $(CPPFLAGS) $(LDFLAGS) -o gateway_test $(TEST_OBJS) $(LDLIBS) + +test: gateway_test + ./gateway_test + +run: build + ./dynagate -port 8080 -csv models.csv + +install: dynagate + install -m 0755 dynagate $(DESTDIR)/usr/local/bin/dynagate + +# Debug build with AddressSanitizer + UndefinedBehaviorSanitizer. +# Rebuilds ./dynagate — run `make clean && make` afterwards to restore the +# optimized build. ASANDIR points the gcc driver (-B) at the runtime libs and +# the replacement libasan_preinit.o, and sets an rpath so they load at run time. +debug: + $(MAKE) CXXFLAGS="-std=c++20 -Wall -Wextra -Wpedantic -D_POSIX_C_SOURCE=200809L -D_DEFAULT_SOURCE -D_DARWIN_C_SOURCE -g -O0 -fsanitize=address,undefined -B$(ASANDIR) -Wl,-rpath,$(ASANDIR)" LDFLAGS="" STRIP=":" dynagate + +# 32-bit build target +# Usage: make 32bit +# This builds a 32-bit binary with large file support enabled. +32bit: + $(MAKE) CXXFLAGS="$(CXXFLAGS) -m32" LDFLAGS="$(LDFLAGS) -m32" dynagate + +clean: + rm -f $(OBJS) gateway_test.o dynagate gateway_test diff --git a/README.md b/README.md new file mode 100644 index 0000000..709f93c --- /dev/null +++ b/README.md @@ -0,0 +1,210 @@ +# Dynagate LLM gateway (C++ port) + +Dynagate is a lightweight, high-performance LLM gateway that acts as a proxy for OpenAI-compatible APIs. This directory is a modern **C++20** port of the Go reference implementation (`dynagate`), using a native RFC 8259 JSON engine (`json.cpp`) and **libcurl** for HTTP networking. + +It provides seamless model routing, automatic key and model failover, live configuration hot-reloading without restarts, token authentication, and custom header injection. + +## Features + +1. **Multi-API Protocol Endpoints (Stateless)**: + - `/v1/models` and `/models` (GET): Dynamically lists unique active model IDs. + - `/v1/chat/completions` and `/chat/completions` (POST): Proxies non-streaming and streaming (`text/event-stream`) completions. + - `/v1/responses` and `/responses` (POST): Stateless-style OpenAI Responses API endpoint (automatically routed to upstream chat completions endpoints). + - `/v1/messages` and `/messages` (POST): Anthropic-compatible Messages API endpoint (automatically routed to upstream chat completions endpoints). + - `/v1/images/generations` and `/images/generations` (POST): Proxies image generation requests. + > **Note** Dynagate operates in a completely stateless manner across all API formats (`/v1/chat/completions`, `/v1/responses`, `/v1/messages`). Dynagate does not store or persist server-side conversation state. Clients must include the full conversation history in each request for multi-turn dialogues. + +2. **Zero-downtime live-reloading**: + - Watches `models.csv` (overridable via command-line flags) continuously using a background thread and automatically reloads configuration updates without dropping active connections. + +3. **Fallback and retry orchestration**: + - Attempts keys configured for the requested model sequentially using Fibonacci backoff delays (`-retry-delay`). + - Cascades automatically to the next model in the priority chain (wrapping around to cover all entries) if all keys for the requested model fail. + - Updates the `"model"` field dynamically in the outgoing request body before proxying, ensuring upstream compatibility. + +4. **Flexible authorization**: + - Optional gateway token authentication via command-line key file. + - Key options (`""`, `"-"`) to strip authorization headers when proxying to local or open upstreams. + - Special `"-blank-"` key config to pass an empty `Authorization: Bearer` header. + +5. **External config updater**: + - Executes a periodic CLI command (`-csv-updater`) at a custom interval (`-csv-update-interval`) to fetch, generate, or download fresh model listings. + +6. **Custom header injection (`extra` field)**: + - Sets or overrides custom headers dynamically per model deployment using a JSON configuration string. + +## Tech stack + +- **Language**: C++20 (`-std=c++20`) +- **Dependencies**: + - `libcurl` (HTTP/HTTPS upstream requests and SSE streaming) + - Native RFC 8259 JSON parser and serializer in `json.cpp` (no extra or vendored third-party JSON dependency) + - POSIX sockets and standard C++20 threads/atomics/filesystem +- **Target Platforms**: Linux, FreeBSD, macOS, Haiku, and any POSIX-compliant OS (64-bit and 32-bit hosts) + +## Repository layout + +```text +dynagate.hpp Shared declarations, data structures, and interfaces +json.cpp Native RFC 8259 JSON parser and serializer +config.cpp RFC 4180 CSV parser, ConfigManager, hot-reloader, CSV updater +handler.cpp Auth, normalization, protocol conversions, libcurl proxying +server.cpp Multi-threaded POSIX HTTP/1.1 server and streaming writer +main.cpp CLI flag parsing, signal handling, server lifecycle +gateway_test.cpp Comprehensive unit and end-to-end test suite +Makefile POSIX-compliant Makefile (bmake, gmake, smake compatible) +models.csv Example model configuration file +free-model-collector.sh Free model discovery and verification script +freekeys.sh Helper script for collecting free keys +``` + +## Prerequisites + +- A C++20 compiler (`c++`, `g++`, or `clang++`) +- Any POSIX `make` (`bmake`, `gmake`, `make`) +- `libcurl` development headers and library: + ```bash + # Debian / Ubuntu + apt-get install build-essential libcurl4-openssl-dev + # Fedora / RHEL + dnf install gcc-c++ libcurl-devel + # FreeBSD + pkg install curl + # macOS (Xcode Command Line Tools include libcurl, or via Homebrew) + xcode-select --install + ``` + +## Getting started + +### 1. Building from source + +```bash +make # builds ./dynagate +make 32bit # builds a 32-bit ./dynagate binary with large file support +make debug # builds ./dynagate with AddressSanitizer + UBSan +make test # builds and runs the test suite +make clean # removes build artifacts +make install # installs to $(DESTDIR)/usr/local/bin/dynagate +``` + +If `libcurl` or custom headers live in a non-standard prefix, override flags on the `make` command line: +```bash +make CXX=clang++ CPPFLAGS="-I/opt/homebrew/include -I." LDLIBS="-L/opt/homebrew/lib -lcurl -pthread" +``` + +### 2. Configure models +Create a `models.csv` file in the same directory. The file supports both header and header-less formats. + +**Example `models.csv` with Header**: +```csv +model,key,endpoint,extra +gpt-4o,sk-proj-key1,https://api.openai.com/v1,"{""Openai-Organization"":""org-123""}" +gpt-4o,sk-proj-key2,https://api.openai.com/v1, +gpt-3.5-turbo,-blank-,https://api.openai.com/v1, +deepseek-chat,sk-ds-key,https://api.deepseek.com/v1,"{""X-Custom-Env"":""production""}" +ollama-llama3,-,http://localhost:11434/v1, +``` + +### 3. Setup gateway key (optional) +To restrict access to your gateway, write one or more plaintext keys (one per line) to a file (e.g. `gatekey.txt`): +```text +my-secure-gateway-token-1 +my-secure-gateway-token-2 +``` + +### 4. Run the server +```bash +./dynagate -port 8080 -csv models.csv -key gatekey.txt +``` + +## Configuration reference + +### Command line flags + +| Flag | Type | Default | Description | +| ---- | ---- | ------- | ----------- | +| `-port` | int | `8080` | Port the gateway listens on. | +| `-csv` | string | `models.csv` | Path to the model configuration CSV. | +| `-key` | string | `""` | Path to a file containing the gateway's access tokens (one per line). If blank, authentication is disabled. | +| `-csv-updater` | string | `""` | Command/script to run periodically to update `models.csv`. | +| `-csv-update-interval`| int | `10` | Frequency in minutes to invoke `-csv-updater`. | +| `-retry-delay` | int | `100` | Base delay in milliseconds for Fibonacci failover retries (0 to disable). | + +### Model CSV format rules +The gateway maps columns dynamically by looking at the header row. If no header is present, it defaults to: +- Column 0: `model` (Model ID) +- Column 1: `key` (API Key / Authorization Token) +- Column 2: `endpoint` (Upstream base URL endpoint) +- Column 3: `extra` (Optional JSON string object containing header mappings) + +#### Auth override key values: +- **`""` or `"-"`**: The gateway will strip the `Authorization` header completely (useful for local LLM endpoints or unauthenticated proxies). +- **`"-blank-"`**: The gateway will attach `Authorization: Bearer` without any key appended. +- **Any other string**: The gateway will attach `Authorization: Bearer `. + +### Stateless Operation & Multi-Turn Conversations + +Dynagate is designed as a lightweight, zero-state proxy gateway. All supported request formats (`/v1/chat/completions`, `/v1/responses`, `/v1/messages`) are processed statelessly without storing session context or conversation state on disk or in memory. + +To maintain context in multi-turn conversations, clients **must include the entire conversation history** in every request payload: +- **OpenAI Chat Completions (`/v1/chat/completions`)**: Pass the full array of system, user, and assistant messages in `messages`. +- **OpenAI Responses API (`/v1/responses`)**: Pass previous conversation turns in `input` (or `messages`) along with `instructions`. +- **Anthropic Messages API (`/v1/messages`)**: Pass the full history of user and assistant messages in `messages` along with `system`. + +### Manual testing with curl + +#### 1. Model listing (authenticated) +```bash +curl -i -H "Authorization: Bearer my-secure-gateway-token" http://localhost:8080/v1/models +``` + +#### 2. Chat completions proxying +```bash +curl -i -X POST http://localhost:8080/v1/chat/completions \ + -H "Authorization: Bearer my-secure-gateway-token" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "gpt-4o", + "messages": [{"role": "user", "content": "Explain gravity in one sentence"}] + }' +``` + +#### 3. Chat completions streaming +```bash +curl -i -X POST http://localhost:8080/v1/chat/completions \ + -H "Authorization: Bearer my-secure-gateway-token" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "gpt-4o", + "stream": true, + "messages": [{"role": "user", "content": "Count from 1 to 5"}] + }' +``` + +#### 4. OpenAI-compatible Responses API (`/v1/responses`) +```bash +curl -i -X POST http://localhost:8080/v1/responses \ + -H "Authorization: Bearer my-secure-gateway-token" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "gpt-4o", + "instructions": "Be concise", + "input": "Explain relativity in one sentence" + }' +``` + +#### 5. Anthropic-compatible Messages API (`/v1/messages`) +```bash +curl -i -X POST http://localhost:8080/v1/messages \ + -H "Authorization: Bearer my-secure-gateway-token" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "claude-3-5-sonnet", + "system": "You are a helpful assistant", + "messages": [{"role": "user", "content": "Hello world"}] + }' +``` + +## Credits + +Created by Luxferre in 2026, released into the public domain with no warranties. diff --git a/config.cpp b/config.cpp new file mode 100644 index 0000000..0514187 --- /dev/null +++ b/config.cpp @@ -0,0 +1,576 @@ +/* + Dynagate config manager, CSV parser, and utilities (C++20 port) + Created by Luxferre in 2026, released into the public domain +*/ + +#ifndef _DARWIN_C_SOURCE +#define _DARWIN_C_SOURCE +#endif + +#include "dynagate.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace { + +std::mutex g_log_mu; + +std::string run_shell_command(const std::string &command, int &exit_status) { + exit_status = -1; + int pipefd[2]; + if (pipe(pipefd) != 0) { + return "pipe() failed: " + std::string(std::strerror(errno)); + } + + pid_t pid = fork(); + if (pid < 0) { + close(pipefd[0]); + close(pipefd[1]); + return "fork() failed: " + std::string(std::strerror(errno)); + } + + if (pid == 0) { + close(pipefd[0]); + dup2(pipefd[1], STDOUT_FILENO); + dup2(pipefd[1], STDERR_FILENO); + close(pipefd[1]); + execl("/bin/sh", "sh", "-c", command.c_str(), static_cast(nullptr)); + _exit(127); + } + + close(pipefd[1]); + std::string output; + char buf[4096]; + for (;;) { + ssize_t n = read(pipefd[0], buf, sizeof(buf)); + if (n > 0) { + output.append(buf, static_cast(n)); + } else if (n == 0) { + break; + } else if (errno != EINTR) { + break; + } + } + close(pipefd[0]); + + int status = 0; + while (waitpid(pid, &status, 0) < 0) { + if (errno != EINTR) { + break; + } + } + if (WIFEXITED(status)) { + exit_status = WEXITSTATUS(status); + } else if (WIFSIGNALED(status)) { + exit_status = 128 + WTERMSIG(status); + } + return output; +} + +} // namespace + +void log_msg(const char *fmt, ...) { + std::lock_guard lock(g_log_mu); + + std::time_t now = std::time(nullptr); + struct tm tm_now; + localtime_r(&now, &tm_now); + char time_buf[32]; + std::strftime(time_buf, sizeof(time_buf), "%Y/%m/%d %H:%M:%S", &tm_now); + + char msg_buf[2048]; + va_list ap; + va_start(ap, fmt); + std::vsnprintf(msg_buf, sizeof(msg_buf), fmt, ap); + va_end(ap); + + std::fprintf(stderr, "%s %s\n", time_buf, msg_buf); + std::fflush(stderr); +} + +std::string str_trim(std::string_view sv) { + size_t start = 0; + while (start < sv.size() && + std::isspace(static_cast(sv[start]))) { + ++start; + } + size_t end = sv.size(); + while (end > start && + std::isspace(static_cast(sv[end - 1]))) { + --end; + } + return std::string(sv.substr(start, end - start)); +} + +std::string str_to_lower(std::string_view sv) { + std::string out(sv); + std::transform(out.begin(), out.end(), out.begin(), [](unsigned char c) { + return static_cast(std::tolower(c)); + }); + return out; +} + +bool str_iequals(std::string_view a, std::string_view b) { + if (a.size() != b.size()) { + return false; + } + for (size_t i = 0; i < a.size(); ++i) { + if (std::tolower(static_cast(a[i])) != + std::tolower(static_cast(b[i]))) { + return false; + } + } + return true; +} + +void HeaderList::add(std::string_view key, std::string_view value) { + items.emplace_back(std::string(key), std::string(value)); +} + +void HeaderList::set(std::string_view key, std::string_view value) { + del(key); + items.emplace_back(std::string(key), std::string(value)); +} + +void HeaderList::del(std::string_view key) { + items.erase( + std::remove_if(items.begin(), items.end(), [&](const auto &kv) { + return str_iequals(kv.first, key); + }), + items.end()); +} + +std::string HeaderList::get(std::string_view key) const { + for (const auto &kv : items) { + if (str_iequals(kv.first, key)) { + return kv.second; + } + } + return ""; +} + +std::vector HeaderList::values(std::string_view key) const { + std::vector res; + for (const auto &kv : items) { + if (str_iequals(kv.first, key)) { + res.push_back(kv.second); + } + } + return res; +} + +bool parse_csv(std::string_view content, + std::vector> &rows, + std::string *err_out) { + rows.clear(); + size_t i = 0; + const size_t n = content.size(); + + while (i < n) { + // Skip empty lines (matching Go encoding/csv behavior) + if (content[i] == '\n') { + ++i; + continue; + } + if (content[i] == '\r' && i + 1 < n && content[i + 1] == '\n') { + i += 2; + continue; + } + + std::vector row; + bool line_done = false; + + while (!line_done) { + std::string field; + if (i < n && content[i] == '"') { + // Quoted field + ++i; + bool closed = false; + while (i < n) { + if (content[i] == '"') { + if (i + 1 < n && content[i + 1] == '"') { + field.push_back('"'); + i += 2; + } else { + ++i; + closed = true; + break; + } + } else { + if (content[i] == '\r' && i + 1 < n && content[i + 1] == '\n') { + field.push_back('\n'); + i += 2; + } else { + field.push_back(content[i]); + ++i; + } + } + } + if (!closed) { + if (err_out) { + *err_out = "parse error: bare or unclosed \" in quoted-field"; + } + return false; + } + if (i < n) { + if (content[i] == ',') { + row.push_back(std::move(field)); + ++i; + if (i == n || content[i] == '\n' || + (content[i] == '\r' && i + 1 < n && content[i + 1] == '\n')) { + row.emplace_back(""); + if (i < n && content[i] == '\r') { + i += 2; + } else if (i < n && content[i] == '\n') { + ++i; + } + line_done = true; + } + } else if (content[i] == '\n') { + row.push_back(std::move(field)); + ++i; + line_done = true; + } else if (content[i] == '\r' && i + 1 < n && + content[i + 1] == '\n') { + row.push_back(std::move(field)); + i += 2; + line_done = true; + } else { + if (err_out) { + *err_out = "parse error: extraneous or missing \" in quoted-field"; + } + return false; + } + } else { + row.push_back(std::move(field)); + line_done = true; + } + } else { + // Unquoted field + while (i < n && content[i] != ',' && content[i] != '\n' && + !(content[i] == '\r' && i + 1 < n && content[i + 1] == '\n')) { + if (content[i] == '"') { + if (err_out) { + *err_out = "parse error: bare \" in non-quoted-field"; + } + return false; + } + field.push_back(content[i]); + ++i; + } + row.push_back(std::move(field)); + if (i < n && content[i] == ',') { + ++i; + if (i == n || content[i] == '\n' || + (content[i] == '\r' && i + 1 < n && content[i + 1] == '\n')) { + row.emplace_back(""); + if (i < n && content[i] == '\r') { + i += 2; + } else if (i < n && content[i] == '\n') { + ++i; + } + line_done = true; + } + } else { + if (i < n && content[i] == '\r' && i + 1 < n && + content[i + 1] == '\n') { + i += 2; + } else if (i < n && content[i] == '\n') { + ++i; + } + line_done = true; + } + } + } + + rows.push_back(std::move(row)); + } + + return true; +} + +std::vector parse_auth_tokens(std::string_view content) { + std::vector tokens; + size_t pos = 0; + while (pos <= content.size()) { + size_t nl = content.find('\n', pos); + std::string_view line = (nl == std::string_view::npos) + ? content.substr(pos) + : content.substr(pos, nl - pos); + std::string token = str_trim(line); + if (!token.empty()) { + tokens.push_back(std::move(token)); + } + if (nl == std::string_view::npos) { + break; + } + pos = nl + 1; + } + return tokens; +} + +ConfigManager::ConfigManager(std::string file_path) + : file_path_(std::move(file_path)) { + std::string err; + if (!load(&err)) { + log_msg( + "Initial configuration load failed or file not found (will retry on change): %s", + err.c_str()); + } +} + +ConfigManager::ConfigManager(std::vector configs, + std::vector unique_models) + : configs_(std::move(configs)), unique_models_(std::move(unique_models)) {} + +bool ConfigManager::load(std::string *err_out) { + std::error_code ec; + auto mod_time = std::filesystem::last_write_time(file_path_, ec); + if (ec) { + if (err_out) { + *err_out = ec.message(); + } + return false; + } + auto fsize = std::filesystem::file_size(file_path_, ec); + if (ec) { + if (err_out) { + *err_out = ec.message(); + } + return false; + } + + std::ifstream ifs(file_path_, std::ios::binary); + if (!ifs.is_open()) { + if (err_out) { + *err_out = "failed to open " + file_path_; + } + return false; + } + + std::ostringstream oss; + oss << ifs.rdbuf(); + std::string content = oss.str(); + + std::vector> rows; + if (!parse_csv(content, rows, err_out)) { + return false; + } + + std::vector configs; + if (rows.empty()) { + update(std::move(configs), mod_time, fsize, true); + return true; + } + + // Detect header row + bool has_header = false; + const auto &first_row = rows[0]; + for (const auto &col : first_row) { + std::string col_lower = str_to_lower(str_trim(col)); + if (col_lower == "model" || col_lower == "key" || col_lower == "endpoint") { + has_header = true; + break; + } + } + + size_t start_idx = 0; + int model_idx = 0; + int key_idx = 1; + int endpoint_idx = 2; + int extra_idx = -1; + + if (has_header) { + start_idx = 1; + model_idx = -1; + key_idx = -1; + endpoint_idx = -1; + extra_idx = -1; + for (size_t i = 0; i < first_row.size(); ++i) { + std::string col_lower = str_to_lower(str_trim(first_row[i])); + if (col_lower == "model") { + model_idx = static_cast(i); + } else if (col_lower == "key") { + key_idx = static_cast(i); + } else if (col_lower == "endpoint") { + endpoint_idx = static_cast(i); + } else if (col_lower == "extra") { + extra_idx = static_cast(i); + } + } + if (model_idx == -1) { + model_idx = 0; + } + if (key_idx == -1) { + key_idx = 1; + } + if (endpoint_idx == -1) { + endpoint_idx = 2; + } + } else { + start_idx = 0; + } + + for (size_t i = start_idx; i < rows.size(); ++i) { + const auto &row = rows[i]; + int max_idx = std::max({model_idx, key_idx, endpoint_idx}); + if (static_cast(row.size()) <= max_idx) { + continue; + } + + std::string m = str_trim(row[static_cast(model_idx)]); + std::string k = str_trim(row[static_cast(key_idx)]); + std::string ep = str_trim(row[static_cast(endpoint_idx)]); + + if (m.empty() || k.empty() || ep.empty()) { + continue; + } + + std::string extra; + if (has_header) { + if (extra_idx != -1 && static_cast(row.size()) > extra_idx) { + extra = str_trim(row[static_cast(extra_idx)]); + } + } else { + if (row.size() > 3) { + extra = str_trim(row[3]); + } + } + + configs.push_back(ModelConfig{ + std::move(m), + std::move(k), + std::move(ep), + std::move(extra), + }); + } + + size_t count = configs.size(); + update(std::move(configs), mod_time, fsize, true); + log_msg("Loaded %zu model configurations from %s", count, file_path_.c_str()); + return true; +} + +void ConfigManager::update(std::vector configs, + std::filesystem::file_time_type mod_time, + std::uintmax_t size, + bool has_stat) { + std::unique_lock lock(mu_); + + configs_ = std::move(configs); + last_mod_time_ = mod_time; + last_size_ = size; + has_stat_ = has_stat; + + std::vector unique_models; + std::unordered_set seen; + for (const auto &cfg : configs_) { + if (seen.insert(cfg.model).second) { + unique_models.push_back(cfg.model); + } + } + unique_models_ = std::move(unique_models); +} + +std::vector ConfigManager::get_configs() const { + std::shared_lock lock(mu_); + return configs_; +} + +std::vector ConfigManager::get_unique_models() const { + std::shared_lock lock(mu_); + return unique_models_; +} + +void ConfigManager::watch(std::atomic &stop_flag) { + while (!stop_flag.load()) { + for (int i = 0; i < 20 && !stop_flag.load(); ++i) { + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } + if (stop_flag.load()) { + return; + } + + std::error_code ec; + auto mod_time = std::filesystem::last_write_time(file_path_, ec); + if (ec) { + continue; + } + auto fsize = std::filesystem::file_size(file_path_, ec); + if (ec) { + continue; + } + + bool changed = false; + { + std::shared_lock lock(mu_); + if (!has_stat_ || mod_time > last_mod_time_ || fsize != last_size_) { + changed = true; + } + } + + if (changed) { + log_msg("Detected changes in %s, reloading...", file_path_.c_str()); + std::string err; + if (!load(&err)) { + log_msg("Error reloading configuration: %s", err.c_str()); + } else { + log_msg("Configuration reloaded successfully"); + } + } + } +} + +void start_csv_updater(std::atomic &stop_flag, + const std::string &command, + std::chrono::milliseconds interval) { + auto run_updater = [&]() { + log_msg("Running CSV updater command: %s", command.c_str()); + int exit_status = 0; + std::string output = run_shell_command(command, exit_status); + if (exit_status != 0) { + std::string trimmed = str_trim(output); + log_msg("CSV updater command failed: exit status %d, output: \"%s\"", + exit_status, trimmed.c_str()); + } else { + log_msg("CSV updater command completed successfully"); + } + }; + + // Run once immediately at startup + run_updater(); + + while (!stop_flag.load()) { + auto start = std::chrono::steady_clock::now(); + while (!stop_flag.load()) { + auto elapsed = std::chrono::steady_clock::now() - start; + if (elapsed >= interval) { + break; + } + auto remaining = std::chrono::duration_cast( + interval - elapsed); + auto sleep_dur = std::min(remaining, std::chrono::milliseconds(50)); + if (sleep_dur.count() > 0) { + std::this_thread::sleep_for(sleep_dur); + } + } + if (stop_flag.load()) { + return; + } + run_updater(); + } +} diff --git a/dynagate.hpp b/dynagate.hpp new file mode 100644 index 0000000..de4798f --- /dev/null +++ b/dynagate.hpp @@ -0,0 +1,278 @@ +/* + Dynagate LLM Gateway: dynamically change models on the fly (C++20 port) + Created by Luxferre in 2026, released into the public domain +*/ + +#ifndef DYNAGATE_HPP +#define DYNAGATE_HPP + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Minimal RFC 8259 JSON value, parser and serializer (json.cpp) +enum class JType { Null, Bool, Int, Float, Str, Arr, Obj }; + +struct Json { + JType type = JType::Null; + bool bval = false; + int64_t ival = 0; + double dval = 0.0; + std::string sval; + std::vector aval; + std::vector> oval; + + Json(); + ~Json(); + Json(const Json &); + Json(Json &&) noexcept; + Json &operator=(const Json &); + Json &operator=(Json &&) noexcept; + + Json(std::nullptr_t); + Json(bool b); + Json(int n); + Json(int64_t n); + Json(double d); + Json(const char *s); + Json(std::string s); + Json(std::string_view s); + + static Json object(); + static Json object( + std::initializer_list> init); + static Json array(); + static Json array(std::initializer_list init); + + bool is_null() const { return type == JType::Null; } + bool is_bool() const { return type == JType::Bool; } + bool is_boolean() const { return type == JType::Bool; } + bool is_number_integer() const { return type == JType::Int; } + bool is_number() const { return type == JType::Int || type == JType::Float; } + bool is_string() const { return type == JType::Str; } + bool is_array() const { return type == JType::Arr; } + bool is_object() const { return type == JType::Obj; } + + bool empty() const; + size_t size() const; + bool contains(std::string_view k) const; + void erase(std::string_view k); + Json &operator[](std::string_view k); + const Json &operator[](std::string_view k) const; + Json &at(size_t i); + const Json &at(size_t i) const; + void push_back(Json v); + + std::string str(std::string_view def = "") const; + bool as_bool(bool def = false) const; + int64_t as_int64(int64_t def = 0) const; + int as_int(int def = 0) const; + + std::string dump(int indent = -1) const; + static bool parse(std::string_view src, Json &out); + static Json parse(std::string_view src); +}; + +struct ModelConfig { + std::string model; + std::string key; + std::string endpoint; + std::string extra; + + bool operator==(const ModelConfig &other) const = default; +}; + +class ConfigManager { +public: + explicit ConfigManager(std::string file_path); + ConfigManager(std::vector configs, + std::vector unique_models); + + bool load(std::string *err_out = nullptr); + std::vector get_configs() const; + std::vector get_unique_models() const; + void watch(std::atomic &stop_flag); + +private: + void update(std::vector configs, + std::filesystem::file_time_type mod_time, + std::uintmax_t size, + bool has_stat); + + std::string file_path_; + mutable std::shared_mutex mu_; + std::vector configs_; + std::vector unique_models_; + std::filesystem::file_time_type last_mod_time_{}; + std::uintmax_t last_size_ = 0; + bool has_stat_ = false; +}; + +struct HeaderList { + std::vector> items; + + void add(std::string_view key, std::string_view value); + void set(std::string_view key, std::string_view value); + void del(std::string_view key); + std::string get(std::string_view key) const; + std::vector values(std::string_view key) const; +}; + +struct HttpRequest { + std::string method; + std::string path; + HeaderList headers; + std::string body; + int client_fd = -1; + std::function is_cancelled; +}; + +class ResponseWriter { +public: + virtual ~ResponseWriter() = default; + virtual HeaderList &header() = 0; + virtual void write_header(int status_code) = 0; + virtual bool write(std::string_view data) = 0; + virtual void flush() = 0; +}; + +class RecorderResponseWriter : public ResponseWriter { +public: + HeaderList &header() override { return headers_; } + void write_header(int status_code) override { + if (!header_written_) { + status_code_ = status_code; + header_written_ = true; + } + } + bool write(std::string_view data) override { + if (!header_written_) { + write_header(200); + } + body_.append(data.data(), data.size()); + return true; + } + void flush() override {} + + int status_code() const { return status_code_; } + const HeaderList &headers() const { return headers_; } + const std::string &body() const { return body_; } + +private: + int status_code_ = 200; + bool header_written_ = false; + HeaderList headers_; + std::string body_; +}; + +// Logging and string utilities (config.cpp) +void log_msg(const char *fmt, ...); +std::string str_trim(std::string_view sv); +std::string str_to_lower(std::string_view sv); +bool str_iequals(std::string_view a, std::string_view b); +bool parse_csv(std::string_view content, + std::vector> &rows, + std::string *err_out = nullptr); +std::vector parse_auth_tokens(std::string_view content); +void start_csv_updater(std::atomic &stop_flag, + const std::string &command, + std::chrono::milliseconds interval); + +// Core gateway logic and handlers (handler.cpp) +void ensure_curl_initialized(); +bool check_auth(const std::vector &expected_tokens, + const HttpRequest &req); +void send_unauthorized(ResponseWriter &w); +std::vector get_trial_configs( + const std::vector &configs, + const std::vector &unique_models, + std::string_view requested_model); +std::string build_url(std::string_view endpoint, std::string_view req_path); +int64_t fibonacci(int n); +bool is_media_content(const Json &m); +Json normalize_content(const Json &content_any); +void autodetect_and_normalize_messages(Json &body_map); +Json convert_anthropic_to_openai(const Json &body_map); +Json convert_anthropic_content_to_openai(const Json &cnt_any); +Json convert_openai_to_anthropic_response( + const Json &openai_resp, + const std::string &requested_model); +Json convert_responses_to_openai(const Json &body_map); +Json convert_openai_to_responses_response( + const Json &openai_resp, + const std::string &requested_model); + +void handle_models(ConfigManager &cm, + const std::vector &expected_tokens, + const HttpRequest &req, + ResponseWriter &w); +void handle_chat_completions( + ConfigManager &cm, + const std::vector &expected_tokens, + std::chrono::milliseconds retry_base_delay, + const HttpRequest &req, + ResponseWriter &w); +void handle_image_generations( + ConfigManager &cm, + const std::vector &expected_tokens, + std::chrono::milliseconds retry_base_delay, + const HttpRequest &req, + ResponseWriter &w); +void handle_messages( + ConfigManager &cm, + const std::vector &expected_tokens, + std::chrono::milliseconds retry_base_delay, + const HttpRequest &req, + ResponseWriter &w); +void handle_responses( + ConfigManager &cm, + const std::vector &expected_tokens, + std::chrono::milliseconds retry_base_delay, + const HttpRequest &req, + ResponseWriter &w); + +// HTTP server (server.cpp) +class GatewayServer { +public: + GatewayServer(int port, + ConfigManager &cm, + std::vector auth_tokens, + std::chrono::milliseconds retry_delay); + ~GatewayServer(); + + GatewayServer(const GatewayServer &) = delete; + GatewayServer &operator=(const GatewayServer &) = delete; + + bool start(std::string *err_out = nullptr); + void serve(); + void stop(); + int bound_port() const { return bound_port_; } + +private: + void handle_client(int client_fd); + + int requested_port_; + int bound_port_ = 0; + int listen_fd_ = -1; + ConfigManager &cm_; + std::vector auth_tokens_; + std::chrono::milliseconds retry_delay_; + std::atomic stopping_{false}; + mutable std::mutex active_mu_; + std::condition_variable active_cv_; + int active_connections_ = 0; + static constexpr int kMaxConcurrentConnections = 256; +}; + +#endif // DYNAGATE_HPP diff --git a/gateway_test.cpp b/gateway_test.cpp new file mode 100644 index 0000000..7d947a6 --- /dev/null +++ b/gateway_test.cpp @@ -0,0 +1,1405 @@ +/* + Dynagate C++20 test suite (port of gateway_test.go) + Created by Luxferre in 2026, released into the public domain +*/ + +#ifndef _DARWIN_C_SOURCE +#define _DARWIN_C_SOURCE +#endif + +#include "dynagate.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace { + +int g_failures = 0; + +void fail_test(const char *test_name, int line, const std::string &msg) { + std::fprintf(stderr, " [FAIL] %s (line %d): %s\n", + test_name, line, msg.c_str()); + ++g_failures; +} + +#define EXPECT_TRUE(cond, msg) \ + do { \ + if (!(cond)) { \ + fail_test(__func__, __LINE__, (msg)); \ + return; \ + } \ + } while (0) + +#define EXPECT_EQ(actual, expected, msg) \ + do { \ + if (!((actual) == (expected))) { \ + std::ostringstream _oss; \ + _oss << (msg); \ + fail_test(__func__, __LINE__, _oss.str()); \ + return; \ + } \ + } while (0) + +bool write_all_fd(int fd, std::string_view data) { + size_t written = 0; + while (written < data.size()) { + ssize_t n = write(fd, data.data() + written, data.size() - written); + if (n > 0) { + written += static_cast(n); + } else if (n < 0 && errno == EINTR) { + continue; + } else { + return false; + } + } + return true; +} + +class MockHttpServer { +public: + using HandlerFunc = std::function; + + explicit MockHttpServer(HandlerFunc handler) : handler_(std::move(handler)) { + fd_ = socket(AF_INET, SOCK_STREAM, 0); + int one = 1; + setsockopt(fd_, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one)); +#ifdef SO_NOSIGPIPE + setsockopt(fd_, SOL_SOCKET, SO_NOSIGPIPE, &one, sizeof(one)); +#endif + + struct sockaddr_in addr; + std::memset(&addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + addr.sin_port = htons(0); + + if (bind(fd_, reinterpret_cast(&addr), sizeof(addr)) < + 0) { + std::perror("MockHttpServer bind"); + std::exit(1); + } + if (listen(fd_, 64) < 0) { + std::perror("MockHttpServer listen"); + std::exit(1); + } + + struct sockaddr_in bound; + socklen_t len = sizeof(bound); + getsockname(fd_, reinterpret_cast(&bound), &len); + port_ = ntohs(bound.sin_port); + url_ = "http://127.0.0.1:" + std::to_string(port_); + + thread_ = std::thread([this]() { serve_loop(); }); + } + + ~MockHttpServer() { + stop_.store(true); + if (fd_ >= 0) { + shutdown(fd_, SHUT_RDWR); + close(fd_); + fd_ = -1; + } + if (thread_.joinable()) { + thread_.join(); + } + } + + const std::string &url() const { return url_; } + + static void send_response(int fd, + int status_code, + std::string_view content_type, + std::string_view body) { + std::string resp = "HTTP/1.1 " + std::to_string(status_code) + " Status\r\n"; + if (!content_type.empty()) { + resp += "Content-Type: " + std::string(content_type) + "\r\n"; + } + resp += "Content-Length: " + std::to_string(body.size()) + "\r\n"; + resp += "Connection: close\r\n\r\n"; + resp.append(body.data(), body.size()); + write_all_fd(fd, resp); + } + +private: + void serve_loop() { + while (!stop_.load()) { + struct pollfd pfd; + pfd.fd = fd_; + pfd.events = POLLIN; + pfd.revents = 0; + int prc = poll(&pfd, 1, 50); + if (prc <= 0 || !(pfd.revents & POLLIN)) { + continue; + } + + struct sockaddr_in caddr; + socklen_t clen = sizeof(caddr); + int cfd = + accept(fd_, reinterpret_cast(&caddr), &clen); + if (cfd < 0) { + continue; + } + + HttpRequest req; + if (read_req(cfd, req)) { + handler_(req, cfd); + } + close(cfd); + } + } + + static bool read_req(int fd, HttpRequest &req) { + std::string buf; + char tmp[4096]; + size_t hend = std::string::npos; + for (;;) { + hend = buf.find("\r\n\r\n"); + if (hend != std::string::npos) { + break; + } + ssize_t n = read(fd, tmp, sizeof(tmp)); + if (n <= 0) { + return false; + } + buf.append(tmp, static_cast(n)); + } + + std::string_view hblock(buf.data(), hend); + size_t fl = hblock.find("\r\n"); + std::string_view rline = + (fl == std::string_view::npos) ? hblock : hblock.substr(0, fl); + size_t sp1 = rline.find(' '); + if (sp1 == std::string_view::npos) { + return false; + } + req.method = std::string(rline.substr(0, sp1)); + std::string_view rest = rline.substr(sp1 + 1); + size_t sp2 = rest.find(' '); + req.path = std::string( + (sp2 == std::string_view::npos) ? rest : rest.substr(0, sp2)); + + size_t pos = (fl == std::string_view::npos) ? hblock.size() : fl + 2; + long clen = 0; + while (pos < hblock.size()) { + size_t le = hblock.find("\r\n", pos); + std::string_view line = + (le == std::string_view::npos) + ? hblock.substr(pos) + : hblock.substr(pos, le - pos); + pos = (le == std::string_view::npos) ? hblock.size() : le + 2; + size_t colon = line.find(':'); + if (colon != std::string_view::npos) { + std::string k = str_trim(line.substr(0, colon)); + std::string v = str_trim(line.substr(colon + 1)); + if (str_iequals(k, "Content-Length")) { + clen = std::atol(v.c_str()); + } + req.headers.add(k, v); + } + } + + size_t boff = hend + 4; + size_t avail = buf.size() - boff; + size_t needed = clen > 0 ? static_cast(clen) : 0; + if (avail > 0) { + req.body.append(buf.data() + boff, std::min(avail, needed)); + } + while (req.body.size() < needed) { + ssize_t n = read(fd, tmp, std::min(sizeof(tmp), needed - req.body.size())); + if (n <= 0) { + break; + } + req.body.append(tmp, static_cast(n)); + } + return true; + } + + int fd_ = -1; + int port_ = 0; + std::string url_; + std::atomic stop_{false}; + HandlerFunc handler_; + std::thread thread_; +}; + +std::filesystem::path make_temp_dir(std::string_view prefix) { + auto base = std::filesystem::temp_directory_path(); + auto now = std::chrono::steady_clock::now().time_since_epoch().count(); + auto path = + base / (std::string(prefix) + "-" + std::to_string(getpid()) + "-" + + std::to_string(now)); + std::filesystem::create_directories(path); + return path; +} + +struct TempDirGuard { + std::filesystem::path path; + explicit TempDirGuard(std::string_view prefix) + : path(make_temp_dir(prefix)) {} + ~TempDirGuard() { + std::error_code ec; + std::filesystem::remove_all(path, ec); + } +}; + +void TestURLBuilding() { + struct TestCase { + std::string endpoint; + std::string req_path; + std::string expected; + }; + + std::vector tests = { + {"https://api.openai.com/v1", "/v1/chat/completions", + "https://api.openai.com/v1/chat/completions"}, + {"https://api.openai.com/v1/", "/v1/chat/completions", + "https://api.openai.com/v1/chat/completions"}, + {"https://api.openai.com", "/v1/chat/completions", + "https://api.openai.com/v1/chat/completions"}, + {"http://localhost:8000/api/v1", "/v1/chat/completions", + "http://localhost:8000/api/v1/chat/completions"}, + {"http://localhost:11434", "/v1/chat/completions", + "http://localhost:11434/v1/chat/completions"}, + {"https://custom.endpoint/v1/custom", "/v1/chat/completions", + "https://custom.endpoint/v1/custom/v1/chat/completions"}, + }; + + for (const auto &tc : tests) { + std::string got = build_url(tc.endpoint, tc.req_path); + EXPECT_EQ(got, tc.expected, + "build_url(" + tc.endpoint + ", " + tc.req_path + ") = " + got + + "; want " + tc.expected); + } +} + +void TestTrialOrdering() { + std::vector configs = { + {"gpt-4", "key-a1", "ep-a", ""}, + {"gpt-4", "key-a2", "ep-a", ""}, + {"gpt-3.5", "key-b1", "ep-b", ""}, + {"claude", "key-c1", "ep-c", ""}, + }; + std::vector unique_models = {"gpt-4", "gpt-3.5", "claude"}; + + // Case 1: "gpt-4" + { + auto trials = get_trial_configs(configs, unique_models, "gpt-4"); + std::vector expected = { + {"gpt-4", "key-a1", "ep-a", ""}, + {"gpt-4", "key-a2", "ep-a", ""}, + {"gpt-3.5", "key-b1", "ep-b", ""}, + {"claude", "key-c1", "ep-c", ""}, + }; + EXPECT_EQ(trials, expected, "Trial ordering for gpt-4 mismatch"); + } + + // Case 2: "gpt-3.5" + { + auto trials = get_trial_configs(configs, unique_models, "gpt-3.5"); + std::vector expected = { + {"gpt-3.5", "key-b1", "ep-b", ""}, + {"claude", "key-c1", "ep-c", ""}, + {"gpt-4", "key-a1", "ep-a", ""}, + {"gpt-4", "key-a2", "ep-a", ""}, + }; + EXPECT_EQ(trials, expected, "Trial ordering for gpt-3.5 mismatch"); + } + + // Case 3: "claude" + { + auto trials = get_trial_configs(configs, unique_models, "claude"); + std::vector expected = { + {"claude", "key-c1", "ep-c", ""}, + {"gpt-4", "key-a1", "ep-a", ""}, + {"gpt-4", "key-a2", "ep-a", ""}, + {"gpt-3.5", "key-b1", "ep-b", ""}, + }; + EXPECT_EQ(trials, expected, "Trial ordering for claude mismatch"); + } + + // Case 4: "non-existent" + { + auto trials = get_trial_configs(configs, unique_models, "non-existent"); + std::vector expected = { + {"gpt-4", "key-a1", "ep-a", ""}, + {"gpt-4", "key-a2", "ep-a", ""}, + {"gpt-3.5", "key-b1", "ep-b", ""}, + {"claude", "key-c1", "ep-c", ""}, + }; + EXPECT_EQ(trials, expected, "Trial ordering for non-existent mismatch"); + } +} + +void TestConfigManagerCSVWatcher() { + TempDirGuard tmp("dynagate-test"); + auto csv_file = (tmp.path / "models.csv").string(); + + std::string initial_content = + "model,key,endpoint,extra\n" + "gpt-4,key-1,http://localhost:8001,\"{\"\"User-Agent\"\":\"\"test/1.0\"\"}\"\n" + "gpt-3.5,key-2,http://localhost:8002,\n"; + + { + std::ofstream ofs(csv_file, std::ios::binary); + ofs << initial_content; + } + + ConfigManager cm(csv_file); + auto configs = cm.get_configs(); + EXPECT_EQ(configs.size(), 2u, "Expected 2 initial configs"); + EXPECT_EQ(configs[0].model, "gpt-4", "Unexpected model 0"); + EXPECT_EQ(configs[0].extra, "{\"User-Agent\":\"test/1.0\"}", + "Unexpected quoted extra JSON"); + EXPECT_EQ(configs[1].model, "gpt-3.5", "Unexpected model 1"); + + std::atomic stop_flag{false}; + std::thread watcher([&]() { cm.watch(stop_flag); }); + + std::this_thread::sleep_for(std::chrono::milliseconds(1100)); + + std::string updated_content = + "model,key,endpoint\n" + "gpt-4,key-1,http://localhost:8001\n" + "gpt-3.5,key-2,http://localhost:8002\n" + "claude,key-3,http://localhost:8003\n"; + + { + std::ofstream ofs(csv_file, std::ios::binary | std::ios::trunc); + ofs << updated_content; + } + + auto deadline = + std::chrono::steady_clock::now() + std::chrono::seconds(3); + std::vector updated_configs; + while (std::chrono::steady_clock::now() < deadline) { + updated_configs = cm.get_configs(); + if (updated_configs.size() == 3) { + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + + stop_flag.store(true); + if (watcher.joinable()) { + watcher.join(); + } + + EXPECT_EQ(updated_configs.size(), 3u, "Hot-reload failed to load 3 configs"); + EXPECT_EQ(updated_configs[2].model, "claude", + "Hot-reload third model mismatch"); +} + +void TestGatewayCompletionsFailover() { + MockHttpServer server_a([](const HttpRequest &, int fd) { + MockHttpServer::send_response( + fd, 500, "application/json", R"({"error": "Internal Server Error A"})"); + }); + + MockHttpServer server_b([](const HttpRequest &, int fd) { + MockHttpServer::send_response( + fd, 401, "application/json", R"({"error": "Unauthorized B"})"); + }); + + MockHttpServer server_c([](const HttpRequest &req, int fd) { + Json body = Json::parse(req.body); + std::string model_sent = body["model"].str(); + std::string resp_body = + R"({"choices":[{"message":{"content":"Hello from )" + model_sent + + R"("}}]})"; + MockHttpServer::send_response(fd, 200, "application/json", resp_body); + }); + + std::vector configs = { + {"model-1", "key-fail-1", server_a.url(), ""}, + {"model-1", "key-fail-2", server_b.url(), ""}, + {"model-2", "key-success", server_c.url(), ""}, + }; + ConfigManager cm(configs, {"model-1", "model-2"}); + + Json req_body_obj = Json::object({ + {"model", "model-1"}, + {"messages", Json::array({ + Json::object({{"role", "user"}, {"content", "Hi"}}), + })}, + }); + + HttpRequest req; + req.method = "POST"; + req.path = "/v1/chat/completions"; + req.headers.set("Content-Type", "application/json"); + req.body = req_body_obj.dump(); + + RecorderResponseWriter w; + handle_chat_completions(cm, {}, std::chrono::milliseconds(10), req, w); + + EXPECT_EQ(w.status_code(), 200, "Expected status 200 after failover"); + + Json resp_obj = Json::parse(w.body()); + EXPECT_TRUE(resp_obj.contains("choices") && resp_obj["choices"].is_array() && + !resp_obj["choices"].empty(), + "Invalid response choices structure"); + std::string content = + resp_obj["choices"].at(0)["message"]["content"].str(); + EXPECT_EQ(content, "Hello from model-2", + "Expected content 'Hello from model-2', got: " + content); +} + +void TestGatewayCompletionsStreamingFailover() { + MockHttpServer server_a([](const HttpRequest &, int fd) { + MockHttpServer::send_response(fd, 500, "text/plain", "Error"); + }); + + MockHttpServer server_b([](const HttpRequest &, int fd) { + std::string headers = + "HTTP/1.1 200 OK\r\n" + "Content-Type: text/event-stream\r\n" + "Cache-Control: no-cache\r\n" + "Connection: close\r\n\r\n"; + write_all_fd(fd, headers); + write_all_fd(fd, "data: chunk1\n\n"); + write_all_fd(fd, "data: chunk2\n\n"); + }); + + std::vector configs = { + {"model-1", "key-fail", server_a.url(), ""}, + {"model-2", "key-success", server_b.url(), ""}, + }; + ConfigManager cm(configs, {"model-1", "model-2"}); + + Json req_body_obj = Json::object({ + {"model", "model-1"}, + {"stream", true}, + {"messages", Json::array({ + Json::object({{"role", "user"}, {"content", "Hi"}}), + })}, + }); + + HttpRequest req; + req.method = "POST"; + req.path = "/v1/chat/completions"; + req.headers.set("Content-Type", "application/json"); + req.body = req_body_obj.dump(); + + RecorderResponseWriter w; + handle_chat_completions(cm, {}, std::chrono::milliseconds(10), req, w); + + EXPECT_EQ(w.status_code(), 200, "Expected status 200 on stream failover"); + EXPECT_TRUE(w.body().find("data: chunk1") != std::string::npos && + w.body().find("data: chunk2") != std::string::npos, + "Expected streaming chunks to be forwarded, got: " + w.body()); +} + +void TestModelsEndpoint() { + std::vector configs = { + {"model-1", "key-1", "ep-1", ""}, + {"model-1", "key-2", "ep-1", ""}, + {"model-2", "key-3", "ep-2", ""}, + }; + ConfigManager cm(configs, {"model-1", "model-2"}); + + HttpRequest req; + req.method = "GET"; + req.path = "/v1/models"; + + RecorderResponseWriter w; + handle_models(cm, {}, req, w); + + EXPECT_EQ(w.status_code(), 200, "Expected status 200 on /v1/models"); + + Json resp_obj = Json::parse(w.body()); + EXPECT_TRUE(resp_obj.contains("data") && resp_obj["data"].is_array() && + resp_obj["data"].size() == 2, + "Expected 2 models in response data"); + EXPECT_EQ(resp_obj["data"].at(0)["id"].str(), "model-1", + "Expected first model to be model-1"); + EXPECT_EQ(resp_obj["data"].at(1)["id"].str(), "model-2", + "Expected second model to be model-2"); +} + +void TestGatewayAuthentication() { + std::vector configs = { + {"model-1", "key-1", "http://127.0.0.1:1", ""}, + }; + ConfigManager cm(configs, {"model-1"}); + std::vector expected_tokens = { + "secret-token-1", + "secret-token-2", + }; + + // Case 1: No Authorization header + { + HttpRequest req; + req.method = "GET"; + req.path = "/v1/models"; + RecorderResponseWriter w; + handle_models(cm, expected_tokens, req, w); + EXPECT_EQ(w.status_code(), 401, + "Expected 401 Unauthorized for missing token"); + } + + // Case 2: Incorrect Authorization header + { + HttpRequest req; + req.method = "GET"; + req.path = "/v1/models"; + req.headers.set("Authorization", "Bearer wrong-token"); + RecorderResponseWriter w; + handle_models(cm, expected_tokens, req, w); + EXPECT_EQ(w.status_code(), 401, + "Expected 401 Unauthorized for wrong token"); + } + + // Case 3: Correct Authorization header (first token) + { + HttpRequest req; + req.method = "GET"; + req.path = "/v1/models"; + req.headers.set("Authorization", "Bearer secret-token-1"); + RecorderResponseWriter w; + handle_models(cm, expected_tokens, req, w); + EXPECT_EQ(w.status_code(), 200, "Expected 200 OK for correct token 1"); + } + + // Case 3b: Correct Authorization header (second token) + { + HttpRequest req; + req.method = "GET"; + req.path = "/v1/models"; + req.headers.set("Authorization", "Bearer secret-token-2"); + RecorderResponseWriter w; + handle_models(cm, expected_tokens, req, w); + EXPECT_EQ(w.status_code(), 200, "Expected 200 OK for correct token 2"); + } + + // Case 4: Completions endpoint with correct token + { + HttpRequest req; + req.method = "POST"; + req.path = "/v1/chat/completions"; + req.headers.set("Authorization", "Bearer secret-token-2"); + req.headers.set("Content-Type", "application/json"); + req.body = R"({"model":"model-1"})"; + RecorderResponseWriter w; + handle_chat_completions(cm, expected_tokens, std::chrono::milliseconds(0), + req, w); + EXPECT_TRUE(w.status_code() != 401, + "Expected completions request to pass authentication"); + } +} + +void TestCSVUpdater() { + TempDirGuard tmp("dynagate-updater-test"); + auto csv_file = (tmp.path / "models.csv").string(); + + { + std::ofstream ofs(csv_file, std::ios::binary); + ofs << "model,key,endpoint\ngpt-4,key-1,http://localhost:8001\n"; + } + + std::string command = + "echo 'gpt-3.5,key-2,http://localhost:8002' >> " + csv_file; + + std::atomic stop_flag{false}; + std::thread updater([&]() { + start_csv_updater(stop_flag, command, std::chrono::milliseconds(100)); + }); + + auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(2); + bool contains_new_model = false; + while (std::chrono::steady_clock::now() < deadline) { + std::ifstream ifs(csv_file); + std::ostringstream oss; + oss << ifs.rdbuf(); + if (oss.str().find("gpt-3.5") != std::string::npos) { + contains_new_model = true; + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } + + stop_flag.store(true); + if (updater.joinable()) { + updater.join(); + } + + EXPECT_TRUE(contains_new_model, + "Expected CSV file to contain updater-appended model"); +} + +void TestGatewayImageGenerations() { + MockHttpServer server([](const HttpRequest &req, int fd) { + if (req.path != "/v1/images/generations") { + MockHttpServer::send_response(fd, 404, "text/plain", "Not Found"); + return; + } + Json body = Json::parse(req.body); + std::string model_sent = body["model"].str(); + std::string resp = + R"({"created":1589478378,"data":[{"url":"http://image-url/generated-by-)" + + model_sent + R"("}]})"; + MockHttpServer::send_response(fd, 200, "application/json", resp); + }); + + std::vector configs = { + {"dall-e-3", "dalle-key", server.url(), ""}, + }; + ConfigManager cm(configs, {"dall-e-3"}); + + Json req_body = Json::object({ + {"prompt", "a beautiful kitten"}, + {"model", "dall-e-3"}, + }); + + HttpRequest req; + req.method = "POST"; + req.path = "/v1/images/generations"; + req.headers.set("Content-Type", "application/json"); + req.body = req_body.dump(); + + RecorderResponseWriter w; + handle_image_generations(cm, {}, std::chrono::milliseconds(0), req, w); + + EXPECT_EQ(w.status_code(), 200, "Expected 200 OK for image generations"); + Json resp_obj = Json::parse(w.body()); + std::string url_val = resp_obj["data"].at(0)["url"].str(); + EXPECT_EQ(url_val, "http://image-url/generated-by-dall-e-3", + "Unexpected image URL: " + url_val); +} + +void TestGatewayNoAuthHeaderOnEmptyKey() { + std::atomic auth_was_empty{true}; + + MockHttpServer server([&](const HttpRequest &req, int fd) { + if (!req.headers.get("Authorization").empty()) { + auth_was_empty.store(false); + MockHttpServer::send_response(fd, 400, "text/plain", "Unexpected Auth"); + return; + } + MockHttpServer::send_response( + fd, 200, "application/json", + R"({"choices":[{"message":{"content":"Hello"}}]})"); + }); + + std::vector configs = { + {"model-no-key", "", server.url(), ""}, + {"model-no-key-2", "-", server.url(), ""}, + }; + ConfigManager cm(configs, {"model-no-key", "model-no-key-2"}); + + // Case 1: Empty key + { + Json req_body = Json::object({ + {"model", "model-no-key"}, + {"messages", Json::array({ + Json::object({{"role", "user"}, {"content", "Hi"}}), + })}, + }); + HttpRequest req; + req.method = "POST"; + req.path = "/v1/chat/completions"; + req.headers.set("Content-Type", "application/json"); + req.headers.set("Authorization", "Bearer some-client-token"); + req.body = req_body.dump(); + + RecorderResponseWriter w; + handle_chat_completions(cm, {}, std::chrono::milliseconds(0), req, w); + EXPECT_EQ(w.status_code(), 200, "Expected status 200 for empty key"); + EXPECT_TRUE(auth_was_empty.load(), + "Expected no Authorization header on upstream request"); + } + + // Case 2: Dash key + { + Json req_body = Json::object({ + {"model", "model-no-key-2"}, + {"messages", Json::array({ + Json::object({{"role", "user"}, {"content", "Hi"}}), + })}, + }); + HttpRequest req; + req.method = "POST"; + req.path = "/v1/chat/completions"; + req.headers.set("Content-Type", "application/json"); + req.headers.set("Authorization", "Bearer some-client-token"); + req.body = req_body.dump(); + + RecorderResponseWriter w; + handle_chat_completions(cm, {}, std::chrono::milliseconds(0), req, w); + EXPECT_EQ(w.status_code(), 200, "Expected status 200 for dash key"); + EXPECT_TRUE(auth_was_empty.load(), + "Expected no Authorization header on upstream request"); + } +} + +void TestGatewayBlankKey() { + std::string received_auth; + std::mutex mu; + + MockHttpServer server([&](const HttpRequest &req, int fd) { + { + std::lock_guard lock(mu); + received_auth = req.headers.get("Authorization"); + } + if (received_auth != "Bearer") { + MockHttpServer::send_response(fd, 400, "text/plain", "Bad Auth"); + return; + } + MockHttpServer::send_response( + fd, 200, "application/json", + R"({"choices":[{"message":{"content":"Hello"}}]})"); + }); + + std::vector configs = { + {"model-blank-key", "-blank-", server.url(), ""}, + }; + ConfigManager cm(configs, {"model-blank-key"}); + + Json req_body = Json::object({ + {"model", "model-blank-key"}, + {"messages", Json::array({ + Json::object({{"role", "user"}, {"content", "Hi"}}), + })}, + }); + HttpRequest req; + req.method = "POST"; + req.path = "/v1/chat/completions"; + req.headers.set("Content-Type", "application/json"); + req.headers.set("Authorization", "Bearer some-client-token"); + req.body = req_body.dump(); + + RecorderResponseWriter w; + handle_chat_completions(cm, {}, std::chrono::milliseconds(0), req, w); + EXPECT_EQ(w.status_code(), 200, "Expected status 200 for -blank- key"); + EXPECT_EQ(received_auth, "Bearer", + "Expected Authorization header 'Bearer', got: " + received_auth); +} + +void TestGatewayExtraHeaders() { + MockHttpServer server([](const HttpRequest &req, int fd) { + if (req.headers.get("X-My-Custom-Header") != "HeaderValue" || + req.headers.get("X-Another-Header") != "123") { + MockHttpServer::send_response(fd, 400, "text/plain", "Missing headers"); + return; + } + MockHttpServer::send_response( + fd, 200, "application/json", + R"({"choices":[{"message":{"content":"Hello"}}]})"); + }); + + std::vector configs = { + {"model-extra-headers", "my-key", server.url(), + R"({"X-My-Custom-Header":"HeaderValue","X-Another-Header":123})"}, + }; + ConfigManager cm(configs, {"model-extra-headers"}); + + Json req_body = Json::object({ + {"model", "model-extra-headers"}, + {"messages", Json::array({ + Json::object({{"role", "user"}, {"content", "Hi"}}), + })}, + }); + HttpRequest req; + req.method = "POST"; + req.path = "/v1/chat/completions"; + req.headers.set("Content-Type", "application/json"); + req.body = req_body.dump(); + + RecorderResponseWriter w; + handle_chat_completions(cm, {}, std::chrono::milliseconds(0), req, w); + EXPECT_EQ(w.status_code(), 200, "Expected status 200 for extra headers"); +} + +void TestMultipleTokensKeyFile() { + std::string key_file_content = "\n token-a \n\ntoken-b\n \ntoken-c\n"; + auto auth_tokens = parse_auth_tokens(key_file_content); + std::vector expected = {"token-a", "token-b", "token-c"}; + EXPECT_EQ(auth_tokens, expected, "Parsed auth tokens mismatch"); + + HttpRequest req_ok; + req_ok.method = "GET"; + req_ok.path = "/v1/models"; + req_ok.headers.set("Authorization", "Bearer token-b"); + EXPECT_TRUE(check_auth(auth_tokens, req_ok), + "Expected token-b to authenticate"); + + HttpRequest req_wrong; + req_wrong.method = "GET"; + req_wrong.path = "/v1/models"; + req_wrong.headers.set("Authorization", "Bearer token-wrong"); + EXPECT_TRUE(!check_auth(auth_tokens, req_wrong), + "Expected token-wrong to fail authentication"); + + auto empty_tokens = parse_auth_tokens("\n \n\n \n"); + EXPECT_TRUE(empty_tokens.empty(), "Expected empty tokens vector"); + HttpRequest req_no_auth; + req_no_auth.method = "GET"; + req_no_auth.path = "/v1/models"; + EXPECT_TRUE(check_auth(empty_tokens, req_no_auth), + "Expected check_auth to return true for empty expected_tokens"); +} + +void TestRequestMessagesAutodetectionAndContentArrays() { + Json received_body; + std::mutex mu; + + MockHttpServer server([&](const HttpRequest &req, int fd) { + { + std::lock_guard lock(mu); + received_body = Json::parse(req.body); + } + MockHttpServer::send_response( + fd, 200, "application/json", + R"({"choices":[{"message":{"content":"OK"}}]})"); + }); + + std::vector configs = { + {"test-model", "test-key", server.url(), ""}, + }; + ConfigManager cm(configs, {"test-model"}); + + // 1. Pi agent format - array content + { + Json req_obj = Json::object({ + {"model", "test-model"}, + {"messages", Json::array({ + Json::object({ + {"role", "user"}, + {"content", Json::array({ + Json::object({ + {"type", "text"}, + {"text", "Hello from Pi agent"}, + }), + })}, + }), + })}, + }); + HttpRequest req; + req.method = "POST"; + req.path = "/v1/chat/completions"; + req.headers.set("Content-Type", "application/json"); + req.body = req_obj.dump(); + + RecorderResponseWriter w; + handle_chat_completions(cm, {}, std::chrono::milliseconds(0), req, w); + EXPECT_EQ(w.status_code(), 200, "Expected 200 OK for Pi agent format"); + EXPECT_EQ(received_body["messages"].size(), 1u, "Expected 1 message"); + EXPECT_EQ(received_body["messages"].at(0)["content"].str(), + "Hello from Pi agent", "Unexpected Pi agent normalized content"); + } + + // 2. Top-level system prompt + { + Json req_obj = Json::object({ + {"model", "test-model"}, + {"system", "You are a helpful coding assistant"}, + {"messages", Json::array({ + Json::object({ + {"role", "user"}, + {"content", "Write a test"}, + }), + })}, + }); + HttpRequest req; + req.method = "POST"; + req.path = "/v1/chat/completions"; + req.headers.set("Content-Type", "application/json"); + req.body = req_obj.dump(); + + RecorderResponseWriter w; + handle_chat_completions(cm, {}, std::chrono::milliseconds(0), req, w); + EXPECT_EQ(received_body["messages"].size(), 2u, "Expected 2 messages"); + EXPECT_EQ(received_body["messages"].at(0)["role"].str(), + "system", "Expected system role first"); + EXPECT_EQ(received_body["messages"].at(0)["content"].str(), + "You are a helpful coding assistant", + "Unexpected system message content"); + EXPECT_EQ(received_body["messages"].at(1)["content"].str(), + "Write a test", "Unexpected user message content"); + EXPECT_TRUE(!received_body.contains("system"), + "Expected top-level system field to be removed"); + } + + // 3. Prompt field autodetection + { + Json req_obj = Json::object({ + {"model", "test-model"}, + {"prompt", "Explain recursion"}, + }); + HttpRequest req; + req.method = "POST"; + req.path = "/v1/chat/completions"; + req.headers.set("Content-Type", "application/json"); + req.body = req_obj.dump(); + + RecorderResponseWriter w; + handle_chat_completions(cm, {}, std::chrono::milliseconds(0), req, w); + EXPECT_EQ(received_body["messages"].size(), 1u, "Expected 1 message"); + EXPECT_EQ(received_body["messages"].at(0)["role"].str(), "user", + "Expected user role"); + EXPECT_EQ(received_body["messages"].at(0)["content"].str(), + "Explain recursion", "Unexpected prompt autodetection content"); + } + + // 4. Multimodal array content preserved + { + Json req_obj = Json::object({ + {"model", "test-model"}, + {"messages", Json::array({ + Json::object({ + {"role", "user"}, + {"content", + Json::array({ + Json::object({ + {"type", "text"}, + {"text", "Describe this image"}, + }), + Json::object({ + {"type", "image_url"}, + {"image_url", + Json::object({ + {"url", "http://example.com/img.png"}, + })}, + }), + })}, + }), + })}, + }); + HttpRequest req; + req.method = "POST"; + req.path = "/v1/chat/completions"; + req.headers.set("Content-Type", "application/json"); + req.body = req_obj.dump(); + + RecorderResponseWriter w; + handle_chat_completions(cm, {}, std::chrono::milliseconds(0), req, w); + EXPECT_TRUE(received_body["messages"].at(0)["content"].is_array() && + received_body["messages"].at(0)["content"].size() == 2, + "Expected multimodal array content to be preserved"); + } +} + +void TestFibonacciBackoff() { + std::vector> expected_seq = { + {0, 0}, {1, 1}, {2, 1}, {3, 2}, {4, 3}, {5, 5}, + {6, 8}, {7, 13}, {8, 21}, {9, 34}, {10, 55}, + }; + for (const auto &p : expected_seq) { + EXPECT_EQ(fibonacci(p.first), p.second, "fibonacci mismatch"); + } +} + +void TestAnthropicMessagesEndpoint() { + Json received_openai_body; + std::mutex mu; + + MockHttpServer server([&](const HttpRequest &req, int fd) { + if (req.path != "/v1/chat/completions") { + MockHttpServer::send_response(fd, 404, "text/plain", "Wrong path"); + return; + } + Json body = Json::parse(req.body); + { + std::lock_guard lock(mu); + received_openai_body = body; + } + bool is_stream = body["stream"].as_bool(false); + if (is_stream) { + std::string headers = + "HTTP/1.1 200 OK\r\n" + "Content-Type: text/event-stream\r\n" + "Connection: close\r\n\r\n"; + write_all_fd(fd, headers); + write_all_fd( + fd, "data: {\"choices\":[{\"delta\":{\"content\":\"Hello \"}}]}\n\n"); + write_all_fd( + fd, + "data: {\"choices\":[{\"delta\":{\"content\":\"Anthropic!\"},\"finish_reason\":\"stop\"}]}\n\n"); + write_all_fd(fd, "data: [DONE]\n\n"); + return; + } + + Json resp = Json::object({ + {"id", "chatcmpl-test"}, + {"model", "claude-model"}, + {"choices", Json::array({ + Json::object({ + {"index", 0}, + {"message", Json::object({ + {"role", "assistant"}, + {"content", "Hello from Anthropic proxy"}, + })}, + {"finish_reason", "stop"}, + }), + })}, + {"usage", Json::object({ + {"prompt_tokens", 10}, + {"completion_tokens", 15}, + {"total_tokens", 25}, + })}, + }); + MockHttpServer::send_response(fd, 200, "application/json", resp.dump()); + }); + + std::vector configs = { + {"claude-model", "test-key", server.url(), ""}, + }; + ConfigManager cm(configs, {"claude-model"}); + + // 1. Non-streaming test + { + Json req_obj = Json::object({ + {"model", "claude-model"}, + {"system", "You are a helpful bot"}, + {"messages", Json::array({ + Json::object({ + {"role", "user"}, + {"content", "Hello"}, + }), + })}, + {"max_tokens", 100}, + }); + HttpRequest req; + req.method = "POST"; + req.path = "/v1/messages"; + req.headers.set("Content-Type", "application/json"); + req.body = req_obj.dump(); + + RecorderResponseWriter w; + handle_messages(cm, {}, std::chrono::milliseconds(0), req, w); + EXPECT_EQ(w.status_code(), 200, "Expected 200 OK for Anthropic messages"); + + Json anthropic_resp = Json::parse(w.body()); + EXPECT_EQ(anthropic_resp["type"].str(), "message", + "Expected type 'message'"); + EXPECT_EQ(anthropic_resp["role"].str(), "assistant", + "Expected role 'assistant'"); + EXPECT_EQ(anthropic_resp["content"].at(0)["text"].str(), + "Hello from Anthropic proxy", "Unexpected Anthropic text"); + EXPECT_EQ(received_openai_body["messages"].size(), 2u, + "Expected 2 messages in OpenAI body"); + EXPECT_EQ(received_openai_body["messages"].at(0)["role"].str(), + "system", "Expected first message to be system"); + } + + // 2. Streaming test + { + Json req_obj = Json::object({ + {"model", "claude-model"}, + {"stream", true}, + {"messages", Json::array({ + Json::object({ + {"role", "user"}, + {"content", "Stream me"}, + }), + })}, + }); + HttpRequest req; + req.method = "POST"; + req.path = "/v1/messages"; + req.headers.set("Content-Type", "application/json"); + req.body = req_obj.dump(); + + RecorderResponseWriter w; + handle_messages(cm, {}, std::chrono::milliseconds(0), req, w); + EXPECT_EQ(w.status_code(), 200, + "Expected 200 OK for streaming Anthropic messages"); + EXPECT_TRUE(w.body().find("event: message_start") != std::string::npos && + w.body().find("event: content_block_delta") != + std::string::npos && + w.body().find("event: message_stop") != std::string::npos, + "Expected Anthropic SSE stream events in response"); + } +} + +void TestOpenAIResponsesEndpoint() { + Json received_openai_body; + std::mutex mu; + + MockHttpServer server([&](const HttpRequest &req, int fd) { + if (req.path != "/v1/chat/completions") { + MockHttpServer::send_response(fd, 404, "text/plain", "Wrong path"); + return; + } + Json body = Json::parse(req.body); + { + std::lock_guard lock(mu); + received_openai_body = body; + } + bool is_stream = body["stream"].as_bool(false); + if (is_stream) { + std::string headers = + "HTTP/1.1 200 OK\r\n" + "Content-Type: text/event-stream\r\n" + "Connection: close\r\n\r\n"; + write_all_fd(fd, headers); + write_all_fd( + fd, + "data: {\"choices\":[{\"delta\":{\"content\":\"Response \"}}]}\n\n"); + write_all_fd( + fd, + "data: {\"choices\":[{\"delta\":{\"content\":\"Chunk\"},\"finish_reason\":\"stop\"}]}\n\n"); + write_all_fd(fd, "data: [DONE]\n\n"); + return; + } + + Json resp = Json::object({ + {"id", "chatcmpl-resp-test"}, + {"model", "gpt-5-model"}, + {"choices", Json::array({ + Json::object({ + {"index", 0}, + {"message", Json::object({ + {"role", "assistant"}, + {"content", "Hello from Responses proxy"}, + })}, + {"finish_reason", "stop"}, + }), + })}, + {"usage", Json::object({ + {"prompt_tokens", 12}, + {"completion_tokens", 18}, + {"total_tokens", 30}, + })}, + }); + MockHttpServer::send_response(fd, 200, "application/json", resp.dump()); + }); + + std::vector configs = { + {"gpt-5-model", "test-key", server.url(), ""}, + }; + ConfigManager cm(configs, {"gpt-5-model"}); + + // 1. Non-streaming test + { + Json req_obj = Json::object({ + {"model", "gpt-5-model"}, + {"instructions", "Be accurate"}, + {"input", "What is 2+2?"}, + }); + HttpRequest req; + req.method = "POST"; + req.path = "/v1/responses"; + req.headers.set("Content-Type", "application/json"); + req.body = req_obj.dump(); + + RecorderResponseWriter w; + handle_responses(cm, {}, std::chrono::milliseconds(0), req, w); + EXPECT_EQ(w.status_code(), 200, "Expected 200 OK for OpenAI Responses"); + + Json resp_obj = Json::parse(w.body()); + EXPECT_EQ(resp_obj["object"].str(), "response", + "Expected object 'response'"); + EXPECT_EQ(resp_obj["status"].str(), "completed", + "Expected status 'completed'"); + EXPECT_EQ( + resp_obj["output"].at(0)["content"].at(0)["text"].str(), + "Hello from Responses proxy", "Unexpected Responses output text"); + EXPECT_EQ(received_openai_body["messages"].size(), 2u, + "Expected 2 messages in OpenAI body"); + EXPECT_EQ(received_openai_body["messages"].at(0)["role"].str(), + "system", "Expected system role for instructions"); + } + + // 2. Streaming test + { + Json req_obj = Json::object({ + {"model", "gpt-5-model"}, + {"stream", true}, + {"input", "Stream test"}, + }); + HttpRequest req; + req.method = "POST"; + req.path = "/v1/responses"; + req.headers.set("Content-Type", "application/json"); + req.body = req_obj.dump(); + + RecorderResponseWriter w; + handle_responses(cm, {}, std::chrono::milliseconds(0), req, w); + EXPECT_EQ(w.status_code(), 200, + "Expected 200 OK for streaming OpenAI Responses"); + EXPECT_TRUE( + w.body().find("event: response.created") != std::string::npos && + w.body().find("event: response.output_text.delta") != + std::string::npos && + w.body().find("event: response.completed") != std::string::npos, + "Expected OpenAI Responses SSE events in response stream"); + } +} + +void TestFullGatewayServerEndToEnd() { + MockHttpServer upstream([](const HttpRequest &req, int fd) { + Json body = Json::parse(req.body); + std::string model_sent = body["model"].str(); + Json resp = Json::object({ + {"id", "chatcmpl-e2e"}, + {"model", model_sent}, + {"choices", Json::array({ + Json::object({ + {"index", 0}, + {"message", Json::object({ + {"role", "assistant"}, + {"content", "E2E OK from " + model_sent}, + })}, + {"finish_reason", "stop"}, + }), + })}, + }); + MockHttpServer::send_response(fd, 200, "application/json", resp.dump()); + }); + + std::vector configs = { + {"e2e-model", "e2e-key", upstream.url(), ""}, + }; + ConfigManager cm(configs, {"e2e-model"}); + + GatewayServer gw(0, cm, {"gw-token"}, std::chrono::milliseconds(0)); + std::string err; + EXPECT_TRUE(gw.start(&err), "GatewayServer start failed: " + err); + std::thread gw_thread([&]() { gw.serve(); }); + + std::string base_url = "http://127.0.0.1:" + std::to_string(gw.bound_port()); + + // Perform a real HTTP GET /v1/models via libcurl + { + CURL *c = curl_easy_init(); + std::string resp_body; + struct curl_slist *hdrs = nullptr; + hdrs = curl_slist_append(hdrs, "Authorization: Bearer gw-token"); + std::string url = base_url + "/v1/models"; + curl_easy_setopt(c, CURLOPT_URL, url.c_str()); + curl_easy_setopt(c, CURLOPT_HTTPHEADER, hdrs); + curl_easy_setopt( + c, CURLOPT_WRITEFUNCTION, + +[](char *p, size_t s, size_t n, void *ud) -> size_t { + static_cast(ud)->append(p, s * n); + return s * n; + }); + curl_easy_setopt(c, CURLOPT_WRITEDATA, &resp_body); + CURLcode rc = curl_easy_perform(c); + long code = 0; + curl_easy_getinfo(c, CURLINFO_RESPONSE_CODE, &code); + curl_slist_free_all(hdrs); + curl_easy_cleanup(c); + + EXPECT_EQ(rc, CURLE_OK, "curl /v1/models failed"); + EXPECT_EQ(code, 200L, "Expected 200 from GatewayServer /v1/models"); + Json j = Json::parse(resp_body); + EXPECT_EQ(j["data"].at(0)["id"].str(), "e2e-model", + "Unexpected model from GatewayServer"); + } + + // Perform a real HTTP POST /v1/chat/completions via libcurl + { + CURL *c = curl_easy_init(); + std::string resp_body; + struct curl_slist *hdrs = nullptr; + hdrs = curl_slist_append(hdrs, "Authorization: Bearer gw-token"); + hdrs = curl_slist_append(hdrs, "Content-Type: application/json"); + std::string payload = + R"({"model":"e2e-model","messages":[{"role":"user","content":"ping"}]})"; + std::string url = base_url + "/v1/chat/completions"; + curl_easy_setopt(c, CURLOPT_URL, url.c_str()); + curl_easy_setopt(c, CURLOPT_POST, 1L); + curl_easy_setopt(c, CURLOPT_POSTFIELDS, payload.c_str()); + curl_easy_setopt(c, CURLOPT_HTTPHEADER, hdrs); + curl_easy_setopt( + c, CURLOPT_WRITEFUNCTION, + +[](char *p, size_t s, size_t n, void *ud) -> size_t { + static_cast(ud)->append(p, s * n); + return s * n; + }); + curl_easy_setopt(c, CURLOPT_WRITEDATA, &resp_body); + CURLcode rc = curl_easy_perform(c); + long code = 0; + curl_easy_getinfo(c, CURLINFO_RESPONSE_CODE, &code); + curl_slist_free_all(hdrs); + curl_easy_cleanup(c); + + EXPECT_EQ(rc, CURLE_OK, "curl /v1/chat/completions failed"); + EXPECT_EQ(code, 200L, + "Expected 200 from GatewayServer /v1/chat/completions"); + Json j = Json::parse(resp_body); + EXPECT_EQ(j["choices"].at(0)["message"]["content"].str(), + "E2E OK from e2e-model", "Unexpected E2E chat completion"); + } + + gw.stop(); + if (gw_thread.joinable()) { + gw_thread.join(); + } +} + +} // namespace + +int main() { + std::signal(SIGPIPE, SIG_IGN); + ensure_curl_initialized(); + + struct TestEntry { + const char *name; + void (*fn)(); + }; + + std::vector tests = { + {"TestURLBuilding", TestURLBuilding}, + {"TestTrialOrdering", TestTrialOrdering}, + {"TestConfigManagerCSVWatcher", TestConfigManagerCSVWatcher}, + {"TestGatewayCompletionsFailover", TestGatewayCompletionsFailover}, + {"TestGatewayCompletionsStreamingFailover", + TestGatewayCompletionsStreamingFailover}, + {"TestModelsEndpoint", TestModelsEndpoint}, + {"TestGatewayAuthentication", TestGatewayAuthentication}, + {"TestCSVUpdater", TestCSVUpdater}, + {"TestGatewayImageGenerations", TestGatewayImageGenerations}, + {"TestGatewayNoAuthHeaderOnEmptyKey", TestGatewayNoAuthHeaderOnEmptyKey}, + {"TestGatewayBlankKey", TestGatewayBlankKey}, + {"TestGatewayExtraHeaders", TestGatewayExtraHeaders}, + {"TestMultipleTokensKeyFile", TestMultipleTokensKeyFile}, + {"TestRequestMessagesAutodetectionAndContentArrays", + TestRequestMessagesAutodetectionAndContentArrays}, + {"TestFibonacciBackoff", TestFibonacciBackoff}, + {"TestAnthropicMessagesEndpoint", TestAnthropicMessagesEndpoint}, + {"TestOpenAIResponsesEndpoint", TestOpenAIResponsesEndpoint}, + {"TestFullGatewayServerEndToEnd", TestFullGatewayServerEndToEnd}, + }; + + for (const auto &t : tests) { + int before = g_failures; + std::printf("=== RUN %s\n", t.name); + t.fn(); + if (g_failures == before) { + std::printf("--- PASS: %s\n", t.name); + } else { + std::printf("--- FAIL: %s\n", t.name); + } + } + + if (g_failures == 0) { + std::printf("PASS\n"); + return 0; + } + std::printf("FAIL (%d failures)\n", g_failures); + return 1; +} diff --git a/handler.cpp b/handler.cpp new file mode 100644 index 0000000..2109c65 --- /dev/null +++ b/handler.cpp @@ -0,0 +1,1887 @@ +/* + Dynagate LLM request proxying handlers (C++20 port) + Created by Luxferre in 2026, released into the public domain +*/ + +#ifndef _DARWIN_C_SOURCE +#define _DARWIN_C_SOURCE +#endif + +#include "dynagate.hpp" + +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace { + +std::once_flag g_curl_init_flag; + +struct ScopedCurl { + CURL *handle = nullptr; + ScopedCurl() : handle(curl_easy_init()) {} + ~ScopedCurl() { + if (handle) { + curl_easy_cleanup(handle); + } + } + ScopedCurl(const ScopedCurl &) = delete; + ScopedCurl &operator=(const ScopedCurl &) = delete; +}; + +struct ScopedCurlSlist { + struct curl_slist *list = nullptr; + ScopedCurlSlist() = default; + ~ScopedCurlSlist() { + if (list) { + curl_slist_free_all(list); + } + } + ScopedCurlSlist(const ScopedCurlSlist &) = delete; + ScopedCurlSlist &operator=(const ScopedCurlSlist &) = delete; + + void append(const std::string &s) { + list = curl_slist_append(list, s.c_str()); + } +}; + +int64_t unix_nano_now() { + auto now = std::chrono::system_clock::now().time_since_epoch(); + return std::chrono::duration_cast(now).count(); +} + +int64_t unix_sec_now() { + auto now = std::chrono::system_clock::now().time_since_epoch(); + return std::chrono::duration_cast(now).count(); +} + +void send_json_error(ResponseWriter &w, + int status_code, + std::string_view message, + std::string_view type, + bool include_code_and_param = false, + std::string_view code = "") { + Json err_obj = Json::object({ + {"message", std::string(message)}, + {"type", std::string(type)}, + }); + if (include_code_and_param) { + err_obj["param"] = nullptr; + err_obj["code"] = std::string(code); + } + Json resp = Json::object({ + {"error", std::move(err_obj)}, + }); + std::string body = resp.dump() + "\n"; + w.header().set("Content-Type", "application/json"); + w.write_header(status_code); + w.write(body); + w.flush(); +} + +bool wait_backoff_or_cancelled(std::chrono::milliseconds delay, + const HttpRequest &req) { + auto start = std::chrono::steady_clock::now(); + while (true) { + if (req.is_cancelled && req.is_cancelled()) { + return false; + } + auto elapsed = std::chrono::steady_clock::now() - start; + if (elapsed >= delay) { + return true; + } + auto remaining = + std::chrono::duration_cast(delay - elapsed); + auto step = std::min(remaining, std::chrono::milliseconds(20)); + if (step.count() > 0) { + std::this_thread::sleep_for(step); + } + } +} + +void build_outgoing_headers(const HttpRequest &req, + const ModelConfig &trial, + size_t trial_num, + ScopedCurlSlist &slist) { + HeaderList out_headers; + for (const auto &kv : req.headers.items) { + std::string k_lower = str_to_lower(kv.first); + if (k_lower == "authorization" || k_lower == "host" || + k_lower == "content-length" || k_lower == "transfer-encoding" || + k_lower == "connection" || k_lower == "expect") { + continue; + } + out_headers.add(kv.first, kv.second); + } + + if (trial.key == "-blank-") { + out_headers.set("Authorization", "Bearer"); + } else if (!trial.key.empty() && trial.key != "-") { + out_headers.set("Authorization", "Bearer " + trial.key); + } + out_headers.set("Content-Type", "application/json"); + + if (!trial.extra.empty()) { + Json extra_headers; + if (!Json::parse(trial.extra, extra_headers)) { + log_msg("Trial %zu: Failed to parse extra headers JSON", trial_num); + } else if (extra_headers.is_object()) { + for (const auto &kv : extra_headers.oval) { + std::string val_str = + kv.second.is_string() ? kv.second.sval : kv.second.dump(); + out_headers.set(kv.first, val_str); + } + } else { + log_msg("Trial %zu: Extra headers JSON is not an object", trial_num); + } + } + + // Disable libcurl's automatic Expect: 100-continue delay + slist.append("Expect:"); + for (const auto &kv : out_headers.items) { + slist.append(kv.first + ": " + kv.second); + } +} + +struct UpstreamTransferCtx { + bool is_stream = false; + int status_code = 0; + bool headers_complete = false; + bool stream_started = false; + bool write_failed = false; + HeaderList resp_headers; + std::string resp_body; + std::string err_body; + std::function on_stream_start; + std::function on_stream_data; +}; + +size_t curl_header_cb(char *buffer, size_t size, size_t nitems, void *userdata) { + size_t total = size * nitems; + auto *ctx = static_cast(userdata); + std::string_view line(buffer, total); + + if (line.size() >= 5 && line.substr(0, 5) == "HTTP/") { + size_t sp = line.find(' '); + if (sp != std::string_view::npos) { + std::string code_str = str_trim(line.substr(sp + 1)); + ctx->status_code = std::atoi(code_str.c_str()); + } + ctx->resp_headers.items.clear(); + ctx->headers_complete = false; + return total; + } + + if (line == "\r\n" || line == "\n") { + if (ctx->status_code >= 200 && ctx->status_code < 300) { + ctx->headers_complete = true; + if (ctx->is_stream && !ctx->stream_started && ctx->on_stream_start) { + ctx->stream_started = true; + ctx->on_stream_start(); + } + } + return total; + } + + if (ctx->status_code >= 100 && ctx->status_code < 200) { + return total; + } + + size_t colon = line.find(':'); + if (colon != std::string_view::npos) { + std::string key = str_trim(line.substr(0, colon)); + std::string val = str_trim(line.substr(colon + 1)); + if (!key.empty()) { + ctx->resp_headers.add(key, val); + } + } + return total; +} + +size_t curl_write_cb(char *ptr, size_t size, size_t nmemb, void *userdata) { + size_t total = size * nmemb; + auto *ctx = static_cast(userdata); + + if (ctx->status_code < 200 || ctx->status_code >= 300) { + if (ctx->err_body.size() < 1024) { + size_t space = 1024 - ctx->err_body.size(); + ctx->err_body.append(ptr, std::min(total, space)); + } + return total; + } + + if (ctx->is_stream) { + if (!ctx->stream_started && ctx->on_stream_start) { + ctx->stream_started = true; + ctx->on_stream_start(); + } + if (ctx->on_stream_data) { + if (!ctx->on_stream_data(std::string_view(ptr, total))) { + ctx->write_failed = true; + return 0; + } + } + return total; + } + + ctx->resp_body.append(ptr, total); + return total; +} + +bool should_skip_upstream_resp_header(std::string_view key, + bool skip_content_type) { + std::string k_lower = str_to_lower(key); + if (k_lower == "transfer-encoding" || k_lower == "content-length" || + k_lower == "connection") { + return true; + } + if (skip_content_type && k_lower == "content-type") { + return true; + } + return false; +} + +} // namespace + +void ensure_curl_initialized() { + std::call_once(g_curl_init_flag, []() { + curl_global_init(CURL_GLOBAL_ALL); + }); +} + +bool check_auth(const std::vector &expected_tokens, + const HttpRequest &req) { + if (expected_tokens.empty()) { + return true; + } + std::string auth_header = req.headers.get("Authorization"); + constexpr std::string_view kBearerPrefix = "Bearer "; + if (auth_header.size() < kBearerPrefix.size() || + std::string_view(auth_header).substr(0, kBearerPrefix.size()) != + kBearerPrefix) { + return false; + } + std::string_view token = + std::string_view(auth_header).substr(kBearerPrefix.size()); + for (const auto &expected : expected_tokens) { + if (token == expected) { + return true; + } + } + return false; +} + +void send_unauthorized(ResponseWriter &w) { + send_json_error(w, 401, "Incorrect API key provided.", + "invalid_request_error", true, "invalid_api_key"); +} + +void handle_models(ConfigManager &cm, + const std::vector &expected_tokens, + const HttpRequest &req, + ResponseWriter &w) { + if (!check_auth(expected_tokens, req)) { + send_unauthorized(w); + return; + } + + if (req.method != "GET") { + send_json_error(w, 405, "Method not allowed", "invalid_request_error"); + return; + } + + auto models = cm.get_unique_models(); + Json data = Json::array(); + for (const auto &m : models) { + data.push_back(Json::object({ + {"id", m}, + {"object", "model"}, + {"created", 1686935002}, + {"owned_by", "dynagate"}, + })); + } + + Json resp = Json::object({ + {"object", "list"}, + {"data", std::move(data)}, + }); + + std::string body = resp.dump() + "\n"; + w.header().set("Content-Type", "application/json"); + w.write_header(200); + w.write(body); + w.flush(); +} + +std::vector get_trial_configs( + const std::vector &configs, + const std::vector &unique_models, + std::string_view requested_model) { + if (configs.empty()) { + return {}; + } + + int req_idx = -1; + for (size_t i = 0; i < unique_models.size(); ++i) { + if (unique_models[i] == requested_model) { + req_idx = static_cast(i); + break; + } + } + + std::vector ordered_models; + ordered_models.reserve(unique_models.size()); + if (req_idx != -1) { + for (size_t i = static_cast(req_idx); i < unique_models.size(); + ++i) { + ordered_models.push_back(unique_models[i]); + } + for (size_t i = 0; i < static_cast(req_idx); ++i) { + ordered_models.push_back(unique_models[i]); + } + } else { + ordered_models = unique_models; + } + + std::vector trial_configs; + trial_configs.reserve(configs.size()); + for (const auto &model_name : ordered_models) { + for (const auto &cfg : configs) { + if (cfg.model == model_name) { + trial_configs.push_back(cfg); + } + } + } + + return trial_configs; +} + +std::string build_url(std::string_view endpoint, std::string_view req_path) { + if (!endpoint.empty() && endpoint.back() == '/') { + endpoint.remove_suffix(1); + } + if (!req_path.empty() && req_path.front() == '/') { + req_path.remove_prefix(1); + } + + if (endpoint.size() >= 3 && + endpoint.substr(endpoint.size() - 3) == "/v1") { + if (req_path.size() >= 3 && req_path.substr(0, 3) == "v1/") { + req_path.remove_prefix(3); + } + } + + return std::string(endpoint) + "/" + std::string(req_path); +} + +int64_t fibonacci(int n) { + if (n <= 0) { + return 0; + } + if (n == 1 || n == 2) { + return 1; + } + int64_t a = 1; + int64_t b = 1; + for (int i = 3; i <= n; ++i) { + int64_t next = a + b; + a = b; + b = next; + } + return b; +} + +bool is_media_content(const Json &m) { + if (!m.is_object()) { + return false; + } + if (m["type"].is_string()) { + std::string t_lower = str_to_lower(m["type"].sval); + if (t_lower == "image_url" || t_lower == "image" || + t_lower == "input_audio" || t_lower == "audio" || + t_lower == "file" || t_lower == "document" || t_lower == "video") { + return true; + } + } + for (const auto &kv : m.oval) { + std::string k_lower = str_to_lower(kv.first); + if (k_lower == "image_url" || k_lower == "input_audio" || + k_lower == "inline_data" || k_lower == "file_data") { + return true; + } + } + return false; +} + +Json normalize_content(const Json &content_any) { + if (content_any.is_null()) { + return nullptr; + } + + if (content_any.is_string()) { + return content_any; + } + + if (content_any.is_array()) { + if (content_any.empty()) { + return ""; + } + bool has_media = false; + for (const auto &item : content_any.aval) { + if (item.is_object() && is_media_content(item)) { + has_media = true; + break; + } + } + + if (has_media) { + Json norm_slice = Json::array(); + for (const auto &item : content_any.aval) { + if (item.is_object()) { + if (item["type"].is_string() && + str_to_lower(item["type"].sval) == "text") { + std::string text_val = item["text"].str(); + norm_slice.push_back(Json::object({ + {"type", "text"}, + {"text", text_val}, + })); + } else { + norm_slice.push_back(item); + } + } else { + norm_slice.push_back(item); + } + } + return norm_slice; + } + + std::vector text_parts; + for (const auto &item : content_any.aval) { + if (item.is_string()) { + text_parts.push_back(item.sval); + } else if (item.is_object()) { + if (item["text"].is_string()) { + text_parts.push_back(item["text"].sval); + } else if (item["content"].is_string()) { + text_parts.push_back(item["content"].sval); + } + } + } + + std::string sb; + for (const auto &part : text_parts) { + if (part.empty()) { + continue; + } + if (!sb.empty()) { + char last_char = sb.back(); + char first_char = part.front(); + if (last_char != '\n' && last_char != ' ' && first_char != '\n' && + first_char != ' ') { + sb.push_back('\n'); + } + } + sb.append(part); + } + return sb; + } + + if (content_any.is_object()) { + if (is_media_content(content_any)) { + return Json::array({content_any}); + } + if (content_any["text"].is_string()) { + return content_any["text"]; + } + if (content_any["content"].is_string()) { + return content_any["content"]; + } + return content_any; + } + + return content_any; +} + +void autodetect_and_normalize_messages(Json &body_map) { + if (!body_map.is_object()) { + return; + } + + Json system_msg = nullptr; + if (body_map.contains("system") && !body_map["system"].is_null()) { + Json sys_text = normalize_content(body_map["system"]); + if (sys_text.is_string() && !sys_text.sval.empty()) { + system_msg = Json::object({ + {"role", "system"}, + {"content", sys_text}, + }); + } else if (sys_text.is_array() && !sys_text.empty()) { + system_msg = Json::object({ + {"role", "system"}, + {"content", sys_text}, + }); + } + body_map.erase("system"); + } + + Json raw_messages = nullptr; + std::string source_key; + + if (body_map.contains("messages") && !body_map["messages"].is_null()) { + raw_messages = body_map["messages"]; + source_key = "messages"; + } else if (body_map.contains("prompt") && !body_map["prompt"].is_null()) { + raw_messages = body_map["prompt"]; + source_key = "prompt"; + body_map.erase("prompt"); + } else if (body_map.contains("contents") && !body_map["contents"].is_null()) { + raw_messages = body_map["contents"]; + source_key = "contents"; + body_map.erase("contents"); + } else if (body_map.contains("input") && !body_map["input"].is_null()) { + raw_messages = body_map["input"]; + source_key = "input"; + body_map.erase("input"); + } + + if (raw_messages.is_null() && system_msg.is_null()) { + return; + } + + std::vector msg_list; + if (raw_messages.is_array()) { + for (const auto &elem : raw_messages.aval) { + if (elem.is_object()) { + msg_list.push_back(elem); + } else if (elem.is_string()) { + msg_list.push_back(Json::object({ + {"role", "user"}, + {"content", elem}, + })); + } + } + } else if (raw_messages.is_object()) { + msg_list.push_back(raw_messages); + } else if (raw_messages.is_string()) { + if (!raw_messages.sval.empty()) { + msg_list.push_back(Json::object({ + {"role", "user"}, + {"content", raw_messages}, + })); + } + } + + Json normalized_list = Json::array(); + bool has_system_in_list = false; + + for (const auto &msg : msg_list) { + std::string role = msg["role"].str(); + if (role.empty()) { + role = "user"; + } + if (role == "system") { + has_system_in_list = true; + } + + Json norm_msg = msg; + norm_msg["role"] = role; + + if (msg.contains("content")) { + norm_msg["content"] = normalize_content(msg["content"]); + } else if (msg.contains("parts")) { + norm_msg["content"] = normalize_content(msg["parts"]); + norm_msg.erase("parts"); + } + + normalized_list.push_back(std::move(norm_msg)); + } + + if (!system_msg.is_null()) { + if (!has_system_in_list) { + normalized_list.aval.insert(normalized_list.aval.begin(), system_msg); + } else if (!normalized_list.empty()) { + auto &first_msg = normalized_list.at(0); + if (first_msg.is_object() && + first_msg.contains("role") && + first_msg["role"].str() == "system") { + Json existing_sys = normalize_content( + first_msg.contains("content") ? first_msg["content"] : Json(nullptr)); + if (system_msg["content"].is_string()) { + std::string sys_str = system_msg["content"].sval; + if (existing_sys.is_string() && !existing_sys.sval.empty()) { + first_msg["content"] = sys_str + "\n" + existing_sys.sval; + } else { + first_msg["content"] = sys_str; + } + } + } + } + } + + if (!normalized_list.empty() || !source_key.empty()) { + body_map["messages"] = std::move(normalized_list); + } +} + +Json convert_anthropic_content_to_openai(const Json &cnt_any) { + if (cnt_any.is_array()) { + Json new_slice = Json::array(); + for (const auto &item : cnt_any.aval) { + if (item.is_object() && + item["type"].str() == "image" && + item["source"].is_object()) { + const auto &src = item["source"]; + std::string media_type = src["media_type"].str(); + if (media_type.empty()) { + media_type = "image/png"; + } + std::string b64_data = src["data"].str(); + std::string data_url = "data:" + media_type + ";base64," + b64_data; + new_slice.push_back(Json::object({ + {"type", "image_url"}, + {"image_url", Json::object({{"url", data_url}})}, + })); + continue; + } + new_slice.push_back(item); + } + return normalize_content(new_slice); + } + return normalize_content(cnt_any); +} + +Json convert_anthropic_to_openai(const Json &body_map) { + Json openai_body = Json::object(); + + if (body_map.contains("model")) { + openai_body["model"] = body_map["model"]; + } + if (body_map["stream"].is_bool()) { + openai_body["stream"] = body_map["stream"]; + } + if (body_map.contains("temperature")) { + openai_body["temperature"] = body_map["temperature"]; + } + if (body_map.contains("top_p")) { + openai_body["top_p"] = body_map["top_p"]; + } + if (body_map.contains("max_tokens")) { + openai_body["max_tokens"] = body_map["max_tokens"]; + } + if (body_map.contains("stop_sequences")) { + openai_body["stop"] = body_map["stop_sequences"]; + } + + Json openai_messages = Json::array(); + + if (body_map.contains("system") && !body_map["system"].is_null()) { + Json sys_content = normalize_content(body_map["system"]); + if (!sys_content.is_null()) { + openai_messages.push_back(Json::object({ + {"role", "system"}, + {"content", sys_content}, + })); + } + } + + if (body_map["messages"].is_array()) { + for (const auto &item : body_map["messages"].aval) { + if (item.is_object()) { + std::string role = item["role"].str(); + if (role.empty()) { + role = "user"; + } + Json cnt = item.contains("content") ? item["content"] : Json(nullptr); + Json norm_cnt = convert_anthropic_content_to_openai(cnt); + openai_messages.push_back(Json::object({ + {"role", role}, + {"content", norm_cnt}, + })); + } + } + } + + openai_body["messages"] = std::move(openai_messages); + autodetect_and_normalize_messages(openai_body); + return openai_body; +} + +Json convert_openai_to_anthropic_response(const Json &openai_resp, + const std::string &requested_model) { + std::string id = openai_resp["id"].str(); + if (id.empty()) { + id = "msg_" + std::to_string(unix_nano_now()); + } else if (id.rfind("msg_", 0) != 0) { + id = "msg_" + id; + } + + std::string model = openai_resp["model"].str(); + if (model.empty()) { + model = requested_model; + } + + std::string text_content; + std::string finish_reason; + + if (openai_resp["choices"].is_array() && !openai_resp["choices"].empty()) { + const auto &choice = openai_resp["choices"].at(0); + if (choice.is_object()) { + finish_reason = choice["finish_reason"].str(); + if (choice["message"].is_object()) { + const auto &msg = choice["message"]; + if (msg["content"].is_string()) { + text_content = msg["content"].sval; + } else if (msg.contains("content")) { + Json cnt_norm = normalize_content(msg["content"]); + if (cnt_norm.is_string()) { + text_content = cnt_norm.sval; + } + } + } + } + } + + std::string stop_reason = "end_turn"; + if (finish_reason == "length") { + stop_reason = "max_tokens"; + } else if (finish_reason == "tool_calls" || + finish_reason == "function_call") { + stop_reason = "tool_use"; + } else if (finish_reason == "stop") { + stop_reason = "end_turn"; + } + + int input_tokens = 0; + int output_tokens = 0; + if (openai_resp["usage"].is_object()) { + const auto &usage = openai_resp["usage"]; + input_tokens = usage["prompt_tokens"].as_int(); + output_tokens = usage["completion_tokens"].as_int(); + } + + return Json::object({ + {"id", id}, + {"type", "message"}, + {"role", "assistant"}, + {"model", model}, + {"content", Json::array({ + Json::object({ + {"type", "text"}, + {"text", text_content}, + }), + })}, + {"stop_reason", stop_reason}, + {"stop_sequence", nullptr}, + {"usage", Json::object({ + {"input_tokens", input_tokens}, + {"output_tokens", output_tokens}, + })}, + }); +} + +Json convert_responses_to_openai(const Json &body_map) { + Json openai_body = Json::object(); + + if (body_map.contains("model")) { + openai_body["model"] = body_map["model"]; + } + if (body_map["stream"].is_bool()) { + openai_body["stream"] = body_map["stream"]; + } + if (body_map.contains("temperature")) { + openai_body["temperature"] = body_map["temperature"]; + } + if (body_map.contains("top_p")) { + openai_body["top_p"] = body_map["top_p"]; + } + if (body_map.contains("max_output_tokens")) { + openai_body["max_tokens"] = body_map["max_output_tokens"]; + } else if (body_map.contains("max_tokens")) { + openai_body["max_tokens"] = body_map["max_tokens"]; + } + + Json openai_messages = Json::array(); + + if (body_map.contains("instructions") && + !body_map["instructions"].is_null()) { + Json sys_content = normalize_content(body_map["instructions"]); + if (!sys_content.is_null()) { + openai_messages.push_back(Json::object({ + {"role", "system"}, + {"content", sys_content}, + })); + } + } + + if (body_map.contains("input") && !body_map["input"].is_null()) { + const auto &inp = body_map["input"]; + if (inp.is_string()) { + openai_messages.push_back(Json::object({ + {"role", "user"}, + {"content", inp}, + })); + } else if (inp.is_array()) { + for (const auto &elem : inp.aval) { + if (elem.is_string()) { + openai_messages.push_back(Json::object({ + {"role", "user"}, + {"content", elem}, + })); + } else if (elem.is_object()) { + std::string role = elem["role"].str(); + if (role.empty()) { + role = "user"; + } + Json cnt = + (elem.contains("content") && !elem["content"].is_null()) + ? elem["content"] + : Json(nullptr); + if (cnt.is_null() && elem["text"].is_string()) { + cnt = elem["text"]; + } + openai_messages.push_back(Json::object({ + {"role", role}, + {"content", normalize_content(cnt)}, + })); + } + } + } else if (inp.is_object()) { + std::string role = inp["role"].str(); + if (role.empty()) { + role = "user"; + } + Json cnt = + (inp.contains("content") && !inp["content"].is_null()) + ? inp["content"] + : Json(nullptr); + if (cnt.is_null() && inp["text"].is_string()) { + cnt = inp["text"]; + } + openai_messages.push_back(Json::object({ + {"role", role}, + {"content", normalize_content(cnt)}, + })); + } + } else if (body_map["messages"].is_array()) { + for (const auto &item : body_map["messages"].aval) { + if (item.is_object()) { + std::string role = item["role"].str(); + if (role.empty()) { + role = "user"; + } + Json cnt = item.contains("content") ? item["content"] : Json(nullptr); + openai_messages.push_back(Json::object({ + {"role", role}, + {"content", normalize_content(cnt)}, + })); + } + } + } + + openai_body["messages"] = std::move(openai_messages); + autodetect_and_normalize_messages(openai_body); + return openai_body; +} + +Json convert_openai_to_responses_response(const Json &openai_resp, + const std::string &requested_model) { + std::string id = openai_resp["id"].str(); + if (id.empty()) { + id = "resp_" + std::to_string(unix_nano_now()); + } else if (id.rfind("resp_", 0) != 0) { + id = "resp_" + id; + } + + std::string model = openai_resp["model"].str(); + if (model.empty()) { + model = requested_model; + } + + std::string text_content; + if (openai_resp["choices"].is_array() && !openai_resp["choices"].empty()) { + const auto &choice = openai_resp["choices"].at(0); + if (choice.is_object() && choice["message"].is_object()) { + const auto &msg = choice["message"]; + if (msg["content"].is_string()) { + text_content = msg["content"].sval; + } else if (msg.contains("content")) { + Json cnt_norm = normalize_content(msg["content"]); + if (cnt_norm.is_string()) { + text_content = cnt_norm.sval; + } + } + } + } + + int64_t created = openai_resp["created"].is_number() + ? openai_resp["created"].as_int64() + : unix_sec_now(); + + int prompt_tokens = 0; + int completion_tokens = 0; + int total_tokens = 0; + if (openai_resp["usage"].is_object()) { + const auto &usage = openai_resp["usage"]; + prompt_tokens = usage["prompt_tokens"].as_int(); + completion_tokens = usage["completion_tokens"].as_int(); + total_tokens = usage["total_tokens"].as_int(); + } + + std::string msg_id = "msg_" + std::to_string(unix_nano_now()); + + return Json::object({ + {"id", id}, + {"object", "response"}, + {"created_at", created}, + {"status", "completed"}, + {"model", model}, + {"output", Json::array({ + Json::object({ + {"id", msg_id}, + {"type", "message"}, + {"status", "completed"}, + {"role", "assistant"}, + {"content", Json::array({ + Json::object({ + {"type", "output_text"}, + {"text", text_content}, + {"annotations", Json::array()}, + {"logprobs", Json::array()}, + }), + })}, + }), + })}, + {"usage", Json::object({ + {"prompt_tokens", prompt_tokens}, + {"completion_tokens", completion_tokens}, + {"total_tokens", total_tokens}, + })}, + }); +} + +void handle_chat_completions( + ConfigManager &cm, + const std::vector &expected_tokens, + std::chrono::milliseconds retry_base_delay, + const HttpRequest &req, + ResponseWriter &w) { + ensure_curl_initialized(); + + if (!check_auth(expected_tokens, req)) { + send_unauthorized(w); + return; + } + + if (req.method != "POST") { + send_json_error(w, 405, "Method not allowed", "invalid_request_error"); + return; + } + + Json body_map; + if (!Json::parse(req.body, body_map) || !body_map.is_object()) { + send_json_error(w, 400, "Invalid JSON in request body", + "invalid_request_error"); + return; + } + + autodetect_and_normalize_messages(body_map); + + std::string requested_model; + if (body_map.contains("model") && body_map["model"].is_string()) { + requested_model = body_map["model"].sval; + } + + auto configs = cm.get_configs(); + auto unique_models = cm.get_unique_models(); + + if (configs.empty()) { + send_json_error(w, 503, "No model configurations loaded", "gateway_error"); + return; + } + + auto trial_configs = + get_trial_configs(configs, unique_models, requested_model); + if (trial_configs.empty()) { + send_json_error(w, 503, "No valid trial configuration candidates", + "gateway_error"); + return; + } + + bool is_stream = false; + if (body_map.contains("stream") && body_map["stream"].is_bool()) { + is_stream = body_map["stream"].as_bool(); + } + + log_msg("Received completion request for model \"%s\" (stream=%s). Found %zu config trials.", + requested_model.c_str(), is_stream ? "true" : "false", + trial_configs.size()); + + for (size_t i = 0; i < trial_configs.size(); ++i) { + const auto &trial = trial_configs[i]; + if (i > 0 && retry_base_delay.count() > 0) { + auto delay = std::chrono::milliseconds( + fibonacci(static_cast(i)) * retry_base_delay.count()); + log_msg("Trial %zu/%zu: Fibonacci backoff delay of %lldms before retry...", + i + 1, trial_configs.size(), + static_cast(delay.count())); + if (!wait_backoff_or_cancelled(delay, req)) { + log_msg("Request context cancelled during retry delay before trial %zu", + i + 1); + send_json_error(w, 499, "Client closed request", "gateway_error"); + return; + } + } + + log_msg("Trial %zu/%zu: model=%s endpoint=%s key_len=%zu", + i + 1, trial_configs.size(), trial.model.c_str(), + trial.endpoint.c_str(), trial.key.size()); + + body_map["model"] = trial.model; + std::string modified_body = body_map.dump(); + std::string target_url = build_url(trial.endpoint, req.path); + + ScopedCurl curl; + if (!curl.handle) { + log_msg("Trial %zu: Failed to initialize curl handle", i + 1); + continue; + } + + ScopedCurlSlist headers_slist; + build_outgoing_headers(req, trial, i + 1, headers_slist); + + UpstreamTransferCtx ctx; + ctx.is_stream = is_stream; + if (is_stream) { + ctx.on_stream_start = [&]() { + log_msg("Trial %zu: Connection established with status %d. Proxying response.", + i + 1, ctx.status_code); + for (const auto &kv : ctx.resp_headers.items) { + if (should_skip_upstream_resp_header(kv.first, false)) { + continue; + } + w.header().add(kv.first, kv.second); + } + w.write_header(ctx.status_code); + w.flush(); + }; + ctx.on_stream_data = [&](std::string_view chunk) -> bool { + if (!w.write(chunk)) { + log_msg("Trial %zu: Client disconnected or write error", i + 1); + return false; + } + w.flush(); + return true; + }; + } + + curl_easy_setopt(curl.handle, CURLOPT_NOSIGNAL, 1L); + curl_easy_setopt(curl.handle, CURLOPT_URL, target_url.c_str()); + curl_easy_setopt(curl.handle, CURLOPT_POST, 1L); + curl_easy_setopt(curl.handle, CURLOPT_POSTFIELDS, modified_body.data()); + curl_easy_setopt(curl.handle, CURLOPT_POSTFIELDSIZE, + static_cast(modified_body.size())); + curl_easy_setopt(curl.handle, CURLOPT_HTTPHEADER, headers_slist.list); + curl_easy_setopt(curl.handle, CURLOPT_HEADERFUNCTION, curl_header_cb); + curl_easy_setopt(curl.handle, CURLOPT_HEADERDATA, &ctx); + curl_easy_setopt(curl.handle, CURLOPT_WRITEFUNCTION, curl_write_cb); + curl_easy_setopt(curl.handle, CURLOPT_WRITEDATA, &ctx); + + CURLcode res = curl_easy_perform(curl.handle); + if (res != CURLE_OK) { + if (ctx.stream_started) { + return; + } + log_msg("Trial %zu: Request to %s failed: %s", + i + 1, target_url.c_str(), curl_easy_strerror(res)); + continue; + } + + if (ctx.status_code < 200 || ctx.status_code >= 300) { + std::string trimmed_err = str_trim(ctx.err_body); + log_msg("Trial %zu: Request to %s returned error status %d: %s", + i + 1, target_url.c_str(), ctx.status_code, trimmed_err.c_str()); + continue; + } + + if (is_stream) { + return; + } + + log_msg("Trial %zu: Connection established with status %d. Proxying response.", + i + 1, ctx.status_code); + for (const auto &kv : ctx.resp_headers.items) { + if (should_skip_upstream_resp_header(kv.first, false)) { + continue; + } + w.header().add(kv.first, kv.second); + } + w.write_header(ctx.status_code); + w.write(ctx.resp_body); + w.flush(); + return; + } + + log_msg("All trials failed. Returning Bad Gateway."); + send_json_error(w, 502, "All configured models and keys failed to respond.", + "gateway_error", true, "all_endpoints_failed"); +} + +void handle_image_generations( + ConfigManager &cm, + const std::vector &expected_tokens, + std::chrono::milliseconds retry_base_delay, + const HttpRequest &req, + ResponseWriter &w) { + ensure_curl_initialized(); + + if (!check_auth(expected_tokens, req)) { + send_unauthorized(w); + return; + } + + if (req.method != "POST") { + send_json_error(w, 405, "Method not allowed", "invalid_request_error"); + return; + } + + Json body_map; + if (!Json::parse(req.body, body_map) || !body_map.is_object()) { + send_json_error(w, 400, "Invalid JSON in request body", + "invalid_request_error"); + return; + } + + if (body_map.contains("prompt") && !body_map["prompt"].is_null()) { + body_map["prompt"] = normalize_content(body_map["prompt"]); + } + + std::string requested_model; + if (body_map.contains("model") && body_map["model"].is_string()) { + requested_model = body_map["model"].sval; + } + + auto configs = cm.get_configs(); + auto unique_models = cm.get_unique_models(); + + if (configs.empty()) { + send_json_error(w, 503, "No model configurations loaded", "gateway_error"); + return; + } + + auto trial_configs = + get_trial_configs(configs, unique_models, requested_model); + if (trial_configs.empty()) { + send_json_error(w, 503, "No valid trial configuration candidates", + "gateway_error"); + return; + } + + log_msg("Received image generation request for model \"%s\". Found %zu config trials.", + requested_model.c_str(), trial_configs.size()); + + for (size_t i = 0; i < trial_configs.size(); ++i) { + const auto &trial = trial_configs[i]; + if (i > 0 && retry_base_delay.count() > 0) { + auto delay = std::chrono::milliseconds( + fibonacci(static_cast(i)) * retry_base_delay.count()); + log_msg("Trial %zu/%zu: Fibonacci backoff delay of %lldms before retry...", + i + 1, trial_configs.size(), + static_cast(delay.count())); + if (!wait_backoff_or_cancelled(delay, req)) { + log_msg("Request context cancelled during retry delay before trial %zu", + i + 1); + send_json_error(w, 499, "Client closed request", "gateway_error"); + return; + } + } + + log_msg("Trial %zu/%zu: model=%s endpoint=%s key_len=%zu", + i + 1, trial_configs.size(), trial.model.c_str(), + trial.endpoint.c_str(), trial.key.size()); + + body_map["model"] = trial.model; + std::string modified_body = body_map.dump(); + std::string target_url = build_url(trial.endpoint, req.path); + + ScopedCurl curl; + if (!curl.handle) { + log_msg("Trial %zu: Failed to initialize curl handle", i + 1); + continue; + } + + ScopedCurlSlist headers_slist; + build_outgoing_headers(req, trial, i + 1, headers_slist); + + UpstreamTransferCtx ctx; + ctx.is_stream = false; + + curl_easy_setopt(curl.handle, CURLOPT_NOSIGNAL, 1L); + curl_easy_setopt(curl.handle, CURLOPT_URL, target_url.c_str()); + curl_easy_setopt(curl.handle, CURLOPT_POST, 1L); + curl_easy_setopt(curl.handle, CURLOPT_POSTFIELDS, modified_body.data()); + curl_easy_setopt(curl.handle, CURLOPT_POSTFIELDSIZE, + static_cast(modified_body.size())); + curl_easy_setopt(curl.handle, CURLOPT_HTTPHEADER, headers_slist.list); + curl_easy_setopt(curl.handle, CURLOPT_HEADERFUNCTION, curl_header_cb); + curl_easy_setopt(curl.handle, CURLOPT_HEADERDATA, &ctx); + curl_easy_setopt(curl.handle, CURLOPT_WRITEFUNCTION, curl_write_cb); + curl_easy_setopt(curl.handle, CURLOPT_WRITEDATA, &ctx); + + CURLcode res = curl_easy_perform(curl.handle); + if (res != CURLE_OK) { + log_msg("Trial %zu: Request to %s failed: %s", + i + 1, target_url.c_str(), curl_easy_strerror(res)); + continue; + } + + if (ctx.status_code < 200 || ctx.status_code >= 300) { + std::string trimmed_err = str_trim(ctx.err_body); + log_msg("Trial %zu: Request to %s returned error status %d: %s", + i + 1, target_url.c_str(), ctx.status_code, trimmed_err.c_str()); + continue; + } + + log_msg("Trial %zu: Connection established with status %d. Proxying response.", + i + 1, ctx.status_code); + for (const auto &kv : ctx.resp_headers.items) { + if (should_skip_upstream_resp_header(kv.first, false)) { + continue; + } + w.header().add(kv.first, kv.second); + } + w.write_header(ctx.status_code); + w.write(ctx.resp_body); + w.flush(); + return; + } + + log_msg("All trials failed. Returning Bad Gateway."); + send_json_error(w, 502, "All configured models and keys failed to respond.", + "gateway_error", true, "all_endpoints_failed"); +} + +void handle_messages( + ConfigManager &cm, + const std::vector &expected_tokens, + std::chrono::milliseconds retry_base_delay, + const HttpRequest &req, + ResponseWriter &w) { + ensure_curl_initialized(); + + if (!check_auth(expected_tokens, req)) { + send_unauthorized(w); + return; + } + + if (req.method != "POST") { + send_json_error(w, 405, "Method not allowed", "invalid_request_error"); + return; + } + + Json body_map; + if (!Json::parse(req.body, body_map) || !body_map.is_object()) { + send_json_error(w, 400, "Invalid JSON in request body", + "invalid_request_error"); + return; + } + + Json openai_body_map = convert_anthropic_to_openai(body_map); + + std::string requested_model; + if (openai_body_map.contains("model") && + openai_body_map["model"].is_string()) { + requested_model = openai_body_map["model"].sval; + } + + auto configs = cm.get_configs(); + auto unique_models = cm.get_unique_models(); + + if (configs.empty()) { + send_json_error(w, 503, "No model configurations loaded", "gateway_error"); + return; + } + + auto trial_configs = + get_trial_configs(configs, unique_models, requested_model); + if (trial_configs.empty()) { + send_json_error(w, 503, "No valid trial configuration candidates", + "gateway_error"); + return; + } + + bool is_stream = false; + if (openai_body_map.contains("stream") && + openai_body_map["stream"].is_bool()) { + is_stream = openai_body_map["stream"].as_bool(); + } + + log_msg("Received Anthropic messages request for model \"%s\" (stream=%s). Found %zu config trials.", + requested_model.c_str(), is_stream ? "true" : "false", + trial_configs.size()); + + for (size_t i = 0; i < trial_configs.size(); ++i) { + const auto &trial = trial_configs[i]; + if (i > 0 && retry_base_delay.count() > 0) { + auto delay = std::chrono::milliseconds( + fibonacci(static_cast(i)) * retry_base_delay.count()); + log_msg("Trial %zu/%zu: Fibonacci backoff delay of %lldms before retry...", + i + 1, trial_configs.size(), + static_cast(delay.count())); + if (!wait_backoff_or_cancelled(delay, req)) { + log_msg("Request context cancelled during retry delay before trial %zu", + i + 1); + send_json_error(w, 499, "Client closed request", "gateway_error"); + return; + } + } + + log_msg("Trial %zu/%zu: model=%s endpoint=%s key_len=%zu", + i + 1, trial_configs.size(), trial.model.c_str(), + trial.endpoint.c_str(), trial.key.size()); + + openai_body_map["model"] = trial.model; + std::string modified_body = openai_body_map.dump(); + std::string target_url = build_url(trial.endpoint, "/v1/chat/completions"); + + ScopedCurl curl; + if (!curl.handle) { + log_msg("Trial %zu: Failed to initialize curl handle", i + 1); + continue; + } + + ScopedCurlSlist headers_slist; + build_outgoing_headers(req, trial, i + 1, headers_slist); + + UpstreamTransferCtx ctx; + ctx.is_stream = is_stream; + + std::string sse_line_buf; + std::string final_stop_reason = "end_turn"; + bool sse_done = false; + + auto process_anthropic_sse_line = [&](std::string_view raw_line) -> bool { + std::string line = str_trim(raw_line); + constexpr std::string_view kDataPrefix = "data: "; + if (line.size() < kDataPrefix.size() || + std::string_view(line).substr(0, kDataPrefix.size()) != kDataPrefix) { + return true; + } + std::string data_str = + str_trim(std::string_view(line).substr(kDataPrefix.size())); + if (data_str == "[DONE]") { + sse_done = true; + return true; + } + Json chunk_mut; + if (Json::parse(data_str, chunk_mut) && chunk_mut.is_object()) { + const Json &chunk_map = chunk_mut; + if (chunk_map["choices"].is_array() && !chunk_map["choices"].empty()) { + const auto &choice = chunk_map["choices"].at(0); + if (choice.is_object()) { + if (choice["finish_reason"].is_string()) { + const std::string &fr = choice["finish_reason"].sval; + if (!fr.empty()) { + if (fr == "length") { + final_stop_reason = "max_tokens"; + } else if (fr == "tool_calls" || fr == "function_call") { + final_stop_reason = "tool_use"; + } else if (fr == "stop") { + final_stop_reason = "end_turn"; + } + } + } + if (choice["delta"].is_object()) { + const auto &delta = choice["delta"]; + if (delta["content"].is_string()) { + const std::string &content_str = delta["content"].sval; + if (!content_str.empty()) { + Json delta_obj = Json::object({ + {"type", "content_block_delta"}, + {"index", 0}, + {"delta", Json::object({ + {"type", "text_delta"}, + {"text", content_str}, + })}, + }); + std::string ev = "event: content_block_delta\ndata: " + + delta_obj.dump() + "\n\n"; + if (!w.write(ev)) { + return false; + } + w.flush(); + } + } + } + } + } + } + return true; + }; + + if (is_stream) { + ctx.on_stream_start = [&]() { + log_msg("Trial %zu: Connection established with status %d. Proxying response.", + i + 1, ctx.status_code); + w.header().set("Content-Type", "text/event-stream"); + w.header().set("Cache-Control", "no-cache"); + w.header().set("Connection", "keep-alive"); + w.write_header(200); + w.flush(); + + std::string msg_id = "msg_" + std::to_string(unix_nano_now()); + Json msg_start_obj = Json::object({ + {"type", "message_start"}, + {"message", Json::object({ + {"id", msg_id}, + {"type", "message"}, + {"role", "assistant"}, + {"model", requested_model}, + {"content", Json::array()}, + {"stop_reason", nullptr}, + {"stop_sequence", nullptr}, + {"usage", Json::object({ + {"input_tokens", 0}, + {"output_tokens", 0}, + })}, + })}, + }); + std::string ev1 = + "event: message_start\ndata: " + msg_start_obj.dump() + "\n\n"; + w.write(ev1); + + Json block_start_obj = Json::object({ + {"type", "content_block_start"}, + {"index", 0}, + {"content_block", Json::object({ + {"type", "text"}, + {"text", ""}, + })}, + }); + std::string ev2 = + "event: content_block_start\ndata: " + block_start_obj.dump() + "\n\n"; + w.write(ev2); + w.flush(); + }; + + ctx.on_stream_data = [&](std::string_view chunk) -> bool { + if (sse_done) { + return true; + } + sse_line_buf.append(chunk.data(), chunk.size()); + for (;;) { + size_t nl = sse_line_buf.find('\n'); + if (nl == std::string::npos) { + break; + } + std::string line = sse_line_buf.substr(0, nl); + sse_line_buf.erase(0, nl + 1); + if (!process_anthropic_sse_line(line)) { + return false; + } + if (sse_done) { + break; + } + } + return true; + }; + } + + curl_easy_setopt(curl.handle, CURLOPT_NOSIGNAL, 1L); + curl_easy_setopt(curl.handle, CURLOPT_URL, target_url.c_str()); + curl_easy_setopt(curl.handle, CURLOPT_POST, 1L); + curl_easy_setopt(curl.handle, CURLOPT_POSTFIELDS, modified_body.data()); + curl_easy_setopt(curl.handle, CURLOPT_POSTFIELDSIZE, + static_cast(modified_body.size())); + curl_easy_setopt(curl.handle, CURLOPT_HTTPHEADER, headers_slist.list); + curl_easy_setopt(curl.handle, CURLOPT_HEADERFUNCTION, curl_header_cb); + curl_easy_setopt(curl.handle, CURLOPT_HEADERDATA, &ctx); + curl_easy_setopt(curl.handle, CURLOPT_WRITEFUNCTION, curl_write_cb); + curl_easy_setopt(curl.handle, CURLOPT_WRITEDATA, &ctx); + + CURLcode res = curl_easy_perform(curl.handle); + if (res != CURLE_OK && !ctx.stream_started) { + log_msg("Trial %zu: Request to %s failed: %s", + i + 1, target_url.c_str(), curl_easy_strerror(res)); + continue; + } + + if (ctx.status_code < 200 || ctx.status_code >= 300) { + std::string trimmed_err = str_trim(ctx.err_body); + log_msg("Trial %zu: Request to %s returned error status %d: %s", + i + 1, target_url.c_str(), ctx.status_code, trimmed_err.c_str()); + continue; + } + + if (is_stream) { + if (!sse_done && !sse_line_buf.empty()) { + process_anthropic_sse_line(sse_line_buf); + sse_line_buf.clear(); + } + Json block_stop_obj = Json::object({ + {"type", "content_block_stop"}, + {"index", 0}, + }); + w.write("event: content_block_stop\ndata: " + block_stop_obj.dump() + + "\n\n"); + + Json msg_delta_obj = Json::object({ + {"type", "message_delta"}, + {"delta", Json::object({ + {"stop_reason", final_stop_reason}, + {"stop_sequence", nullptr}, + })}, + {"usage", Json::object({ + {"output_tokens", 0}, + })}, + }); + w.write("event: message_delta\ndata: " + msg_delta_obj.dump() + "\n\n"); + + Json msg_stop_obj = Json::object({ + {"type", "message_stop"}, + }); + w.write("event: message_stop\ndata: " + msg_stop_obj.dump() + "\n\n"); + w.flush(); + return; + } + + log_msg("Trial %zu: Connection established with status %d. Proxying response.", + i + 1, ctx.status_code); + + Json openai_resp; + if (!Json::parse(ctx.resp_body, openai_resp)) { + log_msg("Trial %zu: Failed to unmarshal upstream response JSON", i + 1); + continue; + } + if (!openai_resp.is_object()) { + log_msg("Trial %zu: Upstream response JSON is not an object", i + 1); + continue; + } + + Json anthropic_resp = + convert_openai_to_anthropic_response(openai_resp, requested_model); + + for (const auto &kv : ctx.resp_headers.items) { + if (should_skip_upstream_resp_header(kv.first, true)) { + continue; + } + w.header().add(kv.first, kv.second); + } + w.header().set("Content-Type", "application/json"); + w.write_header(ctx.status_code); + w.write(anthropic_resp.dump() + "\n"); + w.flush(); + return; + } + + log_msg("All trials failed. Returning Bad Gateway."); + send_json_error(w, 502, "All configured models and keys failed to respond.", + "gateway_error", true, "all_endpoints_failed"); +} + +void handle_responses( + ConfigManager &cm, + const std::vector &expected_tokens, + std::chrono::milliseconds retry_base_delay, + const HttpRequest &req, + ResponseWriter &w) { + ensure_curl_initialized(); + + if (!check_auth(expected_tokens, req)) { + send_unauthorized(w); + return; + } + + if (req.method != "POST") { + send_json_error(w, 405, "Method not allowed", "invalid_request_error"); + return; + } + + Json body_map; + if (!Json::parse(req.body, body_map) || !body_map.is_object()) { + send_json_error(w, 400, "Invalid JSON in request body", + "invalid_request_error"); + return; + } + + Json openai_body_map = convert_responses_to_openai(body_map); + + std::string requested_model; + if (openai_body_map.contains("model") && + openai_body_map["model"].is_string()) { + requested_model = openai_body_map["model"].sval; + } + + auto configs = cm.get_configs(); + auto unique_models = cm.get_unique_models(); + + if (configs.empty()) { + send_json_error(w, 503, "No model configurations loaded", "gateway_error"); + return; + } + + auto trial_configs = + get_trial_configs(configs, unique_models, requested_model); + if (trial_configs.empty()) { + send_json_error(w, 503, "No valid trial configuration candidates", + "gateway_error"); + return; + } + + bool is_stream = false; + if (openai_body_map.contains("stream") && + openai_body_map["stream"].is_bool()) { + is_stream = openai_body_map["stream"].as_bool(); + } + + log_msg("Received OpenAI Responses request for model \"%s\" (stream=%s). Found %zu config trials.", + requested_model.c_str(), is_stream ? "true" : "false", + trial_configs.size()); + + for (size_t i = 0; i < trial_configs.size(); ++i) { + const auto &trial = trial_configs[i]; + if (i > 0 && retry_base_delay.count() > 0) { + auto delay = std::chrono::milliseconds( + fibonacci(static_cast(i)) * retry_base_delay.count()); + log_msg("Trial %zu/%zu: Fibonacci backoff delay of %lldms before retry...", + i + 1, trial_configs.size(), + static_cast(delay.count())); + if (!wait_backoff_or_cancelled(delay, req)) { + log_msg("Request context cancelled during retry delay before trial %zu", + i + 1); + send_json_error(w, 499, "Client closed request", "gateway_error"); + return; + } + } + + log_msg("Trial %zu/%zu: model=%s endpoint=%s key_len=%zu", + i + 1, trial_configs.size(), trial.model.c_str(), + trial.endpoint.c_str(), trial.key.size()); + + openai_body_map["model"] = trial.model; + std::string modified_body = openai_body_map.dump(); + std::string target_url = build_url(trial.endpoint, "/v1/chat/completions"); + + ScopedCurl curl; + if (!curl.handle) { + log_msg("Trial %zu: Failed to initialize curl handle", i + 1); + continue; + } + + ScopedCurlSlist headers_slist; + build_outgoing_headers(req, trial, i + 1, headers_slist); + + UpstreamTransferCtx ctx; + ctx.is_stream = is_stream; + + std::string sse_line_buf; + std::string full_text_buf; + std::string resp_id; + std::string msg_id; + bool sse_done = false; + + auto process_responses_sse_line = [&](std::string_view raw_line) -> bool { + std::string line = str_trim(raw_line); + constexpr std::string_view kDataPrefix = "data: "; + if (line.size() < kDataPrefix.size() || + std::string_view(line).substr(0, kDataPrefix.size()) != kDataPrefix) { + return true; + } + std::string data_str = + str_trim(std::string_view(line).substr(kDataPrefix.size())); + if (data_str == "[DONE]") { + sse_done = true; + return true; + } + Json chunk_mut; + if (Json::parse(data_str, chunk_mut) && chunk_mut.is_object()) { + const Json &chunk_map = chunk_mut; + if (chunk_map["choices"].is_array() && !chunk_map["choices"].empty()) { + const auto &choice = chunk_map["choices"].at(0); + if (choice.is_object() && choice["delta"].is_object()) { + const auto &delta = choice["delta"]; + if (delta["content"].is_string()) { + const std::string &content_str = delta["content"].sval; + if (!content_str.empty()) { + full_text_buf.append(content_str); + Json delta_obj = Json::object({ + {"type", "response.output_text.delta"}, + {"delta", content_str}, + }); + std::string ev = "event: response.output_text.delta\ndata: " + + delta_obj.dump() + "\n\n"; + if (!w.write(ev)) { + return false; + } + w.flush(); + } + } + } + } + } + return true; + }; + + if (is_stream) { + ctx.on_stream_start = [&]() { + log_msg("Trial %zu: Connection established with status %d. Proxying response.", + i + 1, ctx.status_code); + w.header().set("Content-Type", "text/event-stream"); + w.header().set("Cache-Control", "no-cache"); + w.header().set("Connection", "keep-alive"); + w.write_header(200); + w.flush(); + + resp_id = "resp_" + std::to_string(unix_nano_now()); + msg_id = "msg_" + std::to_string(unix_nano_now()); + + Json created_obj = Json::object({ + {"type", "response.created"}, + {"response", Json::object({ + {"id", resp_id}, + {"object", "response"}, + {"status", "in_progress"}, + {"model", requested_model}, + })}, + }); + w.write("event: response.created\ndata: " + created_obj.dump() + + "\n\n"); + + Json part_added_obj = Json::object({ + {"type", "response.content_part.added"}, + {"part", Json::object({ + {"type", "output_text"}, + {"text", ""}, + })}, + }); + w.write("event: response.content_part.added\ndata: " + + part_added_obj.dump() + "\n\n"); + w.flush(); + }; + + ctx.on_stream_data = [&](std::string_view chunk) -> bool { + if (sse_done) { + return true; + } + sse_line_buf.append(chunk.data(), chunk.size()); + for (;;) { + size_t nl = sse_line_buf.find('\n'); + if (nl == std::string::npos) { + break; + } + std::string line = sse_line_buf.substr(0, nl); + sse_line_buf.erase(0, nl + 1); + if (!process_responses_sse_line(line)) { + return false; + } + if (sse_done) { + break; + } + } + return true; + }; + } + + curl_easy_setopt(curl.handle, CURLOPT_NOSIGNAL, 1L); + curl_easy_setopt(curl.handle, CURLOPT_URL, target_url.c_str()); + curl_easy_setopt(curl.handle, CURLOPT_POST, 1L); + curl_easy_setopt(curl.handle, CURLOPT_POSTFIELDS, modified_body.data()); + curl_easy_setopt(curl.handle, CURLOPT_POSTFIELDSIZE, + static_cast(modified_body.size())); + curl_easy_setopt(curl.handle, CURLOPT_HTTPHEADER, headers_slist.list); + curl_easy_setopt(curl.handle, CURLOPT_HEADERFUNCTION, curl_header_cb); + curl_easy_setopt(curl.handle, CURLOPT_HEADERDATA, &ctx); + curl_easy_setopt(curl.handle, CURLOPT_WRITEFUNCTION, curl_write_cb); + curl_easy_setopt(curl.handle, CURLOPT_WRITEDATA, &ctx); + + CURLcode res = curl_easy_perform(curl.handle); + if (res != CURLE_OK && !ctx.stream_started) { + log_msg("Trial %zu: Request to %s failed: %s", + i + 1, target_url.c_str(), curl_easy_strerror(res)); + continue; + } + + if (ctx.status_code < 200 || ctx.status_code >= 300) { + std::string trimmed_err = str_trim(ctx.err_body); + log_msg("Trial %zu: Request to %s returned error status %d: %s", + i + 1, target_url.c_str(), ctx.status_code, trimmed_err.c_str()); + continue; + } + + if (is_stream) { + if (!sse_done && !sse_line_buf.empty()) { + process_responses_sse_line(sse_line_buf); + sse_line_buf.clear(); + } + Json completed_obj = Json::object({ + {"type", "response.completed"}, + {"response", Json::object({ + {"id", resp_id}, + {"object", "response"}, + {"status", "completed"}, + {"model", requested_model}, + {"created_at", unix_sec_now()}, + {"output", Json::array({ + Json::object({ + {"id", msg_id}, + {"type", "message"}, + {"status", "completed"}, + {"role", "assistant"}, + {"content", Json::array({ + Json::object({ + {"type", "output_text"}, + {"text", full_text_buf}, + {"annotations", + Json::array()}, + {"logprobs", + Json::array()}, + }), + })}, + }), + })}, + })}, + }); + w.write("event: response.completed\ndata: " + completed_obj.dump() + + "\n\n"); + w.flush(); + return; + } + + log_msg("Trial %zu: Connection established with status %d. Proxying response.", + i + 1, ctx.status_code); + + Json openai_resp; + if (!Json::parse(ctx.resp_body, openai_resp)) { + log_msg("Trial %zu: Failed to unmarshal upstream response JSON", i + 1); + continue; + } + if (!openai_resp.is_object()) { + log_msg("Trial %zu: Upstream response JSON is not an object", i + 1); + continue; + } + + Json responses_resp = + convert_openai_to_responses_response(openai_resp, requested_model); + + for (const auto &kv : ctx.resp_headers.items) { + if (should_skip_upstream_resp_header(kv.first, true)) { + continue; + } + w.header().add(kv.first, kv.second); + } + w.header().set("Content-Type", "application/json"); + w.write_header(ctx.status_code); + w.write(responses_resp.dump() + "\n"); + w.flush(); + return; + } + + log_msg("All trials failed. Returning Bad Gateway."); + send_json_error(w, 502, "All configured models and keys failed to respond.", + "gateway_error", true, "all_endpoints_failed"); +} diff --git a/json.cpp b/json.cpp new file mode 100644 index 0000000..f042189 --- /dev/null +++ b/json.cpp @@ -0,0 +1,746 @@ +/* + Dynagate native RFC 8259 JSON parser and serializer (C++20 port) + Created by Luxferre in 2026, released into the public domain +*/ + +#include "dynagate.hpp" + +#include +#include +#include +#include +#include +#include +#include + +namespace { + +void str_appendf(std::string &s, const char *fmt, ...) { + va_list ap; + va_start(ap, fmt); + va_list cp; + va_copy(cp, ap); + int need = std::vsnprintf(nullptr, 0, fmt, cp); + va_end(cp); + if (need > 0) { + std::string more; + more.resize(static_cast(need)); + std::vsnprintf(more.data(), static_cast(need) + 1, fmt, ap); + s += more; + } + va_end(ap); +} + +int utf8_decode(const unsigned char *p, size_t avail, size_t *plen) { + if (avail == 0) { + *plen = 0; + return 0xFFFD; + } + unsigned char b0 = p[0]; + if (b0 < 0x80) { + *plen = 1; + return b0; + } + if ((b0 & 0xE0) == 0xC0) { + if (avail < 2 || (p[1] & 0xC0) != 0x80) { + *plen = 1; + return 0xFFFD; + } + int cp = ((b0 & 0x1F) << 6) | (p[1] & 0x3F); + if (cp < 0x80) { + *plen = 1; + return 0xFFFD; + } + *plen = 2; + return cp; + } + if ((b0 & 0xF0) == 0xE0) { + if (avail < 3 || (p[1] & 0xC0) != 0x80 || (p[2] & 0xC0) != 0x80) { + *plen = 1; + return 0xFFFD; + } + int cp = ((b0 & 0x0F) << 12) | ((p[1] & 0x3F) << 6) | (p[2] & 0x3F); + if (cp < 0x800 || (cp >= 0xD800 && cp <= 0xDFFF)) { + *plen = 1; + return 0xFFFD; + } + *plen = 3; + return cp; + } + if ((b0 & 0xF8) == 0xF0) { + if (avail < 4 || (p[1] & 0xC0) != 0x80 || (p[2] & 0xC0) != 0x80 || + (p[3] & 0xC0) != 0x80) { + *plen = 1; + return 0xFFFD; + } + int cp = ((b0 & 0x07) << 18) | ((p[1] & 0x3F) << 12) | + ((p[2] & 0x3F) << 6) | (p[3] & 0x3F); + if (cp < 0x10000 || cp > 0x10FFFF) { + *plen = 1; + return 0xFFFD; + } + *plen = 4; + return cp; + } + *plen = 1; + return 0xFFFD; +} + +const Json kNullJson{}; + +void dump_str(std::string &out, std::string_view s) { + out += '"'; + const auto *p = reinterpret_cast(s.data()); + size_t avail = s.size(); + while (avail > 0) { + unsigned char c = p[0]; + if (c < 0x80) { + p++; + avail--; + switch (c) { + case '"': + out += "\\\""; + break; + case '\\': + out += "\\\\"; + break; + case '\b': + out += "\\b"; + break; + case '\f': + out += "\\f"; + break; + case '\n': + out += "\\n"; + break; + case '\r': + out += "\\r"; + break; + case '\t': + out += "\\t"; + break; + default: + if (c < 0x20) { + str_appendf(out, "\\u%04x", static_cast(c)); + } else { + out += static_cast(c); + } + break; + } + } else { + size_t pl = 0; + int cp = utf8_decode(p, avail, &pl); + if (cp == 0xFFFD && pl == 1) { + out += "\xEF\xBF\xBD"; + p++; + avail--; + } else { + out.append(reinterpret_cast(p), pl); + p += pl; + avail -= pl; + } + } + } + out += '"'; +} + +void dump_rec(std::string &out, const Json &j, int indent, int depth) { + switch (j.type) { + case JType::Null: + out += "null"; + break; + case JType::Bool: + out += (j.bval ? "true" : "false"); + break; + case JType::Int: + str_appendf(out, "%lld", static_cast(j.ival)); + break; + case JType::Float: + if (!std::isfinite(j.dval)) { + out += "null"; + } else { + str_appendf(out, "%.15g", j.dval); + } + break; + case JType::Str: + dump_str(out, j.sval); + break; + case JType::Arr: + if (j.aval.empty()) { + out += "[]"; + } else { + out += '['; + for (size_t i = 0; i < j.aval.size(); i++) { + if (i > 0) { + out += ','; + } + if (indent >= 0) { + out += '\n'; + out.append(static_cast(depth + 1) * + static_cast(indent), + ' '); + } + dump_rec(out, j.aval[i], indent, depth + 1); + } + if (indent >= 0) { + out += '\n'; + out.append(static_cast(depth) * static_cast(indent), + ' '); + } + out += ']'; + } + break; + case JType::Obj: + if (j.oval.empty()) { + out += "{}"; + } else { + out += '{'; + for (size_t i = 0; i < j.oval.size(); i++) { + if (i > 0) { + out += ','; + } + if (indent >= 0) { + out += '\n'; + out.append(static_cast(depth + 1) * + static_cast(indent), + ' '); + } + dump_str(out, j.oval[i].first); + out += (indent >= 0) ? ": " : ":"; + dump_rec(out, j.oval[i].second, indent, depth + 1); + } + if (indent >= 0) { + out += '\n'; + out.append(static_cast(depth) * static_cast(indent), + ' '); + } + out += '}'; + } + break; + } +} + +struct JParser { + std::string_view src; + size_t pos = 0; + + void skip_ws() { + while (pos < src.size()) { + char c = src[pos]; + if (c == ' ' || c == '\t' || c == '\n' || c == '\r') { + pos++; + } else { + break; + } + } + } + + static int hex_val(char c) { + if (c >= '0' && c <= '9') { + return c - '0'; + } + if (c >= 'a' && c <= 'f') { + return c - 'a' + 10; + } + if (c >= 'A' && c <= 'F') { + return c - 'A' + 10; + } + return -1; + } + + bool parse_hex4(uint32_t &out) { + if (pos + 4 > src.size()) { + return false; + } + uint32_t v = 0; + for (int i = 0; i < 4; i++) { + int h = hex_val(src[pos + static_cast(i)]); + if (h < 0) { + return false; + } + v = (v << 4) | static_cast(h); + } + pos += 4; + out = v; + return true; + } + + static void append_utf8(std::string &s, uint32_t cp) { + if (cp <= 0x7F) { + s += static_cast(cp); + } else if (cp <= 0x7FF) { + s += static_cast(0xC0 | (cp >> 6)); + s += static_cast(0x80 | (cp & 0x3F)); + } else if (cp <= 0xFFFF) { + s += static_cast(0xE0 | (cp >> 12)); + s += static_cast(0x80 | ((cp >> 6) & 0x3F)); + s += static_cast(0x80 | (cp & 0x3F)); + } else if (cp <= 0x10FFFF) { + s += static_cast(0xF0 | (cp >> 18)); + s += static_cast(0x80 | ((cp >> 12) & 0x3F)); + s += static_cast(0x80 | ((cp >> 6) & 0x3F)); + s += static_cast(0x80 | (cp & 0x3F)); + } else { + s += "\xEF\xBF\xBD"; + } + } + + bool parse_string(std::string &out) { + if (pos >= src.size() || src[pos] != '"') { + return false; + } + pos++; + out.clear(); + while (pos < src.size()) { + auto c = static_cast(src[pos++]); + if (c == '"') { + return true; + } + if (c < 0x20) { + return false; + } + if (c != '\\') { + out += static_cast(c); + continue; + } + if (pos >= src.size()) { + return false; + } + char esc = src[pos++]; + switch (esc) { + case '"': + out += '"'; + break; + case '\\': + out += '\\'; + break; + case '/': + out += '/'; + break; + case 'b': + out += '\b'; + break; + case 'f': + out += '\f'; + break; + case 'n': + out += '\n'; + break; + case 'r': + out += '\r'; + break; + case 't': + out += '\t'; + break; + case 'u': { + uint32_t cp = 0; + if (!parse_hex4(cp)) { + return false; + } + if (cp >= 0xD800 && cp <= 0xDBFF) { + if (pos + 6 <= src.size() && src[pos] == '\\' && + src[pos + 1] == 'u') { + size_t saved = pos; + pos += 2; + uint32_t low = 0; + if (parse_hex4(low) && low >= 0xDC00 && low <= 0xDFFF) { + cp = 0x10000u + (((cp - 0xD800u) << 10) | (low - 0xDC00u)); + } else { + pos = saved; + cp = 0xFFFDu; + } + } else { + cp = 0xFFFDu; + } + } else if (cp >= 0xDC00 && cp <= 0xDFFF) { + cp = 0xFFFDu; + } + append_utf8(out, cp); + break; + } + default: + return false; + } + } + return false; + } + + bool parse_number(Json &out) { + size_t start = pos; + if (src[pos] == '-') { + pos++; + } + if (pos >= src.size()) { + return false; + } + if (src[pos] == '0') { + pos++; + if (pos < src.size() && src[pos] >= '0' && src[pos] <= '9') { + return false; + } + } else if (src[pos] >= '1' && src[pos] <= '9') { + while (pos < src.size() && src[pos] >= '0' && src[pos] <= '9') { + pos++; + } + } else { + return false; + } + bool is_float = false; + if (pos < src.size() && src[pos] == '.') { + is_float = true; + pos++; + if (pos >= src.size() || src[pos] < '0' || src[pos] > '9') { + return false; + } + while (pos < src.size() && src[pos] >= '0' && src[pos] <= '9') { + pos++; + } + } + if (pos < src.size() && (src[pos] == 'e' || src[pos] == 'E')) { + is_float = true; + pos++; + if (pos < src.size() && (src[pos] == '+' || src[pos] == '-')) { + pos++; + } + if (pos >= src.size() || src[pos] < '0' || src[pos] > '9') { + return false; + } + while (pos < src.size() && src[pos] >= '0' && src[pos] <= '9') { + pos++; + } + } + std::string num_str(src.substr(start, pos - start)); + if (!is_float) { + errno = 0; + char *end = nullptr; + long long iv = std::strtoll(num_str.c_str(), &end, 10); + if (errno == 0 && end == num_str.c_str() + num_str.size()) { + out = Json(static_cast(iv)); + return true; + } + } + errno = 0; + char *end = nullptr; + double dv = std::strtod(num_str.c_str(), &end); + if (end != num_str.c_str() + num_str.size() || !std::isfinite(dv)) { + return false; + } + out = Json(dv); + return true; + } + + bool parse_value(Json &out, int depth) { + if (depth > 256) { + return false; + } + skip_ws(); + if (pos >= src.size()) { + return false; + } + char c = src[pos]; + if (c == 'n') { + if (src.substr(pos, 4) != "null") { + return false; + } + pos += 4; + out = Json(nullptr); + return true; + } + if (c == 't') { + if (src.substr(pos, 4) != "true") { + return false; + } + pos += 4; + out = Json(true); + return true; + } + if (c == 'f') { + if (src.substr(pos, 5) != "false") { + return false; + } + pos += 5; + out = Json(false); + return true; + } + if (c == '"') { + std::string s; + if (!parse_string(s)) { + return false; + } + out = Json(std::move(s)); + return true; + } + if (c == '[') { + pos++; + skip_ws(); + out = Json::array(); + if (pos < src.size() && src[pos] == ']') { + pos++; + return true; + } + while (true) { + Json elem; + if (!parse_value(elem, depth + 1)) { + return false; + } + out.aval.push_back(std::move(elem)); + skip_ws(); + if (pos < src.size() && src[pos] == ',') { + pos++; + continue; + } + if (pos < src.size() && src[pos] == ']') { + pos++; + return true; + } + return false; + } + } + if (c == '{') { + pos++; + skip_ws(); + out = Json::object(); + if (pos < src.size() && src[pos] == '}') { + pos++; + return true; + } + while (true) { + skip_ws(); + std::string key; + if (!parse_string(key)) { + return false; + } + skip_ws(); + if (pos >= src.size() || src[pos] != ':') { + return false; + } + pos++; + Json val; + if (!parse_value(val, depth + 1)) { + return false; + } + out[key] = std::move(val); + skip_ws(); + if (pos < src.size() && src[pos] == ',') { + pos++; + continue; + } + if (pos < src.size() && src[pos] == '}') { + pos++; + return true; + } + return false; + } + } + if (c == '-' || (c >= '0' && c <= '9')) { + return parse_number(out); + } + return false; + } +}; + +} // namespace + +Json::Json() = default; +Json::~Json() = default; +Json::Json(const Json &) = default; +Json::Json(Json &&) noexcept = default; +Json &Json::operator=(const Json &) = default; +Json &Json::operator=(Json &&) noexcept = default; + +Json::Json(std::nullptr_t) : type(JType::Null) {} +Json::Json(bool b) : type(JType::Bool), bval(b) {} +Json::Json(int n) : type(JType::Int), ival(n), dval(n) {} +Json::Json(int64_t n) + : type(JType::Int), ival(n), dval(static_cast(n)) {} +Json::Json(double d) + : type(JType::Float), ival(static_cast(d)), dval(d) {} +Json::Json(const char *s) : type(JType::Str), sval(s ? s : "") {} +Json::Json(std::string s) : type(JType::Str), sval(std::move(s)) {} +Json::Json(std::string_view s) : type(JType::Str), sval(s) {} + +Json Json::object() { + Json j; + j.type = JType::Obj; + return j; +} + +Json Json::object( + std::initializer_list> init) { + Json j; + j.type = JType::Obj; + for (const auto &kv : init) { + j[kv.first] = kv.second; + } + return j; +} + +Json Json::array() { + Json j; + j.type = JType::Arr; + return j; +} + +Json Json::array(std::initializer_list init) { + Json j; + j.type = JType::Arr; + j.aval.assign(init.begin(), init.end()); + return j; +} + +bool Json::empty() const { + if (type == JType::Arr) { + return aval.empty(); + } + if (type == JType::Obj) { + return oval.empty(); + } + if (type == JType::Str) { + return sval.empty(); + } + return true; +} + +size_t Json::size() const { + if (type == JType::Arr) { + return aval.size(); + } + if (type == JType::Obj) { + return oval.size(); + } + if (type == JType::Str) { + return sval.size(); + } + return 0; +} + +bool Json::contains(std::string_view k) const { + if (type != JType::Obj) { + return false; + } + for (const auto &kv : oval) { + if (kv.first == k) { + return true; + } + } + return false; +} + +void Json::erase(std::string_view k) { + if (type != JType::Obj) { + return; + } + for (auto it = oval.begin(); it != oval.end(); ++it) { + if (it->first == k) { + oval.erase(it); + return; + } + } +} + +Json &Json::operator[](std::string_view k) { + if (type != JType::Obj) { + *this = Json::object(); + } + for (auto &kv : oval) { + if (kv.first == k) { + return kv.second; + } + } + oval.emplace_back(std::string(k), Json{}); + return oval.back().second; +} + +const Json &Json::operator[](std::string_view k) const { + if (type != JType::Obj) { + return kNullJson; + } + for (const auto &kv : oval) { + if (kv.first == k) { + return kv.second; + } + } + return kNullJson; +} + +Json &Json::at(size_t i) { + return aval[i]; +} + +const Json &Json::at(size_t i) const { + if (type != JType::Arr || i >= aval.size()) { + return kNullJson; + } + return aval[i]; +} + +void Json::push_back(Json v) { + if (type != JType::Arr) { + *this = Json::array(); + } + aval.push_back(std::move(v)); +} + +std::string Json::str(std::string_view def) const { + if (type == JType::Str) { + return sval; + } + return std::string(def); +} + +bool Json::as_bool(bool def) const { + if (type == JType::Bool) { + return bval; + } + return def; +} + +int64_t Json::as_int64(int64_t def) const { + if (type == JType::Int) { + return ival; + } + if (type == JType::Float) { + return static_cast(dval); + } + return def; +} + +int Json::as_int(int def) const { + if (type != JType::Int && type != JType::Float) { + return def; + } + int64_t v = (type == JType::Int) ? ival : static_cast(dval); + if (v > INT_MAX) { + return INT_MAX; + } + if (v < INT_MIN) { + return INT_MIN; + } + return static_cast(v); +} + +std::string Json::dump(int indent) const { + std::string out; + dump_rec(out, *this, indent, 0); + return out; +} + +bool Json::parse(std::string_view src, Json &out) { + JParser p{src, 0}; + Json tmp; + if (!p.parse_value(tmp, 0)) { + return false; + } + p.skip_ws(); + if (p.pos != src.size()) { + return false; + } + out = std::move(tmp); + return true; +} + +Json Json::parse(std::string_view src) { + Json out; + parse(src, out); + return out; +} diff --git a/main.cpp b/main.cpp new file mode 100644 index 0000000..2ba5ad2 --- /dev/null +++ b/main.cpp @@ -0,0 +1,254 @@ +/* + Dynagate LLM Gateway: dynamically change models on the fly (C++20 port) + Created by Luxferre in 2026, released into the public domain +*/ + +#ifndef _DARWIN_C_SOURCE +#define _DARWIN_C_SOURCE +#endif + +#include "dynagate.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +volatile std::sig_atomic_t g_shutdown_requested = 0; + +void handle_signal(int /*sig*/) { + g_shutdown_requested = 1; +} + +void print_usage(const char *prog) { + std::fprintf(stderr, + "Dynagate LLM gateway by Luxferre (https://luxferre.top)\n\n" + "Usage of %s:\n" + " -csv string\n" + " \tPath to the CSV configuration file (default \"models.csv\")\n" + " -csv-update-interval int\n" + " \tInterval in minutes with which to run the CSV updater command (default 10)\n" + " -csv-updater string\n" + " \tCommand to run to update the configuration CSV file\n" + " -key string\n" + " \tPath to plaintext file containing expected auth token\n" + " -port int\n" + " \tPort to listen on (default 8080)\n" + " -retry-delay int\n" + " \tBase delay in milliseconds for Fibonacci failover retries (0 to disable) (default 100)\n", + prog); +} + +bool parse_int_arg(const std::string &val, int &out) { + if (val.empty()) { + return false; + } + char *endptr = nullptr; + long parsed = std::strtol(val.c_str(), &endptr, 10); + if (endptr == val.c_str() || *endptr != '\0') { + return false; + } + out = static_cast(parsed); + return true; +} + +} // namespace + +int main(int argc, char **argv) { + int port = 8080; + std::string csv_path = "models.csv"; + std::string key_path; + std::string csv_updater; + int csv_update_interval = 10; + int retry_delay_ms = 100; + + for (int i = 1; i < argc; ++i) { + std::string arg = argv[i]; + if (arg == "-h" || arg == "-help" || arg == "--help") { + print_usage(argv[0]); + return 0; + } + + std::string flag_name = arg; + std::string flag_val; + bool has_eq = false; + + if (flag_name.rfind("--", 0) == 0) { + flag_name.erase(0, 1); + } + + size_t eq_pos = flag_name.find('='); + if (eq_pos != std::string::npos) { + flag_val = flag_name.substr(eq_pos + 1); + flag_name.resize(eq_pos); + has_eq = true; + } + + auto next_val = [&](std::string &out) -> bool { + if (has_eq) { + out = flag_val; + return true; + } + if (i + 1 >= argc) { + std::fprintf(stderr, "flag needs an argument: %s\n", arg.c_str()); + print_usage(argv[0]); + return false; + } + out = argv[++i]; + return true; + }; + + if (flag_name == "-port") { + std::string v; + if (!next_val(v) || !parse_int_arg(v, port)) { + std::fprintf(stderr, "invalid value for -port\n"); + print_usage(argv[0]); + return 2; + } + } else if (flag_name == "-csv") { + if (!next_val(csv_path)) { + return 2; + } + } else if (flag_name == "-key") { + if (!next_val(key_path)) { + return 2; + } + } else if (flag_name == "-csv-updater") { + if (!next_val(csv_updater)) { + return 2; + } + } else if (flag_name == "-csv-update-interval") { + std::string v; + if (!next_val(v) || !parse_int_arg(v, csv_update_interval)) { + std::fprintf(stderr, "invalid value for -csv-update-interval\n"); + print_usage(argv[0]); + return 2; + } + } else if (flag_name == "-retry-delay") { + std::string v; + if (!next_val(v) || !parse_int_arg(v, retry_delay_ms)) { + std::fprintf(stderr, "invalid value for -retry-delay\n"); + print_usage(argv[0]); + return 2; + } + } else { + std::fprintf(stderr, "flag provided but not defined: %s\n", arg.c_str()); + print_usage(argv[0]); + return 2; + } + } + + std::signal(SIGPIPE, SIG_IGN); + ensure_curl_initialized(); + + auto retry_delay = std::chrono::milliseconds(retry_delay_ms); + + std::vector auth_tokens; + if (!key_path.empty()) { + std::ifstream ifs(key_path, std::ios::binary); + if (!ifs.is_open()) { + log_msg("Failed to read auth token file: %s", key_path.c_str()); + return 1; + } + std::ostringstream oss; + oss << ifs.rdbuf(); + auto tokens = parse_auth_tokens(oss.str()); + if (!tokens.empty()) { + auth_tokens = std::move(tokens); + log_msg( + "Starting Dynagate LLM Gateway. Config: %s, Port: %d, Auth: Required (%zu tokens loaded)", + csv_path.c_str(), port, auth_tokens.size()); + } else { + log_msg( + "Starting Dynagate LLM Gateway. Config: %s, Port: %d, Auth: None (empty key file)", + csv_path.c_str(), port); + } + } else { + log_msg("Starting Dynagate LLM Gateway. Config: %s, Port: %d, Auth: None", + csv_path.c_str(), port); + } + + // Initialize configuration manager + ConfigManager cm(csv_path); + + std::atomic stop_flag{false}; + + // Start background config watcher + std::thread watcher_thread([&]() { + cm.watch(stop_flag); + }); + + // Start background CSV updater if specified + std::thread updater_thread; + if (!csv_updater.empty()) { + int mins = csv_update_interval; + if (mins <= 0) { + log_msg( + "Warning: csv-update-interval must be positive, defaulting to 10 minutes"); + mins = 10; + } + auto interval = std::chrono::minutes(mins); + log_msg("Starting background CSV updater: command=\"%s\", interval=%dm0s", + csv_updater.c_str(), mins); + updater_thread = std::thread([&, interval]() { + start_csv_updater(stop_flag, csv_updater, interval); + }); + } + + GatewayServer server(port, cm, auth_tokens, retry_delay); + std::string listen_err; + if (!server.start(&listen_err)) { + log_msg("Server listen failed: %s", listen_err.c_str()); + stop_flag.store(true); + if (watcher_thread.joinable()) { + watcher_thread.join(); + } + if (updater_thread.joinable()) { + updater_thread.join(); + } + return 1; + } + + struct sigaction sa; + std::memset(&sa, 0, sizeof(sa)); + sa.sa_handler = handle_signal; + sigemptyset(&sa.sa_mask); + sigaction(SIGINT, &sa, nullptr); + sigaction(SIGTERM, &sa, nullptr); + + std::thread server_thread([&]() { + log_msg("Server listening on port %d", server.bound_port()); + server.serve(); + }); + + while (!g_shutdown_requested) { + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } + + log_msg("Shutting down Dynagate server..."); + + stop_flag.store(true); + server.stop(); + + if (server_thread.joinable()) { + server_thread.join(); + } + if (watcher_thread.joinable()) { + watcher_thread.join(); + } + if (updater_thread.joinable()) { + updater_thread.join(); + } + + log_msg("Server exited cleanly"); + return 0; +} diff --git a/models.csv b/models.csv new file mode 100644 index 0000000..a90ae1d --- /dev/null +++ b/models.csv @@ -0,0 +1,93 @@ +model,key,endpoint,extra +01-ai/yi-large,-,https://zsnonsky-nim-proxy.hf.space/v1, +adept/fuyu-8b,-,https://zsnonsky-nim-proxy.hf.space/v1, +ai21labs/jamba-1.5-large-instruct,-,https://zsnonsky-nim-proxy.hf.space/v1, +aisingapore/sea-lion-7b-instruct,-,https://zsnonsky-nim-proxy.hf.space/v1, +bigcode/starcoder2-15b,-,https://zsnonsky-nim-proxy.hf.space/v1, +big-pickle,-,https://opencode.ai/zen/v1,"{""User-Agent"":""opencode/1.18.30""}" +codestral-latest,-,https://api.llm7.io/v1, +cohere/north-mini-code:free,-,https://api.kilo.ai/api/gateway, +databricks/dbrx-instruct,-,https://zsnonsky-nim-proxy.hf.space/v1, +deepseek-ai/deepseek-coder-6.7b-instruct,-,https://zsnonsky-nim-proxy.hf.space/v1, +deepseek-ai/deepseek-v4-flash-0731,-,https://garystu-openai-nim-proxy.hf.space/v1, +deepseek-ai/deepseek-v4-flash-0731,-,https://zsnonsky-nim-proxy.hf.space/v1, +dots-studio/dots-3-note-preview:free,-,https://api.kilo.ai/api/gateway, +google/codegemma-1.1-7b,-,https://zsnonsky-nim-proxy.hf.space/v1, +google/codegemma-7b,-,https://zsnonsky-nim-proxy.hf.space/v1, +google/deplot,-,https://zsnonsky-nim-proxy.hf.space/v1, +google/gemma-2b,-,https://zsnonsky-nim-proxy.hf.space/v1, +google/gemma-3-12b-it,-,https://zsnonsky-nim-proxy.hf.space/v1, +google/gemma-3-4b-it,-,https://zsnonsky-nim-proxy.hf.space/v1, +google/recurrentgemma-2b,-,https://zsnonsky-nim-proxy.hf.space/v1, +ibm/granite-3.0-3b-a800m-instruct,-,https://zsnonsky-nim-proxy.hf.space/v1, +ibm/granite-3.0-8b-instruct,-,https://zsnonsky-nim-proxy.hf.space/v1, +ibm/granite-34b-code-instruct,-,https://zsnonsky-nim-proxy.hf.space/v1, +ibm/granite-8b-code-instruct,-,https://zsnonsky-nim-proxy.hf.space/v1, +inclusionai/ling-3.0-flash-fin:free,-,https://api.kilo.ai/api/gateway, +inclusionai/ling-3.0-flash-sante:free,-,https://api.kilo.ai/api/gateway, +inclusionai/ling-3.0-flash-vl:free,-,https://api.kilo.ai/api/gateway, +kilo-auto/free,-,https://api.kilo.ai/api/gateway, +ling-3.0-flash-fin-free,-,https://opencode.ai/zen/v1,"{""User-Agent"":""opencode/1.18.30""}" +liquid/lfm-2.5-2.6b:free,-,https://api.kilo.ai/api/gateway, +Lorbus/Qwen3.6-27B-int4-AutoRound,-,https://hermes.ai.unturf.com/v1, +meta/codellama-70b,-,https://zsnonsky-nim-proxy.hf.space/v1, +meta/llama2-70b,-,https://zsnonsky-nim-proxy.hf.space/v1, +meta/llama-3.2-11b-vision-instruct,-,https://zsnonsky-nim-proxy.hf.space/v1, +meta/muse-glimmer-30b,-,https://zsnonsky-nim-proxy.hf.space/v1, +microsoft/kosmos-2,-,https://zsnonsky-nim-proxy.hf.space/v1, +microsoft/phi-3.5-moe-instruct,-,https://zsnonsky-nim-proxy.hf.space/v1, +microsoft/phi-3-vision-128k-instruct,-,https://zsnonsky-nim-proxy.hf.space/v1, +mimo-v2.5-free,-,https://opencode.ai/zen/v1,"{""User-Agent"":""opencode/1.18.30""}" +minimax-m2.7,-,https://api.llm7.io/v1, +mistralai/codestral-22b-instruct-v0.1,-,https://zsnonsky-nim-proxy.hf.space/v1, +mistralai/mistral-7b-instruct-v0.3,-,https://zsnonsky-nim-proxy.hf.space/v1, +mistralai/mistral-large-2-instruct,-,https://zsnonsky-nim-proxy.hf.space/v1, +mistralai/mistral-large,-,https://zsnonsky-nim-proxy.hf.space/v1, +mistralai/mistral-nemotron,-,https://zsnonsky-nim-proxy.hf.space/v1, +mistralai/mixtral-8x22b-v0.1,-,https://zsnonsky-nim-proxy.hf.space/v1, +mistral-Nemo-Instruct-2407,-,https://api.llm7.io/v1, +moonshotai/kimi-k2.6,-,https://zsnonsky-nim-proxy.hf.space/v1, +moonshotai/kimi-k3,-,https://zsnonsky-nim-proxy.hf.space/v1, +nex-agi/nex-n2.5-mini:free,-,https://api.kilo.ai/api/gateway, +nex-agi/nex-n2.5-pro:free,-,https://api.kilo.ai/api/gateway, +nvidia/ai-synthetic-video-detector,-,https://zsnonsky-nim-proxy.hf.space/v1, +nvidia/cosmos-reason2-8b,-,https://zsnonsky-nim-proxy.hf.space/v1, +nvidia/ising-calibration-1.5-31b,-,https://zsnonsky-nim-proxy.hf.space/v1, +nvidia/llama-3.1-nemoguard-8b-content-safety,-,https://zsnonsky-nim-proxy.hf.space/v1, +nvidia/llama-3.1-nemoguard-8b-topic-control,-,https://zsnonsky-nim-proxy.hf.space/v1, +nvidia/llama-3.1-nemotron-51b-instruct,-,https://zsnonsky-nim-proxy.hf.space/v1, +nvidia/llama-3.1-nemotron-70b-instruct,-,https://zsnonsky-nim-proxy.hf.space/v1, +nvidia/llama-3.1-nemotron-safety-guard-8b-v3,-,https://zsnonsky-nim-proxy.hf.space/v1, +nvidia/llama-3.1-nemotron-ultra-253b-v1,-,https://zsnonsky-nim-proxy.hf.space/v1, +nvidia/llama3-chatqa-1.5-70b,-,https://zsnonsky-nim-proxy.hf.space/v1, +nvidia/mistral-nemo-minitron-8b-8k-instruct,-,https://zsnonsky-nim-proxy.hf.space/v1, +nvidia/nemotron-3.5-content-safety:free,-,https://api.kilo.ai/api/gateway, +nvidia/nemotron-3.5-content-safety,-,https://zsnonsky-nim-proxy.hf.space/v1, +nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free,-,https://api.kilo.ai/api/gateway, +nvidia/nemotron-3-nano-omni-30b-a3b-reasoning,-,https://zsnonsky-nim-proxy.hf.space/v1, +nvidia/nemotron-3-super-120b-a12b:free,-,https://api.kilo.ai/api/gateway, +nvidia/nemotron-3-super-120b-a12b,-,https://zsnonsky-nim-proxy.hf.space/v1, +nvidia/nemotron-4-340b-instruct,-,https://zsnonsky-nim-proxy.hf.space/v1, +nvidia/nemotron-4-340b-reward,-,https://zsnonsky-nim-proxy.hf.space/v1, +nvidia/nemotron-nano-3-30b-a3b,-,https://zsnonsky-nim-proxy.hf.space/v1, +nvidia/nemotron-parse-2.0,-,https://zsnonsky-nim-proxy.hf.space/v1, +nvidia/nemotron-parse,-,https://zsnonsky-nim-proxy.hf.space/v1, +nvidia/neva-22b,-,https://zsnonsky-nim-proxy.hf.space/v1, +nvidia/nvclip,-,https://zsnonsky-nim-proxy.hf.space/v1, +nvidia/riva-translate-4b-instruct,-,https://zsnonsky-nim-proxy.hf.space/v1, +nvidia/riva-translate-4b-instruct-v1.1,-,https://zsnonsky-nim-proxy.hf.space/v1, +nvidia/riva-translate-4b-instruct-v2,-,https://zsnonsky-nim-proxy.hf.space/v1, +nvidia/vila,-,https://zsnonsky-nim-proxy.hf.space/v1, +nv-mistralai/mistral-nemo-12b-instruct,-,https://zsnonsky-nim-proxy.hf.space/v1, +openai/gpt-oss-20b,-,https://zsnonsky-nim-proxy.hf.space/v1, +openrouter/free,-,https://api.kilo.ai/api/gateway, +poolside/laguna-s-2.1:free,-,https://api.kilo.ai/api/gateway, +poolside/laguna-xs-2.1:free,-,https://api.kilo.ai/api/gateway, +poolside/laguna-xs-2.1,-,https://zsnonsky-nim-proxy.hf.space/v1, +stepfun/step-3.7-flash:free,-,https://api.kilo.ai/api/gateway, +writer/palmyra-creative-122b,-,https://zsnonsky-nim-proxy.hf.space/v1, +writer/palmyra-fin-70b-32k,-,https://zsnonsky-nim-proxy.hf.space/v1, +writer/palmyra-med-70b-32k,-,https://zsnonsky-nim-proxy.hf.space/v1, +writer/palmyra-med-70b,-,https://zsnonsky-nim-proxy.hf.space/v1, +xortron,-,https://darkc0de-chat.hf.space/v1, +zyphra/zamba2-7b-instruct,-,https://zsnonsky-nim-proxy.hf.space/v1, diff --git a/server.cpp b/server.cpp new file mode 100644 index 0000000..8f9d7bf --- /dev/null +++ b/server.cpp @@ -0,0 +1,624 @@ +/* + Dynagate HTTP/1.1 server implementation (C++20 port) + Created by Luxferre in 2026, released into the public domain +*/ + +#ifndef _DARWIN_C_SOURCE +#define _DARWIN_C_SOURCE +#endif + +#include "dynagate.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace { + +constexpr size_t kMaxHeaderBytes = 65536; +constexpr size_t kMaxBodyBytes = 64 * 1024 * 1024; // 64 MB + +bool write_all(int fd, const char *data, size_t len) { + size_t written = 0; + while (written < len) { + ssize_t n = write(fd, data + written, len - written); + if (n > 0) { + written += static_cast(n); + } else if (n < 0 && errno == EINTR) { + continue; + } else { + return false; + } + } + return true; +} + +const char *status_reason_phrase(int code) { + switch (code) { + case 200: + return "OK"; + case 201: + return "Created"; + case 202: + return "Accepted"; + case 204: + return "No Content"; + case 400: + return "Bad Request"; + case 401: + return "Unauthorized"; + case 403: + return "Forbidden"; + case 404: + return "Not Found"; + case 405: + return "Method Not Allowed"; + case 408: + return "Request Timeout"; + case 413: + return "Payload Too Large"; + case 429: + return "Too Many Requests"; + case 499: + return "Client Closed Request"; + case 500: + return "Internal Server Error"; + case 502: + return "Bad Gateway"; + case 503: + return "Service Unavailable"; + case 504: + return "Gateway Timeout"; + default: + return "Status"; + } +} + +class SocketResponseWriter : public ResponseWriter { +public: + explicit SocketResponseWriter(int fd) : fd_(fd) {} + + HeaderList &header() override { return headers_; } + + void write_header(int status_code) override { + if (!header_written_) { + status_code_ = status_code; + header_written_ = true; + } + } + + bool write(std::string_view data) override { + if (!header_written_) { + write_header(200); + } + write_called_ = true; + if (headers_sent_to_socket_) { + if (data.empty()) { + return !write_error_; + } + if (!write_all(fd_, data.data(), data.size())) { + write_error_ = true; + return false; + } + return true; + } + pending_body_.append(data.data(), data.size()); + return true; + } + + void flush() override { + if (write_error_) { + return; + } + if (!header_written_) { + write_header(200); + } + if (!headers_sent_to_socket_) { + headers_sent_to_socket_ = true; + std::string raw; + raw.reserve(512 + pending_body_.size()); + raw.append("HTTP/1.1 "); + raw.append(std::to_string(status_code_)); + raw.push_back(' '); + raw.append(status_reason_phrase(status_code_)); + raw.append("\r\n"); + + for (const auto &kv : headers_.items) { + raw.append(kv.first); + raw.append(": "); + raw.append(kv.second); + raw.append("\r\n"); + } + + if (write_called_) { + if (headers_.get("Content-Length").empty()) { + raw.append("Content-Length: "); + raw.append(std::to_string(pending_body_.size())); + raw.append("\r\n"); + } + if (headers_.get("Connection").empty()) { + raw.append("Connection: close\r\n"); + } + } else { + if (headers_.get("Connection").empty()) { + raw.append("Connection: close\r\n"); + } + } + + raw.append("\r\n"); + if (!pending_body_.empty()) { + raw.append(pending_body_); + pending_body_.clear(); + } + if (!write_all(fd_, raw.data(), raw.size())) { + write_error_ = true; + } + } + } + +private: + int fd_; + int status_code_ = 200; + bool header_written_ = false; + bool write_called_ = false; + bool headers_sent_to_socket_ = false; + bool write_error_ = false; + HeaderList headers_; + std::string pending_body_; +}; + +bool is_socket_disconnected(int fd) { + if (fd < 0) { + return true; + } + struct pollfd pfd; + pfd.fd = fd; + pfd.events = POLLIN | POLLHUP | POLLERR; + pfd.revents = 0; + int rc = poll(&pfd, 1, 0); + if (rc < 0) { + return errno != EINTR; + } + if (rc == 0) { + return false; + } + if (pfd.revents & (POLLHUP | POLLERR | POLLNVAL)) { + return true; + } + if (pfd.revents & POLLIN) { + char ch = 0; + ssize_t n = recv(fd, &ch, 1, MSG_PEEK); + if (n == 0) { + return true; + } + if (n < 0 && errno != EAGAIN && errno != EWOULDBLOCK && errno != EINTR) { + return true; + } + } + return false; +} + +bool read_chunked_body(int fd, + std::string &buffer, + size_t initial_offset, + std::string &out_body) { + size_t pos = initial_offset; + auto ensure_bytes = [&](size_t needed_end) -> bool { + char tmp[4096]; + while (buffer.size() < needed_end) { + ssize_t n = read(fd, tmp, sizeof(tmp)); + if (n > 0) { + buffer.append(tmp, static_cast(n)); + if (buffer.size() > kMaxBodyBytes + kMaxHeaderBytes) { + return false; + } + } else if (n < 0 && errno == EINTR) { + continue; + } else { + return false; + } + } + return true; + }; + + auto ensure_crlf = [&](size_t from, size_t &crlf_pos) -> bool { + char tmp[4096]; + for (;;) { + crlf_pos = buffer.find("\r\n", from); + if (crlf_pos != std::string::npos) { + return true; + } + ssize_t n = read(fd, tmp, sizeof(tmp)); + if (n > 0) { + buffer.append(tmp, static_cast(n)); + if (buffer.size() > kMaxBodyBytes + kMaxHeaderBytes) { + return false; + } + } else if (n < 0 && errno == EINTR) { + continue; + } else { + return false; + } + } + }; + + for (;;) { + size_t line_end = 0; + if (!ensure_crlf(pos, line_end)) { + return false; + } + std::string size_str = str_trim( + std::string_view(buffer).substr(pos, line_end - pos)); + size_t semi = size_str.find(';'); + if (semi != std::string::npos) { + size_str.resize(semi); + } + if (size_str.empty()) { + return false; + } + char *endptr = nullptr; + unsigned long chunk_len = std::strtoul(size_str.c_str(), &endptr, 16); + if (endptr == size_str.c_str() || chunk_len > kMaxBodyBytes) { + return false; + } + pos = line_end + 2; + if (chunk_len == 0) { + size_t trailer_end = 0; + if (!ensure_crlf(pos, trailer_end)) { + return false; + } + return true; + } + if (out_body.size() + chunk_len > kMaxBodyBytes) { + return false; + } + if (!ensure_bytes(pos + chunk_len + 2)) { + return false; + } + out_body.append(buffer.data() + pos, chunk_len); + pos += chunk_len; + if (buffer.substr(pos, 2) != "\r\n") { + return false; + } + pos += 2; + } +} + +bool read_http_request(int fd, HttpRequest &req, int &err_status) { + err_status = 400; + std::string buffer; + buffer.reserve(4096); + char tmp[4096]; + size_t header_end = std::string::npos; + + for (;;) { + header_end = buffer.find("\r\n\r\n"); + if (header_end != std::string::npos) { + break; + } + if (buffer.size() >= kMaxHeaderBytes) { + return false; + } + ssize_t n = read(fd, tmp, sizeof(tmp)); + if (n > 0) { + buffer.append(tmp, static_cast(n)); + } else if (n < 0 && errno == EINTR) { + continue; + } else { + return false; + } + } + + std::string_view header_block(buffer.data(), header_end); + size_t first_line_end = header_block.find("\r\n"); + std::string_view request_line = + (first_line_end == std::string_view::npos) + ? header_block + : header_block.substr(0, first_line_end); + + size_t sp1 = request_line.find(' '); + if (sp1 == std::string_view::npos) { + return false; + } + req.method = std::string(request_line.substr(0, sp1)); + + std::string_view rest = request_line.substr(sp1 + 1); + size_t sp2 = rest.find(' '); + std::string_view raw_target = + (sp2 == std::string_view::npos) ? rest : rest.substr(0, sp2); + + size_t qpos = raw_target.find('?'); + req.path = std::string( + (qpos == std::string_view::npos) ? raw_target : raw_target.substr(0, qpos)); + + size_t pos = (first_line_end == std::string_view::npos) + ? header_block.size() + : first_line_end + 2; + long content_length = 0; + bool is_chunked = false; + bool expect_continue = false; + + while (pos < header_block.size()) { + size_t le = header_block.find("\r\n", pos); + std::string_view line = + (le == std::string_view::npos) + ? header_block.substr(pos) + : header_block.substr(pos, le - pos); + pos = (le == std::string_view::npos) ? header_block.size() : le + 2; + + if (line.empty()) { + continue; + } + size_t colon = line.find(':'); + if (colon == std::string_view::npos) { + continue; + } + std::string key = str_trim(line.substr(0, colon)); + std::string val = str_trim(line.substr(colon + 1)); + if (key.empty()) { + continue; + } + if (str_iequals(key, "Content-Length")) { + content_length = std::atol(val.c_str()); + } else if (str_iequals(key, "Transfer-Encoding")) { + if (str_to_lower(val).find("chunked") != std::string::npos) { + is_chunked = true; + } + } else if (str_iequals(key, "Expect")) { + if (str_iequals(val, "100-continue")) { + expect_continue = true; + } + } + req.headers.add(key, val); + } + + if (expect_continue) { + const char *cont = "HTTP/1.1 100 Continue\r\n\r\n"; + write_all(fd, cont, std::strlen(cont)); + } + + size_t body_offset = header_end + 4; + if (is_chunked) { + if (!read_chunked_body(fd, buffer, body_offset, req.body)) { + return false; + } + return true; + } + + if (content_length < 0) { + return false; + } + if (static_cast(content_length) > kMaxBodyBytes) { + err_status = 413; + return false; + } + + size_t needed = static_cast(content_length); + size_t avail = buffer.size() - body_offset; + req.body.reserve(needed); + size_t to_copy = std::min(avail, needed); + if (to_copy > 0) { + req.body.append(buffer.data() + body_offset, to_copy); + } + + while (req.body.size() < needed) { + size_t rem = needed - req.body.size(); + ssize_t n = read(fd, tmp, std::min(sizeof(tmp), rem)); + if (n > 0) { + req.body.append(tmp, static_cast(n)); + } else if (n < 0 && errno == EINTR) { + continue; + } else { + return false; + } + } + + return true; +} + +} // namespace + +GatewayServer::GatewayServer(int port, + ConfigManager &cm, + std::vector auth_tokens, + std::chrono::milliseconds retry_delay) + : requested_port_(port), + cm_(cm), + auth_tokens_(std::move(auth_tokens)), + retry_delay_(retry_delay) {} + +GatewayServer::~GatewayServer() { + stop(); +} + +bool GatewayServer::start(std::string *err_out) { + ensure_curl_initialized(); + + int fd = socket(AF_INET, SOCK_STREAM, 0); + if (fd < 0) { + if (err_out) { + *err_out = "socket() failed: " + std::string(std::strerror(errno)); + } + return false; + } + + int one = 1; + setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one)); +#ifdef SO_NOSIGPIPE + setsockopt(fd, SOL_SOCKET, SO_NOSIGPIPE, &one, sizeof(one)); +#endif + + struct sockaddr_in addr; + std::memset(&addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = htonl(INADDR_ANY); + addr.sin_port = htons(static_cast(requested_port_)); + + if (bind(fd, reinterpret_cast(&addr), sizeof(addr)) < 0) { + if (err_out) { + *err_out = "bind() failed: " + std::string(std::strerror(errno)); + } + close(fd); + return false; + } + + if (listen(fd, 128) < 0) { + if (err_out) { + *err_out = "listen() failed: " + std::string(std::strerror(errno)); + } + close(fd); + return false; + } + + struct sockaddr_in bound_addr; + socklen_t addr_len = sizeof(bound_addr); + std::memset(&bound_addr, 0, sizeof(bound_addr)); + if (getsockname(fd, reinterpret_cast(&bound_addr), + &addr_len) == 0) { + bound_port_ = ntohs(bound_addr.sin_port); + } else { + bound_port_ = requested_port_; + } + + listen_fd_ = fd; + stopping_.store(false); + return true; +} + +void GatewayServer::serve() { + while (!stopping_.load()) { + int lfd = listen_fd_; + if (lfd < 0) { + break; + } + + struct pollfd pfd; + pfd.fd = lfd; + pfd.events = POLLIN; + pfd.revents = 0; + + int prc = poll(&pfd, 1, 100); + if (prc < 0) { + if (errno == EINTR) { + continue; + } + break; + } + if (prc == 0 || !(pfd.revents & POLLIN)) { + continue; + } + + struct sockaddr_in client_addr; + socklen_t client_len = sizeof(client_addr); + int cfd = accept(lfd, reinterpret_cast(&client_addr), + &client_len); + if (cfd < 0) { + if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) { + continue; + } + if (stopping_.load()) { + break; + } + continue; + } + +#ifdef SO_NOSIGPIPE + int one = 1; + setsockopt(cfd, SOL_SOCKET, SO_NOSIGPIPE, &one, sizeof(one)); +#endif + + { + std::unique_lock lock(active_mu_); + active_cv_.wait(lock, [&]() { + return stopping_.load() || + active_connections_ < kMaxConcurrentConnections; + }); + if (stopping_.load()) { + close(cfd); + break; + } + ++active_connections_; + } + + std::thread([this, cfd]() { + handle_client(cfd); + close(cfd); + { + std::lock_guard lock(active_mu_); + --active_connections_; + } + active_cv_.notify_all(); + }).detach(); + } +} + +void GatewayServer::stop() { + bool expected = false; + if (!stopping_.compare_exchange_strong(expected, true)) { + return; + } + + if (listen_fd_ >= 0) { + shutdown(listen_fd_, SHUT_RDWR); + close(listen_fd_); + listen_fd_ = -1; + } + + active_cv_.notify_all(); + + std::unique_lock lock(active_mu_); + active_cv_.wait_for(lock, std::chrono::seconds(10), [&]() { + return active_connections_ == 0; + }); +} + +void GatewayServer::handle_client(int client_fd) { + HttpRequest req; + req.client_fd = client_fd; + req.is_cancelled = [this, client_fd]() { + return stopping_.load() || is_socket_disconnected(client_fd); + }; + + SocketResponseWriter w(client_fd); + int err_status = 400; + if (!read_http_request(client_fd, req, err_status)) { + if (err_status == 413) { + w.header().set("Content-Type", "text/plain; charset=utf-8"); + w.write_header(413); + w.write("Payload Too Large\n"); + w.flush(); + } + return; + } + + if (req.path == "/v1/models" || req.path == "/models") { + handle_models(cm_, auth_tokens_, req, w); + } else if (req.path == "/v1/chat/completions" || + req.path == "/chat/completions") { + handle_chat_completions(cm_, auth_tokens_, retry_delay_, req, w); + } else if (req.path == "/v1/images/generations" || + req.path == "/images/generations") { + handle_image_generations(cm_, auth_tokens_, retry_delay_, req, w); + } else if (req.path == "/v1/messages" || req.path == "/messages") { + handle_messages(cm_, auth_tokens_, retry_delay_, req, w); + } else if (req.path == "/v1/responses" || req.path == "/responses") { + handle_responses(cm_, auth_tokens_, retry_delay_, req, w); + } else { + w.header().set("Content-Type", "text/plain; charset=utf-8"); + w.write_header(404); + w.write("404 page not found\n"); + w.flush(); + } +}