From 99f3dc32296f825fec94f202da1e9fede1e78cf9 Mon Sep 17 00:00:00 2001 From: Bernard Ladenthin Date: Mon, 13 Jul 2026 01:58:44 +0200 Subject: [PATCH] server: honour per-request reasoning_budget_tokens in chat completions (#23116) * server: honour per-request reasoning_budget_tokens in chat completions The reasoning-budget block in oaicompat_chat_params_parse read only the server-level default (opt.reasoning_budget, typically -1) and the Anthropic-style alias thinking_budget_tokens, but never the canonical reasoning_budget_tokens field from the request body. Because the key was then written into llama_params before the generic body-copy loop ran, the copy loop found the key already present and silently skipped the caller-supplied value. Any per-request override (e.g. 0 to suppress thinking entirely) was therefore discarded. Fix: read reasoning_budget_tokens from the request body first, so the value that reaches the sampling layer is the one the caller intended. Add a unit test in test-chat.cpp that exercises this path via oaicompat_chat_params_parse with a Qwen3 template (which the autoparser detects as a thinking-capable model) and asserts the returned llama_params carries reasoning_budget_tokens == 0. * server: honour per-request reasoning_budget_message in chat completions The reasoning-budget block in oaicompat_chat_params_parse wrote reasoning_budget_message into llama_params straight from the server-level default (opt.reasoning_budget_message) and never read the canonical reasoning_budget_message field from the request body. Because the key was written before the generic body-copy loop ran, that loop found the key already present and silently skipped the caller-supplied value. Any per-request override of the message injected before the end tag when the budget is exhausted was therefore discarded, even though server-task.cpp already reads reasoning_budget_message from that data. This mirrors the reasoning_budget_tokens bug fixed in the previous commit. Fix: read reasoning_budget_message from the request body first, falling back to the server default, so the value that reaches the sampling layer is the one the caller intended. While here, collapse the adjacent reasoning_budget_tokens override to a single json_value() call; json_value already falls back to the default on a missing/null/wrong-type key, so the explicit body.contains() guard was redundant. No behavioral change. Add a unit test in test-chat.cpp that exercises this path via oaicompat_chat_params_parse with a Qwen3 template (which the autoparser detects as a thinking-capable model) and asserts the returned llama_params carries the per-request reasoning_budget_message rather than the server default. * cleanup --------- Co-authored-by: Xuan Son Nguyen --- tests/test-chat.cpp | 67 ++++++++++++++++++++++++++++++++++ tools/server/server-common.cpp | 5 ++- 2 files changed, 70 insertions(+), 2 deletions(-) diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index e1e0a59e6d..fc8df4fb5e 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -5911,6 +5911,71 @@ static void test_developer_role_to_system_workaround() { } } +static void test_reasoning_budget_tokens_per_request() { + LOG_DBG("%s\n", __func__); + // Use Qwen3 template which has ... reasoning markers. + // The autoparser detects them and sets thinking_start/end_tag, which enables + // the reasoning-budget code path in oaicompat_chat_params_parse. + auto tmpls = read_templates("models/templates/Qwen-Qwen3-0.6B.jinja"); + + server_chat_params opt; + opt.tmpls = std::move(tmpls); + opt.use_jinja = true; + opt.enable_thinking = true; + opt.reasoning_budget = -1; + opt.reasoning_format = COMMON_REASONING_FORMAT_NONE; + + // Body with per-request reasoning_budget_tokens=0 (suppress thinking). + json body = { + {"messages", json::array({json{{"role", "user"}, {"content", "hello"}}})}, + {"reasoning_budget_tokens", 0}, + }; + std::vector out_files; + auto llama_params = oaicompat_chat_params_parse(body, opt, out_files); + + // The per-request value must win over the server default (-1). + if (!llama_params.contains("reasoning_budget_tokens")) { + throw std::runtime_error("reasoning_budget_tokens missing from llama_params (thinking_end_tag may be empty for this template)"); + } + int got = llama_params["reasoning_budget_tokens"].get(); + if (got != 0) { + throw std::runtime_error(std::string("Expected reasoning_budget_tokens=0, got ") + std::to_string(got)); + } +} + +static void test_reasoning_budget_message_per_request() { + LOG_DBG("%s\n", __func__); + // Same code path as test_reasoning_budget_tokens_per_request: the Qwen3 template's + // ... markers enable the reasoning-budget block in oaicompat_chat_params_parse. + auto tmpls = read_templates("models/templates/Qwen-Qwen3-0.6B.jinja"); + + server_chat_params opt; + opt.tmpls = std::move(tmpls); + opt.use_jinja = true; + opt.enable_thinking = true; + opt.reasoning_budget = -1; + opt.reasoning_format = COMMON_REASONING_FORMAT_NONE; + opt.reasoning_budget_message = "server default"; + + // Body with a per-request reasoning_budget_message override. + const std::string per_request_message = "per-request message"; + json body = { + {"messages", json::array({json{{"role", "user"}, {"content", "hello"}}})}, + {"reasoning_budget_message", per_request_message}, + }; + std::vector out_files; + auto llama_params = oaicompat_chat_params_parse(body, opt, out_files); + + // The per-request value must win over the server default. + if (!llama_params.contains("reasoning_budget_message")) { + throw std::runtime_error("reasoning_budget_message missing from llama_params (thinking_end_tag may be empty for this template)"); + } + std::string got = llama_params["reasoning_budget_message"].get(); + if (got != per_request_message) { + throw std::runtime_error("Expected reasoning_budget_message='" + per_request_message + "', got '" + got + "'"); + } +} + static void test_msg_diffs_compute() { LOG_DBG("%s\n", __func__); { @@ -6068,6 +6133,8 @@ int main(int argc, char ** argv) { test_convert_responses_to_chatcmpl(); test_developer_role_to_system_workaround(); test_template_generation_prompt(); + test_reasoning_budget_tokens_per_request(); + test_reasoning_budget_message_per_request(); test_template_output_peg_parsers(detailed_debug); std::cout << "\n[chat] All tests passed!" << '\n'; } diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp index c76ed7cc24..78b5c6819b 100644 --- a/tools/server/server-common.cpp +++ b/tools/server/server-common.cpp @@ -1117,7 +1117,8 @@ json oaicompat_chat_params_parse( // Reasoning budget: pass parameters through to sampling layer { - int reasoning_budget = json_value(body, "thinking_budget_tokens", -1); + int reasoning_budget = json_value(body, "reasoning_budget_tokens", + json_value(body, "thinking_budget_tokens", -1)); if (reasoning_budget == -1) { reasoning_budget = opt.reasoning_budget; } @@ -1126,7 +1127,7 @@ json oaicompat_chat_params_parse( llama_params["reasoning_budget_tokens"] = reasoning_budget; llama_params["reasoning_budget_start_tag"] = chat_params.thinking_start_tag; llama_params["reasoning_budget_end_tag"] = chat_params.thinking_end_tag; - llama_params["reasoning_budget_message"] = opt.reasoning_budget_message; + llama_params["reasoning_budget_message"] = json_value(body, "reasoning_budget_message", opt.reasoning_budget_message); llama_params["reasoning_control"] = json_value(body, "reasoning_control", false); } }