|
| 1 | +package net.ladenthin.llama; |
| 2 | + |
| 3 | +import java.io.File; |
| 4 | +import java.util.Collections; |
| 5 | + |
| 6 | +import net.ladenthin.llama.args.ReasoningFormat; |
| 7 | +import net.ladenthin.llama.json.ChatResponseParser; |
| 8 | +import org.junit.AfterClass; |
| 9 | +import org.junit.Assert; |
| 10 | +import org.junit.Assume; |
| 11 | +import org.junit.BeforeClass; |
| 12 | +import org.junit.Ignore; |
| 13 | +import org.junit.Test; |
| 14 | + |
| 15 | +/** |
| 16 | + * Integration tests for thinking/reasoning mode using Qwen3-0.6B-Q4_K_M. |
| 17 | + * |
| 18 | + * <p>These tests require the Qwen3-0.6B-Q4_K_M model (downloaded by CI). The entire |
| 19 | + * class is skipped when the model file is absent, matching the pattern used by all |
| 20 | + * other model-dependent test classes. |
| 21 | + * |
| 22 | + * <h2>Confirmed behaviour (Qwen3-0.6B, llama.cpp b9151)</h2> |
| 23 | + * <ol> |
| 24 | + * <li><b>Thinking is active by default.</b> Qwen3's built-in chat template injects |
| 25 | + * {@code <think>} into the prompt before generation starts. No extra kwarg is |
| 26 | + * required; the model reasons on every request.</li> |
| 27 | + * <li><b>DEEPSEEK reasoning format correctly extracts thinking tokens.</b> Setting |
| 28 | + * {@code --reasoning-format deepseek} at model load time causes the server to |
| 29 | + * strip the {@code <think>…</think>} block from the response body and surface it |
| 30 | + * in {@code reasoning_content}.</li> |
| 31 | + * <li><b>{@code reasoning_budget_tokens} is NOT enforced for any model when set |
| 32 | + * per-request.</b> The root cause is a bug in |
| 33 | + * {@code tools/server/server-common.cpp}, function |
| 34 | + * {@code oaicompat_chat_params_parse}: the reasoning-budget block writes |
| 35 | + * the model-level default ({@code opt.reasoning_budget}, typically −1) |
| 36 | + * into {@code llama_params["reasoning_budget_tokens"]} before the generic |
| 37 | + * copy loop runs. The copy loop then skips the per-request value from the |
| 38 | + * request body because the key already exists |
| 39 | + * ({@code !llama_params.contains(item.key())} is false). Result: the |
| 40 | + * reasoning-budget sampler is never created (it requires |
| 41 | + * {@code reasoning_budget_tokens ≥ 0}), and any per-request budget |
| 42 | + * has no effect. Parameter serialisation itself is correct — see |
| 43 | + * {@code InferenceParametersTest} and the C++ unit tests.</li> |
| 44 | + * </ol> |
| 45 | + */ |
| 46 | +@ClaudeGenerated( |
| 47 | + purpose = "Integration tests for Qwen3 thinking-mode extraction and reasoning_budget_tokens " + |
| 48 | + "parameter acceptance. Documents the known llama.cpp limitation that budget " + |
| 49 | + "enforcement does not work for prompt-injected thinking models." |
| 50 | +) |
| 51 | +public class ReasoningBudgetTest { |
| 52 | + |
| 53 | + /** |
| 54 | + * Generous token budget: Qwen3-0.6B spends up to ~200 tokens thinking before answering. |
| 55 | + * 500 is enough for thinking + a short answer on all tested platforms. |
| 56 | + */ |
| 57 | + private static final int N_PREDICT = 500; |
| 58 | + |
| 59 | + private static LlamaModel model; |
| 60 | + private final ChatResponseParser parser = new ChatResponseParser(); |
| 61 | + |
| 62 | + @BeforeClass |
| 63 | + public static void setup() { |
| 64 | + Assume.assumeTrue("Reasoning model not found, skipping ReasoningBudgetTest", |
| 65 | + new File(TestConstants.REASONING_MODEL_PATH).exists()); |
| 66 | + int gpuLayers = Integer.getInteger(TestConstants.PROP_TEST_NGL, TestConstants.DEFAULT_TEST_NGL); |
| 67 | + model = new LlamaModel( |
| 68 | + new ModelParameters() |
| 69 | + .setModel(TestConstants.REASONING_MODEL_PATH) |
| 70 | + .setCtxSize(2048) |
| 71 | + .setGpuLayers(gpuLayers) |
| 72 | + .setFit(false) |
| 73 | + .setReasoningFormat(ReasoningFormat.DEEPSEEK) |
| 74 | + .enableLogTimestamps().enableLogPrefix() |
| 75 | + ); |
| 76 | + } |
| 77 | + |
| 78 | + @AfterClass |
| 79 | + public static void tearDown() { |
| 80 | + if (model != null) { |
| 81 | + model.close(); |
| 82 | + } |
| 83 | + } |
| 84 | + |
| 85 | + /** |
| 86 | + * Qwen3 enters thinking mode by default. With {@code reasoning_format=deepseek} set |
| 87 | + * at model level, the thinking tokens must appear in {@code reasoning_content} and |
| 88 | + * the final answer must appear in {@code content}. |
| 89 | + */ |
| 90 | + @Test |
| 91 | + public void testThinkingDefault_reasoningContentAndAnswerPresent() { |
| 92 | + InferenceParameters params = new InferenceParameters("") |
| 93 | + .setMessages(null, Collections.singletonList(new Pair<>("user", "What is 2+2?"))) |
| 94 | + .setNPredict(N_PREDICT); |
| 95 | + |
| 96 | + String json = model.chatComplete(params); |
| 97 | + String reasoningContent = parser.extractChoiceReasoningContent(json); |
| 98 | + String content = parser.extractChoiceContent(json); |
| 99 | + |
| 100 | + Assert.assertFalse( |
| 101 | + "reasoning_content should be non-empty (Qwen3 thinks by default)", |
| 102 | + reasoningContent == null || reasoningContent.trim().isEmpty()); |
| 103 | + Assert.assertFalse( |
| 104 | + "content must not be empty (model must produce an answer after thinking)", |
| 105 | + content == null || content.trim().isEmpty()); |
| 106 | + } |
| 107 | + |
| 108 | + /** |
| 109 | + * {@code reasoning_budget_tokens=0} is accepted by the API and the response |
| 110 | + * completes without error, but the budget is NOT enforced. |
| 111 | + * |
| 112 | + * <p><b>Documents current (broken) behaviour.</b> The per-request value is |
| 113 | + * silently discarded by a bug in {@code tools/server/server-common.cpp} |
| 114 | + * ({@code oaicompat_chat_params_parse}): the reasoning-budget block writes the |
| 115 | + * model-level default (−1) to {@code llama_params["reasoning_budget_tokens"]} |
| 116 | + * before the generic copy loop runs, and the copy loop then skips the user value |
| 117 | + * because the key already exists. The reasoning-budget sampler is therefore never |
| 118 | + * created, and {@code reasoning_content} remains non-empty. |
| 119 | + * |
| 120 | + * <p>This assertion will start <b>failing</b> once the llama.cpp bug is fixed — |
| 121 | + * that is the signal to remove this test and enable |
| 122 | + * {@link #testReasoningBudgetZero_expectedBehavior_suppressesThinking}. |
| 123 | + */ |
| 124 | + @Test |
| 125 | + public void testReasoningBudgetZero_parameterAccepted_thinkingNotSuppressed() { |
| 126 | + InferenceParameters params = new InferenceParameters("") |
| 127 | + .setMessages(null, Collections.singletonList(new Pair<>("user", "What is 2+2?"))) |
| 128 | + .setReasoningBudgetTokens(0) |
| 129 | + .setNPredict(N_PREDICT); |
| 130 | + |
| 131 | + String json = model.chatComplete(params); |
| 132 | + |
| 133 | + Assert.assertNotNull("Response JSON must not be null", json); |
| 134 | + |
| 135 | + String reasoningContent = parser.extractChoiceReasoningContent(json); |
| 136 | + Assert.assertFalse( |
| 137 | + "reasoning_content is expected to be present because the per-request " + |
| 138 | + "budget is not applied (llama.cpp server-common.cpp copy-loop bug). " + |
| 139 | + "If this assertion fails, the bug has been fixed — remove this test and " + |
| 140 | + "enable testReasoningBudgetZero_expectedBehavior_suppressesThinking.", |
| 141 | + reasoningContent == null || reasoningContent.trim().isEmpty()); |
| 142 | + } |
| 143 | + |
| 144 | + /** |
| 145 | + * Expected correct behaviour after the llama.cpp bug is fixed. |
| 146 | + * |
| 147 | + * <p><b>Bug:</b> In {@code tools/server/server-common.cpp}, |
| 148 | + * {@code oaicompat_chat_params_parse} sets |
| 149 | + * {@code llama_params["reasoning_budget_tokens"]} to the model-level default |
| 150 | + * ({@code opt.reasoning_budget}, typically −1) before the generic copy |
| 151 | + * loop runs. The copy loop then skips the per-request value from the request |
| 152 | + * body because the key already exists. Result: the sampler is never created |
| 153 | + * ({@code reasoning_budget_tokens ≥ 0} is required), and budget=0 |
| 154 | + * has no effect. |
| 155 | + * |
| 156 | + * <p><b>Fix (server-common.cpp, reasoning budget block):</b> |
| 157 | + * Read {@code reasoning_budget_tokens} from the request body <em>before</em> |
| 158 | + * writing to {@code llama_params}: |
| 159 | + * <pre> |
| 160 | + * int reasoning_budget = opt.reasoning_budget; |
| 161 | + * if (body.contains("reasoning_budget_tokens")) { |
| 162 | + * reasoning_budget = json_value(body, "reasoning_budget_tokens", reasoning_budget); |
| 163 | + * } |
| 164 | + * if (reasoning_budget == -1 && body.contains("thinking_budget_tokens")) { |
| 165 | + * reasoning_budget = json_value(body, "thinking_budget_tokens", -1); |
| 166 | + * } |
| 167 | + * </pre> |
| 168 | + * |
| 169 | + * <p>Once this fix is applied: remove {@code @Ignore}, confirm this test passes, |
| 170 | + * and remove |
| 171 | + * {@link #testReasoningBudgetZero_parameterAccepted_thinkingNotSuppressed}. |
| 172 | + */ |
| 173 | + @Ignore("llama.cpp bug: per-request reasoning_budget_tokens is overwritten by model default " + |
| 174 | + "in oaicompat_chat_params_parse (server-common.cpp). " + |
| 175 | + "See Javadoc for exact fix location and code.") |
| 176 | + @Test |
| 177 | + public void testReasoningBudgetZero_expectedBehavior_suppressesThinking() { |
| 178 | + InferenceParameters params = new InferenceParameters("") |
| 179 | + .setMessages(null, Collections.singletonList(new Pair<>("user", "What is 2+2?"))) |
| 180 | + .setReasoningBudgetTokens(0) |
| 181 | + .setNPredict(N_PREDICT); |
| 182 | + |
| 183 | + String json = model.chatComplete(params); |
| 184 | + Assert.assertNotNull("Response JSON must not be null", json); |
| 185 | + |
| 186 | + String reasoningContent = parser.extractChoiceReasoningContent(json); |
| 187 | + Assert.assertTrue( |
| 188 | + "reasoning_content should be empty when budget=0 suppresses thinking, " + |
| 189 | + "but was: " + reasoningContent, |
| 190 | + reasoningContent == null || reasoningContent.trim().isEmpty()); |
| 191 | + } |
| 192 | + |
| 193 | + /** |
| 194 | + * A positive {@code reasoning_budget_tokens} value is accepted and the call completes |
| 195 | + * without error. |
| 196 | + * |
| 197 | + * <p>The assertion checks that the model produced a non-empty response — either in |
| 198 | + * {@code reasoning_content} or {@code content}. On slow or constrained hardware the |
| 199 | + * model may exhaust the token budget inside the thinking block and emit an empty |
| 200 | + * {@code content}; checking both fields makes the test robust to that behaviour. |
| 201 | + * |
| 202 | + * <p>See {@link #testReasoningBudgetZero_parameterAccepted_thinkingNotSuppressed} for |
| 203 | + * the note on why the budget count itself is not asserted. |
| 204 | + */ |
| 205 | + @Test |
| 206 | + public void testReasoningBudgetPositive_parameterAccepted() { |
| 207 | + InferenceParameters params = new InferenceParameters("") |
| 208 | + .setMessages(null, Collections.singletonList( |
| 209 | + new Pair<>("user", "Think step by step: what is 3 times 7?"))) |
| 210 | + .setReasoningBudgetTokens(100) |
| 211 | + .setNPredict(N_PREDICT); |
| 212 | + |
| 213 | + String json = model.chatComplete(params); |
| 214 | + Assert.assertNotNull("Response JSON must not be null", json); |
| 215 | + |
| 216 | + String reasoningContent = parser.extractChoiceReasoningContent(json); |
| 217 | + String content = parser.extractChoiceContent(json); |
| 218 | + boolean hasReasoning = reasoningContent != null && !reasoningContent.trim().isEmpty(); |
| 219 | + boolean hasContent = content != null && !content.trim().isEmpty(); |
| 220 | + Assert.assertTrue( |
| 221 | + "model must produce at least some output in reasoning_content or content, " + |
| 222 | + "but both were empty", |
| 223 | + hasReasoning || hasContent); |
| 224 | + } |
| 225 | +} |
0 commit comments