diff --git a/.github/scripts/bench/bench.sh b/.github/scripts/bench/bench.sh new file mode 100755 index 000000000..38f71db9e --- /dev/null +++ b/.github/scripts/bench/bench.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# +# SPDX-License-Identifier: MIT OR Apache-2.0 +# +# Run llama-bench with exactly ONE CPU backend module, so a module the build machine would never pick +# (alderlake on an AVX-512 host, say) can be measured there. docs/benchmarks/moe-mxfp4-cpu-alderlake.md +# is the measurement this was written for, and has the build recipe. +# +# BUILD= bench.sh +# appends one JSON line per test to , tagged with "variant":"" +# BUILD= PROFILE= [PERF_DELAY=] bench.sh - +# one run under `perf record -e cpu-clock` instead; writes .data and a flat +# per-symbol report .sym.txt. PERF_DELAY skips the model load (repacking a 12 GB +# model alone is seconds of samples). +# +# The build must be a GGML_BACKEND_DL one with its libggml-cpu-*.so moved from /bin to +# /cpu-modules: ggml searches the executable's directory for CPU modules and would load the +# best one from there next to the one named here. GGML_BACKEND_PATH then loads the single module. +set -euo pipefail + +if [ $# -lt 2 ]; then + sed -n '6,19p' "$0" + exit 2 +fi +: "${BUILD:?set BUILD to the llama.cpp build directory}" +module=$1 +out=$2 +shift 2 + +lib="$BUILD/cpu-modules/libggml-cpu-$module.so" +if [ ! -f "$lib" ]; then + echo "bench.sh: no module $lib" >&2 + exit 1 +fi +if ls "$BUILD"/bin/libggml-cpu-*.so > /dev/null 2>&1; then + echo "bench.sh: $BUILD/bin still holds CPU modules; move them to $BUILD/cpu-modules first" >&2 + exit 1 +fi +export GGML_BACKEND_PATH="$lib" + +if [ -n "${PROFILE:-}" ]; then + # Ubuntu's /usr/bin/perf is a wrapper that refuses a kernel it has no package for (any VM or + # container kernel); the versioned binary behind it works for user-space sampling regardless + perf= + for p in "$(command -v perf || true)" /usr/lib/linux-tools/*/perf; do + if [ -x "$p" ] && "$p" --version > /dev/null 2>&1; then + perf=$p + break + fi + done + if [ -z "$perf" ]; then + echo "bench.sh: no working perf (Ubuntu: apt-get install linux-tools-generic)" >&2 + exit 1 + fi + "$perf" record -q -D "${PERF_DELAY:-0}" -e cpu-clock -F 1999 -o "$PROFILE.data" -- \ + "$BUILD/bin/llama-bench" -r 1 --no-warmup "$@" + "$perf" report -q -i "$PROFILE.data" --no-children --sort sym --percent-limit 0.3 2> /dev/null \ + | sed 's/ \+$//' > "$PROFILE.sym.txt" + head -25 "$PROFILE.sym.txt" + exit 0 +fi + +"$BUILD/bin/llama-bench" -o jsonl "$@" | sed "s/^{/{\"variant\":\"$module\",/" >> "$out" diff --git a/.github/scripts/bench/ggml-opprof.diff b/.github/scripts/bench/ggml-opprof.diff new file mode 100644 index 000000000..befc76361 --- /dev/null +++ b/.github/scripts/bench/ggml-opprof.diff @@ -0,0 +1,80 @@ +SPDX-FileCopyrightText: 2026 Bernard Ladenthin + +SPDX-License-Identifier: MIT OR Apache-2.0 + +Measurement-only instrumentation of ggml's CPU graph loop: per (op, src0 type) the summed thread time +inside the op and inside the barrier after it, printed at exit as OPPROF lines. Compiled in only with +-DGGML_OPPROF (CMAKE_C_FLAGS), never shipped. Applies to llama.cpp b11887 + llama/patches/ with +`git apply`; docs/benchmarks/moe-mxfp4-cpu-alderlake.md is what it was used for. + +--- a/ggml/src/ggml-cpu/ggml-cpu.c ++++ b/ggml/src/ggml-cpu/ggml-cpu.c +@@ -3108,6 +3108,34 @@ + return 0; + } + ++ ++#ifdef GGML_OPPROF ++// per (op, src0 type): summed thread time in the op and in the barrier after it (benchmark-only) ++#include ++#define OPPROF_N (GGML_OP_COUNT * GGML_TYPE_COUNT) ++static _Atomic int64_t opprof_cmp[OPPROF_N], opprof_wait[OPPROF_N], opprof_cnt[OPPROF_N]; ++static atomic_int opprof_registered; ++static int64_t opprof_now(void) { struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); return (int64_t) ts.tv_sec * 1000000000LL + ts.tv_nsec; } ++static void opprof_dump(void) { ++ int64_t tot = 0; ++ for (int i = 0; i < OPPROF_N; i++) tot += opprof_cmp[i] + opprof_wait[i]; ++ if (!tot) return; ++ int idx[OPPROF_N]; int n = 0; ++ for (int i = 0; i < OPPROF_N; i++) if (opprof_cnt[i]) idx[n++] = i; ++ for (int a = 0; a < n; a++) for (int b = a + 1; b < n; b++) ++ if (opprof_cmp[idx[b]] + opprof_wait[idx[b]] > opprof_cmp[idx[a]] + opprof_wait[idx[a]]) { int t = idx[a]; idx[a] = idx[b]; idx[b] = t; } ++ int64_t wtot = 0; for (int i = 0; i < OPPROF_N; i++) wtot += opprof_wait[i]; ++ fprintf(stderr, "OPPROF total thread-ms %.1f, of it barrier wait %.1f (%.1f%%)\n", tot / 1e6, wtot / 1e6, 100.0 * wtot / tot); ++ fprintf(stderr, "OPPROF %-16s %-8s %10s %9s %9s %7s %7s\n", "op", "src0", "nodes/thr", "cmp-ms", "wait-ms", "cmp%", "wait%"); ++ for (int a = 0; a < n && a < 30; a++) { ++ int i = idx[a]; ++ fprintf(stderr, "OPPROF %-16s %-8s %10lld %9.1f %9.1f %6.1f%% %6.1f%%\n", ggml_op_name((enum ggml_op)(i / GGML_TYPE_COUNT)), ++ ggml_type_name((enum ggml_type)(i % GGML_TYPE_COUNT)), (long long) opprof_cnt[i], opprof_cmp[i] / 1e6, opprof_wait[i] / 1e6, ++ 100.0 * opprof_cmp[i] / tot, 100.0 * opprof_wait[i] / tot); ++ } ++} ++#endif ++ + static thread_ret_t ggml_graph_compute_thread(void * data) { + struct ggml_compute_state * state = (struct ggml_compute_state *) data; + struct ggml_threadpool * tp = state->threadpool; +@@ -3150,12 +3178,20 @@ + + // TODO: move fused-op detection into ggml_graph_plan so fusion decisions are made once at planning time + // Try fused ops, fall back to normal compute ++#ifdef GGML_OPPROF ++ if (!atomic_exchange(&opprof_registered, 1)) atexit(opprof_dump); ++ const int opprof_key = node->op * GGML_TYPE_COUNT + (node->src[0] ? node->src[0]->type : 0); ++ const int64_t opprof_t0 = opprof_now(); ++#endif + const int n_fused = ggml_cpu_try_fuse_ops(cgraph, node_n, ¶ms, cplan); + if (n_fused > 0) { + node_n += n_fused; + } else { + ggml_compute_forward(¶ms, node); + } ++#ifdef GGML_OPPROF ++ const int64_t opprof_t1 = opprof_now(); ++#endif + + if (state->ith == 0 && cplan->abort_callback && + cplan->abort_callback(cplan->abort_callback_data)) { +@@ -3166,6 +3202,12 @@ + if (node_n + 1 < cgraph->n_nodes) { + ggml_barrier(state->threadpool); + } ++#ifdef GGML_OPPROF ++ const int64_t opprof_t2 = opprof_now(); ++ opprof_cmp[opprof_key] += opprof_t1 - opprof_t0; ++ opprof_wait[opprof_key] += opprof_t2 - opprof_t1; ++ opprof_cnt[opprof_key] += 1; ++#endif + } + + #ifdef GGML_USE_OPENMP diff --git a/.github/scripts/bench/mmid-dynamic-gemm.diff b/.github/scripts/bench/mmid-dynamic-gemm.diff new file mode 100644 index 000000000..17aa1f486 --- /dev/null +++ b/.github/scripts/bench/mmid-dynamic-gemm.diff @@ -0,0 +1,131 @@ +SPDX-FileCopyrightText: 2026 Bernard Ladenthin + +SPDX-License-Identifier: MIT OR Apache-2.0 + +PROTOTYPE, not applied to any build of this project: the MUL_MAT_ID (MoE) path of ggml-cpu's repack +buffer, with (a) the work handed out dynamically as (expert, column chunk) units instead of a static +row split per thread, and (b) gemm for every group of four or more tokens routed to the same expert +instead of one gemv per token. Behind -DJLLAMA_MMID_V2 (CMAKE_CXX_FLAGS) so one tree builds both +sides of the A/B. Applies to llama.cpp b11887 + llama/patches/ with `git apply`; the measurement +and the correctness check are in docs/benchmarks/moe-mxfp4-cpu-alderlake.md. + +--- a/ggml/src/ggml-cpu/repack.cpp ++++ b/ggml/src/ggml-cpu/repack.cpp +@@ -4554,7 +4554,19 @@ + + template class tensor_traits : public tensor_traits_base { + +- bool work_size(int /* n_threads */, const struct ggml_tensor * op, size_t & size) override { ++#ifdef JLLAMA_MMID_V2 ++ // MUL_MAT_ID: token rows per gemm call, and the per-thread scratch that call needs ++ static constexpr int64_t MMID_GEMM_ROWS = 16; ++ ++ static size_t mmid_thread_scratch(int64_t ne10, int64_t ne01) { ++ return GGML_PAD(MMID_GEMM_ROWS * ne10 * sizeof(float), 64) + ++ GGML_PAD(MMID_GEMM_ROWS * ggml_row_size(PARAM_TYPE, ne10), 64) + ++ GGML_PAD(MMID_GEMM_ROWS * ne01 * sizeof(float), 64); ++ } ++#endif ++ ++ bool work_size(int n_threads, const struct ggml_tensor * op, size_t & size) override { ++ GGML_UNUSED(n_threads); + // not realy a GGML_TYPE_Q8_0 but same size. + switch (op->op) { + case GGML_OP_MUL_MAT: +@@ -4573,6 +4585,10 @@ + const size_t sizeof_mmid_row_mapping = sizeof(int64_t); + + size += sizeof_mmid_row_mapping*ne02*(ne12 + 1); ++#ifdef JLLAMA_MMID_V2 ++ size += sizeof(int64_t)*(ne02 + 1) + 64; ++ size += (size_t) n_threads * mmid_thread_scratch(op->src[1]->ne[0], op->src[0]->ne[1]); ++#endif + + return true; + } +@@ -4868,6 +4884,77 @@ + } + } + ++#ifdef JLLAMA_MMID_V2 ++ // the experts that got rows, and the unit counter: threads take (expert, column chunk) ++ // units dynamically, so a slower core takes fewer of them instead of holding the barrier ++ int64_t * active = (int64_t *) (matrix_rows + n_as*ne12); ++ if (ith == 0) { ++ int64_t n_active = 0; ++ for (int cur_a = 0; cur_a < n_as; ++cur_a) { ++ if (matrix_row_counts[cur_a] > 0) { ++ active[n_active++] = cur_a; ++ } ++ } ++ active[n_as] = n_active; ++ ggml_threadpool_chunk_set(params->threadpool, nth); ++ } ++ ++ ggml_barrier(params->threadpool); ++ ++ const int64_t n_active = active[n_as]; ++ if (n_active == 0) { ++ return; ++ } ++ const int64_t n_col_blocks = ne01 / NB_COLS; ++ const int64_t nck = MAX(1, MIN(n_col_blocks, (nth * 16 + n_active - 1) / n_active)); ++ const int64_t cols_per = ((n_col_blocks + nck - 1) / nck) * NB_COLS; ++ const int64_t n_units = n_active * nck; ++ ++ char * scratch_base = (char *) (((uintptr_t) (active + n_as + 1) + 63) & ~(uintptr_t) 63); ++ char * scratch = scratch_base + ith * mmid_thread_scratch(ne10, ne01); ++ float * fbuf = (float *) scratch; ++ char * qbuf = scratch + GGML_PAD(MMID_GEMM_ROWS * ne10 * sizeof(float), 64); ++ float * obuf = (float *) (qbuf + GGML_PAD(MMID_GEMM_ROWS * ggml_row_size(PARAM_TYPE, ne10), 64)); ++ GGML_ASSERT((char *) obuf + MMID_GEMM_ROWS * ne01 * sizeof(float) <= (char *) params->wdata + params->wsize); ++ ++ for (int64_t unit = ith; unit < n_units; unit = ggml_threadpool_chunk_add(params->threadpool, 1)) { ++ const int64_t cur_a = active[unit / nck]; ++ const int64_t c0 = (unit % nck) * cols_per; ++ const int64_t c1 = MIN(c0 + cols_per, ne01); ++ if (c0 >= c1) { ++ continue; ++ } ++ const int64_t nc = c1 - c0; ++ const int64_t cne1 = matrix_row_counts[cur_a]; ++ const char * src0_cur = (const char *) src0->data + cur_a*nb02 + c0*nb01; ++ ++ // groups of four or more rows: gather the float rows, interleave-quantize, gemm, scatter ++ int64_t ir1 = 0; ++ while (cne1 - ir1 >= 4) { ++ const int64_t nr = MIN(MMID_GEMM_ROWS, (cne1 - ir1) & ~(int64_t) 3); ++ for (int64_t r = 0; r < nr; r++) { ++ const struct mmid_row_mapping rm = MMID_MATRIX_ROW(cur_a, ir1 + r); ++ memcpy(fbuf + r*ne10, (const char *) src1->data + (rm.i1 % ne11)*nb11 + rm.i2*nb12, ne10*sizeof(float)); ++ } ++ for (int64_t r = 0; r < nr; r += 4) { ++ ggml_quantize_mat_t(fbuf + r*ne10, qbuf + r*nbw1, 4, ne10); ++ } ++ gemm(ne00, obuf, nc, src0_cur, qbuf, nr, nc); ++ for (int64_t r = 0; r < nr; r++) { ++ const struct mmid_row_mapping rm = MMID_MATRIX_ROW(cur_a, ir1 + r); ++ memcpy((float *) ((char *) dst->data + rm.i1*nb1 + rm.i2*nb2) + c0, obuf + r*nc, nc*sizeof(float)); ++ } ++ ir1 += nr; ++ } ++ for (; ir1 < cne1; ir1++) { ++ const struct mmid_row_mapping rm = MMID_MATRIX_ROW(cur_a, ir1); ++ const char * src1_col = (const char *) wdata + ((rm.i1 % ne11) * nbw1 + rm.i2 * nbw2); ++ gemv( ++ ne00, (float *) ((char *) dst->data + (rm.i1 * nb1 + rm.i2 * nb2)) + c0, ne01, ++ src0_cur, src1_col, 1, nc); ++ } ++ } ++#else + ggml_barrier(params->threadpool); + + // compute each matrix multiplication in sequence +@@ -4915,6 +5002,7 @@ + src0_cur + src0_cur_start * nb01, src1_col, 1, src0_cur_end - src0_cur_start); + } + } ++#endif + #undef MMID_MATRIX_ROW + } + diff --git a/.github/scripts/bench/mock_gguf.py b/.github/scripts/bench/mock_gguf.py new file mode 100755 index 000000000..b61fdcba0 --- /dev/null +++ b/.github/scripts/bench/mock_gguf.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# +# SPDX-License-Identifier: MIT OR Apache-2.0 +"""Write a GGUF with random weights in the real shape of a published model, for throughput work. + +The output of such a model is noise; what is real is every tensor's shape and type, i.e. the work +and the bytes a token costs. Tensors are written one at a time, so a 12 GB model needs no 12 GB of +RAM. + + qwen3-0.6b / qwen3-4b F16, to be quantized with llama-quantize (Q4_K_M gives the real mix) + gpt-oss-20b ggml-org's gpt-oss-20b-mxfp4.gguf layout: experts MXFP4, every other + matrix Q8_0 (llama-quantize's MXFP4_MOE rule), norms/biases/router F32 + +Needs gguf-py (PYTHONPATH=/gguf-py) and numpy. +""" + +from __future__ import annotations + +import argparse +import sys + +import numpy as np + +import gguf + +QK = 32 # block size of Q8_0 and MXFP4 + +MODELS = { + # Qwen/Qwen3-0.6B config.json + "qwen3-0.6b": dict(arch="qwen3", n_embd=1024, n_layer=28, n_head=16, n_head_kv=8, head_dim=128, + n_ff=3072, n_vocab=151936, n_ctx=40960, rope_theta=1_000_000.0, eps=1e-6), + # Qwen/Qwen3-4B-Instruct-2507 config.json + "qwen3-4b": dict(arch="qwen3", n_embd=2560, n_layer=36, n_head=32, n_head_kv=8, head_dim=128, + n_ff=9728, n_vocab=151936, n_ctx=262144, rope_theta=5_000_000.0, eps=1e-6), + # openai/gpt-oss-20b config.json + "gpt-oss-20b": dict(arch="gpt-oss", n_embd=2880, n_layer=24, n_head=64, n_head_kv=8, head_dim=64, + n_ff=2880, n_vocab=201088, n_ctx=131072, rope_theta=150_000.0, eps=1e-5, + n_expert=32, n_expert_used=4, sliding_window=128), +} + + +def vocab(n_vocab: int) -> tuple[list[str], list[int]]: + """GPT-2 byte tokens first (any text tokenizes, byte by byte), then unique fillers.""" + bs = list(range(ord("!"), ord("~") + 1)) + list(range(0xA1, 0xAD)) + list(range(0xAE, 0x100)) + cs = bs[:] + n = 0 + for b in range(256): + if b not in bs: + bs.append(b) + cs.append(256 + n) + n += 1 + tokens = [chr(c) for c in cs] + tokens += [f"tok{i}" for i in range(n_vocab - 256 - 2)] + tokens += ["<|bos|>", "<|eos|>"] + types = [1] * (n_vocab - 2) + [3, 3] # normal, control + return tokens, types + + +class Writer: + def __init__(self, path: str, arch: str, seed: int): + self.w = gguf.GGUFWriter(path, arch) + self.rng = np.random.default_rng(seed) + self.pending: list[tuple[str, callable]] = [] + + def f32(self, name, shape, mean=0.0, std=0.02): + n = int(np.prod(shape)) + self.w.add_tensor_info(name, shape, np.dtype(np.float32), n * 4) + self.pending.append((name, lambda: (mean + std * self.rng.standard_normal(n, dtype=np.float32)).reshape(shape))) + + def f16(self, name, shape, std=0.02): + n = int(np.prod(shape)) + self.w.add_tensor_info(name, shape, np.dtype(np.float16), n * 2) + self.pending.append((name, lambda: (std * self.rng.standard_normal(n, dtype=np.float32)).astype(np.float16).reshape(shape))) + + def q8_0(self, name, shape, std=0.02): + # shape in numpy order (..., ne0); block: f16 d, 32 x int8 + *outer, ne0 = shape + nb = int(np.prod(outer)) * ne0 // QK + byte_shape = (*outer, ne0 // QK * 34) + self.w.add_tensor_info(name, byte_shape, np.dtype(np.uint8), nb * 34, raw_dtype=gguf.GGMLQuantizationType.Q8_0) + + def gen(): + blk = np.empty(nb, dtype=[("d", "context, buffer->size)`, exactly what `ggml_backend_cpu_buffer_free_buffer()` does; on Linux that is `free()`, so nothing changes there. **Reached by** the `sapphirerapids` CPU module (the only variant built with `AMX_TILE AMX_INT8`, which `ggml_backend_load_best` picks on Sapphire/Emerald/Granite Rapids; `ggml_amx_init()` is `return true` on `_WIN32`) and a model whose weights have AMX kernels (Q4_0, Q4_1, Q8_0, Q4_K, Q5_K, Q6_K, IQ4_XS -- i.e. every CI model). **Found by the second full run of the modular natives** (38069595101): `Java Tests Windows 2025 x86_64 (clang)` landed on an `Intel(R) Xeon(R) 6973P-C` runner (Granite Rapids), loaded `ggml-cpu-sapphirerapids.dll`, ran `GrammarExample`'s two generations fine and died with exit code -1073740940 at `LlamaModel.close()`; the first run (38065017595) had the same job green on an AMD EPYC 7763 (no AVX-512, `haswell`). **The Windows runners are a CPU lottery**, so which CPU module a Windows job exercises differs per run; before the modular natives the Windows jar was a static Haswell-level build and this code never ran. **Of the six callers of `ggml_aligned_malloc` in the tree this was the only mismatched pair** (`ggml-backend.cpp`, `ggml-cpu.c` twice, `ggml-openvino.cpp`, `ggml-zdnn.cpp` and `ggml.c` all pair correctly), and `ggml-backend.cpp:2464` is the line the fix copies. **Reproduced deliberately rather than waiting for the lottery again**: on a rented Granite Rapids machine, one CPU module staged at a time with the same model and the DLLs CI itself built, `sapphirerapids` crashed **6 of 6** runs -- including at **one** thread, which rules out a data race -- while `cooperlake`, `icelake` and `haswell` were clean 6 of 6, and on Linux with the same CPU all six exit 0 with byte-identical output. The crash lands *after* the inference prints its timings, in cleanup: the signature of a heap corrupted earlier and detected at `free`. With the patch applied the same matrix on the same machine is **clean in all twelve runs**. **The guard is a configure-time check, not a `jllama_test` case**: the AMX code exists only in that one variant of a `GGML_CPU_ALL_VARIANTS` build and executes only on AMX hardware, which the static test build, the local agent's Zen 3 and the cloud sandboxes lack -- so **`llama/cmake/check-aligned-free-pairing.cmake`** fails the configure on **every** platform when a ggml source calls `ggml_aligned_malloc` and never `ggml_aligned_free` (verified red against the unpatched source, naming `amx.cpp`, and green with this patch). A run-time test cannot replace it: without an AMX CPU there is nothing to observe, which is exactly how this shipped. What exercises it: the Windows x86-64 Java test job and the `windows-x86-64` smoke row whenever GitHub hands them an Intel AMX runner; the patch itself is held by `check-patches.py`, applies at b11639, and the patched file compiles with the variant's flags (GCC 13, `-mamx-tile -mamx-int8 -mavx512vnni ...`). Upstream master still carried the `free()` on 2026-10-10; **not yet filed upstream** (nothing before the release). **Drop-check on every bump, note the inversion:** `grep -n "free(buffer->context)" ggml/src/ggml-cpu/amx/amx.cpp` at the new tag -- a **hit** means the bare `free()` is still there and the patch is still needed; **gone** means upstream fixed it, drop the patch. Measured at b11639 (`a0e6dc738`): one hit, line 46. | | `0020-ggml-alloc-validate-leafs-before-reusing-an-allocation.patch` | **An `--embedding` model that is asked to generate crashes the process in `ggml_gallocr_alloc_graph`** (SIGSEGV on Linux, 0xC0000005 with both Windows toolchains), since upstream #30445 (`cb55ffe6e`, "graph: skip the LM head for pooled embeddings", b11574..b11612). `ggml_gallocr_alloc_graph()` assigns every tensor by **position** from the graph it was reserved with, and `ggml_gallocr_needs_realloc()` validates every node and every node's sources but of the **leafs only the count**. Since #30445 the LM head is expanded by `build_logits()` *after* `build_pooling()`, so an embeddings context reserves a graph that expands `t_embd` first and `t_logits` second, while a generation step on it (the server calls `llama_set_embeddings(ctx, false)` for a slot that samples) expands only `t_logits`, whose first source is `output.weight`: same node and leaf counts, every leaf after it shifted by one. An input leaf then lands on the slot of a weight (`buffer_id -1`) and, with the asserts compiled out, `ggml_vbuffer_tensor_alloc()` indexes `galloc->buffers[-1]`; a leaf landing on another input's slot is not size-checked either. The patch validates the leafs exactly as the nodes are validated, so an invalid leaf makes the allocator reserve again (one buffer) or return false, on which `ggml_backend_sched` reserves the graph itself. **Found by** Publish run 38648257387 (2026-10-14, the first full run since b11558): `ErrorHandlingTest` loads its model with `enableEmbedding()` and posts an infill request to it -- its first test in JUnit's order -- and the fork died in all three Java test jobs (`ggml_gallocr_alloc_graph <- ggml_backend_sched_alloc_graph <- llama_context::process_ubatch <- decode <- server_context_impl::update_slots`). **Reproduced locally without a downloaded model** (a random 2-layer llama GGUF with FIM tokens, context 128): infill crashes with `--embedding` and works without; a debug print showed leaf 31 `out_ids` arriving at a slot with no allocation; reverting `cb55ffe6e` makes it pass, and so does this patch -- with greedy sampling the `--embedding` model then returns the same tokens as the model without, run after run. Upstream master (`a0e6dc738`) still lacks the check; **not yet filed upstream**. **Runnable guard: `src/test/cpp/test_gallocr_leafs.cpp`** (two graphs with the same counts and the leafs swapped). **Drop-check on every bump:** read `ggml_gallocr_needs_realloc()` in `ggml/src/ggml-alloc.c` at the new tag -- once it walks `graph->leafs` itself, drop this patch rather than refreshing it. | | `0021-server-clear-the-slot-cache-when-the-server-adapters-change.patch` | **A slot's prompt cache survived a change of the LoRA adapters it was computed with.** `launch_slot_with_task()` (`tools/server/server-context.cpp`) clears the cache when a request names its own adapters and they differ from the slot's (`lora_should_clear_cache`, which keeps it only for aLoRA), but a request that names none takes `params_base.lora_adapters` in the `else` branch without that check. So the cache survived `SET_LORA` (`LlamaModel.setLoraAdapter(s)`, upstream's `POST /lora-adapters`) and the return to the server's adapters after a request with its own, and the slot answered like neither adapter set, silently. **Measured** (J9 of the bug-hunt brief, b11639, upstream's own LoRA test pair `stories15M_MOE-F16.gguf` + `moe_shakespeare15M.gguf`, greedy, one slot): after `setLoraAdapter(0, 0f)` the warm slot answered ". He saw a picture of a bird ..." where an emptied slot with the same scale answers " was a little girl ..."; only `eraseSlot(0)` restored it (F67). The patch applies the per-request rule in the `else` branch too; equal sets keep the cache as before. **A second hunk starts every slot with the server's adapters in `init()`** (`slot.lora = params_base.lora_adapters;`): `slot.lora` used to start empty, so a never-used slot of a server started with `--lora` read as "computed under no adapters", and its first task logged `0 loras -> 1 loras` and cleared a prompt it already held -- a state restored into it (`SLOT_RESTORE`) or loaded from the host-memory prompt cache by `get_available_slot()`, both computed under the server's adapters (F240). Upstream has the same defect on its HTTP route, so the patch is upstream-shaped; **not yet filed upstream**. Not covered, and said in `setLoraAdapters`' Javadoc: the host-memory prompt cache (`--cache-ram`, on by default) stores slot states without their adapters. **Runnable guard: `src/test/cpp/test_lora_cache.cpp`** (a random llama model and a random LoRA adapter on K, V and the FFN output, written by the test; five cases). **Drop-check on every bump, one per hunk:** read the `else` branch of the "process per-request lora adapters" block in `launch_slot_with_task()` at the new tag -- once it compares `params_base.lora_adapters` with `slot.lora` and clears the prompt (or `SET_LORA` clears the slots), drop that hunk rather than refreshing it; and read the slot loop of `init()` -- once it sets `slot.lora` from `params_base.lora_adapters`, drop that hunk. The patch goes when both have. | +| `0022-ggml-cpu-repack-batch-the-tokens-of-an-expert.patch` | **The expert matmuls of a MoE model over repacked CPU weights ran one token at a time on a static thread split.** `forward_mul_mat_id()` (`ggml/src/ggml-cpu/repack.cpp`) -- the path of every MoE model whose experts are repacked at load: MXFP4 (gpt-oss), Q4_0, IQ4_NL and Q4_K on AVX2, their interleaved forms on ARM -- multiplied each token routed to an expert with its own gemv, although the gemm kernel for the type exists, and gave thread `ith` a fixed row range of every expert, so each expert matmul waited for the slowest thread. Measured with a random-weight gpt-oss-20b in ggml-org's MXFP4 layout on the `alderlake` module (the Core Ultra's), 4 threads: 72 % of prompt time was that gemv loop, 13.9 % of generation time the barrier wait after it ([`docs/benchmarks/moe-mxfp4-cpu-alderlake.md`](docs/benchmarks/moe-mxfp4-cpu-alderlake.md)). The patch adds a batched path -- (expert, column chunk) units handed out through `ggml_threadpool_chunk_add`, every group of up to 16 of an expert's tokens quantized interleaved (`ggml_quantize_mat_t`, as `forward_mul_mat` does) and multiplied with `gemm`, a rest of fewer than four by gemv -- and **keeps upstream's path behind a process-wide switch**: `ggml_backend_cpu_set/get_repack_mmid_batched` (`ggml-cpu.h`, and in the CPU backend's `get_proc_address` table, which is how libjllama reaches it in a `GGML_BACKEND_DL` build), on by default, `GGML_CPU_REPACK_MMID_BATCHED=0` in the environment starts it off. Thread 0 reads it once per op and publishes the choice in the work buffer, so a flip during a compute cannot split the threads; `work_size()` reserves the batched path's scratch either way (~420 KB per thread for gpt-oss). Exposed as `LlamaModel.setCpuMoeBatched` / `isCpuMoeBatched`, `jllama_set/get_cpu_moe_batched` and the FFM `Jllama`: the switch exists so a user can measure both paths on their own hardware -- the P-core/E-core effect the static split suggests is unmeasured (TODO.md). **Measured, one binary, the environment variable flipped:** prompt processing +47 % (42.63 -> 62.76 t/s), token generation +26 % (7.90 -> 9.97), logits equal to rounding (`llama-perplexity --kl-divergence`: mean KLD 0.000000, max 0.000002, same top token 100 %). **Runnable guard: `src/test/cpp/test_moe_repack.cpp`** (both paths against ggml's generic `MUL_MAT_ID` over the same weights; the switch resolved through `get_proc_address`). Upstream-shaped without the switch; **not yet filed upstream**. **Drop-check on every bump:** read `forward_mul_mat_id()` in `ggml/src/ggml-cpu/repack.cpp` at the new tag -- once it calls `gemm` for an expert's batched rows and hands its work out through `ggml_threadpool_chunk_add`, drop this patch rather than refreshing it; the switch goes with it, and `setCpuMoeBatched` then throws (`LlamaException`, no entry point), so the Java, C and FFM API needs the decision whether to remove it or keep it as a no-op. | **Dropped patches** (`0004`, `0005`, `0009`, `0010`, `0011`, `0012`, `0013`, `0016`, `0019` -- upstream fixed each defect itself, or for `0016` added the model; `0019` is this project's own fix, merged upstream as #30505) are recorded in [`docs/history/dropped-llama-patches.md`](docs/history/dropped-llama-patches.md): @@ -2321,7 +2322,7 @@ If the local check passes (`BUILD SUCCESS`), the `mvn package` job in - The `server` package is a dedicated top layer in the ArchUnit `layeredArchitecture` rule (the only layer allowed to access the root `Api`); `noInternalJdkImports` carries an explicit exception for the supported `com.sun.net.httpserver` (the exported `jdk.httpserver` module, which `module-info.java` `requires`). See README "OpenAI-compatible HTTP server". **Native layer** (`src/main/cpp/`): -- `jllama.cpp` — JNI implementation bridging Java calls to llama.cpp. 37 native methods (32 `LlamaModel` + 4 `TextToSpeech` + 1 `LlamaQuantizer`) plus `JNI_OnLoad`/`JNI_OnUnload`; since the C API (see "The C API" below) every one of them is a shell over `jllama_api.cpp` / `jllama_engine.cpp`, where the model logic lives. +- `jllama.cpp` — JNI implementation bridging Java calls to llama.cpp. 39 native methods (34 `LlamaModel` + 4 `TextToSpeech` + 1 `LlamaQuantizer`) plus `JNI_OnLoad`/`JNI_OnUnload`; since the C API (see "The C API" below) every one of them is a shell over `jllama_api.cpp` / `jllama_engine.cpp`, where the model logic lives. - `utils.hpp` — Helper utilities (format helpers, argv stripping, token-piece serialisation). - `json_helpers.hpp` — Pure JSON transformation helpers (no JNI, no llama state). Independently unit-testable. - `jni_helpers.hpp` — JNI bridge helpers (handle management + server orchestration). Includes `json_helpers.hpp`. @@ -2447,8 +2448,8 @@ Functions with `_impl` suffix are called directly from `jllama.cpp`. An exception that escapes a native method and unwinds across the JNI boundary is **undefined behaviour and aborts the JVM** on most implementations. **Every `Java_*` entry point must therefore -convert anything that escapes into a Java exception**, and there are 47 of them across four TUs — -`jllama.cpp` (37), `native_server.cpp` (5), `rpc_bridge.cpp` (4), `train_engine.cpp` (1). +convert anything that escapes into a Java exception**, and there are 49 of them across four TUs — +`jllama.cpp` (39), `native_server.cpp` (5), `rpc_bridge.cpp` (4), `train_engine.cpp` (1). The mechanism is `jni_guard_impl(env, exception_class, [&]() -> Ret { … })` (`jni_helpers.hpp`, Layer A). It is **additive**: an entry point that already converts `std::exception` itself keeps @@ -2780,13 +2781,14 @@ ctest --test-dir build --output-on-failure -R "ResultsToJson" | `src/test/cpp/test_wire_contracts.cpp` | 6 | **The same contract for the two quieter surfaces.** `RequestField` against `server_schema::make_llama_cmpl_schema(...)` (5 tests) and `TrainingField` against `jllama_train::config_keys()` (1 test). Both receivers *silently ignore* an unknown key — the schema skips it, `train_engine.cpp` reads with `j.value(key, default)` and falls back — so a dead field produces no error anywhere and every string-mapping test keeps passing. `OAI_LAYER`-declared keys (consumed by `oaicompat_*_params_parse` before the schema) are exempt from the schema check, and are checked **both** ways: still unknown to the schema (the inverted check), and read by at least one upstream reader-shaped site (the configure-time sweep — this is what caught `chat_template`, a key a public builder wrote and nothing read). See [`docs/history/parameter-wire-surface.md`](docs/history/parameter-wire-surface.md). | | `src/test/cpp/test_rpc.cpp` | 28 | **The runnable guard for `patches/0015` and for `rpc_support.hpp`.** The device-selection rules (pure, literal inputs: `--rpc` endpoints accumulate across repeated options, an explicit `--device`/`-dev` is never overridden, stale RPC devices are replaced by the local GPUs chosen the way llama.cpp's default does — duplicates by device id dropped, iGPUs only without a discrete GPU, `none` on a CPU-only host). Then the **real** ggml-rpc client and server over loopback, no model: a `mul_mat` graph computed on the RPC backend equals the local CPU result; `ggml_backend_rpc_stop_server()` ends the server, frees the port and disconnects a client that is still connected; an unreachable or malformed endpoint is `nullptr`/`std::invalid_argument` instead of an abort; a registered server that went away is re-checked on the next registration; its device reports 0/0 memory instead of aborting; and a stale server is kept out of a later load that did not ask for it. The loopback server picks a free port from a range so parallel jobs do not collide, and serves the **CPU** by name — a served GPU that lacks an op aborts the server (see the RPC section). Also: server devices chosen by name (case-insensitive, deduplicated), an unknown name listing the available devices, and a registered RPC device refused; the mmproj device pinned the way clip would choose it minus the stale server; and `exclude_stale_devices()`, the params-level guard for TTS and the trainer. | | `src/test/cpp/test_rpc_comm.cpp` | 6 | **`patches/0015`'s COMM_INIT and stop contract over real TCP, protocol v8.** Each scenario runs the C API's RPC server in a fresh subprocess with a deadline, so a regression fails instead of hanging the runner: a loopback server opens no wildcard peer listener and a network bind stays an explicit opt-in; a stop with no peer closes both listeners and lets the port be bound again; a stop interrupts a stalled capability handshake of rank 0 and of rank 1; and a successful peer setup still works and stops. No model, GPU or external network. | -| `src/test/cpp/test_c_api.cpp` | 54 | **The C API (`jllama_api.h`, layer B) without a JVM.** Writes a tiny random two-layer llama GGUF with the GPT-2 byte tokenizer (the public `gguf` API, like `test_kolibri1.cpp`), loads it through `jllama_model_load` and drives the shipped contract: version and build info, null-argument refusal with a message, a missing model failing to load, the load-progress callback (reported, and a `false` return aborts the load), a chat completion that is an OpenAI `chat.completion` object, an invalid request answered as the caller's fault (`JLLAMA_STATUS_INVALID_REQUEST`), streaming until `stop` with the task released afterwards ("Task not found" on the next receive), a released task gone, and the log callback seeing the server's lines with a null set flushing synchronously. Hermetic, so it runs in every `C++ Tests` job without `restore-models`; the output of a random model is noise, the contract is what is asserted. Two things the fixture does on purpose: the output rows of every non-ASCII and special token are scaled to almost nothing, because a byte above 0x7F is an incomplete UTF-8 piece the server holds back rather than streams (a random model that emits only those sends no partial chunk at all), and the task id is checked `>= 0` -- the server numbers its tasks from zero. Then every other `LlamaModel` operation in turn (the meta, raw and OAI completions, raw streaming, the streamed OAI completion (`text_completion` chunks with one id, probabilities, `usage` on the last), the template, tokenize/detokenize, metrics, slots, LoRA, configure, the refusals of a model loaded without `--embedding`/`--reranking`/FIM tokens, the grammar conversion, a Q8_0 quantisation that loads again, several prompts in one request and several completions of one (`"n"` > 1) refused on the blocking, raw and streamed paths while a token array stays one prompt, and every OpenAI answer and chunk carrying a `chatcmpl-` id and the model name, one id per stream), then the other engines: the embedded server in router mode and attached to the model (which still answers after the server stopped), the RPC server over loopback, text to speech refusing missing models, and a fine-tuning run over the random F32 model that writes a GGUF -- which is why `tts_engine.cpp`, `train_engine.cpp` and `native_server_engine.cpp` are in `jllama_test`'s source list. Four model-free cases (`CApiStatic`) run before any load. `CApiSlotFiles` loads the random model with `--slot-save-path`: a saved slot file lands under that directory by its name and restores, and a name with a directory part, `..`, an absolute path or nothing is `JLLAMA_STATUS_INVALID_REQUEST` with nothing written (red before F251: the name was the path, relative to the working directory); a model without the directory refuses save and restore with `JLLAMA_STATUS_ERROR`. Four `CApiRpc` cases pin `jllama_rpc_server_stop()` around the bind and the next serve: a stop issued before ggml binds the socket -- from the serving thread through the test seam `jllama::rpc::set_serve_hook_for_testing()` (`rpc_support.hpp`, not exported), or from another thread while the serve is held in that seam, where the stop must not return before the serve has -- ends the serve instead of being lost (patches/0015 clears ggml's stop flag at the bind; red before F241), a stop while nothing serves does not stop the next serve, and a stop does not reach a serve started while it waits for the one it is for (the stop seam `set_stop_hook_for_testing()` starts the next serve once the stop is recorded; red before F271: the stop waited 30 s for the newcomer and ended it too). Two `CApiFim` cases load the same random model with FIM tokens (`write_tiny_llama_gguf(prefix, true)`): a raw completion with `input_prefix`/`input_suffix`, blocking and streamed, evaluates exactly the prompt `jllama_infill` builds from the same fields, and one of the two fields alone is refused naming the other (red before F246: the completion path completed the empty prompt and dropped both). Two more `CApi` cases pin `messages` on the raw paths: blocking and streamed, the request evaluates exactly the tokens of a completion of the prompt `jllama_apply_template` renders from them (an empty `prompt` beside them counts as none), and a non-empty `prompt` or the infill fields beside them are refused (red before F247: the field was ignored). Two more: the chat fields the raw paths cannot apply beside `messages` (`response_format`, `tools`, `tool_choice`, `parallel_tool_calls`, `top_logprobs`, `thinking_budget_tokens`) are refused by name on both raw paths, and `CApiTemplateStops` loads the random model with an end-of-generation byte of its own (`write_tiny_llama_gguf(prefix, false, 4)`) and a template upstream adds the stop string `` to, which a grammar forces: the raw paths stop there as the chat path does, also beside a `stop` of the request (red before F248: dropped, `stop_type` `eos` and the string in `content`). And `CApiReasoningBudget` loads it with a template that opens a `` block in its generation prompt: a raw completion with `messages` and `reasoning_budget_tokens` 1 completes the forced `` and then the chat path's answer, blocking and streamed, while an output-format grammar beside it still runs (the generation prompt the budget is primed with is left out there); a template without thinking tags leaves the request unchanged (red before F249: the tags were dropped and the budget did nothing). A third case loads the random model with ``, `` and `` as control tokens (`write_tiny_llama_gguf(prefix, false, 4, special)`): the forced end tag reads as text in the raw content, blocking and streamed, and a request's own `preserved_tokens` stays in force beside the template's (red before F250: the session's preserved tokens were not taken over, and the special end tag was an empty piece). A fourth, without `messages`, names the tags in the request with a logit bias that makes the model write the start tag: a budget of zero forces the end tag after each one, and a `reasoning_budget_message` before it (red before F252: without the message the budget forced nothing). Four cases pin the raw multimodal prompt: `CApiVision` loads the random model with a random llava-style vision projector the shared fixture writes too (`write_tiny_mmproj_gguf()`, `--mmproj`) and finds the server's `media_marker` in the model meta; a raw completion whose `prompt` is upstream's `{"prompt_string": "......", "multimodal_data": [""]}` evaluates the text tokens plus the image's four patch embeddings on the blocking, streamed and OpenAI paths (red before F258: refused as if the model had no projector), while markers that do not match the images, data that is not base64, no data and an image at the very end (an abort of the process in `update_slots`) are refused, and the same object on the model without a projector is the caller's fault. `CApiVocabOnly` loads the random model with `--vocab-only`: the meta, tokenize and detokenize calls answer, and a parameterized case per other C function on a model (22 instances: every completion, chat and stream path, the template, embeddings, rerank, infill, metrics, slots, LoRA, configure, systemone, the attached server) is `JLLAMA_STATUS_ERROR` with "`jllama_` is not available in vocab-only mode" (red before F266: a `SIGSEGV` in `llama_vocab::token_bos()` or a wait that never ended). Two pin the `capabilities` object of the model meta (b11901, from upstream's server meta): the random model chats and neither generates nor streams audio, with the vision projector loaded as well (`CApiVision`), while a vocab-only load claims none; and `CApiTts` refuses `jllama_tts_supports_speaker_reference()` with a null handle or out-parameter. | +| `src/test/cpp/test_c_api.cpp` | 55 | **The C API (`jllama_api.h`, layer B) without a JVM.** Writes a tiny random two-layer llama GGUF with the GPT-2 byte tokenizer (the public `gguf` API, like `test_kolibri1.cpp`), loads it through `jllama_model_load` and drives the shipped contract: version and build info, null-argument refusal with a message, a missing model failing to load, the load-progress callback (reported, and a `false` return aborts the load), a chat completion that is an OpenAI `chat.completion` object, an invalid request answered as the caller's fault (`JLLAMA_STATUS_INVALID_REQUEST`), streaming until `stop` with the task released afterwards ("Task not found" on the next receive), a released task gone, and the log callback seeing the server's lines with a null set flushing synchronously. Hermetic, so it runs in every `C++ Tests` job without `restore-models`; the output of a random model is noise, the contract is what is asserted. Two things the fixture does on purpose: the output rows of every non-ASCII and special token are scaled to almost nothing, because a byte above 0x7F is an incomplete UTF-8 piece the server holds back rather than streams (a random model that emits only those sends no partial chunk at all), and the task id is checked `>= 0` -- the server numbers its tasks from zero. Then every other `LlamaModel` operation in turn (the meta, raw and OAI completions, raw streaming, the streamed OAI completion (`text_completion` chunks with one id, probabilities, `usage` on the last), the template, tokenize/detokenize, metrics, slots, LoRA, configure, the refusals of a model loaded without `--embedding`/`--reranking`/FIM tokens, the grammar conversion, a Q8_0 quantisation that loads again, several prompts in one request and several completions of one (`"n"` > 1) refused on the blocking, raw and streamed paths while a token array stays one prompt, and every OpenAI answer and chunk carrying a `chatcmpl-` id and the model name, one id per stream), then the other engines: the embedded server in router mode and attached to the model (which still answers after the server stopped), the RPC server over loopback, text to speech refusing missing models, and a fine-tuning run over the random F32 model that writes a GGUF -- which is why `tts_engine.cpp`, `train_engine.cpp` and `native_server_engine.cpp` are in `jllama_test`'s source list. Five model-free cases (`CApiStatic`) run before any load. `CApiSlotFiles` loads the random model with `--slot-save-path`: a saved slot file lands under that directory by its name and restores, and a name with a directory part, `..`, an absolute path or nothing is `JLLAMA_STATUS_INVALID_REQUEST` with nothing written (red before F251: the name was the path, relative to the working directory); a model without the directory refuses save and restore with `JLLAMA_STATUS_ERROR`. Four `CApiRpc` cases pin `jllama_rpc_server_stop()` around the bind and the next serve: a stop issued before ggml binds the socket -- from the serving thread through the test seam `jllama::rpc::set_serve_hook_for_testing()` (`rpc_support.hpp`, not exported), or from another thread while the serve is held in that seam, where the stop must not return before the serve has -- ends the serve instead of being lost (patches/0015 clears ggml's stop flag at the bind; red before F241), a stop while nothing serves does not stop the next serve, and a stop does not reach a serve started while it waits for the one it is for (the stop seam `set_stop_hook_for_testing()` starts the next serve once the stop is recorded; red before F271: the stop waited 30 s for the newcomer and ended it too). Two `CApiFim` cases load the same random model with FIM tokens (`write_tiny_llama_gguf(prefix, true)`): a raw completion with `input_prefix`/`input_suffix`, blocking and streamed, evaluates exactly the prompt `jllama_infill` builds from the same fields, and one of the two fields alone is refused naming the other (red before F246: the completion path completed the empty prompt and dropped both). Two more `CApi` cases pin `messages` on the raw paths: blocking and streamed, the request evaluates exactly the tokens of a completion of the prompt `jllama_apply_template` renders from them (an empty `prompt` beside them counts as none), and a non-empty `prompt` or the infill fields beside them are refused (red before F247: the field was ignored). Two more: the chat fields the raw paths cannot apply beside `messages` (`response_format`, `tools`, `tool_choice`, `parallel_tool_calls`, `top_logprobs`, `thinking_budget_tokens`) are refused by name on both raw paths, and `CApiTemplateStops` loads the random model with an end-of-generation byte of its own (`write_tiny_llama_gguf(prefix, false, 4)`) and a template upstream adds the stop string `` to, which a grammar forces: the raw paths stop there as the chat path does, also beside a `stop` of the request (red before F248: dropped, `stop_type` `eos` and the string in `content`). And `CApiReasoningBudget` loads it with a template that opens a `` block in its generation prompt: a raw completion with `messages` and `reasoning_budget_tokens` 1 completes the forced `` and then the chat path's answer, blocking and streamed, while an output-format grammar beside it still runs (the generation prompt the budget is primed with is left out there); a template without thinking tags leaves the request unchanged (red before F249: the tags were dropped and the budget did nothing). A third case loads the random model with ``, `` and `` as control tokens (`write_tiny_llama_gguf(prefix, false, 4, special)`): the forced end tag reads as text in the raw content, blocking and streamed, and a request's own `preserved_tokens` stays in force beside the template's (red before F250: the session's preserved tokens were not taken over, and the special end tag was an empty piece). A fourth, without `messages`, names the tags in the request with a logit bias that makes the model write the start tag: a budget of zero forces the end tag after each one, and a `reasoning_budget_message` before it (red before F252: without the message the budget forced nothing). Four cases pin the raw multimodal prompt: `CApiVision` loads the random model with a random llava-style vision projector the shared fixture writes too (`write_tiny_mmproj_gguf()`, `--mmproj`) and finds the server's `media_marker` in the model meta; a raw completion whose `prompt` is upstream's `{"prompt_string": "......", "multimodal_data": [""]}` evaluates the text tokens plus the image's four patch embeddings on the blocking, streamed and OpenAI paths (red before F258: refused as if the model had no projector), while markers that do not match the images, data that is not base64, no data and an image at the very end (an abort of the process in `update_slots`) are refused, and the same object on the model without a projector is the caller's fault. `CApiVocabOnly` loads the random model with `--vocab-only`: the meta, tokenize and detokenize calls answer, and a parameterized case per other C function on a model (22 instances: every completion, chat and stream path, the template, embeddings, rerank, infill, metrics, slots, LoRA, configure, systemone, the attached server) is `JLLAMA_STATUS_ERROR` with "`jllama_` is not available in vocab-only mode" (red before F266: a `SIGSEGV` in `llama_vocab::token_bos()` or a wait that never ended). Two pin the `capabilities` object of the model meta (b11901, from upstream's server meta): the random model chats and neither generates nor streams audio, with the vision projector loaded as well (`CApiVision`), while a vocab-only load claims none; and `CApiTts` refuses `jllama_tts_supports_speaker_reference()` with a null handle or out-parameter. | | `src/test/cpp/test_native_server_shutdown.cpp` | 3 | **The runnable guard for the shutdown-handler guard in `patches/0006`/`0007`.** Runs the real `llama_server()` in router mode over an empty `--models-dir` on an ephemeral loopback port (no model, no worker) and stops it the way `native_server.cpp` does. Pins a clean stop (exit code 0), that `llama_server_request_shutdown()` **after** the server returned is a no-op (the deterministic form of the CI `SIGSEGV` — it crashed every run before the fix), and that requests hammered from another thread while the server tears down are safe (5 rounds). Compiled on non-Android only, like `server.cpp` itself. | | `src/test/cpp/test_gallocr_leafs.cpp` | 1 | **The runnable guard for `patches/0020`.** Two graphs with the same two nodes in the same order (`n1 = add(x, x)`, `n2 = mul(w, n1)`) and only the leafs moved: one expands `n1` before `n2` and meets the input first, the other expands only `n2`, whose first source is the weight -- the shape of the llama.cpp case, where every node and node source still matches its slot, so the old check passed it. Reserved with the first, the second must get an allocation for the input inside the allocator's buffer and compute the right product on it. No model, every platform. **Verified both ways:** with the fix removed from the source tree the test dies with SIGSEGV (rc 139); a first version that swapped the operands of one `add` passed without the fix, because a swapped *node source* was already caught -- a guard has to move only the leafs. | | `src/test/cpp/test_lora_cache.cpp` | 5 | **The runnable guard for `patches/0021`.** Writes a random llama model (the shared `tiny_llama_gguf.hpp`, also used by `test_c_api.cpp`) and a random LoRA adapter on K, V and the FFN output of every layer, loads them with one slot and `--cache-ram 0`, and answers the same prompt greedily. After `jllama_set_lora_adapters` (scale 1 -> 0 and back) and after a request with its own adapters, the warm slot must answer exactly like an emptied one. One case guards the others against passing vacuously: the adapter changes the answer, and a repeated request reuses the cache (`timings.cache_n > 0`; not `tokens_cached`, which counts every token the slot holds and is positive after any request). After each adapter change the next request must take nothing from the cache (`timings.cache_n == 0`), so those three cases are red without the patch whether or not the random model happens to answer differently. **Verified both ways:** without the patch 3 of 4 red (the warm slot answered from the old scales), with it 4 of 4 green in three runs; no model, every platform. A second fixture (`LoraCacheFreshSlot`: two slots, `--slot-save-path`) saves slot 0 (by its file name inside that directory) after a request and restores it into the never-used slot 1, whose request must take all of the prompt but at most two tokens from it and answer like slot 0 -- red before the `init()` hunk (`Expected: (prompt_n) <= (2), actual: 77 vs 2`), green with it (F240). | +| `src/test/cpp/test_moe_repack.cpp` | 2 | **The runnable guard for `patches/0022`.** `MUL_MAT_ID` over weights in the `CPU_REPACK` buffer -- MXFP4, Q4_0, IQ4_NL and Q4_K, random, quantized with `ggml_quantize_chunk` -- against ggml's generic `MUL_MAT_ID` over the same bytes in plain memory (independent code, the same activation quantization), for both settings of the switch on 1, 3 and 4 threads (3 does not divide the 9 column blocks, 1 leaves a unit past the last column). Routings: experts with 0, 1, 3, 4, 15 and 17 tokens (the gemv rest alone, one gemm group, a full 16-row group plus a rest), the same with one slot of -1 (its row stays zero), and a single token; each with one input row per token (gate/up) and one per slot (down): 24 instances. A type the running CPU does not repack is skipped, not passed. The second test resolves the switch through the CPU backend's `get_proc_address`, as libjllama does, and round-trips it, so a bump that drops the patch reds this file. | -**Current total: 678 tests** (`TEST`/`TEST_F`/`TEST_P` definitions; ctest counts 699 cases, because -the parameterized `CApiVocabOnly` case expands to 22 instances). +**Current total: 681 tests** (`TEST`/`TEST_F`/`TEST_P` definitions; ctest counts 725 cases, because +the parameterized `CApiVocabOnly` case expands to 22 instances and `MoeRepack` to 24). #### Upstream source location (in CMake build tree) diff --git a/README.md b/README.md index f1adc43bd..b926f30ae 100644 --- a/README.md +++ b/README.md @@ -1678,6 +1678,29 @@ The `LogLevel` enum values passed to the callback correspond to the native llama | `WARN` | Non-fatal warnings | | `ERROR` | Errors that may affect inference results | +### Mixture-of-experts models on the CPU + +A mixture-of-experts model (gpt-oss, Qwen3-MoE, ...) whose expert weights the CPU backend repacks at +load -- MXFP4, Q4_0, IQ4_NL and Q4_K experts on an AVX2 or ARM CPU -- runs its expert +multiplications through a path this project changes (`llama/patches/0022`): the tokens routed to one +expert are multiplied together, and the work is handed out to the threads in small units, so a slower +core takes fewer of them. Upstream llama.cpp multiplies one token at a time and gives every thread a +fixed share. The results are the same up to rounding; measured on gpt-oss-20b in a sandbox, prompt +processing was 47 % faster and token generation 26 % +([`docs/benchmarks/moe-mxfp4-cpu-alderlake.md`](docs/benchmarks/moe-mxfp4-cpu-alderlake.md)). + +Both paths stay available, so the difference can be measured on your own machine: + +```java +LlamaModel.setCpuMoeBatched(false); // upstream's path +LlamaModel.setCpuMoeBatched(true); // the batched path (the default) +boolean batched = LlamaModel.isCpuMoeBatched(); +``` + +The switch is process-wide and takes effect from the next computation on, for `NativeServer` as well. +`GGML_CPU_REPACK_MMID_BATCHED=0` in the environment starts a process on upstream's path without a +line of code, e.g. for the server launched from the command line. + ## Importing in Android > [!IMPORTANT] diff --git a/TODO.md b/TODO.md index 0f7c9b5a1..3fd5bf427 100644 --- a/TODO.md +++ b/TODO.md @@ -150,6 +150,7 @@ Kept as a list because each entry names the machine it needs -- the harness is |---|---|---| | **OpenVINO on an Intel GPU/NPU** | suitable Intel device, unavailable to these sessions | the 21.7x/336x figures are CPU-plugin fallback only; they do not prove GPU/NPU execution | | the two **SME** aarch64 modules | Linux aarch64 with SME, absent from current CI/cloud machines | macOS's static Metal library does not exercise these modules | +| **gpt-oss-20b on both MoE paths of `llama/patches/0022`**, on the **Core Ultra 7 165U** (`-t 6/8/10`) | the laptop; switch with `LlamaModel.setCpuMoeBatched` or `GGML_CPU_REPACK_MMID_BATCHED=0` ([`docs/benchmarks/moe-mxfp4-cpu-alderlake.md`](docs/benchmarks/moe-mxfp4-cpu-alderlake.md)) | upstream's static work split meets real P-, E- and LP-E-cores only there; the sandbox could emulate one slow core for prompt processing (−49 % upstream's path, −24 % batched) but not for generation | | [`docs/local-test-plan.md`](docs/local-test-plan.md) parts A–D on the **Core Ultra 7 165U** | the laptop the Windows guide (#575) is written for | part A -- which CPU module ggml picks there and what it gains -- is what CI cannot measure; the only report so far is the Zen 3 one (`docs/handover/local-agent-report-b11538-windows.md`). One result file per machine, unmeasured values marked as such | ## Open — jllama-specific @@ -368,8 +369,8 @@ because each is a design decision in code this project does not own. And say pla downstream instead (the opt-in, the non-aborting registration, the `-w` on `ggml-hip`), so a maintainer can see what a merged change would let this project delete. -There are **twelve** patches today (`0001`–`0003`, `0006`–`0008`, `0014`, `0015`, -`0017`, `0018`, `0020`, `0021`). Each +There are **thirteen** patches today (`0001`–`0003`, `0006`–`0008`, `0014`, `0015`, +`0017`, `0018`, `0020`, `0021`, `0022`). Each accepted upstream change deletes a patch from the bump checklist once the pin is bumped past it. - **Submittable as they are** (self-contained, no jllama specifics): `0001` (Windows arg-parse embed @@ -383,7 +384,10 @@ accepted upstream change deletes a patch from the bump checklist once the pin is the clearest submission of the set, since the defect reproduces on any AMX machine and the fix copies the line `ggml-backend.cpp` already has), `0020` (validate leaf allocations before reusing a graph allocation), `0021` (invalidate the slot prompt cache when server LoRA adapters - change; both hunks have independent drop-checks in `CLAUDE.md`). + change; both hunks have independent drop-checks in `CLAUDE.md`), `0022` (batch an expert's tokens + through gemm in the repack `MUL_MAT_ID` and hand its work out dynamically -- submitted **without** + its switch, which exists for measurements here; the evidence is + `docs/benchmarks/moe-mxfp4-cpu-alderlake.md`). - **Permanent:** `0003` carries upstream PR #22393, which upstream closed without merging. - **Two upstream-only defects worth an issue on that day, no patch here:** a lost RPC server mid-inference still aborts the process (the ggml backend interface has no error return for a lost diff --git a/docs/benchmarks/README.md b/docs/benchmarks/README.md index 47f2572e4..5e891c23f 100644 --- a/docs/benchmarks/README.md +++ b/docs/benchmarks/README.md @@ -14,6 +14,7 @@ themselves live in [`../../TODO.md`](../../TODO.md). | File | What it answers | |---|---| | [`cpu-backend-modules.md`](cpu-backend-modules.md) | What each of the 22 shipped CPU backend modules is worth, per microarchitecture — and whether `ggml_backend_load_best` picks the fastest one | +| [`moe-mxfp4-cpu-alderlake.md`](moe-mxfp4-cpu-alderlake.md) | Where the `alderlake` module (Core Ultra) loses time on Qwen3 and gpt-oss-20b, measured with random-weight models in the real shapes -- and what the fix to ggml's MoE path in `llama/patches/0022` is worth (+47 % prompt, +26 % generation; switchable for A/B) | | [`gpu-module-jars.md`](gpu-module-jars.md) | What happens when a GPU module jar is actually *used* -- two findings that contradict the "a module that cannot help is skipped" premise | | [`arm-fp16-conversion.md`](arm-fp16-conversion.md) | Every shipped ARM binary converts fp32 to fp16 in software, ours and upstream's: 26 of 26 disassembled, 8.9x to 13.3x on the function, and 0.03 % of cycles so inference does not move | | [`upstream-f16c-30270.md`](upstream-f16c-30270.md) | An isolation of one upstream commit: what it does, that it changes nothing measurable here, and a compile-time guard proven red against the defect | diff --git a/docs/benchmarks/moe-mxfp4-cpu-alderlake.md b/docs/benchmarks/moe-mxfp4-cpu-alderlake.md new file mode 100644 index 000000000..a57417127 --- /dev/null +++ b/docs/benchmarks/moe-mxfp4-cpu-alderlake.md @@ -0,0 +1,383 @@ + + +# gpt-oss-20b on the `alderlake` CPU module: where the time goes + +**The question.** The Windows guide [`../windows-intel-core-ultra.md`](../windows-intel-core-ultra.md) +puts a Core Ultra 7 165U on `ggml-cpu-alderlake` (AVX2 + AVX-VNNI, no AVX-512, no AMX). Where does +that module lose compute, on which models, and what would make gpt-oss-20b faster there? + +**The answer, measured 2026-10-26/27 on b11887 + `llama/patches/`, the module staged alone:** + +1. **Dense Q4_K_M models lose little.** Qwen3-4B spends 89-92 % of its thread time in the Q4_K / Q6_K + matmuls, Qwen3-0.6B 75-86 % (plus 16 % flash attention in prompt processing); both wait 3-8 % at + barriers. There is no lever there short of the kernels themselves. +2. **gpt-oss-20b loses a lot, in one place: ggml-cpu's MoE path for repacked weights** + (`forward_mul_mat_id` in `ggml/src/ggml-cpu/repack.cpp`, identical on upstream master on + 2026-10-26). It has two defects: + - **Prompt processing computes every token on its own** (one GEMV per token per expert, never the + GEMM kernel that exists for the same type). 72 % of prompt time is that loop. + - **Its work is split statically**, one fixed row range per thread. Token generation therefore + waits for the slowest thread after every expert matmul: 13.9 % of all thread time in token + generation is that wait. Every other matmul hands out chunks dynamically and waits 1-2 %. +3. **A fix for both** (dynamic work units plus GEMM for groups of ≥ 4 tokens per expert), first as a + prototype ([`../../.github/scripts/bench/mmid-dynamic-gemm.diff`](../../.github/scripts/bench/mmid-dynamic-gemm.diff)), + then as [`llama/patches/0022`](../patches.md#0022-ggml-cpu-repack-batch-the-tokens-of-an-expertpatch), + gives **+46-47 % prompt processing and +23-26 % token generation** at 4 threads, **+59 % prompt + processing single-threaded**, with logits identical to rounding. With one core emulated at half + speed, prompt processing on upstream's path halves (−49 %), on the batched path it loses 24 %. + That emulation is the closest this machine gets to a P-core/E-core laptop. +4. **No configuration knob makes it faster, and one makes it much slower.** Repack on and flash + attention on (both defaults) are already the fastest settings, the sampler chain costs nothing + measurable, and the server path (the code `NativeServer` runs) is within 3-5 % of `llama-bench`. + But a **quantized KV cache** (`-ctk q8_0 -ctv q8_0`, as in the README's gpt-oss-20b example) cuts + prompt processing at an 8k context to **a quarter** (30.76 → 7.45 t/s): ggml's fast + flash-attention paths need an f16/f32 cache with K and V of one type. + +The patch keeps upstream's path behind a **process-wide switch**, on by default, so the comparison +can be repeated on any machine without a second build: `LlamaModel.setCpuMoeBatched(boolean)` in +Java, `jllama_set_cpu_moe_batched()` in C, or `GGML_CPU_REPACK_MMID_BATCHED=0` in the environment +of any process ([Patch 0022 and its switch](#patch-0022-and-its-switch)). + +## What this run can and cannot show + +**Can:** which code `alderlake` runs, where its time goes, and what a change to that code is worth on +the same machine. The kernels are the module's own: `-mavx2 -mavxvnni -mfma -mf16c -mbmi2`, built with +plain clang like the shipped `cpu-windows-x86-64` jar. + +**Cannot:** + +- **The laptop's numbers.** The host is a 4-vCPU KVM guest on an Intel Xeon (family 6, model 173, + Granite Rapids generation) with 15 GiB RAM. Single-threaded it reads 15.4 GB/s, with 4 threads + 55.7 GB/s (a 2 GiB AVX2 read probe). The 165U has a hybrid core set (2 P + 8 E + 2 LP-E), + LPDDR5 shared with the iGPU and a 15 W budget. **Absolute t/s here are not a forecast for it.** + The ratios between code variants on one machine are what transfers, and even those need the + device to confirm them. +- **Linux, clang 18.1.3.** The shipped Windows jar is built by clang 20 (the runner's LLVM) and runs + on Windows' scheduler. +- **Real routing.** The mock's router has random weights, so tokens spread evenly over the 32 + experts (≈ 64 per expert in a 512-token batch). A trained router is less even. That changes how + many tokens each expert's GEMM gets, not whether GEMM applies. +- **Hybrid cores, properly.** The slow-core emulation (a busy loop sharing one pinned core) halves + that core's throughput, but for token generation the OS's time slices dominate. A thread that is + descheduled for a whole slice stalls hundreds of barriers, which a slower but continuously running + E-core does not do. Only the prompt-processing half of that table means what it says. +- **Vulkan / the Arc iGPU.** No GPU here. + +## Setup + +### The build + +llama.cpp at the pinned commit (`6050b467`, b11887) with `llama/patches/*.patch` applied in order, +built the way the modular library jars are: `GGML_BACKEND_DL=ON`, `GGML_CPU_ALL_VARIANTS=ON`, +`GGML_NATIVE=OFF`, `GGML_OPENMP=OFF`, `BUILD_SHARED_LIBS=ON`, Release (`-O3`), plain clang. On top, +upstream's own tools: `llama-bench`, `llama-quantize`, `llama-perplexity`, `llama-server`. + +```bash +git clone --depth 1 --branch b11887 https://github.com/ggml-org/llama.cpp && cd llama.cpp +for p in $(ls /llama/patches/*.patch | sort); do git apply "$p"; done +cmake -B build -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ \ + -DBUILD_SHARED_LIBS=ON -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON -DGGML_NATIVE=OFF -DGGML_OPENMP=OFF \ + -DLLAMA_BUILD_TOOLS=ON -DLLAMA_BUILD_SERVER=ON -DLLAMA_OPENSSL=OFF +cmake --build build -j --target llama-bench llama-quantize llama-perplexity llama-server +mkdir build/cpu-modules && mv build/bin/libggml-cpu-*.so build/cpu-modules/ +``` + +**Forcing one module.** ggml loads the best CPU module it finds next to the executable, which on this +host is `sapphirerapids`. With every module moved out of `bin/`, `GGML_BACKEND_PATH` names the one to +load. [`../../.github/scripts/bench/bench.sh`](../../.github/scripts/bench/bench.sh) does that and +refuses a `bin/` that still holds modules. The load line confirms it: +`load_backend: loaded CPU backend from .../libggml-cpu-alderlake.so`. + +### The models: random weights in the real shapes + +Hugging Face is not reachable from the machine, and nothing here depends on what a model *says*. +Throughput is set by every tensor's shape and type: the work and the bytes a token costs. +[`../../.github/scripts/bench/mock_gguf.py`](../../.github/scripts/bench/mock_gguf.py) writes those +shapes with random weights, one tensor at a time (a 12 GB file needs no 12 GB of RAM): + +| mock | written as | size | real file | +|---|---|--:|--:| +| Qwen3-0.6B (`Qwen/Qwen3-0.6B` config) | F16 → `llama-quantize Q4_K_M` | 393.9 MB | ~397 MB | +| Qwen3-4B-Instruct-2507 (config of that name) | F16 → `llama-quantize Q4_K_M` | 2,494 MB | ~2.5 GB | +| gpt-oss-20b (`openai/gpt-oss-20b` config) | directly in ggml-org's layout: experts MXFP4, every other matrix Q8_0 (llama-quantize's `MXFP4_MOE` rule), norms/biases/router/sinks F32 | 12,100.7 MB, 20.91 B params | `gpt-oss-20b-mxfp4.gguf`, 12.1 GB | + +The Qwen mocks go through `llama-quantize`, so their tensor-type mix *is* the real Q4_K_M mix +(Q6_K for the tied embedding/head and half of `attn_v`/`ffn_down`). The gpt-oss layout follows +`llama-quant.cpp`'s `MXFP4_MOE` branch, and the size check confirms it: Q8_0 for the non-expert +matrices adds up to 12.07 GB, BF16 would be 13.75 GB (the size of the published F16 file). The +vocabularies are the real sizes (151,936 and 201,088), since the output head is a large part of +every token. + +```bash +export PYTHONPATH=/gguf-py # gguf-py of the same tag +python3 .github/scripts/bench/mock_gguf.py gpt-oss-20b gpt-oss-20b-mxfp4.gguf # 67 s, 12.1 GB +python3 .github/scripts/bench/mock_gguf.py qwen3-4b qwen3-4b-f16.gguf +llama-quantize qwen3-4b-f16.gguf qwen3-4b-q4_k_m.gguf Q4_K_M +# --layers N writes fewer layers with the same per-layer work, for quick iterations +``` + +### Bytes per generated token + +What one generated token has to read, which sets the memory-bound ceiling: + +| model | bytes / token | of which | +|---|--:|---| +| Qwen3-0.6B Q4_K_M | 391 MB | the whole model, a third of it the Q6_K head (151,936 × 1,024) | +| Qwen3-4B Q4_K_M | 2,494 MB | the whole model | +| gpt-oss-20b MXFP4 | 2,570 MB | 4 of 32 experts × 24 layers 1,269 MB, attention (Q8_0) 677 MB, **output head (Q8_0, 201,088 × 2,880) 615 MB** | + +## Baseline: `alderlake`, unchanged code + +`llama-bench`, defaults otherwise (repack on, flash attention auto, f16 KV cache), Qwen3-0.6B `-r 5`, +the others `-r 3`. t/s ± standard deviation: + +| model | threads | pp512 | tg128 | +|---|--:|--:|--:| +| Qwen3-0.6B Q4_K_M | 1 | 127.33 ± 0.68 | 18.75 ± 0.35 | +| | 2 | 244.09 ± 4.95 | 31.75 ± 0.16 | +| | 4 | 452.17 ± 17.92 | 59.70 ± 1.01 | +| Qwen3-4B Q4_K_M | 1 | 18.00 ± 0.12 | 3.01 ± 0.02 | +| | 2 | 35.15 ± 0.43 | 5.39 ± 0.05 | +| | 4 | 65.65 ± 1.13 | 9.91 ± 0.07 | +| gpt-oss-20b MXFP4 | 1 | 10.22 ± 0.02 | 3.06 ± 0.04 | +| | 2 | 20.47 ± 0.22 | 5.24 ± 0.05 | +| | 4 | 43.79 ± 0.69 | 7.72 ± 0.03 | + +Two things stand out before any profile: + +- **Token generation is not memory-bound here.** Single-threaded, every model reads ~7.5 GB/s of + weights, half of what one core can stream (15.4). The AVX2 kernels are instruction-bound on this + host, which is also why AVX-512 changes so much (below). +- **gpt-oss scales worst.** From 1 to 4 threads its token generation rises 2.52×, Qwen3-4B's 3.29×, + Qwen3-0.6B's 3.18×. Its prompt processing runs at 62 GFLOP/s per thread (6.06 GFLOP of active + weights per token × 10.22 t/s), Qwen3-4B's at 131. + +## Where the time goes + +Two instruments. `perf record -e cpu-clock` (this VM exposes no hardware counters) for a flat +per-function profile. And a small instrumentation of ggml's graph loop, +[`../../.github/scripts/bench/ggml-opprof.diff`](../../.github/scripts/bench/ggml-opprof.diff) +(compiled in only with `-DGGML_OPPROF`). It sums, per op and source type, each thread's time *inside* +the op and *in the barrier after it*. The barrier time after an op is that op's load imbalance: +the time the faster threads wait for the slowest. + +4 threads, share of all thread time: + +| model / phase | biggest op | 2nd | 3rd | barrier wait (of which after the biggest op) | +|---|---|---|---|--:| +| gpt-oss tg | MoE MXFP4 `MUL_MAT_ID` 42.7 % | Q8_0 `MUL_MAT` 38.4 % | flash attn 0.8 % | **17.0 % (13.9 %)** | +| gpt-oss pp | MoE MXFP4 `MUL_MAT_ID` **72.3 %** | Q8_0 `MUL_MAT` 18.6 % | flash attn 2.6 % | 3.9 % (2.5 %) | +| Qwen3-4B tg | Q4_K 68.9 % | Q6_K 23.5 % | flash attn 1.4 % | 5.5 % (4.0 %) | +| Qwen3-4B pp | Q4_K 77.9 % | Q6_K 11.2 % | flash attn 6.3 % | 2.7 % (2.1 %) | +| Qwen3-0.6B tg | Q4_K 51.3 % | Q6_K (the head) 35.1 % | flash attn 4.2 % | 7.9 % (4.6 %) | +| Qwen3-0.6B pp | Q4_K 64.3 % | flash attn 16.2 % | Q6_K 10.6 % | 4.7 % (3.0 %) | + +The perf profile names the functions behind the two gpt-oss rows: in prompt processing **65 % of all +samples are `ggml_gemv_mxfp4_8x8_q8_0`**, the *single-row* kernel, and 16 % the llamafile Q8_0 GEMM +(attention). In token generation it is 39 % that GEMV, 35 % `ggml_vec_dot_q8_0_q8_0` (attention and +the head) and 18 % `ggml_graph_compute_thread`, which is threads spinning at a barrier. + +### Why: `forward_mul_mat_id` in the repack buffer + +MXFP4 expert weights are repacked at load into the 8×8 interleaved layout (`mxfp4_8x8_q8_0`, chosen +for every AVX2 CPU). For `MUL_MAT_ID` the repack buffer's `forward_mul_mat_id`: + +1. quantizes the activations, groups the token rows by expert, and passes one barrier; +2. gives **thread `ith` the fixed rows `[ith·n/nth, (ith+1)·n/nth)`** of every expert, and +3. loops over the expert's tokens calling **`gemv` once per token**. + +The plain `MUL_MAT` path of the same buffer (`forward_mul_mat`) does neither: it calls `gemm` for +every four rows and hands out chunks through `ggml_threadpool_chunk_add`. The repack `gemm` for +MXFP4 (`ggml_gemm_mxfp4_8x8_q8_0`, AVX2 with a 16-row tile) exists and is simply never reached from +the MoE path. In ggml's generic `MUL_MAT_ID` the new tiled kernels take over for batches, but they +cover K-quants and IQ types, not MXFP4. Upstream master's `forward_mul_mat_id` is byte-identical to +b11887's (checked 2026-10-26). + +What the static split costs depends on how equal the threads are. On this VM's identical vCPUs it +is the 13.9 % above. On a P-core + E-core machine every expert matmul runs at the E-cores' pace. + +### Configuration does not reach it + +gpt-oss, 4 threads, `-r 3`: + +| repack | flash attn | pp512 | tg128 | +|---|---|--:|--:| +| off | off | 31.41 ± 0.11 | 7.16 ± 0.07 | +| off | on | 32.58 ± 0.19 | 7.27 ± 0.01 | +| **on** | off | 40.39 ± 1.22 | **7.99 ± 0.02** | +| **on** | **on** | **43.41 ± 0.42** | 7.73 ± 0.05 | + +Repack (the default) is the faster path despite the per-token GEMV. Without it the generic +`vec_dot` path is slower still. Flash attention is a wash at an empty context (off at a long context +was not measured). **The server path** (`llama-server`, i.e. the +upstream `llama_server` that `NativeServer` runs in-process; `-c 4096 -np 1`, three 512-token +prompts with 128 tokens each) measured pp 42.61 ± 0.18 and tg 7.45 ± 0.04. That is 3 % and 5 % under +`llama-bench`, and identical with greedy sampling (7.45 ± 0.02), so neither the sampler chain over +201,088 logits nor the server loop is where time goes. + +### The KV cache type, at a long context + +An agent re-sends a growing conversation, so what matters there is a prompt *on top of* a filled +cache. `-d 8192` fills 8,192 tokens first, then measures. Unchanged code, flash attention on, +4 threads, `-r 3` (q8_0/q8_0: `-r 2`): + +| K / V cache | pp512 at depth 8192 | tg32 at depth 8192 | +|---|--:|--:| +| **f16 / f16** (default) | **30.76 ± 0.37** | 4.57 ± 0.04 | +| q8_0 / f16 | 11.21 ± 0.30 (−64 %) | 5.01 ± 0.08 (+10 %) | +| q8_0 / q8_0 | **7.45 ± 0.02 (−76 %)** | *not measured on this build* | + +On the prototype build (`-r 2`, generation only) the four combinations gave tg32 4.88 ± 0.21 (f16/f16), +4.72 ± 0.02 (f16/q8_0), 5.65 ± 0.09 (q8_0/f16) and 4.91 ± 0.11 (q8_0/q8_0). + +The cause is a type check, not the arithmetic. `ggml_compute_forward_flash_attn_ext` in +`ggml/src/ggml-cpu/ops.cpp` takes its tiled path for a batch of queries and its split-KV path for a +single query only when K and V are F32 or F16 **and of the same type** (`kv_is_f32_or_f16`, +`k->type == v->type`). Any quantized or mixed cache falls back to the per-row path. A quantized K +saves some bandwidth in generation, and the prompt side pays for it four times over. **On the CPU, +leave the cache at f16.** For gpt-oss the memory is not a reason either: at 65,536 tokens the f16 cache +of the 12 full-attention layers is 1.6 GB, and the 12 sliding-window layers keep only their +128-token window (plus one batch). + +(`llama-bench` caches the filled context by depth only and then fails to restore an f16 state into a +q8_0 cache, `incompatible key rotation`, and refills. The numbers are unaffected, the runs just take +longer.) + +### The instruction set + +Same machine, same build, one module at a time, 4 threads: + +| module | gpt-oss pp512 | gpt-oss tg128 | Qwen3-4B pp512 | Qwen3-4B tg128 | +|---|--:|--:|--:|--:| +| `haswell` (AVX2) | 38.41 ± 0.77 | 7.36 ± 0.03 | 66.74 ± 0.87 | 10.17 ± 0.08 | +| `alderlake` (+ AVX-VNNI) | 43.79 ± 0.69 | 7.72 ± 0.03 | 65.65 ± 1.13 | 9.91 ± 0.07 | +| `sapphirerapids` (AVX-512, AMX) | 50.73 ± 1.10 | 11.60 ± 0.07 | 146.29 ± 4.42 | 12.68 ± 0.08 | + +AVX-VNNI is worth 14 % / 5 % on gpt-oss (its MXFP4 and Q8_0 kernels use `vpdpbusd`) and nothing on +Q4_K. The Core Ultra's module is `alderlake` and cannot be anything else: AVX-512 is fused off in +Core Ultra series 1. The `sapphirerapids` row shows how much the 256-bit kernels leave on the table +on a core that is not memory-starved: 50 % of gpt-oss's token generation. + +## The fix, first as a prototype + +[`mmid-dynamic-gemm.diff`](../../.github/scripts/bench/mmid-dynamic-gemm.diff) changes only +`forward_mul_mat_id` and its `work_size`, behind the compile-time `-DJLLAMA_MMID_V2`, so one tree built +both sides of the A/B below. `llama/patches/0022` is the same code with the choice moved to run time +(next section): + +- **Work units.** After the grouping barrier, thread 0 lists the experts that received tokens. The + work becomes `n_active × nck` units of (expert, column chunk), with `nck` chosen for ~16 units per + thread. Threads take units from `ggml_threadpool_chunk_add`, as `forward_mul_mat` does. +- **GEMM.** Within a unit, every group of up to 16 of the expert's tokens (in multiples of 4) is + gathered from `src1`, interleave-quantized with `ggml_quantize_mat_t` (the same quantizer + `forward_mul_mat` uses for its GEMM rows), multiplied with `gemm` into a per-thread tile, and + scattered back to the token rows. Fewer than four leftover tokens keep the `gemv` path. +- **Cost:** ~420 KB of scratch per thread for gpt-oss's shapes, plus one list of `n_expert` integers. + +### Correctness + +`llama-perplexity` over 4 × 512 tokens of text, unchanged code with `--kl-divergence-base`, then the +prototype with `--kl-divergence`: + +| | | +|---|---| +| PPL, unchanged / prototype | 321390.708865 / 321390.684826 (ratio 1.000000 ± 0.000023) | +| mean / max KL divergence | 0.000000 / 0.000002 | +| same top token | **100.000 %** | + +The two paths quantize the activations identically and differ only in the order of the float +accumulation, which is what a KLD of 10⁻⁶ is. (A random model's perplexity is meaningless as a +number. Its agreement between two builds is the check.) + +### Speed + +Unchanged code and prototype alternated, `alderlake`, gpt-oss: + +| threads | test | unchanged | prototype | change | +|--:|---|--:|--:|--:| +| 1 | pp256 | 10.53 ± 0.03 | 16.74 ± 0.16 | **+59 %** | +| 1 | tg32 | 3.09 ± 0.01 | 3.23 ± 0.00 | +5 % | +| 2 | pp512 | 20.38 ± 0.23 | 32.39 ± 0.55 | **+59 %** | +| 2 | tg128 | 5.32 ± 0.12 | 5.58 ± 0.12 | +5 % | +| 4 | pp512 | 43.94 ± 0.25 | 64.10 ± 0.44 | **+46 %** | +| 4 | tg128 | 7.84 ± 0.05 | 9.63 ± 0.09 | **+23 %** | + +(1 thread `-r 2`, otherwise `-r 2` twice, n = 4 samples.) The single-threaded prompt row isolates +GEMM against GEMV: the same work, 37 % less time. The 4-thread generation row is the balance: the +barrier wait after `MUL_MAT_ID` falls from 13.9 % to 1.7 % of thread time and the total from 17.0 % +to 5.2 % (same instrumentation). Prompt processing now runs at 101 GFLOP/s per thread, three +quarters of the dense Q4_K model's 131. + +**One core at half speed**, all four threads pinned (`-C F --cpu-strict 1`), a busy loop sharing +core 3, `-r 3`: + +| | unchanged | prototype | +|---|--:|--:| +| pp256, no load | 43.50 ± 0.36 | 60.20 ± 4.10 | +| pp256, core 3 shared | **22.19 ± 0.79** (−49 %) | **45.77 ± 0.05** (−24 %) | +| tg64, no load | 7.96 ± 0.01 | 9.60 ± 0.13 | +| tg64, core 3 shared | 4.65 ± 0.02 | 5.49 ± 0.02 | + +With the static split the whole prompt runs at the slow core's pace. With work units the other +three take over its share, and what is lost is about that core's missing half. The generation rows +fall alike in both builds. That is the time-slice artefact described above, not a property of either +build, which is why they are not read as a hybrid-core result. + +### After the fix + +The prototype's profile at 4 threads. Token generation: MoE 47.0 %, **Q8_0 GEMV 45.5 %**, barrier 5.2 %. +Prompt processing: MoE 62.0 %, Q8_0 GEMM 27.4 %, flash attention 3.9 %, barrier 2.7 %. + +At a long context the picture shifts. With 8,192 tokens already in the cache (`-d 8192`, flash +attention on) the prototype measures pp512 **39.28** and tg32 **4.99**, against 64.10 and 9.63 at an +empty context, and flash attention becomes the second-largest op. + +## Patch 0022 and its switch + +`llama/patches/0022` is the prototype with the `#ifdef` replaced by a run-time switch, reachable from +every layer: + +| layer | how | +|---|---| +| ggml | `ggml_backend_cpu_set_repack_mmid_batched(bool)` / `..._get_...()` in `ggml-cpu.h`, and in the CPU backend's `get_proc_address` table -- in a `GGML_BACKEND_DL` build the CPU backend is a module loaded at run time, and that table is how anything outside it reaches the switch | +| C API | `jllama_set_cpu_moe_batched(bool)`, `jllama_get_cpu_moe_batched(bool *)` | +| Java | `LlamaModel.setCpuMoeBatched(boolean)`, `LlamaModel.isCpuMoeBatched()`; FFM: `Jllama.setCpuMoeBatched` / `isCpuMoeBatched` | +| any process | `GGML_CPU_REPACK_MMID_BATCHED=0` in the environment makes upstream's path the initial value -- also for `llama-bench` and a server started from the command line | + +It is process-wide. Thread 0 reads it once per `MUL_MAT_ID`, while it groups the tokens, and +publishes the choice in the work buffer before the barrier, so a flip during a running computation +cannot send the threads down different paths; the computation finishes on the path it started with. +`work_size()` reserves the batched path's scratch whatever the switch says, since it may flip between +planning a graph and running it. + +The same measurement as above, now **with one binary**, the switch flipped through the environment +(b11887 + all patches, `alderlake`, gpt-oss, 4 threads, `-r 2` twice, n = 4): + +| | upstream's path (`=0`) | batched (default) | change | +|---|--:|--:|--:| +| pp512 | 42.63 ± 1.16 | 62.76 ± 2.76 | **+47 %** | +| tg128 | 7.90 ± 0.07 | 9.97 ± 0.06 | **+26 %** | + +The guard is `llama/src/test/cpp/test_moe_repack.cpp`: both paths against ggml's generic +`MUL_MAT_ID` over the same weights (MXFP4, Q4_0, IQ4_NL, Q4_K), on 1, 3 and 4 threads, with experts +that get 0, 1, 3, 4, 15 and 17 tokens, an empty slot and a single token. + +## Next steps + +1. **Upstream.** The change applies to every repacked expert type, not only MXFP4, and is + upstream-shaped without the switch. The release policy for upstream submissions ("nothing before + the release") no longer blocks it. +2. **Measure on the Core Ultra 7 165U**, both paths through the switch, `-t 6/8/10`: that is where the + static split meets real P-, E- and LP-E-cores, and the hybrid half of this report can only be + settled there. The KV-cache table is worth repeating there too, at the context an agent session + actually reaches. +3. **The README's gpt-oss-20b `NativeServer` example** runs on the CPU (`-ngl 0`) with + `-ctk q8_0 -ctv q8_0`, the configuration the KV table above measures at a quarter of the prompt + speed. Whether the example keeps those flags, for a reason not visible here, is the owner's call. +4. **The next candidates, unmeasured as fixes:** the Q8_0 GEMV for token generation (45.5 % after the + fix; x86 has no repacked Q8_0 path, ARM has one), and flash attention at long contexts (64 query + heads over 8 KV heads). diff --git a/docs/c-api.md b/docs/c-api.md index 970316be6..20687d898 100644 --- a/docs/c-api.md +++ b/docs/c-api.md @@ -54,6 +54,7 @@ Every operation of `LlamaModel` and `LlamaQuantizer` is in it (the roadmap's "B. | `jllama_json_schema_to_grammar()` | a JSON schema as GBNF (no model needed) | | `jllama_quantize()` | `llama_model_quantize` over a GGUF file (no model needed) | | `jllama_set_log_callback()` | every log line of llama.cpp, ggml and the server to a callback; null restores the console after a synchronous flush | +| `jllama_set_cpu_moe_batched()`, `jllama_get_cpu_moe_batched()` | how the CPU backend runs the expert matmuls of a MoE model over repacked weights: batched per expert with the work handed out dynamically (the default), or upstream's one gemv per token over a static split (`llama/patches/0022`); process-wide, `GGML_CPU_REPACK_MMID_BATCHED=0` in the environment starts it off | | `jllama_server_start()`, `jllama_server_start_attached()`, `jllama_server_stop()`, `jllama_server_is_running()`, `jllama_server_set_worker_command()` | the embedded llama-server (`NativeServer`): the full upstream server from an argv, WebUI included, or the HTTP frontend attached to a loaded model; router mode's worker command. Not on Android (server.cpp does not build there) | | `jllama_rpc_server_serve()`, `jllama_rpc_server_stop()`, `jllama_rpc_server_listening()`, `jllama_rpc_server_devices()`, `jllama_strings_free()` | the RPC server (`RpcServer`): this process's devices served to llama.cpp RPC clients, one per process | | `jllama_tts_load()`, `jllama_tts_synthesize()`, `jllama_tts_supports_speaker_reference()`, `jllama_tts_free()`, `jllama_bytes_free()` | text to speech (`TextToSpeech`): a Qwen3-TTS backbone and mmproj, a WAV per call; whether the pair clones a voice from a speaker reference (a pair that does not refuses one) | diff --git a/docs/patches.md b/docs/patches.md index f32a4389f..ce23672ad 100644 --- a/docs/patches.md +++ b/docs/patches.md @@ -18,7 +18,7 @@ upstream-shaped, and dropped here the moment upstream carries the fix (the drop- in its `jllama-build.txt` (`patches=`) and a hash over the set (`patches.sha256=`), so a consumer can match a binary to this page. -**12 patches.** Ids as `jllama-build.txt` lists them under `patches=`. +**13 patches.** Ids as `jllama-build.txt` lists them under `patches=`. | Id | Patch | Title | Files | Hunks | |---|---|---|---:|---:| @@ -34,6 +34,7 @@ match a binary to this page. | 0018 | [`0018-ggml-amx-free-the-aligned-buffer.patch`](#0018-ggml-amx-free-the-aligned-bufferpatch) | ggml-cpu(amx): free the AMX weight buffer with ggml_aligned_free(), not free() | 1 | 1 | | 0020 | [`0020-ggml-alloc-validate-leafs-before-reusing-an-allocation.patch`](#0020-ggml-alloc-validate-leafs-before-reusing-an-allocationpatch) | ggml-alloc : validate the leafs before reusing a graph's allocation | 1 | 1 | | 0021 | [`0021-server-clear-the-slot-cache-when-the-server-adapters-change.patch`](#0021-server-clear-the-slot-cache-when-the-server-adapters-changepatch) | server : clear a slot's cache when the server's LoRA adapters change | 1 | 2 | +| 0022 | [`0022-ggml-cpu-repack-batch-the-tokens-of-an-expert.patch`](#0022-ggml-cpu-repack-batch-the-tokens-of-an-expertpatch) | ggml-cpu : batch the tokens of an expert in the repack MUL_MAT_ID, and hand its work out dynamically | 3 | 7 | ## 0001-win32-arg-parse-embed-guard.patch @@ -514,3 +515,65 @@ params_base.lora_adapters with slot.lora and clears the prompt (or SET_LORA clea drop that hunk rather than refreshing it; and read the slot loop of init() -- once it sets slot.lora from params_base.lora_adapters, drop that hunk. The patch goes when both have. ``` + +## 0022-ggml-cpu-repack-batch-the-tokens-of-an-expert.patch + +**Files (3):** `ggml/include/ggml-cpu.h`, `ggml/src/ggml-cpu/ggml-cpu.cpp`, `ggml/src/ggml-cpu/repack.cpp` + +The patch's own header, verbatim: + +```text +ggml-cpu : batch the tokens of an expert in the repack MUL_MAT_ID, and hand its work out dynamically + +forward_mul_mat_id() of the CPU_REPACK buffer (ggml/src/ggml-cpu/repack.cpp) runs the expert +matmuls of every MoE model whose expert weights are repacked at load -- MXFP4 (gpt-oss), Q4_0, +IQ4_NL and Q4_K on AVX2, and their interleaved forms on ARM -- and it pays two costs that +forward_mul_mat() of the same buffer does not: + +1. One gemv per token. The tokens routed to an expert are multiplied one at a time, so the 4-bit + weights are unpacked once per token, and the gemm kernel that exists for the same type is never + reached from here. In ggml's generic MUL_MAT_ID the tiled kernels take over for batches, but they + cover K-quants and IQ types, not MXFP4. +2. A static split. Thread ith gets the fixed rows [ith*n/nth, (ith+1)*n/nth) of every expert, so + each expert matmul ends when the slowest thread does -- on a hybrid CPU, at the pace of its + slowest core. forward_mul_mat() hands its chunks out through ggml_threadpool_chunk_add(). + +The patch keeps that path and adds a second one; thread 0 picks one per op, before the grouping +barrier, from a process-wide switch, and publishes the choice in the work buffer, so a flip during a +compute cannot send the threads down different paths: + + ggml_backend_cpu_set_repack_mmid_batched(bool) / ggml_backend_cpu_get_repack_mmid_batched() + (ggml-cpu.h, and in the CPU backend's get_proc_address table for a GGML_BACKEND_DL build). + On by default; GGML_CPU_REPACK_MMID_BATCHED=0 in the environment makes off the initial value. + +The batched path: thread 0 lists the experts that got rows; the work is (expert, column chunk) +units, about 16 per thread, taken through ggml_threadpool_chunk_add(). In a unit, every group of up +to 16 of the expert's tokens (in fours) is gathered from src1, quantized interleaved with +ggml_quantize_mat_t() -- the quantizer forward_mul_mat() uses for its gemm rows --, multiplied with +gemm() into a per-thread tile and scattered back to the token rows; fewer than four leftover tokens +take the gemv path from the rows quantized before. work_size() reserves the per-op state, the +expert list and the per-thread scratch (16 float rows, their quantized form and a 16-row output +tile; ~420 KB per thread for gpt-oss-20b) whatever the switch says, since it may flip before the +graph runs. + +Measured in java-llama.cpp (docs/benchmarks/moe-mxfp4-cpu-alderlake.md): a random-weight GGUF in +gpt-oss-20b's exact shape and ggml-org's MXFP4 layout, the alderlake CPU module (AVX2 + AVX-VNNI) +staged alone, 4 threads: prompt processing +46 % (43.9 -> 64.1 t/s), token generation +23 % +(7.84 -> 9.63 t/s), single-threaded prompt processing +59 %; the barrier wait after MUL_MAT_ID +fell from 13.9 % to 1.7 % of all thread time. llama-perplexity with --kl-divergence against the +unchanged code: mean KLD 0.000000, max 0.000002, same top token 100 %. + +The switch is this project's addition, for A/B measurements on a user's own machine +(LlamaModel.setCpuMoeBatched, jllama_set_cpu_moe_batched, the environment variable); an upstream +submission would offer the batched path alone. + +The guard is src/test/cpp/test_moe_repack.cpp: MUL_MAT_ID over repacked weights of four types, with +both settings of the switch and 1, 3 and 4 threads, against the generic path over the same weights +in a plain buffer, for routings that give an expert 0, 1, 3, 4 and 17 tokens, a slot of -1, one +token alone and one input row per slot; and the switch is resolved through the CPU backend's +get_proc_address, so a bump that drops the patch reds the test at once. + +Drop-check on every bump: read forward_mul_mat_id() in ggml/src/ggml-cpu/repack.cpp at the new tag +-- once upstream calls gemm for an expert's batched rows and hands the work out through +ggml_threadpool_chunk_add(), drop this patch (and with it the switch) rather than refreshing it. +``` diff --git a/docs/windows-intel-core-ultra.md b/docs/windows-intel-core-ultra.md index c52171a98..b323758aa 100644 --- a/docs/windows-intel-core-ultra.md +++ b/docs/windows-intel-core-ultra.md @@ -313,6 +313,53 @@ Measure `-t 6`, `-t 8` and `-t 10` and take the best. `-tb` sets the prompt-proc separately and tolerates a higher value, since prompt processing has enough work per barrier to hide the synchronisation cost. +## gpt-oss-20b + +ggml-org's `gpt-oss-20b-mxfp4.gguf` (12.1 GB: the experts in MXFP4, everything else Q8_0) runs on +the CPU modules. It needs about 13 GB of RAM plus the KV cache, so a 32 GB machine holds it next to +Windows. The notes below were **measured in a Linux sandbox with the `alderlake` module staged alone, +not on this laptop**. [`benchmarks/moe-mxfp4-cpu-alderlake.md`](benchmarks/moe-mxfp4-cpu-alderlake.md) +has the numbers and their limits: + +- **Leave the KV cache at f16**, i.e. no `-ctk`/`-ctv`. With `-ctk q8_0 -ctv q8_0`, prompt processing + at an 8k context fell to a quarter (30.8 → 7.5 t/s): ggml's fast flash-attention paths need an f16 + cache with K and V of the same type. For an agent, which re-sends the whole conversation every + turn, that is the number that counts. The f16 cache costs 1.6 GB at `-c 65536`. +- **Repack and flash attention stay at their defaults**, which were the fastest. +- **Measure `-tb` for this model separately.** Upstream llama.cpp (and the 6.0.0 jar) splits the + expert matmuls statically across threads, in prompt processing as well, so every thread waits for + the slowest one. With one core at half speed, prompt processing halved. The LP-E cores are exactly + such cores, so the `-tb` that suits a dense model may be too high here. + +```bat +mvn -o exec:java -Dexec.args="-m C:\models\gpt-oss-20b-mxfp4.gguf --host 127.0.0.1 --port 9931 -c 32768 -ngl 0 -t 8 -tb 8 --jinja" +``` + +`-ngl 0` keeps it on the CPU modules, the configuration measured. Through Vulkan on the Arc iGPU +(`-ngl 99`) it is unmeasured for this model. + +### Both expert paths, measured on this machine + +From the release after 6.0.0 on, the natives carry `llama/patches/0022` (6.0.0 itself runs upstream's +path only): the tokens routed to one expert are multiplied +together and the work goes to the threads in small units, so the E- and LP-E-cores take less of it. +In the sandbox that gave +47 % prompt processing and +26 % token generation. Upstream's path stays +switchable, so the difference can be measured here, where the cores really differ: +`LlamaModel.setCpuMoeBatched(false)` in your own code, or for the server the environment variable, +set in the window **before** `mvn` starts: + +```bat +:: upstream's path; leave the line out (or open a new window) for the batched default +set GGML_CPU_REPACK_MMID_BATCHED=0 +mvn -o exec:java -Dexec.args="-m C:\models\gpt-oss-20b-mxfp4.gguf --port 9931 -c 32768 -ngl 0 -t 8 -tb 8 --jinja" +``` + +Then send the same long prompt to each run and compare the `timings` of the response, e.g. +`prompt_per_second` and `predicted_per_second` from +`curl.exe http://127.0.0.1:9931/completion -d "{\"prompt\":\"...\",\"n_predict\":128,\"cache_prompt\":false}"`. +A second request with the same prompt and `cache_prompt` on would take the prompt from the cache and +measure nothing. + ## Verifying what loaded The start-up log names the library, the modules put in place and the devices ggml ended up with: diff --git a/llama-ffm/src/main/java/net/ladenthin/llama/ffm/Jllama.java b/llama-ffm/src/main/java/net/ladenthin/llama/ffm/Jllama.java index 1c19474da..37bed438b 100644 --- a/llama-ffm/src/main/java/net/ladenthin/llama/ffm/Jllama.java +++ b/llama-ffm/src/main/java/net/ladenthin/llama/ffm/Jllama.java @@ -3,6 +3,7 @@ // SPDX-License-Identifier: MIT package net.ladenthin.llama.ffm; +import static java.lang.foreign.ValueLayout.JAVA_BOOLEAN; import static net.ladenthin.llama.ffm.JllamaNative.call; import static net.ladenthin.llama.ffm.JllamaNative.cString; import static net.ladenthin.llama.ffm.JllamaNative.invoke; @@ -91,6 +92,36 @@ public static synchronized void setLogCallback(BiConsumer call } } + /** + * Chooses how the CPU backend runs the expert matrix multiplications of a mixture-of-experts model + * over repacked weights: batched per expert with the work handed out dynamically ({@code true}, the + * default) or as upstream llama.cpp does, one multiplication per token over a fixed split + * ({@code false}). Process-wide and the same switch as {@code LlamaModel.setCpuMoeBatched} of the + * core, whose documentation has the details; {@code GGML_CPU_REPACK_MMID_BATCHED=0} in the + * environment makes {@code false} the initial value. + * + * @param batched {@code true} for the batched path, {@code false} for upstream's + * @throws LlamaException if the loaded library was built without the switch + */ + public static void setCpuMoeBatched(boolean batched) { + call(JllamaNative.SET_CPU_MOE_BATCHED, batched); + } + + /** + * Whether the CPU backend runs the expert matrix multiplications batched; see + * {@link #setCpuMoeBatched(boolean)}. + * + * @return {@code true} for the batched path (the default), {@code false} for upstream's + * @throws LlamaException if the loaded library was built without the switch + */ + public static boolean isCpuMoeBatched() { + try (Arena arena = Arena.ofConfined()) { + MemorySegment out = arena.allocate(JAVA_BOOLEAN); + call(JllamaNative.GET_CPU_MOE_BATCHED, out); + return out.get(JAVA_BOOLEAN, 0); + } + } + @SuppressWarnings("unused") // the upcall target of setLogCallback private static void onLog(BiConsumer callback, int level, MemorySegment text, MemorySegment userData) { diff --git a/llama-ffm/src/main/java/net/ladenthin/llama/ffm/JllamaNative.java b/llama-ffm/src/main/java/net/ladenthin/llama/ffm/JllamaNative.java index 17a62cd2f..a7f020183 100644 --- a/llama-ffm/src/main/java/net/ladenthin/llama/ffm/JllamaNative.java +++ b/llama-ffm/src/main/java/net/ladenthin/llama/ffm/JllamaNative.java @@ -83,6 +83,10 @@ final class JllamaNative { static final MethodHandle QUANTIZE = downcall( "jllama_quantize", FunctionDescriptor.of(JAVA_INT, ADDRESS, ADDRESS, JAVA_INT, JAVA_INT, JAVA_BOOLEAN)); static final MethodHandle FINETUNE = downcall("jllama_finetune", FunctionDescriptor.of(JAVA_INT, ADDRESS)); + static final MethodHandle SET_CPU_MOE_BATCHED = + downcall("jllama_set_cpu_moe_batched", FunctionDescriptor.of(JAVA_INT, JAVA_BOOLEAN)); + static final MethodHandle GET_CPU_MOE_BATCHED = + downcall("jllama_get_cpu_moe_batched", FunctionDescriptor.of(JAVA_INT, ADDRESS)); // -- the model ---------------------------------------------------------------------------------- static final MethodHandle MODEL_LOAD = downcall( diff --git a/llama-ffm/src/test/java/net/ladenthin/llama/ffm/JllamaNativeTest.java b/llama-ffm/src/test/java/net/ladenthin/llama/ffm/JllamaNativeTest.java index f377c4484..9dad4e65a 100644 --- a/llama-ffm/src/test/java/net/ladenthin/llama/ffm/JllamaNativeTest.java +++ b/llama-ffm/src/test/java/net/ladenthin/llama/ffm/JllamaNativeTest.java @@ -103,6 +103,23 @@ void takeStringsDoesNotFreeANullArray() { } } + /** + * llama/patches/0022's switch through the FFM binding: it reaches the CPU backend's entry points + * (the same process-wide switch as {@code LlamaModel.setCpuMoeBatched}) and round-trips. + */ + @Test + void cpuMoeBatchedSwitchRoundTrips() { + boolean initial = Jllama.isCpuMoeBatched(); + try { + Jllama.setCpuMoeBatched(false); + assertThat(Jllama.isCpuMoeBatched(), is(false)); + Jllama.setCpuMoeBatched(true); + assertThat(Jllama.isCpuMoeBatched(), is(true)); + } finally { + Jllama.setCpuMoeBatched(initial); + } + } + @Test void apiVersionIsTheVersionThisModuleWasBuiltAs() { String expected = System.getProperty("net.ladenthin.llama.ffm.expected.version"); diff --git a/llama/CMakeLists.txt b/llama/CMakeLists.txt index 188f66767..3878923db 100644 --- a/llama/CMakeLists.txt +++ b/llama/CMakeLists.txt @@ -1367,6 +1367,7 @@ if(BUILD_TESTING) src/test/cpp/test_rpc_comm.cpp src/test/cpp/test_gallocr_leafs.cpp src/test/cpp/test_lora_cache.cpp + src/test/cpp/test_moe_repack.cpp ${llama.cpp_SOURCE_DIR}/tools/server/server-common.cpp ${llama.cpp_SOURCE_DIR}/tools/server/server-chat.cpp ${llama.cpp_SOURCE_DIR}/tools/server/server-context.cpp diff --git a/llama/patches/0022-ggml-cpu-repack-batch-the-tokens-of-an-expert.patch b/llama/patches/0022-ggml-cpu-repack-batch-the-tokens-of-an-expert.patch new file mode 100644 index 000000000..3da1f2a27 --- /dev/null +++ b/llama/patches/0022-ggml-cpu-repack-batch-the-tokens-of-an-expert.patch @@ -0,0 +1,254 @@ +ggml-cpu : batch the tokens of an expert in the repack MUL_MAT_ID, and hand its work out dynamically + +forward_mul_mat_id() of the CPU_REPACK buffer (ggml/src/ggml-cpu/repack.cpp) runs the expert +matmuls of every MoE model whose expert weights are repacked at load -- MXFP4 (gpt-oss), Q4_0, +IQ4_NL and Q4_K on AVX2, and their interleaved forms on ARM -- and it pays two costs that +forward_mul_mat() of the same buffer does not: + +1. One gemv per token. The tokens routed to an expert are multiplied one at a time, so the 4-bit + weights are unpacked once per token, and the gemm kernel that exists for the same type is never + reached from here. In ggml's generic MUL_MAT_ID the tiled kernels take over for batches, but they + cover K-quants and IQ types, not MXFP4. +2. A static split. Thread ith gets the fixed rows [ith*n/nth, (ith+1)*n/nth) of every expert, so + each expert matmul ends when the slowest thread does -- on a hybrid CPU, at the pace of its + slowest core. forward_mul_mat() hands its chunks out through ggml_threadpool_chunk_add(). + +The patch keeps that path and adds a second one; thread 0 picks one per op, before the grouping +barrier, from a process-wide switch, and publishes the choice in the work buffer, so a flip during a +compute cannot send the threads down different paths: + + ggml_backend_cpu_set_repack_mmid_batched(bool) / ggml_backend_cpu_get_repack_mmid_batched() + (ggml-cpu.h, and in the CPU backend's get_proc_address table for a GGML_BACKEND_DL build). + On by default; GGML_CPU_REPACK_MMID_BATCHED=0 in the environment makes off the initial value. + +The batched path: thread 0 lists the experts that got rows; the work is (expert, column chunk) +units, about 16 per thread, taken through ggml_threadpool_chunk_add(). In a unit, every group of up +to 16 of the expert's tokens (in fours) is gathered from src1, quantized interleaved with +ggml_quantize_mat_t() -- the quantizer forward_mul_mat() uses for its gemm rows --, multiplied with +gemm() into a per-thread tile and scattered back to the token rows; fewer than four leftover tokens +take the gemv path from the rows quantized before. work_size() reserves the per-op state, the +expert list and the per-thread scratch (16 float rows, their quantized form and a 16-row output +tile; ~420 KB per thread for gpt-oss-20b) whatever the switch says, since it may flip before the +graph runs. + +Measured in java-llama.cpp (docs/benchmarks/moe-mxfp4-cpu-alderlake.md): a random-weight GGUF in +gpt-oss-20b's exact shape and ggml-org's MXFP4 layout, the alderlake CPU module (AVX2 + AVX-VNNI) +staged alone, 4 threads: prompt processing +46 % (43.9 -> 64.1 t/s), token generation +23 % +(7.84 -> 9.63 t/s), single-threaded prompt processing +59 %; the barrier wait after MUL_MAT_ID +fell from 13.9 % to 1.7 % of all thread time. llama-perplexity with --kl-divergence against the +unchanged code: mean KLD 0.000000, max 0.000002, same top token 100 %. + +The switch is this project's addition, for A/B measurements on a user's own machine +(LlamaModel.setCpuMoeBatched, jllama_set_cpu_moe_batched, the environment variable); an upstream +submission would offer the batched path alone. + +The guard is src/test/cpp/test_moe_repack.cpp: MUL_MAT_ID over repacked weights of four types, with +both settings of the switch and 1, 3 and 4 threads, against the generic path over the same weights +in a plain buffer, for routings that give an expert 0, 1, 3, 4 and 17 tokens, a slot of -1, one +token alone and one input row per slot; and the switch is resolved through the CPU backend's +get_proc_address, so a bump that drops the patch reds the test at once. + +Drop-check on every bump: read forward_mul_mat_id() in ggml/src/ggml-cpu/repack.cpp at the new tag +-- once upstream calls gemm for an expert's batched rows and hands the work out through +ggml_threadpool_chunk_add(), drop this patch (and with it the switch) rather than refreshing it. + +diff --git a/ggml/include/ggml-cpu.h b/ggml/include/ggml-cpu.h +index dc6453c..2330c96 100644 +--- a/ggml/include/ggml-cpu.h ++++ b/ggml/include/ggml-cpu.h +@@ -138,6 +138,13 @@ extern "C" { + + GGML_BACKEND_API void ggml_backend_cpu_set_use_ref(ggml_backend_t backend_cpu, bool use_ref); + ++ // MUL_MAT_ID over weights in the CPU_REPACK buffer: hand the work out as (expert, column chunk) ++ // units and run every group of four or more tokens routed to one expert through gemm (true, the ++ // default), or split the rows statically and run one gemv per token (false). Process-wide; the ++ // environment variable GGML_CPU_REPACK_MMID_BATCHED=0 makes false the initial value. ++ GGML_BACKEND_API void ggml_backend_cpu_set_repack_mmid_batched(bool batched); ++ GGML_BACKEND_API bool ggml_backend_cpu_get_repack_mmid_batched(void); ++ + GGML_BACKEND_API ggml_backend_reg_t ggml_backend_cpu_reg(void); + + GGML_BACKEND_API void ggml_cpu_fp32_to_fp32(const float *, float *, int64_t); +diff --git a/ggml/src/ggml-cpu/ggml-cpu.cpp b/ggml/src/ggml-cpu/ggml-cpu.cpp +index 81ff5d7..05738c6 100644 +--- a/ggml/src/ggml-cpu/ggml-cpu.cpp ++++ b/ggml/src/ggml-cpu/ggml-cpu.cpp +@@ -679,6 +679,12 @@ static void * ggml_backend_cpu_get_proc_address(ggml_backend_reg_t reg, const ch + if (strcmp(name, "ggml_backend_cpu_set_use_ref") == 0) { + return (void *)ggml_backend_cpu_set_use_ref; + } ++ if (strcmp(name, "ggml_backend_cpu_set_repack_mmid_batched") == 0) { ++ return (void *)ggml_backend_cpu_set_repack_mmid_batched; ++ } ++ if (strcmp(name, "ggml_backend_cpu_get_repack_mmid_batched") == 0) { ++ return (void *)ggml_backend_cpu_get_repack_mmid_batched; ++ } + + // threadpool - TODO: move to ggml-base + if (strcmp(name, "ggml_threadpool_new") == 0) { +diff --git a/ggml/src/ggml-cpu/repack.cpp b/ggml/src/ggml-cpu/repack.cpp +index 9282ca2..bdeb684 100644 +--- a/ggml/src/ggml-cpu/repack.cpp ++++ b/ggml/src/ggml-cpu/repack.cpp +@@ -15,9 +15,29 @@ + #include + #include + #include // for GGML_ASSERT ++#include ++#include + + #include "repack.h" + ++// MUL_MAT_ID over repacked weights: batched (work units + gemm) or one gemv per token, see ++// ggml_backend_cpu_set_repack_mmid_batched in ggml-cpu.h ++static std::atomic & repack_mmid_batched() { ++ static std::atomic batched{[] { ++ const char * env = std::getenv("GGML_CPU_REPACK_MMID_BATCHED"); ++ return env == nullptr || strcmp(env, "0") != 0; ++ }()}; ++ return batched; ++} ++ ++void ggml_backend_cpu_set_repack_mmid_batched(bool batched) { ++ repack_mmid_batched().store(batched); ++} ++ ++bool ggml_backend_cpu_get_repack_mmid_batched(void) { ++ return repack_mmid_batched().load(); ++} ++ + #if defined(__GNUC__) + #pragma GCC diagnostic ignored "-Woverlength-strings" + #endif +@@ -4554,7 +4574,16 @@ class tensor_traits_base : public ggml::cpu::tensor_traits { + + template class tensor_traits : public tensor_traits_base { + +- bool work_size(int /* n_threads */, const struct ggml_tensor * op, size_t & size) override { ++ // MUL_MAT_ID, batched path: token rows per gemm call, and the scratch each thread needs for one ++ static constexpr int64_t MMID_GEMM_ROWS = 16; ++ ++ static size_t mmid_thread_scratch(int64_t ne10, int64_t ne01) { ++ return GGML_PAD(MMID_GEMM_ROWS * ne10 * sizeof(float), 64) + ++ GGML_PAD(MMID_GEMM_ROWS * ggml_row_size(PARAM_TYPE, ne10), 64) + ++ GGML_PAD(MMID_GEMM_ROWS * ne01 * sizeof(float), 64); ++ } ++ ++ bool work_size(int n_threads, const struct ggml_tensor * op, size_t & size) override { + // not realy a GGML_TYPE_Q8_0 but same size. + switch (op->op) { + case GGML_OP_MUL_MAT: +@@ -4574,6 +4603,11 @@ template src[1]->ne[0], op->src[0]->ne[1]); ++ + return true; + } + default: +@@ -4845,6 +4879,9 @@ template 0) { ++ mmid_state[2 + n_active++] = cur_a; ++ } ++ } ++ mmid_state[1] = n_active; ++ ggml_threadpool_chunk_set(params->threadpool, nth); ++ } + } + + ggml_barrier(params->threadpool); + ++ if (mmid_state[0]) { ++ // Batched: threads take (expert, column chunk) units dynamically, so a slower core takes ++ // fewer of them instead of holding the barrier, and every group of four or more tokens ++ // routed to the same expert goes through one gemm instead of a gemv per token. ++ const int64_t n_active = mmid_state[1]; ++ const int64_t * active = mmid_state + 2; ++ const int64_t n_col_blocks = ne01 / NB_COLS; ++ if (n_active == 0) { ++ return; ++ } ++ const int64_t nck = MAX(1, MIN(n_col_blocks, (nth * 16 + n_active - 1) / n_active)); ++ const int64_t cols_per = ((n_col_blocks + nck - 1) / nck) * NB_COLS; ++ const int64_t n_units = n_active * nck; ++ ++ char * scratch = (char *) (((uintptr_t) (mmid_state + 2 + n_as) + 63) & ~(uintptr_t) 63) + ++ ith * mmid_thread_scratch(ne10, ne01); ++ float * fbuf = (float *) scratch; ++ char * qbuf = scratch + GGML_PAD(MMID_GEMM_ROWS * ne10 * sizeof(float), 64); ++ float * obuf = (float *) (qbuf + GGML_PAD(MMID_GEMM_ROWS * nbw1, 64)); ++ GGML_ASSERT((char *) (obuf + MMID_GEMM_ROWS * ne01) <= (char *) params->wdata + params->wsize); ++ ++ for (int64_t unit = ith; unit < n_units; unit = ggml_threadpool_chunk_add(params->threadpool, 1)) { ++ const int64_t cur_a = active[unit / nck]; ++ const int64_t c0 = (unit % nck) * cols_per; ++ const int64_t c1 = MIN(c0 + cols_per, ne01); ++ if (c0 >= c1) { ++ continue; ++ } ++ const int64_t nc = c1 - c0; ++ const int64_t cne1 = matrix_row_counts[cur_a]; ++ const char * src0_cur = (const char *) src0->data + cur_a*nb02 + c0*nb01; ++ ++ // groups of four or more rows: gather the float rows, quantize them interleaved as the ++ // gemm of forward_mul_mat does, multiply, and scatter the rows back ++ int64_t ir1 = 0; ++ while (cne1 - ir1 >= 4) { ++ const int64_t nr = MIN(MMID_GEMM_ROWS, (cne1 - ir1) & ~(int64_t) 3); ++ for (int64_t r = 0; r < nr; r++) { ++ const struct mmid_row_mapping rm = MMID_MATRIX_ROW(cur_a, ir1 + r); ++ memcpy(fbuf + r*ne10, (const char *) src1->data + (rm.i1 % ne11)*nb11 + rm.i2*nb12, ++ ne10*sizeof(float)); ++ } ++ for (int64_t r = 0; r < nr; r += 4) { ++ ggml_quantize_mat_t(fbuf + r*ne10, qbuf + r*nbw1, 4, ne10); ++ } ++ gemm(ne00, obuf, nc, src0_cur, qbuf, nr, nc); ++ for (int64_t r = 0; r < nr; r++) { ++ const struct mmid_row_mapping rm = MMID_MATRIX_ROW(cur_a, ir1 + r); ++ memcpy((float *) ((char *) dst->data + rm.i1*nb1 + rm.i2*nb2) + c0, obuf + r*nc, ++ nc*sizeof(float)); ++ } ++ ir1 += nr; ++ } ++ // the rest, fewer than four rows: one gemv each, from the rows quantized above ++ for (; ir1 < cne1; ir1++) { ++ const struct mmid_row_mapping rm = MMID_MATRIX_ROW(cur_a, ir1); ++ const char * src1_col = (const char *) wdata + ((rm.i1 % ne11) * nbw1 + rm.i2 * nbw2); ++ gemv( ++ ne00, (float *) ((char *) dst->data + (rm.i1 * nb1 + rm.i2 * nb2)) + c0, ne01, ++ src0_cur, src1_col, 1, nc); ++ } ++ } ++ return; ++ } ++ + // compute each matrix multiplication in sequence + for (int cur_a = 0; cur_a < n_as; ++cur_a) { + const int64_t cne1 = matrix_row_counts[cur_a]; diff --git a/llama/src/main/cpp/jllama.cpp b/llama/src/main/cpp/jllama.cpp index 90bb4749a..50132bb10 100644 --- a/llama/src/main/cpp/jllama.cpp +++ b/llama/src/main/cpp/jllama.cpp @@ -1223,6 +1223,31 @@ JNIEXPORT void JNICALL Java_net_ladenthin_llama_LlamaModel_setLogger(JNIEnv *env }); } +// Reach LlamaModel.setCpuMoeBatched / isCpuMoeBatched (patches/0022's switch, process-wide). +JNIEXPORT void JNICALL Java_net_ladenthin_llama_LlamaModel_setCpuMoeBatched(JNIEnv *env, jclass clazz, + jboolean batched) { + return jni_guard_impl(env, c_llama_error, [&]() -> void { + (void)clazz; + const jllama_status status = jllama_set_cpu_moe_batched(batched == JNI_TRUE); + if (status != JLLAMA_STATUS_OK) { + throw_api_status(env, status); + } + }); +} + +JNIEXPORT jboolean JNICALL Java_net_ladenthin_llama_LlamaModel_isCpuMoeBatched(JNIEnv *env, jclass clazz) { + return jni_guard_impl(env, c_llama_error, [&]() -> jboolean { + (void)clazz; + bool batched = false; + const jllama_status status = jllama_get_cpu_moe_batched(&batched); + if (status != JLLAMA_STATUS_OK) { + throw_api_status(env, status); + return JNI_FALSE; + } + return batched ? JNI_TRUE : JNI_FALSE; + }); +} + JNIEXPORT jbyteArray JNICALL Java_net_ladenthin_llama_LlamaModel_jsonSchemaToGrammarBytes(JNIEnv *env, jclass clazz, jstring j_schema) { return jni_guard_impl(env, c_llama_error, [&]() -> jbyteArray { diff --git a/llama/src/main/cpp/jllama.h b/llama/src/main/cpp/jllama.h index 29109b4dd..006c07672 100644 --- a/llama/src/main/cpp/jllama.h +++ b/llama/src/main/cpp/jllama.h @@ -91,6 +91,20 @@ JNIEXPORT void JNICALL Java_net_ladenthin_llama_LlamaModel_delete(JNIEnv *, jobj */ JNIEXPORT void JNICALL Java_net_ladenthin_llama_LlamaModel_releaseTask(JNIEnv *, jobject, jint); +/* + * Class: net_ladenthin_llama_LlamaModel + * Method: setCpuMoeBatched + * Signature: (Z)V + */ +JNIEXPORT void JNICALL Java_net_ladenthin_llama_LlamaModel_setCpuMoeBatched(JNIEnv *, jclass, jboolean); + +/* + * Class: net_ladenthin_llama_LlamaModel + * Method: isCpuMoeBatched + * Signature: ()Z + */ +JNIEXPORT jboolean JNICALL Java_net_ladenthin_llama_LlamaModel_isCpuMoeBatched(JNIEnv *, jclass); + /* * Class: net_ladenthin_llama_LlamaModel * Method: jsonSchemaToGrammarBytes diff --git a/llama/src/main/cpp/jllama_api.cpp b/llama/src/main/cpp/jllama_api.cpp index bd5b360a7..f2aff2e99 100644 --- a/llama/src/main/cpp/jllama_api.cpp +++ b/llama/src/main/cpp/jllama_api.cpp @@ -867,6 +867,33 @@ jllama_status jllama_finetune(const char *config_json) { } } +jllama_status jllama_set_cpu_moe_batched(bool batched) { + try { + jllama::engine::set_cpu_moe_batched(batched); + t_last_error.clear(); + return JLLAMA_STATUS_OK; + } catch (const std::exception &e) { + return fail(JLLAMA_STATUS_ERROR, e.what()); + } catch (...) { + return fail(JLLAMA_STATUS_ERROR, "unknown C++ exception"); + } +} + +jllama_status jllama_get_cpu_moe_batched(bool *batched) { + if (batched == nullptr) { + return fail(JLLAMA_STATUS_INVALID_ARGUMENT, "batched must not be null"); + } + try { + *batched = jllama::engine::cpu_moe_batched(); + t_last_error.clear(); + return JLLAMA_STATUS_OK; + } catch (const std::exception &e) { + return fail(JLLAMA_STATUS_ERROR, e.what()); + } catch (...) { + return fail(JLLAMA_STATUS_ERROR, "unknown C++ exception"); + } +} + jllama_status jllama_set_log_callback(jllama_log_fn callback, void *user_data) { try { if (callback == nullptr) { diff --git a/llama/src/main/cpp/jllama_api.h b/llama/src/main/cpp/jllama_api.h index bb866611c..79313a52d 100644 --- a/llama/src/main/cpp/jllama_api.h +++ b/llama/src/main/cpp/jllama_api.h @@ -161,6 +161,20 @@ JLLAMA_API jllama_status jllama_task_release(jllama_model *model, int task_id); */ JLLAMA_API jllama_status jllama_set_log_callback(jllama_log_fn callback, void *user_data); +/** + * Whether ggml's CPU backend runs the expert matmuls of a MoE model over repacked weights batched -- + * each expert's tokens through one gemm, the work handed out dynamically across the threads (true, + * the default) -- or as upstream llama.cpp does, one gemv per token over a static split of the rows + * (false); llama/patches/0022. Process-wide, from the next graph computed on; the environment + * variable GGML_CPU_REPACK_MMID_BATCHED=0 makes false the initial value. It needs the CPU backend + * loaded, which a GGML_BACKEND_DL build does at its first model load (or, under Java, when the + * library is loaded); JLLAMA_STATUS_ERROR before that. + */ +JLLAMA_API jllama_status jllama_set_cpu_moe_batched(bool batched); + +/** Reads the switch of jllama_set_cpu_moe_batched into `*batched`. */ +JLLAMA_API jllama_status jllama_get_cpu_moe_batched(bool *batched); + /* ------------------------------------------------------------------------------------------------ * The rest of LlamaModel. Request and response bodies are the ones llama.cpp's HTTP server reads * and writes for the route of the same name; every out-string is released with jllama_string_free, diff --git a/llama/src/main/cpp/jllama_engine.cpp b/llama/src/main/cpp/jllama_engine.cpp index 11a5e3210..b7756af30 100644 --- a/llama/src/main/cpp/jllama_engine.cpp +++ b/llama/src/main/cpp/jllama_engine.cpp @@ -606,6 +606,28 @@ void set_log_sink(common_log_callback callback, void *user_data) { } } +namespace { +// patches/0022's switch; decltype ties the expected signature to ggml-cpu.h's declaration +template Fn cpu_proc(const char *name) { + ggml_backend_reg_t reg = ggml_backend_reg_by_name("CPU"); + void *fn = reg != nullptr ? ggml_backend_reg_get_proc_address(reg, name) : nullptr; + if (fn == nullptr) { + throw std::runtime_error(reg == nullptr ? std::string("no CPU backend is loaded yet") + : std::string("the CPU backend does not provide ") + name + + " (a library built without llama/patches/0022)"); + } + return reinterpret_cast(fn); +} +} // namespace + +void set_cpu_moe_batched(bool batched) { + cpu_proc("ggml_backend_cpu_set_repack_mmid_batched")(batched); +} + +bool cpu_moe_batched() { + return cpu_proc("ggml_backend_cpu_get_repack_mmid_batched")(); +} + // --------------------------------------------------------------------------- // The rest of LlamaModel (B.3). // --------------------------------------------------------------------------- diff --git a/llama/src/main/cpp/jllama_engine.hpp b/llama/src/main/cpp/jllama_engine.hpp index af2e0c1df..728f9de98 100644 --- a/llama/src/main/cpp/jllama_engine.hpp +++ b/llama/src/main/cpp/jllama_engine.hpp @@ -156,6 +156,17 @@ void task_release(jllama_context *jctx, int id_task); */ void set_log_sink(common_log_callback callback, void *user_data); +/** + * The process-wide switch of patches/0022: whether ggml's CPU backend runs MUL_MAT_ID over repacked + * weights (the expert matmuls of a MoE model) batched -- each expert's tokens through gemm, the work + * handed out dynamically across threads -- or as upstream does, one gemv per token over a static row + * split. It lives in the CPU backend, a module of its own in a GGML_BACKEND_DL build, so it is + * resolved through the backend's proc-address table like rpc_proc<>; throws when the CPU backend is + * not loaded yet or lacks the entry points (a library built without the patch). + */ +void set_cpu_moe_batched(bool batched); +[[nodiscard]] bool cpu_moe_batched(); + // --------------------------------------------------------------------------- // The rest of LlamaModel, operation by operation (TODO.md item 7, B.3). Every function takes and // returns JSON text or plain C++ values, wakes the server from idle sleep where it touches the diff --git a/llama/src/main/java/net/ladenthin/llama/LlamaModel.java b/llama/src/main/java/net/ladenthin/llama/LlamaModel.java index f11861776..d2603b9a0 100644 --- a/llama/src/main/java/net/ladenthin/llama/LlamaModel.java +++ b/llama/src/main/java/net/ladenthin/llama/LlamaModel.java @@ -455,6 +455,40 @@ public String decode(int... tokens) { */ public static native void setLogger(LogFormat format, BiConsumer callback); + /** + * Chooses how the CPU backend runs the expert matrix multiplications of a mixture-of-experts model + * (gpt-oss, Qwen3-MoE, ...) whose expert weights were repacked at load, which on an AVX2 or ARM CPU + * is the case for MXFP4, Q4_0, IQ4_NL and Q4_K experts. + * + *

{@code true}, the default: the tokens routed to one expert are multiplied together, and the + * work is handed out to the threads in small units, so a slower core (an efficiency core) takes + * fewer of them. {@code false}: upstream llama.cpp's way, one multiplication per token and a fixed + * share of every expert per thread. The results are the same up to floating-point rounding; the + * switch exists to measure the difference on a given machine (llama/patches/0022). Prompt processing + * gains from both halves, token generation (one token at a time) from the work distribution only; + * dense models and layers offloaded to a GPU are not affected.

+ * + *

Process-wide, for every model, {@code NativeServer} included, from the next computation on; + * a computation already running finishes on the path it started with. Setting the environment + * variable {@code GGML_CPU_REPACK_MMID_BATCHED=0} before the JVM starts makes {@code false} the + * initial value, also for a process that never calls this method.

+ * + * @param batched {@code true} for the batched path, {@code false} for upstream's + * @throws net.ladenthin.llama.exception.LlamaException if the loaded native library was built + * without the switch + */ + public static native void setCpuMoeBatched(boolean batched); + + /** + * Whether the CPU backend runs the expert matrix multiplications of a mixture-of-experts model + * batched; see {@link #setCpuMoeBatched(boolean)}. + * + * @return {@code true} for the batched path (the default), {@code false} for upstream's + * @throws net.ladenthin.llama.exception.LlamaException if the loaded native library was built + * without the switch + */ + public static native boolean isCpuMoeBatched(); + /** * Returns the upstream llama.cpp build identifier actually linked into the loaded native * library, composed as {@code "b-"} (e.g. {@code "b9959-0badc06ab"}). diff --git a/llama/src/test/cpp/test_c_api.cpp b/llama/src/test/cpp/test_c_api.cpp index 6bbab1737..89aafd055 100644 --- a/llama/src/test/cpp/test_c_api.cpp +++ b/llama/src/test/cpp/test_c_api.cpp @@ -1501,6 +1501,23 @@ TEST(CApiStatic, JsonSchemaToGrammarNeedsNoModel) { EXPECT_EQ(jllama_json_schema_to_grammar(nullptr, &grammar.s), JLLAMA_STATUS_INVALID_ARGUMENT); } +// patches/0022's switch through the C API: it reaches the CPU backend's entry points, round-trips, +// and refuses a null out-pointer. What the two paths compute is test_moe_repack.cpp's business. +TEST(CApiStatic, CpuMoeBatchedSwitchRoundTrips) { + bool initial = false; + ASSERT_EQ(jllama_get_cpu_moe_batched(&initial), JLLAMA_STATUS_OK) << jllama_last_error(); + bool now = true; + ASSERT_EQ(jllama_set_cpu_moe_batched(false), JLLAMA_STATUS_OK) << jllama_last_error(); + ASSERT_EQ(jllama_get_cpu_moe_batched(&now), JLLAMA_STATUS_OK); + EXPECT_FALSE(now); + ASSERT_EQ(jllama_set_cpu_moe_batched(true), JLLAMA_STATUS_OK); + ASSERT_EQ(jllama_get_cpu_moe_batched(&now), JLLAMA_STATUS_OK); + EXPECT_TRUE(now); + EXPECT_EQ(jllama_get_cpu_moe_batched(nullptr), JLLAMA_STATUS_INVALID_ARGUMENT); + EXPECT_NE(std::strlen(jllama_last_error()), 0u); + EXPECT_EQ(jllama_set_cpu_moe_batched(initial), JLLAMA_STATUS_OK); +} + TEST_F(CApi, QuantizeWritesAModelThatLoads) { const std::string out = path_ + ".q8_0.gguf"; ASSERT_EQ(jllama_quantize(path_.c_str(), out.c_str(), /*LLAMA_FTYPE_MOSTLY_Q8_0*/ 7, 1, false), JLLAMA_STATUS_OK) diff --git a/llama/src/test/cpp/test_moe_repack.cpp b/llama/src/test/cpp/test_moe_repack.cpp new file mode 100644 index 000000000..812be1ae8 --- /dev/null +++ b/llama/src/test/cpp/test_moe_repack.cpp @@ -0,0 +1,270 @@ +// SPDX-FileCopyrightText: 2026 Bernard Ladenthin +// +// SPDX-License-Identifier: MIT +// +// Runnable guard for patches/0022 (ggml-cpu: batch the tokens of an expert in the repack MUL_MAT_ID, +// and hand its work out dynamically). +// +// MUL_MAT_ID is the expert matmul of a mixture-of-experts model. With the expert weights in the +// CPU_REPACK buffer it runs through forward_mul_mat_id() of ggml-cpu's repack code, which the patch +// splits into two paths behind a process-wide switch: upstream's (one gemv per token, a static row +// split per thread) and the batched one (every group of four or more tokens of an expert through one +// gemm, the work handed out as (expert, column chunk) units). Both must compute what ggml's generic +// MUL_MAT_ID computes over the same weights in a plain buffer -- that path is independent code +// (ggml-cpu.c), quantizes the activations the same way, and is the reference here. +// +// The routings give an expert 0, 1, 3, 4 and 17 tokens (the gemv tail alone, one gemm group, a full +// 16-row gemm group plus a gemv tail), one slot of -1 (no expert: the output row stays zero), and a +// single token (token generation). Each runs with one input row per token (the gate/up projections) +// and one per slot (the down projection), on 1, 3 and 4 threads -- 3 does not divide the column blocks +// evenly, and 1 thread leaves units past the last column. Four weight types reach the repack path on +// AVX2 (MXFP4 of gpt-oss, Q4_0, IQ4_NL, Q4_K with its Q8_K activations); a type the running CPU does +// not repack is skipped, not passed. +// +// The switch is resolved through the CPU backend's get_proc_address, the way libjllama reaches it in a +// GGML_BACKEND_DL build, so a llama.cpp bump that drops the patch reds this file at once. + +#include "ggml-alloc.h" +#include "ggml-backend.h" +#include "ggml-cpu.h" +#include "ggml.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +constexpr int64_t K = 256; // row length: one Q4_K/Q8_K super-block, eight 32-blocks +constexpr int64_t N = 72; // rows per expert: 9 column blocks of 8, not a multiple of 3 or 4 +constexpr int64_t N_EXPERT = 6; // experts + +using switch_set_fn = void (*)(bool); +using switch_get_fn = bool (*)(); + +template Fn cpu_proc(const char *name) { + return reinterpret_cast(ggml_backend_reg_get_proc_address(ggml_backend_cpu_reg(), name)); +} + +void set_batched(bool batched) { + auto set = cpu_proc("ggml_backend_cpu_set_repack_mmid_batched"); + ASSERT_NE(set, nullptr); + set(batched); +} + +bool get_batched() { + auto get = cpu_proc("ggml_backend_cpu_get_repack_mmid_batched"); + return get != nullptr && get(); +} + +ggml_backend_buffer_type_t repack_buffer_type() { + ggml_backend_dev_t dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU); + auto get_extra = reinterpret_cast( + ggml_backend_reg_get_proc_address(ggml_backend_cpu_reg(), "ggml_backend_dev_get_extra_bufts")); + if (dev == nullptr || get_extra == nullptr) { + return nullptr; + } + for (ggml_backend_buffer_type_t *buft = get_extra(dev); buft != nullptr && *buft != nullptr; ++buft) { + if (std::string(ggml_backend_buft_name(*buft)) == "CPU_REPACK") { + return *buft; + } + } + return nullptr; +} + +// ids[token][slot] -> expert, -1 for none +struct Routing { + const char *name; + int n_used; + std::vector> ids; +}; + +// 20 tokens, 2 experts each: e3 gets 17 tokens, e5 15, e2 4, e1 3, e0 1, e4 none +Routing spread(bool with_empty_slot) { + Routing r{with_empty_slot ? "spread_with_an_empty_slot" : "spread", 2, {}}; + for (int t = 0; t < 20; ++t) { + const int32_t first = t <= 16 ? 3 : 5; + const int32_t second = t <= 11 ? 5 : t <= 15 ? 2 : t == 16 ? 0 : 1; + r.ids.push_back({first, second}); + } + if (with_empty_slot) { + r.ids[19][1] = -1; + } + return r; +} + +Routing single_token() { return Routing{"single_token", 4, {{5, 0, 3, 2}}}; } + +struct Case { + ggml_type type; + Routing routing; + bool row_per_slot; // src1 has one row per slot (down projection) instead of one per token +}; + +std::string case_name(const testing::TestParamInfo &info) { + return std::string(ggml_type_name(info.param.type)) + "_" + info.param.routing.name + + (info.param.row_per_slot ? "_row_per_slot" : "_row_per_token"); +} + +class MoeRepack : public testing::TestWithParam { + protected: + void SetUp() override { initial_ = get_batched(); } + void TearDown() override { set_batched(initial_); } + + private: + bool initial_ = true; +}; + +// out = mul_mat_id(w, x, ids) on n_threads; w lives in the given buffer, or in the context when null +std::vector run(ggml_tensor *w, const std::vector &x_data, const Routing &routing, int64_t ne11, + int n_threads) { + const int64_t n_tokens = (int64_t)routing.ids.size(); + ggml_init_params ip = {64u * 1024 * 1024, nullptr, false}; + ggml_context *ctx = ggml_init(ip); + ggml_tensor *x = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, K, ne11, n_tokens); + std::memcpy(x->data, x_data.data(), ggml_nbytes(x)); + ggml_tensor *ids = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, routing.n_used, n_tokens); + for (int64_t t = 0; t < n_tokens; ++t) { + for (int s = 0; s < routing.n_used; ++s) { + ((int32_t *)ids->data)[t * routing.n_used + s] = routing.ids[t][s]; + } + } + ggml_tensor *out = ggml_mul_mat_id(ctx, w, x, ids); + ggml_cgraph *gf = ggml_new_graph(ctx); + ggml_build_forward_expand(gf, out); + EXPECT_EQ(ggml_graph_compute_with_ctx(ctx, gf, n_threads), GGML_STATUS_SUCCESS); + std::vector result((const float *)out->data, (const float *)out->data + ggml_nelements(out)); + ggml_free(ctx); + return result; +} + +TEST_P(MoeRepack, BothPathsComputeWhatTheGenericPathComputes) { + const Case &c = GetParam(); + ggml_backend_buffer_type_t repack = repack_buffer_type(); + if (repack == nullptr) { + GTEST_SKIP() << "this build has no CPU_REPACK buffer"; + } + + // random weights, quantized to the type, the same bytes in both buffers + std::mt19937 rng(22); + std::normal_distribution dist(0.0f, 1.0f); + std::vector w_f32(K * N * N_EXPERT); + for (float &v : w_f32) { + v = 0.1f * dist(rng); + } + std::vector w_q(ggml_row_size(c.type, K) * N * N_EXPERT); + ggml_quantize_chunk(c.type, w_f32.data(), w_q.data(), 0, N * N_EXPERT, K, nullptr); + + // the reference: the weights in plain memory, ggml's generic MUL_MAT_ID + ggml_init_params plain_ip = {w_q.size() + 4096, nullptr, false}; + ggml_context *plain_ctx = ggml_init(plain_ip); + ggml_tensor *w_plain = ggml_new_tensor_3d(plain_ctx, c.type, K, N, N_EXPERT); + std::memcpy(w_plain->data, w_q.data(), w_q.size()); + + // the subject: the same weights repacked + ggml_init_params repack_ip = {ggml_tensor_overhead() * 2, nullptr, true}; + ggml_context *repack_ctx = ggml_init(repack_ip); + ggml_tensor *w_repack = ggml_new_tensor_3d(repack_ctx, c.type, K, N, N_EXPERT); + ggml_backend_buffer_t buffer = ggml_backend_alloc_ctx_tensors_from_buft(repack_ctx, repack); + ASSERT_NE(buffer, nullptr); + if (w_repack->extra == nullptr) { + ggml_backend_buffer_free(buffer); + ggml_free(repack_ctx); + ggml_free(plain_ctx); + GTEST_SKIP() << ggml_type_name(c.type) << " is not repacked on this CPU"; + } + ggml_backend_tensor_set(w_repack, w_q.data(), 0, w_q.size()); + + const int64_t ne11 = c.row_per_slot ? c.routing.n_used : 1; + std::vector x(K * ne11 * c.routing.ids.size()); + for (float &v : x) { + v = dist(rng); + } + + const std::vector expected = run(w_plain, x, c.routing, ne11, 1); + float scale = 0.0f; + for (float v : expected) { + scale = std::max(scale, std::fabs(v)); + } + ASSERT_GT(scale, 0.0f); + + for (bool batched : {true, false}) { + set_batched(batched); + for (int n_threads : {1, 3, 4}) { + const std::vector actual = run(w_repack, x, c.routing, ne11, n_threads); + ASSERT_EQ(actual.size(), expected.size()); + size_t worst = 0; + float worst_diff = 0.0f; + for (size_t i = 0; i < actual.size(); ++i) { + if (!std::isfinite(actual[i])) { + worst_diff = INFINITY; + worst = i; + break; + } + const float diff = std::fabs(actual[i] - expected[i]); + if (diff > worst_diff) { + worst_diff = diff; + worst = i; + } + } + EXPECT_LE(worst_diff, 1e-3f * scale) + << (batched ? "batched" : "upstream's") << " path, " << n_threads << " thread(s): element " << worst + << " is " << actual[worst] << ", the generic path gives " << expected[worst]; + } + } + // a slot routed to no expert stays zero on every path (the generic path's reference included) + for (size_t t = 0; t < c.routing.ids.size(); ++t) { + for (int s = 0; s < c.routing.n_used; ++s) { + if (c.routing.ids[t][s] == -1) { + for (int64_t n = 0; n < N; ++n) { + EXPECT_EQ(expected[(t * c.routing.n_used + s) * N + n], 0.0f); + } + } + } + } + + ggml_backend_buffer_free(buffer); + ggml_free(repack_ctx); + ggml_free(plain_ctx); +} + +std::vector cases() { + std::vector out; + for (ggml_type type : {GGML_TYPE_MXFP4, GGML_TYPE_Q4_0, GGML_TYPE_IQ4_NL, GGML_TYPE_Q4_K}) { + for (const Routing &routing : {spread(false), spread(true), single_token()}) { + for (bool row_per_slot : {false, true}) { + out.push_back({type, routing, row_per_slot}); + } + } + } + return out; +} + +INSTANTIATE_TEST_SUITE_P(Types, MoeRepack, testing::ValuesIn(cases()), case_name); + +TEST(MoeRepackSwitch, IsReachableThroughTheCpuBackendAndRoundTrips) { + auto set = cpu_proc("ggml_backend_cpu_set_repack_mmid_batched"); + auto get = cpu_proc("ggml_backend_cpu_get_repack_mmid_batched"); + ASSERT_NE(set, nullptr) << "the CPU backend has no switch: llama/patches/0022 is not applied"; + ASSERT_NE(get, nullptr) << "the CPU backend has no switch: llama/patches/0022 is not applied"; + const bool initial = get(); + const char *env = std::getenv("GGML_CPU_REPACK_MMID_BATCHED"); + if (env == nullptr) { + // nothing in this process has flipped it yet: the test cases restore what they found + EXPECT_TRUE(initial) << "the batched path is the default"; + } + set(false); + EXPECT_FALSE(get()); + set(true); + EXPECT_TRUE(get()); + set(initial); +} + +} // namespace diff --git a/llama/src/test/java/net/ladenthin/llama/CpuMoeBatchedTest.java b/llama/src/test/java/net/ladenthin/llama/CpuMoeBatchedTest.java new file mode 100644 index 000000000..876f0d887 --- /dev/null +++ b/llama/src/test/java/net/ladenthin/llama/CpuMoeBatchedTest.java @@ -0,0 +1,53 @@ +// SPDX-FileCopyrightText: 2026 Bernard Ladenthin +// +// SPDX-License-Identifier: MIT + +package net.ladenthin.llama; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import net.ladenthin.llama.loader.NativeLibraryPresence; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +@ClaudeGenerated( + purpose = "Model-free guard that LlamaModel.setCpuMoeBatched / isCpuMoeBatched reach the CPU backend's " + + "switch of llama/patches/0022 through JNI and the C API (it lives in a separately loaded CPU " + + "module in the shipped builds), round-trip both values, and start on the batched path. What " + + "the two paths compute is pinned by the C++ guard test_moe_repack.cpp.") +class CpuMoeBatchedTest { + + private boolean initial; + + @BeforeEach + void rememberTheSwitch() { + assumeTrue(NativeLibraryPresence.onClasspath(), "libjllama not on classpath — skipping the switch guard"); + initial = LlamaModel.isCpuMoeBatched(); + } + + @AfterEach + void restoreTheSwitch() { + if (NativeLibraryPresence.onClasspath()) { + LlamaModel.setCpuMoeBatched(initial); + } + } + + @Test + void theBatchedPathIsTheDefault() { + assumeTrue( + System.getenv("GGML_CPU_REPACK_MMID_BATCHED") == null, + "GGML_CPU_REPACK_MMID_BATCHED is set for this JVM, so the default is not observable"); + assertThat(initial, is(true)); + } + + @Test + void theSwitchRoundTripsBothValues() { + LlamaModel.setCpuMoeBatched(false); + assertThat(LlamaModel.isCpuMoeBatched(), is(false)); + LlamaModel.setCpuMoeBatched(true); + assertThat(LlamaModel.isCpuMoeBatched(), is(true)); + } +}