Batch the tokens of an expert in ggml-cpu's MoE path (patch 0022, +47 % prompt / +26 % generation for gpt-oss-20b on the Core Ultra module), switchable from Java

.github/scripts/bench/bench.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: 2026 Bernard Ladenthin <bernard.ladenthin@gmail.com>
+#
+# SPDX-License-Identifier: MIT OR Apache-2.0
+#
+# Run llama-bench with exactly ONE CPU backend module, so a module the build machine would never pick
+# (alderlake on an AVX-512 host, say) can be measured there. docs/benchmarks/moe-mxfp4-cpu-alderlake.md
+# is the measurement this was written for, and has the build recipe.
+#
+#   BUILD=<llama.cpp build dir> bench.sh <module> <out.jsonl> <llama-bench args...>
+#       appends one JSON line per test to <out.jsonl>, tagged with "variant":"<module>"
+#   BUILD=<dir> PROFILE=<out-prefix> [PERF_DELAY=<ms>] bench.sh <module> - <llama-bench args...>
+#       one run under `perf record -e cpu-clock` instead; writes <out-prefix>.data and a flat
+#       per-symbol report <out-prefix>.sym.txt. PERF_DELAY skips the model load (repacking a 12 GB
+#       model alone is seconds of samples).
+#
+# The build must be a GGML_BACKEND_DL one with its libggml-cpu-*.so moved from <BUILD>/bin to
+# <BUILD>/cpu-modules: ggml searches the executable's directory for CPU modules and would load the
+# best one from there next to the one named here. GGML_BACKEND_PATH then loads the single module.
+set -euo pipefail
+
+if [ $# -lt 2 ]; then
+    sed -n '6,19p' "$0"
+    exit 2
+fi
+: "${BUILD:?set BUILD to the llama.cpp build directory}"
+module=$1
+out=$2
+shift 2
+
+lib="$BUILD/cpu-modules/libggml-cpu-$module.so"
+if [ ! -f "$lib" ]; then
+    echo "bench.sh: no module $lib" >&2
+    exit 1
+fi
+if ls "$BUILD"/bin/libggml-cpu-*.so > /dev/null 2>&1; then
+    echo "bench.sh: $BUILD/bin still holds CPU modules; move them to $BUILD/cpu-modules first" >&2
+    exit 1
+fi
+export GGML_BACKEND_PATH="$lib"
+
+if [ -n "${PROFILE:-}" ]; then
+    # Ubuntu's /usr/bin/perf is a wrapper that refuses a kernel it has no package for (any VM or
+    # container kernel); the versioned binary behind it works for user-space sampling regardless
+    perf=
+    for p in "$(command -v perf || true)" /usr/lib/linux-tools/*/perf; do
+        if [ -x "$p" ] && "$p" --version > /dev/null 2>&1; then
+            perf=$p
+            break
+        fi
+    done
+    if [ -z "$perf" ]; then
+        echo "bench.sh: no working perf (Ubuntu: apt-get install linux-tools-generic)" >&2
+        exit 1
+    fi
+    "$perf" record -q -D "${PERF_DELAY:-0}" -e cpu-clock -F 1999 -o "$PROFILE.data" -- \
+        "$BUILD/bin/llama-bench" -r 1 --no-warmup "$@"
+    "$perf" report -q -i "$PROFILE.data" --no-children --sort sym --percent-limit 0.3 2> /dev/null \
+        | sed 's/ \+$//' > "$PROFILE.sym.txt"
+    head -25 "$PROFILE.sym.txt"
+    exit 0
+fi
+
+"$BUILD/bin/llama-bench" -o jsonl "$@" | sed "s/^{/{\"variant\":\"$module\",/" >> "$out"

.github/scripts/bench/ggml-opprof.diff

@@ -0,0 +1,80 @@
+SPDX-FileCopyrightText: 2026 Bernard Ladenthin <bernard.ladenthin@gmail.com>
+
+SPDX-License-Identifier: MIT OR Apache-2.0
+
+Measurement-only instrumentation of ggml's CPU graph loop: per (op, src0 type) the summed thread time
+inside the op and inside the barrier after it, printed at exit as OPPROF lines. Compiled in only with
+-DGGML_OPPROF (CMAKE_C_FLAGS), never shipped. Applies to llama.cpp b11887 + llama/patches/ with
+`git apply`; docs/benchmarks/moe-mxfp4-cpu-alderlake.md is what it was used for.
+
+--- a/ggml/src/ggml-cpu/ggml-cpu.c
++++ b/ggml/src/ggml-cpu/ggml-cpu.c
+@@ -3108,6 +3108,34 @@
+     return 0;
+ }
+
++
++#ifdef GGML_OPPROF
++// per (op, src0 type): summed thread time in the op and in the barrier after it (benchmark-only)
++#include <stdatomic.h>
++#define OPPROF_N (GGML_OP_COUNT * GGML_TYPE_COUNT)
++static _Atomic int64_t opprof_cmp[OPPROF_N], opprof_wait[OPPROF_N], opprof_cnt[OPPROF_N];
++static atomic_int opprof_registered;
++static int64_t opprof_now(void) { struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); return (int64_t) ts.tv_sec * 1000000000LL + ts.tv_nsec; }
++static void opprof_dump(void) {
++    int64_t tot = 0;
++    for (int i = 0; i < OPPROF_N; i++) tot += opprof_cmp[i] + opprof_wait[i];
++    if (!tot) return;
++    int idx[OPPROF_N]; int n = 0;
++    for (int i = 0; i < OPPROF_N; i++) if (opprof_cnt[i]) idx[n++] = i;
++    for (int a = 0; a < n; a++) for (int b = a + 1; b < n; b++)
++        if (opprof_cmp[idx[b]] + opprof_wait[idx[b]] > opprof_cmp[idx[a]] + opprof_wait[idx[a]]) { int t = idx[a]; idx[a] = idx[b]; idx[b] = t; }
++    int64_t wtot = 0; for (int i = 0; i < OPPROF_N; i++) wtot += opprof_wait[i];
++    fprintf(stderr, "OPPROF total thread-ms %.1f, of it barrier wait %.1f (%.1f%%)\n", tot / 1e6, wtot / 1e6, 100.0 * wtot / tot);
++    fprintf(stderr, "OPPROF %-16s %-8s %10s %9s %9s %7s %7s\n", "op", "src0", "nodes/thr", "cmp-ms", "wait-ms", "cmp%", "wait%");
++    for (int a = 0; a < n && a < 30; a++) {
++        int i = idx[a];
++        fprintf(stderr, "OPPROF %-16s %-8s %10lld %9.1f %9.1f %6.1f%% %6.1f%%\n", ggml_op_name((enum ggml_op)(i / GGML_TYPE_COUNT)),
++                ggml_type_name((enum ggml_type)(i % GGML_TYPE_COUNT)), (long long) opprof_cnt[i], opprof_cmp[i] / 1e6, opprof_wait[i] / 1e6,
++                100.0 * opprof_cmp[i] / tot, 100.0 * opprof_wait[i] / tot);
++    }
++}
++#endif
++
+ static thread_ret_t ggml_graph_compute_thread(void * data) {
+     struct ggml_compute_state * state = (struct ggml_compute_state *) data;
+     struct ggml_threadpool    * tp    = state->threadpool;
+@@ -3150,12 +3178,20 @@
+
+         // TODO: move fused-op detection into ggml_graph_plan so fusion decisions are made once at planning time
+         // Try fused ops, fall back to normal compute
++#ifdef GGML_OPPROF
++        if (!atomic_exchange(&opprof_registered, 1)) atexit(opprof_dump);
++        const int opprof_key = node->op * GGML_TYPE_COUNT + (node->src[0] ? node->src[0]->type : 0);
++        const int64_t opprof_t0 = opprof_now();
++#endif
+         const int n_fused = ggml_cpu_try_fuse_ops(cgraph, node_n, &params, cplan);
+         if (n_fused > 0) {
+             node_n += n_fused;
+         } else {
+             ggml_compute_forward(&params, node);
+         }
++#ifdef GGML_OPPROF
++        const int64_t opprof_t1 = opprof_now();
++#endif
+
+         if (state->ith == 0 && cplan->abort_callback &&
+                 cplan->abort_callback(cplan->abort_callback_data)) {
+@@ -3166,6 +3202,12 @@
+         if (node_n + 1 < cgraph->n_nodes) {
+             ggml_barrier(state->threadpool);
+         }
++#ifdef GGML_OPPROF
++        const int64_t opprof_t2 = opprof_now();
++        opprof_cmp[opprof_key] += opprof_t1 - opprof_t0;
++        opprof_wait[opprof_key] += opprof_t2 - opprof_t1;
++        opprof_cnt[opprof_key] += 1;
++#endif
+     }
+
+ #ifdef GGML_USE_OPENMP

.github/scripts/bench/mmid-dynamic-gemm.diff

@@ -0,0 +1,131 @@
+SPDX-FileCopyrightText: 2026 Bernard Ladenthin <bernard.ladenthin@gmail.com>
+
+SPDX-License-Identifier: MIT OR Apache-2.0
+
+PROTOTYPE, not applied to any build of this project: the MUL_MAT_ID (MoE) path of ggml-cpu's repack
+buffer, with (a) the work handed out dynamically as (expert, column chunk) units instead of a static
+row split per thread, and (b) gemm for every group of four or more tokens routed to the same expert
+instead of one gemv per token. Behind -DJLLAMA_MMID_V2 (CMAKE_CXX_FLAGS) so one tree builds both
+sides of the A/B. Applies to llama.cpp b11887 + llama/patches/ with `git apply`; the measurement
+and the correctness check are in docs/benchmarks/moe-mxfp4-cpu-alderlake.md.
+
+--- a/ggml/src/ggml-cpu/repack.cpp
++++ b/ggml/src/ggml-cpu/repack.cpp
+@@ -4554,7 +4554,19 @@
+
+ template <typename BLOC_TYPE, int64_t INTER_SIZE, int64_t NB_COLS, ggml_type PARAM_TYPE> class tensor_traits : public tensor_traits_base {
+
+-    bool work_size(int /* n_threads */, const struct ggml_tensor * op, size_t & size) override {
++#ifdef JLLAMA_MMID_V2
++    // MUL_MAT_ID: token rows per gemm call, and the per-thread scratch that call needs
++    static constexpr int64_t MMID_GEMM_ROWS = 16;
++
++    static size_t mmid_thread_scratch(int64_t ne10, int64_t ne01) {
++        return GGML_PAD(MMID_GEMM_ROWS * ne10 * sizeof(float), 64) +
++               GGML_PAD(MMID_GEMM_ROWS * ggml_row_size(PARAM_TYPE, ne10), 64) +
++               GGML_PAD(MMID_GEMM_ROWS * ne01 * sizeof(float), 64);
++    }
++#endif
++
++    bool work_size(int n_threads, const struct ggml_tensor * op, size_t & size) override {
++        GGML_UNUSED(n_threads);
+         // not realy a GGML_TYPE_Q8_0 but same size.
+         switch (op->op) {
+             case GGML_OP_MUL_MAT:
+@@ -4573,6 +4585,10 @@
+                     const size_t sizeof_mmid_row_mapping = sizeof(int64_t);
+
+                     size += sizeof_mmid_row_mapping*ne02*(ne12 + 1);
++#ifdef JLLAMA_MMID_V2
++                    size += sizeof(int64_t)*(ne02 + 1) + 64;
++                    size += (size_t) n_threads * mmid_thread_scratch(op->src[1]->ne[0], op->src[0]->ne[1]);
++#endif
+
+                     return true;
+                 }
+@@ -4868,6 +4884,77 @@
+             }
+         }
+
++#ifdef JLLAMA_MMID_V2
++        // the experts that got rows, and the unit counter: threads take (expert, column chunk)
++        // units dynamically, so a slower core takes fewer of them instead of holding the barrier
++        int64_t * active = (int64_t *) (matrix_rows + n_as*ne12);
++        if (ith == 0) {
++            int64_t n_active = 0;
++            for (int cur_a = 0; cur_a < n_as; ++cur_a) {
++                if (matrix_row_counts[cur_a] > 0) {
++                    active[n_active++] = cur_a;
++                }
++            }
++            active[n_as] = n_active;
++            ggml_threadpool_chunk_set(params->threadpool, nth);
++        }
++
++        ggml_barrier(params->threadpool);
++
++        const int64_t n_active = active[n_as];
++        if (n_active == 0) {
++            return;
++        }
++        const int64_t n_col_blocks = ne01 / NB_COLS;
++        const int64_t nck          = MAX(1, MIN(n_col_blocks, (nth * 16 + n_active - 1) / n_active));
++        const int64_t cols_per     = ((n_col_blocks + nck - 1) / nck) * NB_COLS;
++        const int64_t n_units      = n_active * nck;
++
++        char * scratch_base = (char *) (((uintptr_t) (active + n_as + 1) + 63) & ~(uintptr_t) 63);
++        char * scratch      = scratch_base + ith * mmid_thread_scratch(ne10, ne01);
++        float * fbuf = (float *) scratch;
++        char *  qbuf = scratch + GGML_PAD(MMID_GEMM_ROWS * ne10 * sizeof(float), 64);
++        float * obuf = (float *) (qbuf + GGML_PAD(MMID_GEMM_ROWS * ggml_row_size(PARAM_TYPE, ne10), 64));
++        GGML_ASSERT((char *) obuf + MMID_GEMM_ROWS * ne01 * sizeof(float) <= (char *) params->wdata + params->wsize);
++
++        for (int64_t unit = ith; unit < n_units; unit = ggml_threadpool_chunk_add(params->threadpool, 1)) {
++            const int64_t cur_a = active[unit / nck];
++            const int64_t c0    = (unit % nck) * cols_per;
++            const int64_t c1    = MIN(c0 + cols_per, ne01);
++            if (c0 >= c1) {
++                continue;
++            }
++            const int64_t nc   = c1 - c0;
++            const int64_t cne1 = matrix_row_counts[cur_a];
++            const char * src0_cur = (const char *) src0->data + cur_a*nb02 + c0*nb01;
++
++            // groups of four or more rows: gather the float rows, interleave-quantize, gemm, scatter
++            int64_t ir1 = 0;
++            while (cne1 - ir1 >= 4) {
++                const int64_t nr = MIN(MMID_GEMM_ROWS, (cne1 - ir1) & ~(int64_t) 3);
++                for (int64_t r = 0; r < nr; r++) {
++                    const struct mmid_row_mapping rm = MMID_MATRIX_ROW(cur_a, ir1 + r);
++                    memcpy(fbuf + r*ne10, (const char *) src1->data + (rm.i1 % ne11)*nb11 + rm.i2*nb12, ne10*sizeof(float));
++                }
++                for (int64_t r = 0; r < nr; r += 4) {
++                    ggml_quantize_mat_t<INTER_SIZE, PARAM_TYPE>(fbuf + r*ne10, qbuf + r*nbw1, 4, ne10);
++                }
++                gemm<BLOC_TYPE, INTER_SIZE, NB_COLS, PARAM_TYPE>(ne00, obuf, nc, src0_cur, qbuf, nr, nc);
++                for (int64_t r = 0; r < nr; r++) {
++                    const struct mmid_row_mapping rm = MMID_MATRIX_ROW(cur_a, ir1 + r);
++                    memcpy((float *) ((char *) dst->data + rm.i1*nb1 + rm.i2*nb2) + c0, obuf + r*nc, nc*sizeof(float));
++                }
++                ir1 += nr;
++            }
++            for (; ir1 < cne1; ir1++) {
++                const struct mmid_row_mapping rm = MMID_MATRIX_ROW(cur_a, ir1);
++                const char * src1_col = (const char *) wdata + ((rm.i1 % ne11) * nbw1 + rm.i2 * nbw2);
++                gemv<BLOC_TYPE, INTER_SIZE, NB_COLS, PARAM_TYPE>(
++                    ne00, (float *) ((char *) dst->data + (rm.i1 * nb1 + rm.i2 * nb2)) + c0, ne01,
++                    src0_cur, src1_col, 1, nc);
++            }
++        }
++#else
+         ggml_barrier(params->threadpool);
+
+         // compute each matrix multiplication in sequence
+@@ -4915,6 +5002,7 @@
+                     src0_cur + src0_cur_start * nb01, src1_col, 1, src0_cur_end - src0_cur_start);
+             }
+         }
++#endif
+ #undef MMID_MATRIX_ROW
+     }
+