@@ -886,24 +886,25 @@ jobs:
886886 path : ${{ github.workspace }}/llama/src/main/natives/net/ladenthin/llama/
887887
888888 build-linux-s390x :
889- name : Build and Test Linux s390x (big-endian, qemu )
889+ name : Build Linux s390x (big-endian, CPU variants )
890890 needs : [startgate, build-webui]
891- # Cross-compile for IBM Z (s390x, BIG-ENDIAN), then run the full C++ unit suite under qemu-user --
892- # a real big-endian correctness gate for our helpers and serializers (esp. the little-endian WAV
893- # writer, JSON/token/embedding transforms). The BUILD is native speed: clang on the x86-64 runner
894- # against the manylinux_2_28 s390x image unpacked as the sysroot (.github/manylinux/
891+ # Cross-compile the cpu-linux-s390x library jar for IBM Z (s390x, BIG-ENDIAN) as a modular build
892+ # (JLLAMA_BACKEND_DL, CLAUDE.md "Linux s390x"): libjllama, libllama, libmtmd, ggml and one CPU
893+ # module per IBM Z level -- ggml-cpu-z13 (scalar, the baseline patches/0023 adds), -z15 (VXE2)
894+ # and -z16 (VXE2 + NNPA) -- of which ggml loads the best for the running machine. Up to 6.0.x
895+ # the jar was one static, scalar library: 4.5x slower prompt processing and 4x slower generation
896+ # on a z16 (measured on cfarm191, patches/0023). The BUILD is native speed: clang on the x86-64
897+ # runner against the manylinux_2_28 s390x image unpacked as the sysroot (.github/manylinux/
895898 # manylinux_2_28-s390x-sysroot and s390x-toolchain.cmake), linked by binutils-s390x-linux-gnu, so
896899 # the jar has the floor of the other Linux CPU jars -- glibc 2.28, GLIBCXX_3.4.25 -- which
897900 # verify-native-deps.py holds and smoke-glibc-floor proves on RHEL 8. (Until 2026-10-25 the job
898901 # used the runner's GCC 13 cross toolchain and its glibc 2.39 sysroot: the 6.0.0 jar needed glibc
899- # 2.38 and GCC 13's libstdc++, CLAUDE.md "Linux s390x".) Only the test binary is emulated, and it
900- # runs against the sysroot's glibc 2.28 and GCC 8 libstdc++.so.6, the oldest runtime the jar
901- # promises. s390x is a CPU platform like aarch64 and ships as the cpu-linux-s390x natives jar.
902- # Model-backed Java tests are NOT run under emulation (a JVM + GGUF inference under qemu-user is
903- # slow/flaky); the C++ gate covers the actual byte-order risk since the Java<->JNI boundary uses
904- # host-native array copies. GGML_OPENMP=OFF avoids cross-libgomp issues (ggml uses its own
905- # std::thread pool). The toolchain's CMAKE_CROSSCOMPILING_EMULATOR runs the s390x executables
906- # through qemu with QEMU_LD_PREFIX pointing at the sysroot.
902+ # 2.38 and GCC 13's libstdc++, CLAUDE.md "Linux s390x".) build.sh runs the plain-C test c_abi
903+ # through the toolchain's qemu (CMAKE_CROSSCOMPILING_EMULATOR, QEMU_LD_PREFIX at the sysroot):
904+ # upstream's C API out of the shipped directory, with ggml scoring the IBM Z modules against
905+ # qemu's CPU model. The C++ unit suite cannot link against a modular build; it runs in
906+ # test-cpp-linux-s390x, under qemu as well. GGML_OPENMP=OFF avoids cross-libgomp issues (ggml
907+ # uses its own std::thread pool).
907908 # Pinned to ubuntu-24.04: the toolchain names its clang-18.
908909 runs-on : ubuntu-24.04
909910 env :
@@ -930,22 +931,26 @@ jobs:
930931 .github/manylinux/manylinux_2_28-s390x-sysroot "$RUNNER_TEMP/s390x-sysroot"
931932 echo "JLLAMA_S390X_SYSROOT=$RUNNER_TEMP/s390x-sysroot" >> "$GITHUB_ENV"
932933 # NOTE: GGML_NATIVE=OFF is required here (an x86 build host must not bake -march=native into an
933- # s390x artifact), and it has a non-obvious second effect: ggml declares
934- # `option(GGML_VXE "ggml: enable vxe" ${GGML_NATIVE})`, so VXE is off too. No `-mvx -mzvector` is
935- # passed, `__VEC__` stays undefined, and ggml-cpu-impl.h's `#if defined(__s390x__) && defined(__VEC__)`
936- # never self-defines `__VXE__`/`__VXE2__`. This job therefore builds a SCALAR s390x binary -- which is
937- # exactly right for what it is (a big-endian correctness gate for our own layer, not a perf target).
938- # Do NOT "fix" a VXE-related compile error by adding -DGGML_VXE=ON: that define sets __VXE__ and
939- # __VXE2__ together while -march stays at z13 (arch11, s390x-toolchain.cmake), so every z14+
940- # builtin is rejected. See the `0013` note in docs/history/dropped-llama-patches.md for the measured
941- # comparison (the patch itself is gone -- upstream merged it as ggml-org/llama.cpp#28775).
934+ # s390x artifact). The vector code comes from GGML_CPU_ALL_VARIANTS instead (set by
935+ # JLLAMA_BACKEND_DL): each variant module carries its own -march=z15/z16 -mvx -mzvector, while
936+ # ggml-base, libllama and libjllama stay at the toolchain's -march=z13. Do NOT pass
937+ # -DGGML_VXE=ON: ggml-cpu/CMakeLists.txt adds VXE2 to every variant when it is set, so the z13
938+ # module would become a VXE2 one, score 0 without VXE2, and leave a z13/z14 without any CPU
939+ # backend.
942940 - name : Build libraries (cross-compile s390x)
943941 shell : bash
944942 run : |
945943 mvn --no-transfer-progress -f llama/pom.xml compile
946- .github/build.sh "-DGGML_NATIVE=OFF -DGGML_OPENMP=OFF -DBUILD_TESTING=ON -DCMAKE_TOOLCHAIN_FILE=$GITHUB_WORKSPACE/.github/manylinux/s390x-toolchain.cmake -DOS_NAME=Linux -DOS_ARCH=s390x"
947- - name : Run C++ unit tests under qemu-s390x (big-endian gate)
948- run : ctest --test-dir llama/build --no-tests=error --output-on-failure
944+ .github/build.sh "-DGGML_NATIVE=OFF -DGGML_OPENMP=OFF -DJLLAMA_BACKEND_DL=ON -DCMAKE_TOOLCHAIN_FILE=$GITHUB_WORKSPACE/.github/manylinux/s390x-toolchain.cmake -DOS_NAME=Linux -DOS_ARCH=s390x"
945+ # A module ggml could not build is a missing file, not a failure ggml reports: the configure
946+ # skips a variant whose flag the compiler rejects (as it does z17 with clang 18). z13 is the
947+ # only one every IBM Z the jar supports can load, and comes from patches/0023.
948+ - name : Check the IBM Z CPU modules
949+ run : |
950+ d=llama/src/main/natives/net/ladenthin/llama/Linux/s390x/cpu
951+ for v in z13 z15 z16; do
952+ test -f "$d/libggml-cpu-$v.so" || { echo "::error::$d has no libggml-cpu-$v.so"; ls -la "$d"; exit 1; }
953+ done
949954 # The same check `package` runs over the merged tree, here so a lost floor fails in this job:
950955 # GLIBC_2.28, GLIBCXX_3.4.25, CXXABI_1.3.11, the exact dependency list, and every typeinfo name
951956 # at an even address (s390x-typeinfo-names.ld + -fdata-sections; the link alone fails only when
@@ -2706,6 +2711,48 @@ jobs:
27062711 - name : Run C++ unit tests
27072712 run : ctest --test-dir llama/build --no-tests=error --output-on-failure
27082713
2714+ test-cpp-linux-s390x :
2715+ name : C++ Tests Linux s390x (big-endian, qemu)
2716+ needs : startgate
2717+ # The full C++ unit suite on IBM Z (BIG-ENDIAN) under qemu-user -- the big-endian correctness
2718+ # gate for our helpers and serializers (the little-endian WAV writer, the JSON/token/embedding
2719+ # transforms, the JNI helpers). Same toolchain as build-linux-s390x (clang-18 against the
2720+ # manylinux_2_28 sysroot, s390x-toolchain.cmake), but the default static build: the shipped CPU
2721+ # variants configuration cannot host jllama_test (its CPU and RPC backends are modules), the
2722+ # same split as test-cpp-linux-x86_64 / -aarch64. The test binary runs against the sysroot's
2723+ # glibc 2.28 and GCC 8 libstdc++.so.6, the oldest runtime the jar promises. Model-backed Java
2724+ # tests are NOT run under emulation (a JVM + GGUF inference under qemu-user is slow and flaky);
2725+ # the Java<->JNI boundary uses host-native array copies, so this gate covers the byte-order risk.
2726+ # Until 6.1.0 the suite rode along in build-linux-s390x, whose library was static then.
2727+ runs-on : ubuntu-24.04
2728+ env :
2729+ SCCACHE_WEBDAV_TOKEN : ${{ secrets.DEPOT_TOKEN }}
2730+ steps :
2731+ - uses : actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
2732+ - name : Display CPU Info
2733+ uses : bernardladenthin/actions/common/print-host-info@44390bd464395f560f4170d2c4bfb7a6348541b3 # v1.1.0
2734+ - uses : actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6
2735+ with :
2736+ distribution : ' temurin'
2737+ java-version-file : .java-version
2738+ - name : Install clang, the s390x binutils and qemu-user
2739+ run : |
2740+ sudo apt-get update
2741+ sudo apt-get install -y clang-18 binutils-s390x-linux-gnu qemu-user-static
2742+ - name : Unpack the manylinux_2_28 s390x sysroot
2743+ run : |
2744+ .github/manylinux/manylinux_2_28-s390x-sysroot "$RUNNER_TEMP/s390x-sysroot"
2745+ echo "JLLAMA_S390X_SYSROOT=$RUNNER_TEMP/s390x-sysroot" >> "$GITHUB_ENV"
2746+ - name : Build libraries (cross-compile s390x)
2747+ shell : bash
2748+ run : |
2749+ mvn --no-transfer-progress -f llama/pom.xml compile
2750+ .github/build.sh "-DGGML_NATIVE=OFF -DGGML_OPENMP=OFF -DBUILD_TESTING=ON -DCMAKE_TOOLCHAIN_FILE=$GITHUB_WORKSPACE/.github/manylinux/s390x-toolchain.cmake -DOS_NAME=Linux -DOS_ARCH=s390x"
2751+ - name : Verify llama.cpp patches are applied
2752+ run : .github/verify-patches-applied.sh
2753+ - name : Run C++ unit tests under qemu-s390x (big-endian gate)
2754+ run : ctest --test-dir llama/build --no-tests=error --output-on-failure
2755+
27092756 test-cpp-windows-x86_64 :
27102757 name : C++ Tests Windows 2025 x86_64 (clang)
27112758 needs : startgate
@@ -3140,6 +3187,7 @@ jobs:
31403187 - build-windows-x86_64-openvino
31413188 - test-cpp-linux-x86_64
31423189 - test-cpp-linux-aarch64
3190+ - test-cpp-linux-s390x
31433191 - test-cpp-windows-x86_64
31443192 - test-cpp-windows-arm64
31453193 - build-macos-arm64-metal-15
0 commit comments