GH-49321: [C++][Python] Add pixi build for Arrow and PyArrow

ci/docker/pixi.dockerfile

@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+ARG arch=amd64
+FROM ${arch}/ubuntu:24.04
+
+# install build essentials
+RUN export DEBIAN_FRONTEND=noninteractive && \
+    apt-get update -y -q && \
+    apt-get install -y -q \
+    curl \
+    gdb \
+    libc6-dbg \
+    tzdata \
+    wget && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# install pixi
+RUN curl -fsSL https://pixi.sh/install.sh -o /tmp/install-pixi.sh
+RUN PIXI_HOME=/opt/pixi PIXI_NO_PATH_UPDATE=1 sh /tmp/install-pixi.sh && rm /tmp/install-pixi.sh
+ENV PATH=/opt/pixi/bin:$PATH
+
+# Verify pixi installation
+RUN pixi --version

ci/pixi/.gitignore

@@ -0,0 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+**/.pixi
+**/pixi.lock

ci/pixi/cpp/pixi.toml

@@ -0,0 +1,90 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[workspace]
+channels = ["https://prefix.dev/conda-forge"]
+platforms = ["linux-64", "linux-aarch64", "osx-arm64"]
+preview = ["pixi-build"]
+
+[environments]
+default = []
+test    = ["test"]
+
+[feature.test.activation.env]                                                                                                                                                                                       
+ARROW_TEST_DATA = "/arrow/testing/data"
+PARQUET_TEST_DATA = "/arrow/cpp/submodules/parquet-testing/data"
+
+[feature.test.dependencies]
+binutils = "*"
+cmake = "*"
+python = "*"
+
+[feature.test.tasks]
+# This path mangling feels extremely finicky. Investigate alternatives to make this more robust.
+test = "cd $(ls -d /arrow/ci/pixi/cpp/.pixi/build/work/arrow-cpp-*/work/build | head -n 1) && ctest --output-on-failure"
+
+[package]
+name = "arrow-cpp"
+# TODO: Incorporate this to the bump version script and tests
+version = "25.0.0.dev"
+authors = ["Apache Arrow"]
+
+[package.build]
+source.path = "../../../cpp"
+
+[package.build.backend]
+name = "pixi-build-cmake"
+version = "*"
+
+[package.build.config]
+extra-args = [
+  "-DCMAKE_BUILD_TYPE=Release",
+  "-DARROW_BUILD_TESTS=ON",
+  "-DARROW_COMPUTE=ON",
+  "-DARROW_CSV=ON",
+  "-DARROW_DATASET=ON",
+  "-DARROW_FILESYSTEM=ON",
+  "-DARROW_JSON=ON",
+  "-DARROW_PARQUET=ON",
+  "-DARROW_WITH_BROTLI=ON",
+  "-DARROW_WITH_BZ2=ON",
+  "-DARROW_WITH_LZ4=ON",
+  "-DARROW_WITH_SNAPPY=ON",
+  "-DARROW_WITH_ZLIB=ON",
+  "-DARROW_WITH_ZSTD=ON",
+  # libutf8proc on package-host-dependencies seems to fail.
+  # More investigation needed. In the meantime use bundled.
+  "-Dutf8proc_SOURCE=BUNDLED",
+  "-DOPENSSL_ROOT_DIR=$PREFIX",
+]
+
+[package.host-dependencies]
+brotli = "*"
+bzip2 = "*"
+gflags = "*"
+gmock  = ">=1.10.0"
+gtest  = ">=1.10.0"
+libboost-devel = "*"
+lz4-c = "*"
+openssl = "*"
+rapidjson = "*"
+re2 = "*"
+snappy = "*"
+thrift-cpp = ">=0.11.0"
+xsimd = ">=14.0"
+zlib = "*"
+zstd = "*"

ci/pixi/python/pixi.toml

@@ -0,0 +1,85 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[workspace]
+channels = ["https://prefix.dev/conda-forge"]
+platforms = ["linux-64", "linux-aarch64", "osx-arm64"]
+preview = ["pixi-build"]
+
+[environments]
+default = []
+test    = ["test"]
+
+[feature.test]
+platforms = ["linux-64"]
+
+[feature.test.activation.env]
+ARROW_TEST_DATA = "/arrow/testing/data"
+PARQUET_TEST_DATA = "/arrow/cpp/submodules/parquet-testing/data"
+
+[feature.test.dependencies]
+hypothesis = "*"
+pyarrow = { path = "." }
+pytest = "*"
+pytest-timeout = "*"
+
+[feature.test.tasks]
+test = "pytest -r s --pyargs pyarrow --timeout=60"
+
+[package]
+name = "pyarrow"
+# TODO: Incorporate this to the bump version script and tests
+version = "25.0.0.dev"
+authors = ["Apache Arrow"]
+
+[package.build]
+source.path = "../../../python"
+
+[package.build.backend]
+name = "pixi-build-python"
+version = "*"
+
+[package.build.config]
+noarch = false
+compilers = ["c", "cxx"]
+env = {
+  PYARROW_BUNDLE_ARROW_CPP = "0",
+  CMAKE_GENERATOR = "Ninja",
+  ARROW_HOME = "$PREFIX",
+}
+
+[package.host-dependencies]
+arrow-cpp = { path = "../cpp" }   # source-dep on the arrow-cpp manifest
+cmake = "*"
+cython = ">=3.0"
+gcc_linux-64 = "*"
+git = "*"
+gxx_linux-64 = "*"
+ninja = "*"
+numpy = "*"
+pip = "*"
+pkg-config = "*"
+python = ">=3.10,<3.14"
+scikit-build-core = "*"
+setuptools-scm = "*"
+sysroot_linux-64 = "*"
+zlib = "*"
+
+[package.run-dependencies]
+arrow-cpp = { path = "../cpp" }
+python = ">=3.10,<3.14"
+numpy = "*"

compose.yaml

@@ -138,6 +138,8 @@ x-hierarchy:
   - fedora-cpp:
     - fedora-python
   - fedora-r-clang
+  - pixi-cpp
+  - pixi-python
   - python-sdist
   - ubuntu-cpp:
     - ubuntu-cpp-static
@@ -197,6 +199,10 @@ volumes:
     name: ${ARCH}-fedora-${FEDORA}-ccache
   maven-cache:
     name: maven-cache
+  pixi-cpp-build:
+    name: ${ARCH}-pixi-cpp-build
+  pixi-python-build:
+    name: ${ARCH}-pixi-python-build
   python-wheel-manylinux-2-28-ccache:
     name: python-wheel-manylinux-2-28-ccache
   python-wheel-musllinux-1-2-ccache:
@@ -1158,6 +1164,50 @@ services:
     volumes: *ubuntu-volumes
     command: *python-command
 
+  ############################  Pixi builds  ##################################
+
+  pixi-cpp:
+    # C++ build with pixi
+    #
+    # Usage:
+    #   docker compose build pixi-cpp
+    #   docker compose run --rm pixi-cpp
+    # Parameters:
+    #   ARCH: amd64, arm32v7
+    image: ${REPO}:${ARCH}-pixi
+    build:
+      context: .
+      dockerfile: ci/docker/pixi.dockerfile
+      args:
+        arch: ${ARCH}
+    volumes:
+      - .:/arrow:delegated
+      - ${DOCKER_VOLUME_PREFIX}pixi-cpp-build:/arrow/ci/pixi/cpp/.pixi:delegated
+    command:  >
+      /bin/sh -c "pixi build --path /arrow/ci/pixi/cpp &&
+        pixi run --environment test --manifest-path /arrow/ci/pixi/cpp test"
+
+  pixi-python:
+    # CPython build with pixi
+    #
+    # Usage:
+    #   docker compose build pixi-python
+    #   docker compose run --rm pixi-python
+    # Parameters:
+    #   ARCH: amd64, arm32v7
+    image: ${REPO}:${ARCH}-pixi
+    build:
+      context: .
+      dockerfile: ci/docker/pixi.dockerfile
+      args:
+        arch: ${ARCH}
+    volumes:
+      - .:/arrow:delegated
+      - ${DOCKER_VOLUME_PREFIX}pixi-python-build:/arrow/ci/pixi/python/.pixi:delegated
+    command: >
+      /bin/sh -c "pixi build --path /arrow/ci/pixi/python &&
+        pixi run --environment test --manifest-path /arrow/ci/pixi/python test"
+
   ############################ Python wheels ##################################
 
   # See available versions at:

dev/tasks/tasks.yml

@@ -364,6 +364,18 @@ tasks:
     params:
       image: conda-cpp-valgrind
 
+  test-pixi-cpp:
+    ci: github
+    template: docker-tests/github.linux.yml
+    params:
+      image: pixi-cpp
+
+  test-pixi-python:
+    ci: github
+    template: docker-tests/github.linux.yml
+    params:
+      image: pixi-python
+
   test-ubuntu-22.04-cpp:
     ci: github
     template: docker-tests/github.linux.yml