#16: Make docker build python setup script

.github/workflows/build-docker-image.yml

@@ -7,6 +7,10 @@ on:
   pull_request:
     branches: "*"
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   get-matrix:
     runs-on: ubuntu-latest

ci/build-setup.py

@@ -74,8 +74,7 @@ def build(self):
             setup_script = setup_script.replace("%DEPS_INSTALL%", '\n'.join(instructions))
 
             setup_filename = f"setup-{setup_id}.sh"
-            setup_filepath = os.path.join(os.path.dirname(__file__),
-                                          "shared", "scripts", setup_filename)
+            setup_filepath = os.path.join(os.path.dirname(__file__), setup_filename)
 
             with open(setup_filepath, "w+", encoding="utf-8") as f:
                 f.write(setup_script)

ci/docker/base.dockerfile

@@ -1,5 +1,6 @@
 # Base dockerfile to build images used in Darma testing.
 
+ARG REPO=lifflander1/vt
 ARG ARCH=amd64
 ARG BASE=ubuntu:22.04
 
@@ -37,8 +38,6 @@ ADD ci/shared/scripts/deps ${WF_TMP_DIR}/deps
 # Setup environment variables
 ENV DEBIAN_FRONTEND=noninteractive
 
-ENV SCRIPTS_INSTALL_DIR=${SCRIPTS_INSTALL_DIR}
-
 ENV WF_DOCKER=1 \
     WF_SETUP_ID=${SETUP_ID}
 
@@ -73,8 +72,17 @@ ENV CMPLR_ROOT=$CMPLR_ROOT \
     LIBRARY_PATH=$LIBRARY_PATH \
     LD_LIBRARY_PATH=$LD_LIBRARY_PATH
 
-COPY ci/shared/scripts/setup-${SETUP_ID}.sh ${WF_TMP_DIR}/setup.sh
-RUN chmod +x ${WF_TMP_DIR}/setup.sh && . ${WF_TMP_DIR}/setup.sh
+
+COPY ci/setup-basic.sh ${WF_TMP_DIR}
+RUN chmod +x ${WF_TMP_DIR}/setup-basic.sh && . ${WF_TMP_DIR}/setup-basic.sh
+
+COPY ci/config.yaml ${WF_TMP_DIR}
+COPY ci/build-setup.py ${WF_TMP_DIR}
+COPY ci/util.py ${WF_TMP_DIR}
+COPY ci/setup-template.sh ${WF_TMP_DIR}
+RUN python3 ${WF_TMP_DIR}/build-setup.py ${REPO}:wf-${SETUP_ID}
+
+RUN chmod +x ${WF_TMP_DIR}/setup-${SETUP_ID}.sh && . ${WF_TMP_DIR}/setup-${SETUP_ID}.sh
 
 # Clean
 RUN rm -rf $WF_TMP_DIR

ci/shared/scripts/setup-arm64-macos-14-clang-14-cpp.sh → ci/setup-basic.sh

@@ -1,14 +1,6 @@
 #!/usr/bin/env sh
 
-#
-# macos-14-arm64, clang-14, mpich - Installation
-# Note: requires
-# - `git` and `wget` installed.
-# - run as root
-
-#
-# IMPORTANT: This file has been generated by a script. Please do not edit !
-#
+# Basic universal setup script
 
 WF_TMP_DIR=${WF_TMP_DIR:-"/opt/workflows"}
 WF_DEPS_URL=${WF_DEPS_URL:-"https://raw.githubusercontent.com/DARMA-tasking/workflows/refs/heads/master/ci/shared/scripts/deps"}
@@ -51,85 +43,15 @@ echo "-- Installing Core packages ($OS_NAME)..."
 if [ "$OS_NAME" = "Ubuntu" ]
 then
     apt-get update -y -q
-    apt-get install -y -q --no-install-recommends ca-certificates wget git
+    apt-get install -y -q --no-install-recommends ca-certificates wget git python3 python3-yaml
 elif [ "$OS_NAME" = "Alpine Linux" ]
 then
     apk update
-    apk add --no-cache wget git bash
+    apk add --no-cache wget git bash python3 python3-yaml
 elif [ "$OS_NAME" = "macOS" ]
 then
     brew update
 else
     echo "Error. Please implement the pre-setup instructions for OS=$OS_NAME"
     exit 1
 fi
-
-echo "--"
-echo "-- Core packages installed !"
-echo "--"
-
-set -e
-
-### SETUP DEPENDENCIES
-
-echo "--"
-echo "-- Installing dependencies..."
-echo "--"
-
-if [ "$WF_DOCKER" = "1" ]; then
-    # If a docker image is being built then deps are already available.
-    cd $WF_TMP_DIR/deps
-else
-    # if setup is ru directly (for example by a CI runner) then fetch dependencies
-    # trigger deps download from the workflows repo.
-    mkdir -p $WF_TMP_DIR
-    mkdir -p $WF_TMP_DIR/deps
-    # 1. Download dependency installation script
-    cd $WF_TMP_DIR/deps
-    wget $WF_DEPS_URL/packages.sh
-fi
-
-chmod u+x *.sh
-ls -l
-./packages.sh "ccache" "coreutils" "ninja" "mpich"
-
-# Remove install scripts
-rm -rf $WF_SCRIPTS_DIR
-
-if [ "$WF_DOCKER" = "1" ]; then
-    rm -rf /var/lib/apt/lists/*
-fi
-
-echo "--"
-echo "-- Dependencies installed !"
-echo "--"
-
-### CLEAN-UP
-if [ "$OS_NAME" = "Ubuntu" ]
-then
-    rm -rf /var/lib/apt/lists/*
-elif [ "$OS_NAME" = "Alpine Linux" ]
-then
-    :
-elif [ "$OS_NAME" = "macOS" ]
-then
-   :
-else
-    echo "No cleanup instructions defined for OS=$OS."
-fi
-
-echo "---------- Setup OK ! ------------"
-echo "--"
-echo "Operating system: $OS_NAME / Version: $OS_VERSION"
-echo "--"
-echo "Setup id: $WF_SETUP_ID"
-echo "Docker: $WF_DOCKER"
-echo "--"
-echo "Environment:"
-echo "  CC=$CC"
-echo "  CXX=$CXX"
-echo "  FC=$FC"
-echo "  PATH=$PATH"
-echo "--"
-echo "-------- Ready to test ! ---------"
-echo "--"

ci/shared/scripts/deps/mpich.sh

@@ -19,7 +19,7 @@ echo "${mpich_name}"
 echo "${mpich_tar_name}"
 echo "${make_flags}"
 
-wget http://www.mpich.org/static/downloads/${mpich_version}/${mpich_tar_name}
+wget --no-check-certificate http://www.mpich.org/static/downloads/${mpich_version}/${mpich_tar_name}
 tar xzf ${mpich_tar_name}
 rm ${mpich_tar_name}
 cd ${mpich_name}