Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .github/workflows/beaker_acceptance.yml
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,7 @@ jobs:
acceptance-working-dir: ${{ steps.set-defaults.outputs.acceptance_working_dir }}
acceptance-pre-suite: ${{ steps.set-defaults.outputs.acceptance_pre_suite }}
acceptance-tests: ${{ steps.set-defaults.outputs.acceptance_tests }}
acceptance-post-suite: ${{ steps.set-defaults.outputs.acceptance_post_suite }}
beaker-options: ${{ steps.set-defaults.outputs.beaker_options }}
steps:
- name: Checkout platform config
Expand Down Expand Up @@ -353,6 +354,8 @@ jobs:
${{ needs.set-project-defaults.outputs.acceptance-pre-suite || '[]' }}
TESTS_ARRAY: |-
${{ needs.set-project-defaults.outputs.acceptance-tests || '[]' }}
POST_SUITE_ARRAY: |-
${{ needs.set-project-defaults.outputs.acceptance-post-suite || '[]' }}
TAGS: ${{ matrix.tags && toJson(matrix.tags) || '[]' }}
BEAKER_OPTIONS: |-
${{ needs.set-project-defaults.outputs.beaker-options || '{}' }}
Expand All @@ -375,6 +378,7 @@ jobs:
"type": "aio",
"pre_suite": ${PRE_SUITE_ARRAY},
"tests": ${TESTS_ARRAY},
"post_suite": ${POST_SUITE_ARRAY},
"test_tag_and": ${TAGS}
},
${BEAKER_OPTIONS}
Expand Down
129 changes: 112 additions & 17 deletions .github/workflows/build_ezbake.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,12 @@ on:
Branch/tag from ezbake that will be used for openvoxdb/server builds.
type: string
required: false
ezbake-ver:
description: 'The version specified in project.clj in the given ezbake-ref. Will default to the version found in project.clj in the repo if not specified.'
ezbake-2-ver:
description: 'Version for the ezbake 2 build group used for older OSes.'
type: string
required: false
ezbake-4-ver:
description: 'Version for the ezbake 4 build group used for newer OSes.'
type: string
required: false
branch:
Expand All @@ -38,17 +42,15 @@ env:
# https://github.com/boto/boto3/issues/4398#issuecomment-2619946229
AWS_REQUEST_CHECKSUM_CALCULATION: "WHEN_REQUIRED"
AWS_RESPONSE_CHECKSUM_VALIDATION: "WHEN_REQUIRED"
EZBAKE_BRANCH: ${{ inputs.ezbake-ref }}

jobs:
build:
runs-on: ubuntu-24.04
build-ezbake-2:
name: ezbake 2.x build for legacy OSes
runs-on: ubuntu-26.04
timeout-minutes: 600
strategy:
fail-fast: false
steps:
# This checks out the release branch, because the vox:build task will
# check out the tag itself.
- name: Checkout code
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
Expand All @@ -58,19 +60,30 @@ jobs:
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: OpenVoxProject/shared-actions
ref: main
ref: ezbake4 # switch to main before merge!
path: .shared-actions
sparse-checkout: platforms.json
sparse-checkout-cone-mode: false

- name: Resolve platform lists
- name: Resolve platform lists for ezbake 2
env:
DEB_PLATFORMS: ${{ inputs.deb_platform_list }}
RPM_PLATFORMS: ${{ inputs.rpm_platform_list }}
BRANCH: ${{ inputs.branch }}
EZBAKE_2_VER: ${{ inputs.ezbake-2-ver }}
run: |
echo "DEB_PLATFORMS=${DEB_PLATFORMS:-$(jq -r --arg branch "${BRANCH}" '.[$branch]."ezbake-deb" | join(",")' .shared-actions/platforms.json)}" >> $GITHUB_ENV
echo "RPM_PLATFORMS=${RPM_PLATFORMS:-$(jq -r --arg branch "${BRANCH}" '.[$branch]."ezbake-rpm" | join(",")' .shared-actions/platforms.json)}" >> $GITHUB_ENV
EZBAKE_2_VER="${EZBAKE_2_VER:-$(jq -r --arg branch "${BRANCH}" '.[$branch]."ezbake2-version"' .shared-actions/platforms.json)}"

echo "EZBAKE_BRANCH=${{ inputs.ezbake-ref }}" >> $GITHUB_ENV
echo "EZBAKE_VERSION=${EZBAKE_2_VER}" >> $GITHUB_ENV
DEB_PLATFORMS="${DEB_PLATFORMS:-$(jq -r --arg branch "${BRANCH}" '.[$branch]."ezbake2-deb" | join(",")' .shared-actions/platforms.json)}"
echo "DEB_PLATFORMS=$DEB_PLATFORMS" >> $GITHUB_ENV
RPM_PLATFORMS="${RPM_PLATFORMS:-$(jq -r --arg branch "${BRANCH}" '.[$branch]."ezbake2-rpm" | join(",")' .shared-actions/platforms.json)}"
echo "RPM_PLATFORMS=$RPM_PLATFORMS" >> $GITHUB_ENV

echo "::notice title=ezbake 2 EZBAKE_BRANCH::${{ inputs.ezbake-ref }}"
echo "::notice title=ezbake 2 EZBAKE_VERSION::$EZBAKE_2_VER"
echo "::notice title=ezbake 2 PLATFORMS::$DEB_PLATFORMS,$RPM_PLATFORMS"

- name: Install Ruby
uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0
Expand All @@ -82,7 +95,7 @@ jobs:
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: /tmp/ezbake-builder.tar
key: docker-ezbake-${{ hashFiles('Dockerfile') }}
key: docker-ezbake-${{ hashFiles('Dockerfile') }}-legacy # same key as in the fips workflow

- name: Load or build Docker image
run: |
Expand All @@ -97,17 +110,99 @@ jobs:
if: always() && steps.docker-cache.outputs.cache-hit != 'true'
with:
path: /tmp/ezbake-builder.tar
key: docker-ezbake-${{ hashFiles('Dockerfile') }}
key: docker-ezbake-${{ hashFiles('Dockerfile') }}-legacy

- name: Update awscli
run: |
python -m pip install --upgrade awscli

- name: Set EZBAKE_VERSION if provided
if: ${{ inputs.ezbake-ver != '' }}
- name: Run build script
env:
BUILD_REF: ${{ inputs.ref }}
run: |
rm -rf output
bundle exec rake "vox:build[${BUILD_REF}]"

- name: Configure OSUOSL CA Certificate for S3
uses: $/.github/actions/setup-osuosl-ca

- name: Upload output to S3
run: |
bundle exec rake vox:upload

build-ezbake-4:
name: ezbake 4.x build for modern OSes
runs-on: ubuntu-26.04
if: inputs.branch == 'main'
timeout-minutes: 600
strategy:
fail-fast: false
steps:
- name: Checkout code
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ inputs.branch }}

- name: Checkout platform config
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: OpenVoxProject/shared-actions
ref: ezbake4 # switch to main before merge!
path: .shared-actions
sparse-checkout: platforms.json
sparse-checkout-cone-mode: false

- name: Resolve platform lists for ezbake 4
env:
EZBAKE_VER: ${{ inputs.ezbake-ver }}
run: echo "EZBAKE_VERSION=${EZBAKE_VER}" >> $GITHUB_ENV
DEB_PLATFORMS: ${{ inputs.deb_platform_list }}
RPM_PLATFORMS: ${{ inputs.rpm_platform_list }}
BRANCH: ${{ inputs.branch }}
EZBAKE_4_VER: ${{ inputs.ezbake-4-ver }}
run: |
EZBAKE_4_VER="${EZBAKE_4_VER:-$(jq -r --arg branch "${BRANCH}" '.[$branch]."ezbake4-version"' .shared-actions/platforms.json)}"

echo "EZBAKE_BRANCH=${{ inputs.ezbake-ref }}" >> $GITHUB_ENV
echo "EZBAKE_VERSION=${EZBAKE_4_VER}" >> $GITHUB_ENV
DEB_PLATFORMS="${DEB_PLATFORMS:-$(jq -r --arg branch "${BRANCH}" '.[$branch]."ezbake4-deb" | join(",")' .shared-actions/platforms.json)}"
echo "DEB_PLATFORMS=$DEB_PLATFORMS" >> $GITHUB_ENV
RPM_PLATFORMS="${RPM_PLATFORMS:-$(jq -r --arg branch "${BRANCH}" '.[$branch]."ezbake4-rpm" | join(",")' .shared-actions/platforms.json)}"
echo "RPM_PLATFORMS=$RPM_PLATFORMS" >> $GITHUB_ENV

echo "::notice title=ezbake 4 EZBAKE_BRANCH::${{ inputs.ezbake-ref }}"
echo "::notice title=ezbake 4 EZBAKE_VERSION::$EZBAKE_4_VER"
echo "::notice title=ezbake 4 PLATFORMS::$DEB_PLATFORMS,$RPM_PLATFORMS"

- name: Install Ruby
uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0
with:
ruby-version: '4.0'
bundler-cache: true

# we need to cache the container per-ezbake version because ezbake is checked out inside of that container
- id: docker-cache
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: /tmp/ezbake-builder.tar
key: docker-ezbake-${{ hashFiles('Dockerfile') }}-modern

- name: Load or build Docker image
run: |
if [[ -f /tmp/ezbake-builder.tar ]]; then
docker load -i /tmp/ezbake-builder.tar
else
docker build -t ezbake-builder .
docker save ezbake-builder -o /tmp/ezbake-builder.tar
fi

- uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
if: always() && steps.docker-cache.outputs.cache-hit != 'true'
with:
path: /tmp/ezbake-builder.tar
key: docker-ezbake-${{ hashFiles('Dockerfile') }}-modern

- name: Update awscli
run: |
python -m pip install --upgrade awscli

- name: Run build script
env:
Expand Down
27 changes: 15 additions & 12 deletions .github/workflows/build_ezbake_fips.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ on:
type: string
required: false
ezbake-ver:
description: 'The version specified in project.clj in the given ezbake-ref. Will default to the version found in project.clj in the repo if not specified.'
description: 'Version for the ezbake 2 build group used for older OSes.'
type: string
required: false
branch:
Expand All @@ -39,7 +39,7 @@ env:

jobs:
build:
runs-on: ubuntu-24.04
runs-on: ubuntu-26.04
timeout-minutes: 600
strategy:
fail-fast: false
Expand All @@ -55,7 +55,7 @@ jobs:
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: OpenVoxProject/shared-actions
ref: main
ref: ezbake4 # rename to main before merge
path: .shared-actions
sparse-checkout: platforms.json
sparse-checkout-cone-mode: false
Expand All @@ -65,7 +65,16 @@ jobs:
RPM_PLATFORMS: ${{ inputs.rpm_platform_list }}
BRANCH: ${{ inputs.branch }}
run: |
echo "RPM_PLATFORMS=${RPM_PLATFORMS:-$(jq -r --arg branch "${BRANCH}" '.[$branch]."ezbake-fips-rpm" | join(",")' .shared-actions/platforms.json)}" >> $GITHUB_ENV
EZBAKE_VERSION="${EZBAKE_VER:-$(jq -r --arg branch "${BRANCH}" '.[$branch]."ezbake2-version"' .shared-actions/platforms.json)}"

echo "EZBAKE_BRANCH=${{ inputs.ezbake-ref }}" >> $GITHUB_ENV
echo "EZBAKE_VERSION=${EZBAKE_VERSION}" >> $GITHUB_ENV
RPM_PLATFORMS="${RPM_PLATFORMS:-$(jq -r --arg branch "${BRANCH}" '.[$branch]."ezbake-fips-rpm" | join(",")' .shared-actions/platforms.json)}"
echo "RPM_PLATFORMS=$RPM_PLATFORMS" >> $GITHUB_ENV

echo "::notice title=ezbake 2 EZBAKE_BRANCH::${{ inputs.ezbake-ref }}"
echo "::notice title=ezbake 2 EZBAKE_VERSION::$EZBAKE_VERSION"
echo "::notice title=ezbake 2 PLATFORMS::$RPM_PLATFORMS"

- name: Install Ruby
uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0
Expand All @@ -77,7 +86,7 @@ jobs:
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: /tmp/ezbake-builder.tar
key: docker-ezbake-${{ hashFiles('Dockerfile') }}
key: docker-ezbake-${{ hashFiles('Dockerfile') }}-legacy # same key as in non-fips ezbake 2

- name: Load or build Docker image
run: |
Expand All @@ -92,18 +101,12 @@ jobs:
if: always() && steps.docker-cache.outputs.cache-hit != 'true'
with:
path: /tmp/ezbake-builder.tar
key: docker-ezbake-${{ hashFiles('Dockerfile') }}
key: docker-ezbake-${{ hashFiles('Dockerfile') }}-legacy

- name: Update awscli
run: |
python -m pip install --upgrade awscli

- name: Set EZBAKE_VERSION if provided
if: ${{ inputs.ezbake-ver != '' }}
env:
EZBAKE_VER: ${{ inputs.ezbake-ver }}
run: echo "EZBAKE_VERSION=${EZBAKE_VER}" >> $GITHUB_ENV

- name: Run build script
env:
BUILD_REF: ${{ inputs.ref }}
Expand Down
28 changes: 18 additions & 10 deletions platforms.json
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
{
"main": {
"ezbake2-version": "2.9.1",
"ezbake4-version": "4.0.0",
"vanagon": [
"amazon-2023-aarch64",
"amazon-2023-x86_64",
Expand Down Expand Up @@ -35,20 +37,24 @@
"ubuntu-26.04-armhf",
"windows-msys2-x64"
],
"ezbake-deb": [
"ubuntu-22.04",
"ubuntu-24.04",
"ubuntu-26.04",
"debian-13"
"ezbake2-deb": [
"ubuntu-22.04"
],
"ezbake-rpm": [
"ezbake2-rpm": [
"amazon-2023",
"el-8",
"el-9",
"sles-15"
],
"ezbake4-deb": [
"ubuntu-24.04",
"ubuntu-26.04",
"debian-13"
],
"ezbake4-rpm": [
"el-10",
"fedora-43",
"fedora-44",
"sles-15",
"sles-16"
],
"ezbake-fips-rpm": [
Expand All @@ -57,6 +63,8 @@
]
},
"8.x": {
"ezbake2-version": "2.9.1",
"ezbake4-version": "4.0.0",
"vanagon": [
"amazon-2-aarch64",
"amazon-2-x86_64",
Expand Down Expand Up @@ -102,16 +110,16 @@
"ubuntu-26.04-armhf",
"windows-all-x64"
],
"ezbake-deb": [
"ezbake2-deb": [
"ubuntu-22.04",
"ubuntu-24.04",
"ubuntu-25.04",
"ubuntu-26.04",
"debian-11",
"debian-12",
"ubuntu-26.04",
"debian-13"
],
"ezbake-rpm": [
"ezbake2-rpm": [
"amazon-2",
"amazon-2023",
"el-8",
Expand Down
14 changes: 11 additions & 3 deletions scripts/project-config.sh
Original file line number Diff line number Diff line change
Expand Up @@ -78,18 +78,26 @@ map_platforms='
intersect='. as $uni | $uni | map(select(. as $u | ($set | any(. == $u))))'

# Which build list gates this suite: the agent (vanagon) list for the agent
# suites, or the combined server/db (ezbake) list otherwise.
# suites, or the combined server/db (ezbake) list otherwise. The platform file
# uses both the legacy ezbake2 keys and the current ezbake4 keys depending on
# the branch, so we must accept both shapes instead of assuming one stale name.
case "$suite" in
openvox|openvox-agent)
build_list=$(jq -c --arg b "$branch" '.[$b].vanagon' "$platforms") ;;
openvox-server|openvoxdb)
build_list=$(jq -c --arg b "$branch" '(.[$b]["ezbake-deb"] + .[$b]["ezbake-rpm"])' "$platforms") ;;
build_list=$(jq -c --arg b "$branch" '
(.[$b]["ezbake4-deb"] // [] + .[$b]["ezbake4-rpm"] // [] +
.[$b]["ezbake2-deb"] // [] + .[$b]["ezbake2-rpm"] // [])
' "$platforms") ;;
*)
echo "Unknown suite '$suite'" >&2; exit 1 ;;
esac
# The server/db build list is always needed to decide when a fall back is needed
# to test an agent platform that does not have a corresponding server build.
server_list=$(jq -c --arg b "$branch" '(.[$b]["ezbake-deb"] + .[$b]["ezbake-rpm"])' "$platforms")
server_list=$(jq -c --arg b "$branch" '
(.[$b]["ezbake4-deb"] // [] + .[$b]["ezbake4-rpm"] // [] +
.[$b]["ezbake2-deb"] // [] + .[$b]["ezbake2-rpm"] // [])
' "$platforms")

# Translate platforms.json OS names to nested_vms tuples.
build_os=$(jq -c "$map_platforms" <<<"$build_list")
Expand Down
3 changes: 3 additions & 0 deletions scripts/project_defaults.json
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@
"acceptance-tests": [
"tests"
],
"acceptance-post-suite": [
"teardown/common/099_Archive_Logs.rb"
],
"acceptance-shard-tags": [
"shard:group1",
"shard:group2",
Expand Down