Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
107 changes: 107 additions & 0 deletions .github/workflows/docker.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
name: Docker image

# Nothing else in this repository builds the benchmark image, so its clone ref and
# its cmake flags go stale unnoticed — which is how it came to build a release
# branch while the jar it runs is generated from the catalog of master. Building it
# IS the test.
#
# It runs on a change to the image, to what the image copies, or to this workflow;
# on a push to main; and weekly. The weekly run is the one that matters: the image
# tracks MobilityDB master, so it can start failing with nothing here having changed.

on:
pull_request:
paths:
- benchmark/Dockerfile
- benchmark/run_query.sh
- benchmark/wait-for-it.sh
- .github/workflows/docker.yml
push:
branches: [main]
paths:
- benchmark/Dockerfile
- benchmark/run_query.sh
- benchmark/wait-for-it.sh
- .github/workflows/docker.yml
schedule:
- cron: "37 5 * * 1"
workflow_dispatch:

jobs:
image:
name: Build the benchmark image and check what it carries
runs-on: ubuntu-latest
# The image compiles MobilityDB from source with every family on, on top of the
# chain that produces the application jar it copies in.
timeout-minutes: 120

steps:
- uses: actions/checkout@v4

- name: Set up Java 21
uses: actions/setup-java@v4
with:
distribution: temurin
java-version: "21"
cache: maven

# The image COPYs benchmark/target/flink-kafka2postgres-1.0-SNAPSHOT.jar, so the
# jar has to exist before docker build runs — which means the same chain the build
# job uses. Reaching it through the shared action rather than restating it keeps
# this job from drifting away from that one.
- name: Provision MEOS catalog + libmeos
id: provision
uses: MobilityDB/MEOS-API/.github/actions/provision-meos@master
with:
mobilitydb-ref: master
build-libmeos: "true"

- name: Stage the derived MEOS catalog
run: |
cp "${{ steps.provision.outputs.catalog-path }}" tools/meos-idl.json
echo "LD_LIBRARY_PATH=/usr/local/lib" >> "$GITHUB_ENV"

- name: Check out JMEOS at main
uses: actions/checkout@v4
with:
repository: MobilityDB/JMEOS
ref: main
path: jmeos

- name: Stage the JVM generator JMEOS owns
run: cp jmeos/tools/codegen_jvm.py jmeos/tools/codegen_spark_udfs.py tools/

- name: Build + install the JMEOS jar as org.jmeos:meos:1.0
run: |
CATALOG="${{ steps.provision.outputs.catalog-path }}" \
LIBMEOS=/usr/local/lib/libmeos.so \
jmeos/tools/regen-from-catalog.sh
mvn -B install:install-file \
-Dfile=jmeos/jar/JMEOS.jar \
-DgroupId=org.jmeos -DartifactId=meos -Dversion=1.0 -Dpackaging=jar

- name: Build the application jar the image copies
run: mvn -B -Dmeos.lib.dir=/usr/local/lib -Dmeos.enabled=true -pl benchmark -am package

- name: Build the image
run: docker build --progress=plain -f benchmark/Dockerfile -t flink-benchmark-ci:${{ github.sha }} benchmark/

# A build that succeeds still proves nothing about WHICH library the image holds,
# and a narrower one is the failure this guards: a libmeos configured without
# -DALL=ON carries no S2CELL and no POSECHAIN entry, while the jar generated from
# the catalog of master names both. `temporal_merge` is the positive control — were
# it missing too, the check would be reading the wrong file rather than a narrow
# build. The jar assertion is the other half: the shaded jar carries JMEOS itself,
# which is why the image needs no separate copy of it on the classpath.
- name: The image carries the application jar and an all-families libmeos
run: |
docker run --rm --entrypoint bash flink-benchmark-ci:${{ github.sha }} -euc '
test -f /app/flink-kafka2postgres.jar
grep -ac "functions/GeneratedFunctions.class" /app/flink-kafka2postgres.jar
L=/usr/local/lib/libmeos.so
for s in ts2cell_in tposechain_in temporal_merge; do
n=$(grep -ac "$s" "$L" || true)
echo "$s: $n"
[ "${n:-0}" -ge 1 ] || { echo "::error::$L carries no $s"; exit 1; }
done
'
11 changes: 8 additions & 3 deletions benchmark/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,8 +1,13 @@
FROM debian:bookworm-slim
FROM ubuntu:24.04

# Install necessary packages
# The MEOS build dependencies are the set MobilityDB's own MEOS workflow installs
# (.github/workflows/meos.yml), on the distribution it installs them on: `libh3-dev`
# is what -DALL=ON needs for the H3 family and Debian bookworm has no such package,
# while Ubuntu 24.04 carries it. A -DMEOS=ON build needs neither the PostgreSQL
# server headers nor GSL nor PostGIS, so none of the three is installed.
RUN apt-get update \
&& apt-get install -y git curl gnupg build-essential tree vim cmake postgresql-server-dev-15 libproj-dev libjson-c-dev libgsl-dev libgeos-dev postgis \
&& apt-get install -y git curl gnupg build-essential cmake \
libgeos-dev libproj-dev libjson-c-dev libgdal-dev libh3-dev libxml2-dev zlib1g-dev \
&& export GNUPGHOME="$(mktemp -d)" \
&& curl -fL https://apt.corretto.aws/corretto.key | gpg --batch --import \
&& gpg --batch --export '6DC3636DAE534049C8B94623A122542AB04F24E3' > /usr/share/keyrings/corretto.gpg \
Expand Down
Loading