From 47bf8168e58b958e2279155ffeeb34f7aa6fb170 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Esteban=20Zim=C3=A1nyi?= Date: Mon, 31 Aug 2026 22:40:04 +0200 Subject: [PATCH] Build the benchmark images from the master the chain tracks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Dockerfile` built MobilityDB from `stable-1.3` while the jar it runs is generated from the catalog of master, and it built without `-DALL=ON`, so the library answered for a narrower family list than the surface the generator emits. It now clones master and passes `-DALL=ON`, the flag `CMakeLists.txt` turns into the family list. Its JMEOS step cloned the branch `fix-tests-using-docker` into `/usr/local/jmeos`, swapped a library into it and deleted its jars. Nothing in the image ever read that directory — the branch was pinned for a checkout no stage consumed. `COPY jar/JMEOS.jar` and the classpath entry beside it are the `systemPath` model `pom.xml` itself has commented out: the shade plugin includes `*:*`, and the 33 MB application jar carries `functions/GeneratedFunctions.class`. Both go, and `jar/` need not be assembled by hand before a build. `Dockerfile_q8_meos_kalman` existed to build `marianaGarcez/MobilityDB` and patch `temporal_ext_kalman_filter` into `functions.java` with two `sed` scripts. That entry is declared in `meos/include/meos.h` on upstream master, so the generator emits it and `meos/examples/ais_ekf_clean.c` is upstream as well. The image differed from the base one in nothing else, and `run_query.sh` already patches the entrypoint class it needed, so query 8 runs on the base image and the file goes with its `QUERY_DOCKERFILE` entry. The two `Query8_V2_Main` classes name the public MEOS entry rather than a fork. `mvn test` is green over the module: 12 tests, 0 skipped. --- benchmark/Dockerfile | 23 +++-- benchmark/Dockerfile_q8_meos_kalman | 86 ------------------- benchmark/run_query.sh | 4 +- .../src/main/java/aisdata/Query8_V2_Main.java | 5 +- .../main/java/sncbdata/Query8_V2_Main.java | 7 +- 5 files changed, 19 insertions(+), 106 deletions(-) delete mode 100644 benchmark/Dockerfile_q8_meos_kalman diff --git a/benchmark/Dockerfile b/benchmark/Dockerfile index a961ac3..cc2f2c1 100644 --- a/benchmark/Dockerfile +++ b/benchmark/Dockerfile @@ -12,13 +12,17 @@ RUN apt-get update \ && apt-get update \ && apt-get install -y java-21-amazon-corretto-jdk -# Build MobilityDB with MEOS -RUN git clone --depth 1 https://github.com/MobilityDB/MobilityDB.git -b stable-1.3 /usr/local/src/MobilityDB -RUN mkdir -p /usr/local/src/MobilityDB/build +# Build MobilityDB with MEOS. The jar this image runs is generated from the +# catalog of MobilityDB master with every family on, so the library must answer +# for the same surface: -DALL=ON is what CMakeLists.txt turns into the family +# list, and a narrower build leaves the jar naming symbols libmeos lacks. +RUN git clone --depth 1 https://github.com/MobilityDB/MobilityDB.git -b master /usr/local/src/MobilityDB +RUN mkdir -p /usr/local/src/MobilityDB/build RUN cd /usr/local/src/MobilityDB/build && \ - cmake -DMEOS=ON .. && \ + cmake -DMEOS=ON -DALL=ON .. && \ make -j$(nproc) && \ - make install + make install && \ + ldconfig # Set up Maven ENV MAVEN_HOME /usr/share/maven @@ -28,19 +32,12 @@ COPY --from=maven:3.9.6-eclipse-temurin-11 /usr/local/bin/mvn-entrypoint.sh /usr COPY --from=maven:3.9.6-eclipse-temurin-11 /usr/share/maven/ref/settings-docker.xml /usr/share/maven/ref/settings-docker.xml RUN ln -s ${MAVEN_HOME}/bin/mvn /usr/bin/mvn -# Set up MobilityDB-JMEOS -RUN git clone --depth 1 --branch fix-tests-using-docker https://github.com/MobilityDB/JMEOS /usr/local/jmeos -RUN rm /usr/local/jmeos/src/main/resources/lib/libmeos.so -RUN cp /usr/local/lib/libmeos.so /usr/local/jmeos/src/main/resources/lib/libmeos.so -RUN rm /usr/local/jmeos/jar/* - # Clean up RUN rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*lists/* RUN apt-get remove --purge --autoremove -y curl gnupg # Copy project-specific files COPY target/flink-kafka2postgres-1.0-SNAPSHOT.jar /app/flink-kafka2postgres.jar -COPY jar/JMEOS.jar /usr/local/jar/JMEOS.jar COPY wait-for-it.sh /app/wait-for-it.sh # Make the wait-for-it script executable @@ -53,4 +50,4 @@ ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH WORKDIR /app # Set entrypoint with the correct classpath -ENTRYPOINT ["/app/wait-for-it.sh", "kafka:29092", "--", "java", "-cp", "/app/flink-kafka2postgres.jar:/usr/local/jar/JMEOS.jar", "--add-opens", "java.base/java.lang=ALL-UNNAMED", "aisdata.Query9_Main"] +ENTRYPOINT ["/app/wait-for-it.sh", "kafka:29092", "--", "java", "-cp", "/app/flink-kafka2postgres.jar", "--add-opens", "java.base/java.lang=ALL-UNNAMED", "aisdata.Query9_Main"] diff --git a/benchmark/Dockerfile_q8_meos_kalman b/benchmark/Dockerfile_q8_meos_kalman deleted file mode 100644 index d6a4f61..0000000 --- a/benchmark/Dockerfile_q8_meos_kalman +++ /dev/null @@ -1,86 +0,0 @@ -FROM debian:bookworm-slim - -# Install necessary packages -RUN apt-get update \ - && apt-get install -y git curl gnupg build-essential tree vim cmake postgresql-server-dev-15 libproj-dev libjson-c-dev libgsl-dev libgeos-dev postgis \ - && export GNUPGHOME="$(mktemp -d)" \ - && curl -fL https://apt.corretto.aws/corretto.key | gpg --batch --import \ - && gpg --batch --export '6DC3636DAE534049C8B94623A122542AB04F24E3' > /usr/share/keyrings/corretto.gpg \ - && rm -r "$GNUPGHOME" \ - && unset GNUPGHOME \ - && echo "deb [signed-by=/usr/share/keyrings/corretto.gpg] https://apt.corretto.aws stable main" > /etc/apt/sources.list.d/corretto.list \ - && apt-get update \ - && apt-get install -y java-21-amazon-corretto-jdk - -# Build MobilityDB with MEOS — marianaGarcez fork (PR #749, native EKF) -RUN git clone --depth 1 https://github.com/marianaGarcez/MobilityDB.git \ - -b master /usr/local/src/MobilityDB -RUN mkdir -p /usr/local/src/MobilityDB/build -RUN cd /usr/local/src/MobilityDB/build && \ - cmake -DMEOS=ON .. && \ - make -j$(nproc) && \ - make install && \ - ldconfig - -# Set up Maven -ENV MAVEN_HOME=/usr/share/maven -ENV MAVEN_CONFIG="/root/.m2" -COPY --from=maven:3.9.6-eclipse-temurin-11 ${MAVEN_HOME} ${MAVEN_HOME} -COPY --from=maven:3.9.6-eclipse-temurin-11 /usr/local/bin/mvn-entrypoint.sh /usr/local/bin/mvn-entrypoint.sh -COPY --from=maven:3.9.6-eclipse-temurin-11 /usr/share/maven/ref/settings-docker.xml /usr/share/maven/ref/settings-docker.xml -RUN ln -s ${MAVEN_HOME}/bin/mvn /usr/bin/mvn - -# Set up MobilityDB-JMEOS -# We clone and rebuild the jar here so that the libmeos.so bundled inside it -# (in resources/lib/) is the marianaGarcez EKF-capable one, not the upstream. -# Copying a pre-built JMEOS.jar would silently embed the wrong libmeos.so, -# since JarLibraryLoader extracts and loads the one from inside the jar at runtime. -RUN git clone --branch fix-tests-using-docker https://github.com/MobilityDB/JMEOS /usr/local/jmeos -RUN rm /usr/local/jmeos/src/main/resources/lib/libmeos.so -RUN cp /usr/local/lib/libmeos.so /usr/local/jmeos/src/main/resources/lib/libmeos.so -RUN rm /usr/local/jmeos/jar/* - -# Patch functions.java to expose temporal_ext_kalman_filter via JNR-FFI. -RUN FPATH=/usr/local/jmeos/src/main/java/functions/functions.java && \ - sed -i \ - 's/Pointer tpoint_trajectory(Pointer temp, boolean unary_union);/Pointer tpoint_trajectory(Pointer temp, boolean unary_union);\n\t\tPointer temporal_ext_kalman_filter(Pointer temp, double gate, double q, double r, boolean dropOutliers);/' \ - "$FPATH" && \ - sed -z -i \ - 's/\(public static Pointer tpoint_trajectory(Pointer temp, boolean unary_union) {\n\t\tvar _result = MeosLibrary.meos.tpoint_trajectory(temp, unary_union);\n\t\tMeosErrorHandler.checkError();\n\t\treturn _result;\n\t}\)/\1\n\n\tpublic static Pointer temporal_ext_kalman_filter(Pointer temp, double gate, double q, double r, boolean dropOutliers) {\n\t\tvar _result = MeosLibrary.meos.temporal_ext_kalman_filter(temp, gate, q, r, dropOutliers);\n\t\tMeosErrorHandler.checkError();\n\t\treturn _result;\n\t}/' \ - "$FPATH" - -# Build JMEOS jar (with EKF-capable libmeos.so embedded + patched functions.java). -# Jars are output to /usr/local/jmeos/jar/ (not target/). -RUN cd /usr/local/jmeos && mvn clean package \ - && mkdir -p /usr/local/jar \ - && cp /usr/local/jmeos/jar/*.jar /usr/local/jar/ - -# Clean up -RUN rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*lists/* -RUN apt-get remove --purge --autoremove -y curl gnupg - -# Copy application sources and compile inside Docker. -# This is necessary because temporal_ext_kalman_filter only exists in the -# patched JMEOS built above thus the application cannot be compiled on the host. -COPY src /build/src -COPY pom.xml /build/pom.xml -RUN mkdir -p /build/jar && cp /usr/local/jar/JMEOS.jar /build/jar/JMEOS.jar -RUN cd /build \ - && mvn clean package -DqueryMainClass="aisdata.Query8_V2_Main" -q \ - && mkdir -p /app \ - && cp target/flink-kafka2postgres-1.0-SNAPSHOT.jar /app/flink-kafka2postgres.jar - -COPY wait-for-it.sh /app/wait-for-it.sh - -# Make the wait-for-it script executable -RUN chmod +x /app/wait-for-it.sh - -# Set the library path -ENV LD_LIBRARY_PATH=/usr/local/lib - -# Set working directory -WORKDIR /app - -# Set entrypoint — /usr/local/jar/* expands to all jars copied above -#ENTRYPOINT ["/app/wait-for-it.sh", "kafka:29092", "--", "java", "-cp", "/app/flink-kafka2postgres.jar:/usr/local/jar/*", "--add-opens", "java.base/java.lang=ALL-UNNAMED", "sncbdata.Query8_V2_Main"] -ENTRYPOINT ["/app/wait-for-it.sh", "kafka:29092", "--", "java", "-cp", "/app/flink-kafka2postgres.jar:/usr/local/jar/*", "--add-opens", "java.base/java.lang=ALL-UNNAMED", "aisdata.Query8_V2_Main"] diff --git a/benchmark/run_query.sh b/benchmark/run_query.sh index ce12d0e..8f4120d 100644 --- a/benchmark/run_query.sh +++ b/benchmark/run_query.sh @@ -33,12 +33,12 @@ declare -A QUERY_DESC=( [5]="Trajectory Creation and High-Speed Alert" [6]="Positional Divergence for a Device" [7]="Global Closest Device Pairs (Top-k)" - [8]="Trajectory Denoising — MEOS native EKF (marianaGarcez fork)" + [8]="Trajectory Denoising — MEOS native EKF" [9]="Windowed Per-Device kNN Join" ) declare -A QUERY_DOCKERFILE=( - [1]="" [2]="" [3]="" [4]="" [5]="" [6]="" [7]="" [8]="Dockerfile_q8_meos_kalman" [9]="" + [1]="" [2]="" [3]="" [4]="" [5]="" [6]="" [7]="" [8]="" [9]="" ) QUERY_NUM="${1:-}" diff --git a/benchmark/src/main/java/aisdata/Query8_V2_Main.java b/benchmark/src/main/java/aisdata/Query8_V2_Main.java index 9594ec4..d700807 100644 --- a/benchmark/src/main/java/aisdata/Query8_V2_Main.java +++ b/benchmark/src/main/java/aisdata/Query8_V2_Main.java @@ -33,7 +33,8 @@ * Query 8 V2 - Trajectory Denoising with MEOS native EKF (AIS dataset) * *

Uses the Extended Kalman Filter implemented natively in MEOS via - * {@code temporal_ext_kalman_filter()} (PR #749: marianaGarcez/MobilityDB fork). + * {@code temporal_ext_kalman_filter()}, a public entry of {@code meos.h} that + * the generated JMEOS surface carries. * *

Original MobilityNebula pseudocode: * 

@@ -51,7 +52,7 @@ public class Query8_V2_Main {
 
     private static final Logger logger = LoggerFactory.getLogger(Query8_V2_Main.class);
 
-    // EKF parameters: degree² units (C reference example defaults: https://github.com/marianaGarcez/MobilityDB/blob/e3319c1e6fc9157d19cb0580e097224cd42a8214/meos/examples/ais_ekf_clean.c)
+    // EKF parameters: degree² units (C reference example defaults: meos/examples/ais_ekf_clean.c)
     private static final double  GATE          = 8.0;
     private static final double  Q             = 5e-10;
     private static final double  R             = 4e-6;
diff --git a/benchmark/src/main/java/sncbdata/Query8_V2_Main.java b/benchmark/src/main/java/sncbdata/Query8_V2_Main.java
index b49c187..73444db 100644
--- a/benchmark/src/main/java/sncbdata/Query8_V2_Main.java
+++ b/benchmark/src/main/java/sncbdata/Query8_V2_Main.java
@@ -33,7 +33,8 @@
  * Query 8 V2 - Trajectory Denoising with MEOS native EKF (SNCB dataset)
  *
  * 
Uses the Extended Kalman Filter implemented natively in MEOS via
- * {@code temporal_ext_kalman_filter()} (PR #749: marianaGarcez/MobilityDB fork).
+ * {@code temporal_ext_kalman_filter()}, a public entry of {@code meos.h} that
+ * the generated JMEOS surface carries.
  * This replaces the Apache Commons Math Kalman Filter used in V1.
  *
  * 
Original MobilityNebula pseudocode:
@@ -55,7 +56,7 @@
  * 
  *
  * 
Build requirement

- * 
Requires {@code libmeos.so} compiled from the {@code marianaGarcez/MobilityDB}
+ * 
Requires {@code libmeos.so} compiled from {@code MobilityDB/MobilityDB}
  * fork (branch {@code master}) and {@code GeneratedFunctions.java} patched to expose
  * {@code temporal_ext_kalman_filter}, both handled by {@code Dockerfile.q8v2}.
  *
@@ -66,7 +67,7 @@ public class Query8_V2_Main {
 
     private static final Logger logger = LoggerFactory.getLogger(Query8_V2_Main.class);
 
-    // EKF parameters: degree² units (C reference example defaults: https://github.com/marianaGarcez/MobilityDB/blob/e3319c1e6fc9157d19cb0580e097224cd42a8214/meos/examples/ais_ekf_clean.c)
+    // EKF parameters: degree² units (C reference example defaults: meos/examples/ais_ekf_clean.c)
     private static final double GATE = 3.0;
 
     private static final double Q = 5e-10;