From 065d2d1f60bede64758af922939126900a5d3867 Mon Sep 17 00:00:00 2001 From: Esteban Zimanyi Date: Wed, 22 Jul 2026 21:23:45 +0200 Subject: [PATCH 1/2] Align the batch benchmark queries to the canonical BerlinMOD schema MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The shared batch queries used a divergent vehId / Query* schema that does not match the schema the canonical loaders produce. Rename the table and column references to the canonical names the merged loaders (berlinmod_load.sql and mobilityduck_load.sql) create — VehicleId, TripId, and the Instants, Periods, Points, Regions and Licences parameter relations — so the single query source is consistent with the canonical per-tool loaders every runner uses and with the classic berlinmod_r_queries_portable.sql query set. --- BerlinMOD/benchmarks/batch/bench/queries.sql | 138 +++++++++---------- 1 file changed, 69 insertions(+), 69 deletions(-) diff --git a/BerlinMOD/benchmarks/batch/bench/queries.sql b/BerlinMOD/benchmarks/batch/bench/queries.sql index e9f0b5c..0df2b66 100644 --- a/BerlinMOD/benchmarks/batch/bench/queries.sql +++ b/BerlinMOD/benchmarks/batch/bench/queries.sql @@ -7,7 +7,7 @@ -- @query q01 --- BerlinMOD Q1: Models of vehicles with licences from QueryLicences. +-- BerlinMOD Q1: Models of vehicles with licences from Licences. -- -- Portable: works unchanged on MobilityDB/PostgreSQL, MobilityDuck/DuckDB, -- and MobilitySpark/Spark SQL. @@ -15,7 +15,7 @@ -- Temporal operations used: none (pure relational join — baseline portability test). SELECT l.licence, v.model -FROM QueryLicences l +FROM Licences l JOIN Vehicles v ON v.licence = l.licence ORDER BY l.licence; @@ -39,8 +39,8 @@ ORDER BY l.licence; SELECT DISTINCT v.licence FROM Vehicles v -JOIN Trips t ON t.vehId = v.vehId -JOIN QueryRegions r ON +JOIN Trips t ON t.VehicleId = v.VehicleId +JOIN Regions r ON eEq(geoToH3IndexSet(r.geom, 7), t.trip_h3) AND eIntersects(t.trip, r.geom) ORDER BY v.licence; @@ -58,16 +58,16 @@ ORDER BY v.licence; -- (temporal_as_hexwkb, variant 0 = little-endian NDR) so the output is -- byte-for-byte identical across platforms. -SELECT v.vehId AS vehid, +SELECT v.VehicleId AS VehicleId, v.licence, i.instantId AS instantid, asHexWKB(atTime(t.trip, i.instant)) AS pos -FROM QueryLicences l +FROM Licences l JOIN Vehicles v ON v.licence = l.licence -JOIN Trips t ON t.vehId = v.vehId -JOIN QueryInstants i ON true +JOIN Trips t ON t.VehicleId = v.VehicleId +JOIN Instants i ON true WHERE atTime(t.trip, i.instant) IS NOT NULL -ORDER BY v.vehId, i.instantId; +ORDER BY v.VehicleId, i.instantId; -- @query q04 @@ -96,8 +96,8 @@ ORDER BY v.vehId, i.instantId; SELECT DISTINCT v.licence FROM Vehicles v -JOIN Trips t ON t.vehId = v.vehId -JOIN QueryPoints p ON +JOIN Trips t ON t.VehicleId = v.VehicleId +JOIN Points p ON COALESCE(eEq(geoToH3Cell(p.geom, 7), t.trip_h3), TRUE) AND eIntersects(t.trip, p.geom) ORDER BY v.licence; @@ -124,9 +124,9 @@ WITH LicTrips AS ( SELECT l.licence, l.licenceId, array_agg(t.trip) AS trips - FROM QueryLicences l + FROM Licences l JOIN Vehicles v ON v.licence = l.licence - JOIN Trips t ON t.vehId = v.vehId + JOIN Trips t ON t.VehicleId = v.VehicleId GROUP BY l.licence, l.licenceId ) SELECT a.licence AS licence1, b.licence AS licence2, @@ -156,7 +156,7 @@ WITH TruckTrips AS ( SELECT array_agg(t.trip) AS trips, array_agg(v.licence) AS lic FROM Vehicles v - JOIN Trips t ON t.vehId = v.vehId + JOIN Trips t ON t.VehicleId = v.VehicleId WHERE v.type = 'truck' ) SELECT DISTINCT g.lic[p.i] AS licence1, g.lic[p.j] AS licence2 FROM TruckTrips g, @@ -176,16 +176,16 @@ ORDER BY licence1, licence2; -- produced by asHexWKB(). All three platforms call the same MEOS C function -- so the output is byte-for-byte identical across platforms. -SELECT v.vehId AS vehid, +SELECT v.VehicleId AS VehicleId, v.licence, p.periodId AS periodid, asHexWKB(atTime(t.trip, p.period)) AS pos -FROM QueryLicences l +FROM Licences l JOIN Vehicles v ON v.licence = l.licence -JOIN Trips t ON t.vehId = v.vehId -JOIN QueryPeriods p ON true +JOIN Trips t ON t.VehicleId = v.VehicleId +JOIN Periods p ON true WHERE atTime(t.trip, p.period) IS NOT NULL -ORDER BY v.vehId, p.periodId, t.tripId; +ORDER BY v.VehicleId, p.periodId, t.TripId; -- @query q08 @@ -200,10 +200,10 @@ ORDER BY v.vehId, p.periodId, t.tripId; -- and MobilitySpark's trajectory() UDF produces the same format via -- geo_as_hexewkb(), so the output is byte-for-byte identical across platforms. -SELECT tripId AS tripid, +SELECT TripId AS TripId, trajectory(trip) AS traj FROM Trips -ORDER BY tripId; +ORDER BY TripId; -- @query qrt @@ -218,15 +218,15 @@ ORDER BY tripId; -- little-endian NDR) — the same C function on all three platforms. -- The hex-WKB strings must be byte-for-byte identical across platforms. -SELECT tripId AS tripid, +SELECT TripId AS TripId, asHexWKB(trip) AS trip_hexwkb FROM Trips -ORDER BY tripId; +ORDER BY TripId; -- @query q09 -- BerlinMOD Q9: What is the longest distance travelled by a vehicle during --- each of the periods from QueryPeriods? +-- each of the periods from Periods? -- -- Portable: works unchanged on MobilityDB/PostgreSQL, MobilityDuck/DuckDB, -- and MobilitySpark/Spark SQL. @@ -236,11 +236,11 @@ ORDER BY tripId; -- length(tgeompoint) → float8 (Euclidean path length) WITH Distances AS ( - SELECT p.periodId, p.period, t.vehId, + SELECT p.periodId, p.period, t.VehicleId, SUM(length(atTime(t.trip, p.period))) AS dist - FROM Trips t, QueryPeriods p + FROM Trips t, Periods p WHERE t.trip && p.period - GROUP BY p.periodId, p.period, t.vehId + GROUP BY p.periodId, p.period, t.VehicleId ) SELECT periodId, period, ROUND(MAX(dist)::numeric, 3) AS maxDist FROM Distances @@ -249,7 +249,7 @@ ORDER BY periodId; -- @query q10 --- BerlinMOD Q10: When did the vehicles with licences from QueryLicences meet +-- BerlinMOD Q10: When did the vehicles with licences from Licences meet -- other vehicles (within 3 m) and what are the other vehicle IDs? -- -- Portable: works unchanged on MobilityDB/PostgreSQL, MobilityDuck/DuckDB, @@ -269,13 +269,13 @@ ORDER BY periodId; WITH LicTrips AS ( SELECT array_agg(t1.trip) AS trips, array_agg(l.licence) AS lic, - array_agg(t1.vehId) AS veh - FROM QueryLicences l + array_agg(t1.VehicleId) AS veh + FROM Licences l JOIN Vehicles v1 ON v1.licence = l.licence - JOIN Trips t1 ON t1.vehId = v1.vehId ), + JOIN Trips t1 ON t1.VehicleId = v1.VehicleId ), AllTrips AS ( SELECT array_agg(t2.trip) AS trips, - array_agg(t2.vehId) AS veh + array_agg(t2.VehicleId) AS veh FROM Trips t2 ) SELECT a.lic[p.i] AS licence1, b.veh[p.j] AS car2Id, p.periods AS periods FROM LicTrips a, AllTrips b, @@ -285,8 +285,8 @@ ORDER BY licence1, car2Id; -- @query q11 --- BerlinMOD Q11: Which vehicles passed a point from QueryPoints at one of --- the instants from QueryInstants? +-- BerlinMOD Q11: Which vehicles passed a point from Points at one of +-- the instants from Instants? -- -- Portable: works unchanged on MobilityDB/PostgreSQL, MobilityDuck/DuckDB, -- and MobilitySpark/Spark SQL. @@ -296,20 +296,20 @@ ORDER BY licence1, car2Id; -- stbox(geometry, timestamptz) → stbox (index pre-filter constructor) WITH Temp AS ( - SELECT p.pointId, p.geom, p.geomWKT, i.instantId, i.instant, t.vehId - FROM Trips t, QueryPoints p, QueryInstants i + SELECT p.pointId, p.geom, p.geomWKT, i.instantId, i.instant, t.VehicleId + FROM Trips t, Points p, Instants i WHERE t.trip && stbox(p.geom, i.instant) AND valueAtTimestamp(t.trip, i.instant) = p.geom ) SELECT t.pointId, t.geomWKT AS geom, t.instantId, t.instant, v.licence FROM Temp t -JOIN Vehicles v ON t.vehId = v.vehId +JOIN Vehicles v ON t.VehicleId = v.VehicleId ORDER BY t.pointId, t.instantId, v.licence; -- @query q12 -- BerlinMOD Q12: Which pairs of vehicles were at the same point from --- QueryPoints at the same instant from QueryInstants? +-- Points at the same instant from Instants? -- -- Portable: works unchanged on MobilityDB/PostgreSQL, MobilityDuck/DuckDB, -- and MobilitySpark/Spark SQL. @@ -319,8 +319,8 @@ ORDER BY t.pointId, t.instantId, v.licence; -- stbox(geometry, timestamptz) → stbox (index pre-filter constructor) WITH Temp AS ( - SELECT DISTINCT p.pointId, p.geom, p.geomWKT, i.instantId, i.instant, t.vehId - FROM Trips t, QueryPoints p, QueryInstants i + SELECT DISTINCT p.pointId, p.geom, p.geomWKT, i.instantId, i.instant, t.VehicleId + FROM Trips t, Points p, Instants i WHERE t.trip && stbox(p.geom, i.instant) AND valueAtTimestamp(t.trip, i.instant) = p.geom ) @@ -328,23 +328,23 @@ SELECT DISTINCT t1.pointId, t1.geomWKT AS geom, t1.instantId, t1.instant, v1.licence AS licence1, v2.licence AS licence2 FROM Temp t1 -JOIN Vehicles v1 ON t1.vehId = v1.vehId -JOIN Temp t2 ON t1.vehId < t2.vehId +JOIN Vehicles v1 ON t1.VehicleId = v1.VehicleId +JOIN Temp t2 ON t1.VehicleId < t2.VehicleId AND t1.pointId = t2.pointId AND t1.instantId = t2.instantId -JOIN Vehicles v2 ON t2.vehId = v2.vehId +JOIN Vehicles v2 ON t2.VehicleId = v2.VehicleId ORDER BY t1.pointId, t1.instantId, licence1, licence2; -- @query q13 --- BerlinMOD Q13: Which vehicles travelled within a region from QueryRegions --- during a period from QueryPeriods? +-- BerlinMOD Q13: Which vehicles travelled within a region from Regions +-- during a period from Periods? -- -- Portable: works unchanged on MobilityDB/PostgreSQL, MobilityDuck/DuckDB, -- and MobilitySpark/Spark SQL. -- -- Scale note: the original BerlinMOD uses 10-item subsets for each dimension; --- applying all 100 QueryRegions × 100 QueryPeriods is ~100× more expensive. +-- applying all 100 Regions × 100 Periods is ~100× more expensive. -- This query mirrors the original by using only the first 10 regions and 10 periods. -- -- Temporal operations used: @@ -353,21 +353,21 @@ ORDER BY t1.pointId, t1.instantId, licence1, licence2; -- stbox(geometry, tstzspan) → stbox (GiST index pre-filter constructor) WITH Temp AS ( - SELECT DISTINCT r.regionId, p.periodId, p.period, t.vehId - FROM Trips t, QueryRegions r, QueryPeriods p + SELECT DISTINCT r.regionId, p.periodId, p.period, t.VehicleId + FROM Trips t, Regions r, Periods p WHERE r.regionId <= 10 AND p.periodId <= 10 AND t.trip && stbox(r.geom, p.period) AND eIntersects(atTime(t.trip, p.period), r.geom) ) SELECT DISTINCT t.regionId, t.periodId, t.period, v.licence FROM Temp t, Vehicles v -WHERE t.vehId = v.vehId +WHERE t.VehicleId = v.VehicleId ORDER BY t.regionId, t.periodId, v.licence; -- @query q14 --- BerlinMOD Q14: Which vehicles were inside a region from QueryRegions at --- one of the instants from QueryInstants? +-- BerlinMOD Q14: Which vehicles were inside a region from Regions at +-- one of the instants from Instants? -- -- Portable: works unchanged on MobilityDB/PostgreSQL, MobilityDuck/DuckDB, -- and MobilitySpark/Spark SQL. @@ -377,26 +377,26 @@ ORDER BY t.regionId, t.periodId, v.licence; -- stbox(geometry, timestamptz) → stbox (index pre-filter constructor) WITH Temp AS ( - SELECT DISTINCT r.regionId, i.instantId, i.instant, t.vehId - FROM Trips t, QueryRegions r, QueryInstants i + SELECT DISTINCT r.regionId, i.instantId, i.instant, t.VehicleId + FROM Trips t, Regions r, Instants i WHERE t.trip && stbox(r.geom, i.instant) AND ST_Contains(r.geom, valueAtTimestamp(t.trip, i.instant)) ) SELECT DISTINCT t.regionId, t.instantId, t.instant, v.licence FROM Temp t -JOIN Vehicles v ON t.vehId = v.vehId +JOIN Vehicles v ON t.VehicleId = v.VehicleId ORDER BY t.regionId, t.instantId, v.licence; -- @query q15 --- BerlinMOD Q15: Which vehicles passed a point from QueryPoints during a --- period from QueryPeriods? +-- BerlinMOD Q15: Which vehicles passed a point from Points during a +-- period from Periods? -- -- Portable: works unchanged on MobilityDB/PostgreSQL, MobilityDuck/DuckDB, -- and MobilitySpark/Spark SQL. -- -- Scale note: the original BerlinMOD uses 10-item subsets for each dimension; --- applying all 100 QueryPoints × 100 QueryPeriods is ~100× more expensive. +-- applying all 100 Points × 100 Periods is ~100× more expensive. -- This query mirrors the original by using only the first 10 points and 10 periods. -- -- Temporal operations used: @@ -405,28 +405,28 @@ ORDER BY t.regionId, t.instantId, v.licence; -- stbox(geometry, tstzspan) → stbox (GiST index pre-filter constructor) WITH Temp AS ( - SELECT DISTINCT pt.pointId, pt.geom, pt.geomWKT, pr.periodId, pr.period, t.vehId - FROM Trips t, QueryPoints pt, QueryPeriods pr + SELECT DISTINCT pt.pointId, pt.geom, pt.geomWKT, pr.periodId, pr.period, t.VehicleId + FROM Trips t, Points pt, Periods pr WHERE pt.pointId <= 10 AND pr.periodId <= 10 AND t.trip && stbox(pt.geom, pr.period) AND eIntersects(atTime(t.trip, pr.period), pt.geom) ) SELECT DISTINCT t.pointId, t.geomWKT AS geom, t.periodId, t.period, v.licence FROM Temp t, Vehicles v -WHERE t.vehId = v.vehId +WHERE t.VehicleId = v.VehicleId ORDER BY t.pointId, t.periodId, v.licence; -- @query q16 -- BerlinMOD Q16: Which pairs of query-licence vehicles were both within a --- region from QueryRegions during a period from QueryPeriods, but never at +-- region from Regions during a period from Periods, but never at -- the same location at the same time (always disjoint)? -- -- Portable: works unchanged on MobilityDB/PostgreSQL, MobilityDuck/DuckDB, -- and MobilitySpark/Spark SQL. -- -- Scale note: the original BerlinMOD uses 10-item subsets for each dimension; --- applying all 100 QueryLicences × 100 QueryPeriods × 100 QueryRegions is +-- applying all 100 Licences × 100 Periods × 100 Regions is -- ~10,000× more expensive. This query mirrors the original by using only the -- first 10 licences, 10 periods, and 10 regions. -- @@ -449,11 +449,11 @@ WITH PR AS ( array_agg(atTime(t.trip, p.period)) AS trips, array_agg(l.licence) AS lic, array_agg(l.licenceId) AS lid - FROM QueryLicences l + FROM Licences l JOIN Vehicles v ON v.licence = l.licence - JOIN Trips t ON t.vehId = v.vehId - JOIN QueryPeriods p ON true - JOIN QueryRegions r ON true + JOIN Trips t ON t.VehicleId = v.VehicleId + JOIN Periods p ON true + JOIN Regions r ON true WHERE l.licenceId <= 10 AND p.periodId <= 10 AND r.regionId <= 10 AND t.trip && stbox(r.geom, p.period) AND eIntersects(atTime(t.trip, p.period), r.geom) @@ -467,7 +467,7 @@ ORDER BY g.periodId, g.regionId, licence1, licence2; -- @query q17 --- BerlinMOD Q17: Which point(s) from QueryPoints have been visited by the +-- BerlinMOD Q17: Which point(s) from Points have been visited by the -- maximum number of distinct vehicles? -- -- Portable: works unchanged on MobilityDB/PostgreSQL, MobilityDuck/DuckDB, @@ -477,8 +477,8 @@ ORDER BY g.periodId, g.regionId, licence1, licence2; -- eIntersects(tgeompoint, geometry) → bool (avoids trajectory() override) WITH PointCount AS ( - SELECT p.pointId, COUNT(DISTINCT t.vehId) AS hits - FROM Trips t, QueryPoints p + SELECT p.pointId, COUNT(DISTINCT t.VehicleId) AS hits + FROM Trips t, Points p WHERE eIntersects(t.trip, p.geom) GROUP BY p.pointId ) From c39a713f4cadb3b997de4aab1e0e824f7894c4bc Mon Sep 17 00:00:00 2001 From: Esteban Zimanyi Date: Wed, 22 Jul 2026 22:24:45 +0200 Subject: [PATCH 2/2] Wire the batch runners to the canonical BerlinMOD loaders bench_mbdb.sh and bench_mduck.sh read a load_*.sql loader and per-query qNN.sql files that are absent from the repository, so the batch benchmark cannot run. Wire each runner to reuse its canonical merged loader instead of a bespoke one: bench_mbdb.sh runs berlinmod_load.sql (SELECT berlinmod_load(dir, true)) and berlinmod_th3index_setup.sql; bench_mduck.sh runs mobilityduck_load.sql. Both read the shared queries by splitting the single queries.sql on its `-- @query` markers. The MobilityDuck query loop drops the search_path override, since the canonical loader creates its tables in the default (main) schema. --- .../benchmarks/batch/bench/bench_mbdb.sh | 34 ++++++++++++++----- .../benchmarks/batch/bench/bench_mduck.sh | 30 ++++++++++++---- 2 files changed, 48 insertions(+), 16 deletions(-) diff --git a/BerlinMOD/benchmarks/batch/bench/bench_mbdb.sh b/BerlinMOD/benchmarks/batch/bench/bench_mbdb.sh index 3ec7411..2306440 100755 --- a/BerlinMOD/benchmarks/batch/bench/bench_mbdb.sh +++ b/BerlinMOD/benchmarks/batch/bench/bench_mbdb.sh @@ -29,6 +29,8 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" BERLINMOD_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +BMROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" # repo BerlinMOD/ dir (canonical loaders) +QUERIES_SQL="${SCRIPT_DIR}/queries.sql" # single canonical query source # ── defaults ────────────────────────────────────────────────────────────────── DBNAME="berlinmod_bench" @@ -99,11 +101,11 @@ _psql() { psql -d "$DBNAME" -q "$@"; } if $LOAD; then echo "=== Creating database: $DBNAME ===" createdb "$DBNAME" 2>/dev/null || true - LOADER=$(mktemp --suffix=.sql) - trap 'rm -f "$LOADER"' EXIT - sed "s|DATADIR|${DATADIR}|g" "${BERLINMOD_DIR}/load_mbdb.sql" > "$LOADER" - echo "=== Loading data ===" - _psql -f "$LOADER" + echo "=== Loading data (canonical berlinmod_load + th3index setup) ===" + _psql -v ON_ERROR_STOP=1 -c "CREATE EXTENSION IF NOT EXISTS MobilityDB CASCADE;" + _psql -v ON_ERROR_STOP=1 -f "${BMROOT}/berlinmod_load.sql" + _psql -v ON_ERROR_STOP=1 -c "SELECT berlinmod_load('${DATADIR}/', true);" + _psql -v ON_ERROR_STOP=1 -f "${BMROOT}/berlinmod_th3index_setup.sql" echo " done." fi @@ -152,15 +154,29 @@ echo "=== Runs : ${RUNS} per query (queries: ${QUERIES_MSG}) ===" echo "" TIMEFILE=$(mktemp) -trap 'rm -f "$TIMEFILE"' EXIT +QDIR=$(mktemp -d) +trap 'rm -f "$TIMEFILE"; rm -rf "$QDIR"' EXIT +# Split the single canonical queries.sql on its `-- @query ` markers. for Q in "${QUERIES[@]}"; do - QFILE="${BERLINMOD_DIR}/${Q}.sql" - [[ -f "$QFILE" ]] || { echo " [skip] ${Q} — SQL file not found"; continue; } + awk -v s="-- @query ${Q}" ' + index($0, s)==1 { grab=1; next } + grab && /^-- @query / { exit } + grab { print } + ' "$QUERIES_SQL" > "${QDIR}/${Q}.sql" +done + +for Q in "${QUERIES[@]}"; do + QFILE="${QDIR}/${Q}.sql" + [[ -s "$QFILE" ]] || { echo " [skip] ${Q} — not in queries.sql"; continue; } + if ! ERR=$(_psql -v ON_ERROR_STOP=1 -o /dev/null -f "$QFILE" 2>&1); then + echo " [FAIL] ${Q} — ${ERR##*ERROR: }" + continue + fi printf " timing %-6s: " "$Q" for RUN in $(seq 1 "$RUNS"); do T0=$(date +%s%3N) - _psql -o /dev/null -f "$QFILE" 2>/dev/null || true + _psql -o /dev/null -f "$QFILE" >/dev/null 2>&1 T1=$(date +%s%3N) ELAPSED=$((T1 - T0)) printf "%d " "$ELAPSED" diff --git a/BerlinMOD/benchmarks/batch/bench/bench_mduck.sh b/BerlinMOD/benchmarks/batch/bench/bench_mduck.sh index d07ee28..ed537c3 100755 --- a/BerlinMOD/benchmarks/batch/bench/bench_mduck.sh +++ b/BerlinMOD/benchmarks/batch/bench/bench_mduck.sh @@ -33,6 +33,8 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" BERLINMOD_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +BMROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" # repo BerlinMOD/ dir (canonical loaders) +QUERIES_SQL="${SCRIPT_DIR}/queries.sql" # single canonical query source # ── defaults ────────────────────────────────────────────────────────────────── DUCKDB="${DUCKDB:-duckdb}" @@ -114,8 +116,8 @@ _duck_q() { "$DUCKDB" "$DBFILE" -noheader -list -c "$1" 2>/dev/null; } if $LOAD; then echo "=== Loading data into: $DBFILE ===" rm -f "$DBFILE" - LOAD_BODY="$(sed '/^SET VARIABLE DATADIR/d' "${BERLINMOD_DIR}/load_mduck.sql")" - LOAD_SQL="${MOBILITY_LOAD} SET VARIABLE DATADIR='${DATADIR}/'; ${LOAD_BODY}" + LOAD_BODY="$(sed '/^SET VARIABLE data_dir/d' "${BMROOT}/mobilityduck_load.sql")" + LOAD_SQL="${MOBILITY_LOAD} SET VARIABLE data_dir='${DATADIR}/'; ${LOAD_BODY}" "$DUCKDB" "$DBFILE" -c "$LOAD_SQL" echo " done." fi @@ -165,15 +167,29 @@ echo "" TIMEFILE=$(mktemp) trap 'rm -f "$TIMEFILE"' EXIT +# Split the single canonical queries.sql on its `-- @query ` markers. +QDIR=$(mktemp -d) +trap 'rm -f "$TIMEFILE"; rm -rf "$QDIR"' EXIT for Q in "${QUERIES[@]}"; do - QFILE="${BERLINMOD_DIR}/${Q}.sql" - [[ -f "$QFILE" ]] || { echo " [skip] ${Q} — SQL file not found"; continue; } - QSQL=$(grep -v '^\s*--' "$QFILE" | tr '\n' ' ') + awk -v s="-- @query ${Q}" ' + index($0, s)==1 { grab=1; next } + grab && /^-- @query / { exit } + grab { print } + ' "$QUERIES_SQL" | grep -v '^\s*--' | tr '\n' ' ' > "${QDIR}/${Q}.sql" +done + +for Q in "${QUERIES[@]}"; do + QSQL=$(cat "${QDIR}/${Q}.sql") + [[ -n "${QSQL// /}" ]] || { echo " [skip] ${Q} — not in queries.sql"; continue; } + # mobilityduck_load creates its tables in the default (main) schema. + if ! ERR=$("$DUCKDB" "$DBFILE" -c "${MOBILITY_LOAD} ${QSQL}" 2>&1 >/dev/null); then + echo " [FAIL] ${Q} — $(echo "$ERR" | grep -iE 'error' | head -1)" + continue + fi printf " timing %-6s: " "$Q" for RUN in $(seq 1 "$RUNS"); do T0=$(date +%s%3N) - "$DUCKDB" "$DBFILE" -c "${MOBILITY_LOAD} SET search_path='portable,main'; ${QSQL}" \ - > /dev/null 2>&1 || true + "$DUCKDB" "$DBFILE" -c "${MOBILITY_LOAD} ${QSQL}" > /dev/null 2>&1 || true T1=$(date +%s%3N) ELAPSED=$((T1 - T0)) printf "%d " "$ELAPSED"