TimeVary 2026 summer

containers/database/Dockerfile

@@ -6,7 +6,7 @@
 
 # Dockerfile for OED Postgres server.
 # Use a pinned version
-FROM postgres:15.3
+FROM timescale/timescaledb:2.27.2-pg17
 
 # All SQL files in the build context
 # get copied into the container and

containers/database/TimescaleDB/timevary_hourly_continuous_aggregate.sql

@@ -0,0 +1,241 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+/*
+ * This file sets up the necessary database objects to benchmark TimescaleDB
+ * continuous aggregates against the existing PostgreSQL materialized views
+ * for hourly meter readings on the timeVary branch.
+ *
+ * It creates a readings hypertable (readings_hypertable) as a direct copy of
+ * the readings table and a second hypertable (hypertable_hourly_split) that
+ * splits raw meter readings into hourly intervals and applies the time varying
+ * cik_vary conversions at split time. A continuous aggregate
+ * (meter_hourly_readings_unit_cagg) is then built on top of hypertable_hourly_split
+ * to replicate the meter_hourly_readings_unit materialized view as accurately
+ * as possible.
+ *
+ * A separate hypertable was needed because TimescaleDB continuous aggregates
+ * can only aggregate data, not expand it. Since meter_hourly_readings_unit splits
+ * readings that span multiple hours into one row per hour, this splitting had to
+ * be done in advance and stored in a separate hypertable before the continuous
+ * aggregate could work with it. The cik_vary conversion (slope/intercept) is also
+ * applied at split time and stored in the hypertable so the continuous aggregate
+ * does not need to join to any other tables at query time.
+ *
+ * The continuous aggregate sits on top of hypertable_hourly_split and stores
+ * pre-computed aggregated results from it, similar to how a materialized view
+ * works in regular PostgreSQL. The underlying data always lives in the hypertable.
+ *
+ * The goal is to compare query speeds between the original materialized views
+ * and TimescaleDB continuous aggregates to determine whether migrating to
+ * TimescaleDB is worth the effort for the timeVary branch.
+ *
+ * Setup steps:
+ * 1. Ensure the TimescaleDB extension is enabled in the oed database
+ * 2. Ensure test data is populated (npm run testData)
+ * 3. Run this file in pgAdmin against the oed database
+ *
+ * To tear down all objects created by this file:
+ *     DROP MATERIALIZED VIEW IF EXISTS meter_hourly_readings_unit_cagg;
+ *     DROP TABLE IF EXISTS hypertable_hourly_split;
+ *     DROP TABLE IF EXISTS readings_hypertable;
+ */
+
+-- 1. Verify test data loaded correctly before proceeding.
+SELECT COUNT(*) FROM readings;
+SELECT COUNT(*) FROM meters;
+
+-- 2. Create readings_hypertable as a separate copy of the readings table.
+-- This keeps the original readings table untouched so other parts of the
+-- codebase that query it directly are not affected.
+CREATE TABLE readings_hypertable (
+    meter_id INTEGER NOT NULL,
+    reading FLOAT NOT NULL,
+    start_timestamp TIMESTAMP NOT NULL,
+    end_timestamp TIMESTAMP NOT NULL,
+    CHECK (start_timestamp < end_timestamp),
+    PRIMARY KEY (meter_id, start_timestamp)
+);
+
+-- 3. Convert readings_hypertable into a TimescaleDB hypertable, partitioned
+-- by start_timestamp. This enables time-series optimizations and allows
+-- continuous aggregates to be built on top.
+SELECT create_hypertable('readings_hypertable', 'start_timestamp');
+
+-- 4. Populate readings_hypertable with all data from the readings table.
+INSERT INTO readings_hypertable SELECT * FROM readings;
+
+-- 5. Verify readings_hypertable populated correctly.
+-- Both counts should be identical.
+SELECT COUNT(*) FROM readings_hypertable;
+SELECT COUNT(*) FROM readings;
+
+-- 6. Create hypertable_hourly_split to store readings split into hourly intervals
+-- with cik_vary conversions applied at split time.
+-- slope, intercept, and graphic_unit_id are included so the continuous aggregate
+-- can apply the time varying conversion without needing to join to cik_vary at
+-- query time. unit_represent and sec_in_rate are included to correctly handle
+-- quantity, flow, and raw readings differently during aggregation.
+CREATE TABLE hypertable_hourly_split (
+    meter_id INTEGER NOT NULL,
+    reading FLOAT NOT NULL,          -- Reading rate scaled by overlap duration
+    start_timestamp TIMESTAMP NOT NULL,
+    end_timestamp TIMESTAMP NOT NULL,
+    unit_represent unit_represent_type NOT NULL,
+    sec_in_rate FLOAT NOT NULL,
+    slope FLOAT NOT NULL,            -- From cik_vary, used to convert reading to graphic unit
+    intercept FLOAT NOT NULL,        -- From cik_vary, used to convert reading to graphic unit
+    graphic_unit_id INTEGER NOT NULL -- The destination unit this conversion applies to
+);
+
+-- 7. Convert hypertable_hourly_split into a TimescaleDB hypertable.
+SELECT create_hypertable('hypertable_hourly_split', 'start_timestamp');
+
+-- 8. Populate hypertable_hourly_split by splitting each raw reading from
+-- readings_hypertable into one row per hour it spans, and joining to cik_vary
+-- to store the applicable conversion for each slice.
+-- For example, a reading spanning 3 hours with 2 applicable conversions becomes
+-- 6 rows — one per hour per conversion.
+-- Quantity readings are converted to a rate per hour and scaled by overlap duration.
+-- Flow/raw readings are already a rate, normalized to per hour using sec_in_rate,
+-- and also scaled by overlap duration.
+-- The slope and intercept from cik_vary are stored directly so the continuous
+-- aggregate can apply them without any additional joins.
+INSERT INTO hypertable_hourly_split
+SELECT
+    r.meter_id,
+    CASE WHEN u.unit_represent = 'quantity'::unit_represent_type THEN
+        -- Convert to rate per hour and scale by overlap duration within this hour slice
+        (r.reading * 3600 / extract(EPOCH FROM (r.end_timestamp - r.start_timestamp))) *
+        extract(EPOCH FROM (
+            least(r.end_timestamp, gen.interval_start + INTERVAL '1 hour')
+            - greatest(r.start_timestamp, gen.interval_start)
+        ))
+    WHEN (u.unit_represent = 'flow'::unit_represent_type OR u.unit_represent = 'raw'::unit_represent_type) THEN
+        -- Normalize flow/raw to per hour and scale by overlap duration
+        (r.reading * 3600 / u.sec_in_rate) *
+        extract(EPOCH FROM (
+            least(r.end_timestamp, gen.interval_start + INTERVAL '1 hour')
+            - greatest(r.start_timestamp, gen.interval_start)
+        ))
+    END AS reading,
+    -- Clamp start/end timestamps to the hour boundary
+    greatest(r.start_timestamp, gen.interval_start) AS start_timestamp,
+    least(r.end_timestamp, gen.interval_start + INTERVAL '1 hour') AS end_timestamp,
+    u.unit_represent,
+    u.sec_in_rate,
+    c.slope,
+    c.intercept,
+    c.destination_id AS graphic_unit_id
+FROM readings_hypertable r
+INNER JOIN meters m ON r.meter_id = m.id
+INNER JOIN units u ON m.unit_id = u.id
+-- Join to cik_vary using a time range overlap to get all applicable conversions
+-- for each reading. The exclusive bounds '()' ensure no two conversions overlap.
+INNER JOIN cik_vary c ON c.source_id = m.unit_id
+    AND tsrange(c.start_time, c.end_time, '()') && tsrange(r.start_timestamp, r.end_timestamp, '[]')
+-- Generate one row per hour that the reading spans
+CROSS JOIN LATERAL generate_series(
+    date_trunc('hour', r.start_timestamp),
+    -- Subtract 1 hour because generate_series is end-inclusive
+    date_trunc_up('hour', r.end_timestamp) - INTERVAL '1 hour',
+    INTERVAL '1 hour'
+) gen(interval_start);
+
+-- 9. Verify row count of the hourly split hypertable.
+-- Will be higher than meter_hourly_readings_unit since each raw reading
+-- can produce multiple rows (one per hour per cik_vary conversion) before aggregation.
+SELECT COUNT(*) FROM hypertable_hourly_split;
+SELECT * FROM hypertable_hourly_split LIMIT 10;
+
+-- 10. Check how many distinct (meter, hour, graphic_unit) combinations exist
+-- in the hypertable. Should match the count from meter_hourly_readings_unit below.
+SELECT COUNT(*) FROM (
+    SELECT DISTINCT meter_id, date_trunc('hour', start_timestamp), graphic_unit_id
+    FROM hypertable_hourly_split
+) s;
+
+-- 11. Check how many distinct (meter, hour, graphic_unit) combinations exist
+-- in the materialized view. Should match the count from hypertable_hourly_split above.
+SELECT COUNT(*) FROM (
+    SELECT DISTINCT meter_id, lower(time_interval), graphic_unit_id
+    FROM meter_hourly_readings_unit
+) s;
+
+-- 12. Create a continuous aggregate on top of hypertable_hourly_split.
+-- This replicates the logic of meter_hourly_readings_unit but uses TimescaleDB's
+-- continuous aggregate mechanism instead of a regular materialized view.
+-- The reading in hypertable_hourly_split is already scaled by overlap duration,
+-- so the weighted average is computed by dividing by duration to get the rate,
+-- applying slope/intercept, then weighting by duration and normalizing.
+-- This matches the two-step logic in meter_hourly_readings_unit:
+--   Step 1 (base_hourly CTE): splits and computes raw reading rate
+--   Step 2 (outer SELECT): applies cik_vary conversion
+CREATE MATERIALIZED VIEW meter_hourly_readings_unit_cagg
+WITH (timescaledb.continuous) AS
+SELECT
+    meter_id,
+    graphic_unit_id,
+    time_bucket('1 hour', start_timestamp) AS bucket,
+    -- Weighted average reading rate by slice duration, with cik_vary conversion applied.
+    -- Dividing reading by duration recovers the rate, applying slope/intercept converts it,
+    -- then multiplying by duration and dividing by total duration gives the weighted average.
+    sum(
+        (reading / extract(EPOCH FROM (end_timestamp - start_timestamp)) * slope + intercept)
+        * extract(EPOCH FROM (end_timestamp - start_timestamp))
+    ) / sum(extract(EPOCH FROM (end_timestamp - start_timestamp))) AS reading_rate,
+    -- Max and min rates with cik_vary conversion applied
+    max(reading / extract(EPOCH FROM (end_timestamp - start_timestamp)) * slope + intercept) AS max_rate,
+    min(reading / extract(EPOCH FROM (end_timestamp - start_timestamp)) * slope + intercept) AS min_rate,
+    unit_represent,
+    sec_in_rate
+FROM hypertable_hourly_split
+GROUP BY meter_id, graphic_unit_id, bucket, unit_represent, sec_in_rate;
+
+-- 13. Verify row count of the continuous aggregate.
+-- Should match the count from meter_hourly_readings_unit.
+SELECT COUNT(*) FROM meter_hourly_readings_unit_cagg;
+SELECT * FROM meter_hourly_readings_unit_cagg LIMIT 10;
+
+-- 14. Compare reading_rate values between the original materialized view and
+-- the continuous aggregate. Results are ordered by largest difference first
+-- to surface any inaccuracies. Differences should be zero or extremely close
+-- to zero (floating point rounding only).
+SELECT
+    mv.meter_id,
+    lower(mv.time_interval) AS mv_time,
+    mv.reading_rate AS mv_reading_rate,
+    cagg.bucket AS cagg_time,
+    cagg.reading_rate AS cagg_reading_rate,
+    mv.reading_rate - cagg.reading_rate AS difference
+FROM meter_hourly_readings_unit mv
+INNER JOIN meter_hourly_readings_unit_cagg cagg
+    ON mv.meter_id = cagg.meter_id
+    AND lower(mv.time_interval) = cagg.bucket
+    AND mv.graphic_unit_id = cagg.graphic_unit_id
+ORDER BY abs(mv.reading_rate - cagg.reading_rate) DESC
+LIMIT 20;
+
+-- 15. Verify accuracy of max_rate and min_rate between the original materialized
+-- view (meter_hourly_readings_unit) and the continuous aggregate
+-- (meter_hourly_readings_unit_cagg). Results are ordered by largest max_rate
+-- difference first to surface any inaccuracies. Differences should be zero or
+-- extremely close to zero (floating point rounding only).
+SELECT
+    mv.meter_id,
+    lower(mv.time_interval) AS mv_time,
+    mv.max_rate AS mv_max_rate,
+    cagg.max_rate AS cagg_max_rate,
+    mv.max_rate - cagg.max_rate AS max_difference,
+    mv.min_rate AS mv_min_rate,
+    cagg.min_rate AS cagg_min_rate,
+    mv.min_rate - cagg.min_rate AS min_difference
+FROM meter_hourly_readings_unit mv
+INNER JOIN meter_hourly_readings_unit_cagg cagg
+    ON mv.meter_id = cagg.meter_id
+    AND lower(mv.time_interval) = cagg.bucket
+    AND mv.graphic_unit_id = cagg.graphic_unit_id
+ORDER BY abs(mv.max_rate - cagg.max_rate) DESC
+LIMIT 20;

docker-compose.yml

@@ -16,13 +16,14 @@ services:
             - POSTGRES_PASSWORD=pleaseChange # default postgres password that should be changed for security.
         volumes:
             - ./postgres-data:/var/lib/postgresql/data/pgdata
+            - ./src/server/tmp/timescaleddb:/benchmarkscripts:ro
         healthcheck:
             test: ["CMD-SHELL", "pg_isready -U postgres"]
             interval: 10s
             timeout: 10s
             retries: 3
-        # ports:
-        #    - "5432:5432"
+        ports:
+           - "5432:5432"
         # Uncomment the above lines to enable access to the PostgreSQL server
         # from the host machine.
         # Web service runs Node
@@ -111,4 +112,3 @@ services:
         /bin/sh -c "
         rm -f /tmp/.X99-lock &&
         Xvfb :99 -screen 0 1024x768x16"
-

package.json

@@ -16,8 +16,8 @@
 		"check:typescript": "./src/scripts/checkTypescript.sh",
 		"check:types": "tsc -p . --noEmit",
 		"check:lint": "eslint --ignore-path .eslintignore --ext .ts,.tsx,.js,.jsx ./src/client/app",
-		"test": "mocha --timeout 15000 \"src/server/test/**/*.js\"",
-		"testsome": "mocha --timeout 15000",
+		"test": "mocha --timeout 5000 \"src/server/test/**/*.js\"",
+		"testsome": "mocha --timeout 5000",
 		"createdb": "node ./src/server/services/createDB.js",
 		"developerdb": "node -e 'require(\"./src/server/util/developer.js\").createShiftReadingsFunction()'",
 		"migratedb": "node ./src/server/services/migrateDB.js",
@@ -28,11 +28,12 @@
 		"editUser": "node ./src/server/services/user/editUser.js",
 		"sendLogEmail": "node ./src/server/services/sendLogEmail.js",
 		"refreshReadingViews": "node -e 'require(\"./src/server/services/refreshReadingViews\").refreshReadingViews()'",
-		"refreshAllReadingViews": "node -e 'require(\"./src/server/services/refreshAllReadingViews\").refreshAllReadingViews()'",
+		"refreshAllReadingViews": "node -e 'require(\"./src/server/services/refreshAllReadingViews\")()'",
+		"rebuildAllReadingViews": "node -e 'require(\"./src/server/services/refreshAllReadingViews\")({ rebuild: true })'",
 		"refreshDailyReadingViews": "node -e 'require(\"./src/server/services/refreshReadingViews\").refreshReadingViews()'",
 		"refreshHourlyReadingViews": "node -e 'require(\"./src/server/services/refreshHourlyReadingViews\").refreshHourlyReadingViews()'",
 		"refreshGroupsDeepMeters": "node -e 'require(\"./src/server/services/refreshGroupsDeepMetersView\").refreshGroupsDeepMetersView()'",
-		"updateCikAndViews": "node -e 'require(\"./src/server/services/graph/redoCik.js\").updateCikAndViews()'",
+		"updateCikAndViews": "node -e 'require(\"./src/server/services/graph/redoCik.js\").updateCikVaryAndViews()'",
 		"obvius:showConfigfiles": "node ./src/server/services/obvius/showConfigfiles.js",
 		"obvius:purgeConfigfiles": "node ./src/server/services/obvius/purgeConfigfiles.js",
 		"generateFourDayTestingData": "node -e 'require(\"./src/server/data/automatedTestingData\").generateFourDayTestingData()'",

src/client/app/types/redux/ciks.ts

@@ -5,6 +5,4 @@
 export interface CikData {
 	meterUnitId: number;
 	nonMeterUnitId: number;
-	slope: number;
-	intercept: number;
 }

src/server/data/automatedTestingData.js

@@ -8,11 +8,11 @@
 
 const { generateSine, generateCosine } = require('./generateTestingData');
 const Unit = require('../models/Unit');
+const Group = require('../models/Group');
 const { insertUnits, insertStandardUnits, insertConversions, insertStandardConversions, insertMeters, insertGroups } = require('../util/insertData');
 const { getConnection } = require('../db');
-const { redoCik } = require('../services/graph/redoCik');
-const { refreshAllReadingViews } = require('../services/refreshAllReadingViews');
-const fs = require('fs').promises;
+const { redoCikVary } = require('../services/graph/redoCik');
+const refreshAllReadingViews = require('../services/refreshAllReadingViews');
 
 // Define the start and end date for data generation.
 const DEFAULT_OPTIONS = {
@@ -1335,18 +1335,20 @@ async function insertSpecialUnitsConversionsMetersGroups() {
 	await insertSpecialConversions(conn);
 	// Recreate the Cik entries since changed units/conversions.
 	// Do now since needed to insert meters with suffix units.
-	await redoCik(conn);
+	await redoCikVary(conn);
 	// Generate the mathematical test data needed.
 	console.log(`Start loading each set of test data into OED meters, may take minutes):\n`);
 	// This is very fast so wait since simpler and easier to see if this part fails.
 	await insertMeters(specialMeters, conn);
 	// Now do the large dataset generation.
 	await testData();
 	// Recreate the Cik entries since changed meters.
-	await redoCik(conn);
-	// Refresh the readings since added new ones.
-	await refreshAllReadingViews();
+	await redoCikVary(conn);
 	await insertGroups(specialGroups, conn);
+	// Refresh groups deep meters view after adding new groups.
+	await Group.refreshGroupsDeepMetersView(conn);
+	// Refresh the readings since added new ones & groups.
+	await refreshAllReadingViews();
 }
 
 /*

src/server/data/websiteData.js

@@ -11,8 +11,8 @@
 
 const Unit = require('../models/Unit');
 const Group = require('../models/Group');
-const { redoCik } = require('../services/graph/redoCik');
-const { refreshAllReadingViews } = require('../services/refreshAllReadingViews');
+const { redoCikVary } = require('../services/graph/redoCik');
+const refreshAllReadingViews = require('../services/refreshAllReadingViews');
 const { getConnection } = require('../db');
 const { insertUnits, insertStandardUnits, insertConversions, insertStandardConversions, insertMeters, insertGroups } = require('../util/insertData');
 const { shiftReadings } = require('../util/developer');
@@ -753,12 +753,12 @@ async function insertWebsiteData() {
 	await insertConversions(conversions, conn);
 	// Recreate the Cik entries since changed units/conversions.
 	// Do now since needed to insert meters with suffix units.
-	await redoCik(conn);
+	await redoCikVary(conn);
 	console.log(`Start loading each set of test data into OED meters (${meters.length} files of varying length, may take minutes):`);
 	// await Meter.insertMany(meters, conn);
     await insertMeters(meters, conn);
 	// Recreate the Cik entries since changed meters.
-	await redoCik(conn);
+	await redoCikVary(conn);
 	await insertGroups(groups, conn);
 	// Refresh groups deep meters view after adding new groups.
 	await Group.refreshGroupsDeepMetersView(conn);

src/server/migrations/2.0.0-3.0.0/index.js

@@ -0,0 +1,19 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+const database = require('../../models/database');
+const sqlFile = database.sqlFile;
+
+module.exports = {
+    fromVersion: '2.0.0',
+    toVersion: '3.0.0',
+    up: async db => {
+        await db.none(sqlFile('../migrations/2.0.0-3.0.0/sql/cik/alter_cik_table.sql'));
+        await db.none(sqlFile('../migrations/2.0.0-3.0.0/sql/readings/drop_old_views.sql'));
+        await db.none(sqlFile('../migrations/2.0.0-3.0.0/sql/readings/create_reading_views.sql'));
+        await db.none(sqlFile('../migrations/2.0.0-3.0.0/sql/readings/create_function_get_3d_readings.sql'));
+        await db.none(sqlFile('../migrations/2.0.0-3.0.0/sql/readings/create_function_get_compare_readings.sql'));
+    }
+};