diff --git a/containers/database/Dockerfile b/containers/database/Dockerfile index ec446505dd..fd9053f971 100644 --- a/containers/database/Dockerfile +++ b/containers/database/Dockerfile @@ -6,7 +6,7 @@ # Dockerfile for OED Postgres server. # Use a pinned version -FROM postgres:15.3 +FROM timescale/timescaledb:2.27.2-pg17 # All SQL files in the build context # get copied into the container and diff --git a/containers/database/TimescaleDB/timevary_hourly_continuous_aggregate.sql b/containers/database/TimescaleDB/timevary_hourly_continuous_aggregate.sql new file mode 100644 index 0000000000..e04c447232 --- /dev/null +++ b/containers/database/TimescaleDB/timevary_hourly_continuous_aggregate.sql @@ -0,0 +1,241 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +/* + * This file sets up the necessary database objects to benchmark TimescaleDB + * continuous aggregates against the existing PostgreSQL materialized views + * for hourly meter readings on the timeVary branch. + * + * It creates a readings hypertable (readings_hypertable) as a direct copy of + * the readings table and a second hypertable (hypertable_hourly_split) that + * splits raw meter readings into hourly intervals and applies the time varying + * cik_vary conversions at split time. A continuous aggregate + * (meter_hourly_readings_unit_cagg) is then built on top of hypertable_hourly_split + * to replicate the meter_hourly_readings_unit materialized view as accurately + * as possible. + * + * A separate hypertable was needed because TimescaleDB continuous aggregates + * can only aggregate data, not expand it. Since meter_hourly_readings_unit splits + * readings that span multiple hours into one row per hour, this splitting had to + * be done in advance and stored in a separate hypertable before the continuous + * aggregate could work with it. The cik_vary conversion (slope/intercept) is also + * applied at split time and stored in the hypertable so the continuous aggregate + * does not need to join to any other tables at query time. + * + * The continuous aggregate sits on top of hypertable_hourly_split and stores + * pre-computed aggregated results from it, similar to how a materialized view + * works in regular PostgreSQL. The underlying data always lives in the hypertable. + * + * The goal is to compare query speeds between the original materialized views + * and TimescaleDB continuous aggregates to determine whether migrating to + * TimescaleDB is worth the effort for the timeVary branch. + * + * Setup steps: + * 1. Ensure the TimescaleDB extension is enabled in the oed database + * 2. Ensure test data is populated (npm run testData) + * 3. Run this file in pgAdmin against the oed database + * + * To tear down all objects created by this file: + * DROP MATERIALIZED VIEW IF EXISTS meter_hourly_readings_unit_cagg; + * DROP TABLE IF EXISTS hypertable_hourly_split; + * DROP TABLE IF EXISTS readings_hypertable; + */ + +-- 1. Verify test data loaded correctly before proceeding. +SELECT COUNT(*) FROM readings; +SELECT COUNT(*) FROM meters; + +-- 2. Create readings_hypertable as a separate copy of the readings table. +-- This keeps the original readings table untouched so other parts of the +-- codebase that query it directly are not affected. +CREATE TABLE readings_hypertable ( + meter_id INTEGER NOT NULL, + reading FLOAT NOT NULL, + start_timestamp TIMESTAMP NOT NULL, + end_timestamp TIMESTAMP NOT NULL, + CHECK (start_timestamp < end_timestamp), + PRIMARY KEY (meter_id, start_timestamp) +); + +-- 3. Convert readings_hypertable into a TimescaleDB hypertable, partitioned +-- by start_timestamp. This enables time-series optimizations and allows +-- continuous aggregates to be built on top. +SELECT create_hypertable('readings_hypertable', 'start_timestamp'); + +-- 4. Populate readings_hypertable with all data from the readings table. +INSERT INTO readings_hypertable SELECT * FROM readings; + +-- 5. Verify readings_hypertable populated correctly. +-- Both counts should be identical. +SELECT COUNT(*) FROM readings_hypertable; +SELECT COUNT(*) FROM readings; + +-- 6. Create hypertable_hourly_split to store readings split into hourly intervals +-- with cik_vary conversions applied at split time. +-- slope, intercept, and graphic_unit_id are included so the continuous aggregate +-- can apply the time varying conversion without needing to join to cik_vary at +-- query time. unit_represent and sec_in_rate are included to correctly handle +-- quantity, flow, and raw readings differently during aggregation. +CREATE TABLE hypertable_hourly_split ( + meter_id INTEGER NOT NULL, + reading FLOAT NOT NULL, -- Reading rate scaled by overlap duration + start_timestamp TIMESTAMP NOT NULL, + end_timestamp TIMESTAMP NOT NULL, + unit_represent unit_represent_type NOT NULL, + sec_in_rate FLOAT NOT NULL, + slope FLOAT NOT NULL, -- From cik_vary, used to convert reading to graphic unit + intercept FLOAT NOT NULL, -- From cik_vary, used to convert reading to graphic unit + graphic_unit_id INTEGER NOT NULL -- The destination unit this conversion applies to +); + +-- 7. Convert hypertable_hourly_split into a TimescaleDB hypertable. +SELECT create_hypertable('hypertable_hourly_split', 'start_timestamp'); + +-- 8. Populate hypertable_hourly_split by splitting each raw reading from +-- readings_hypertable into one row per hour it spans, and joining to cik_vary +-- to store the applicable conversion for each slice. +-- For example, a reading spanning 3 hours with 2 applicable conversions becomes +-- 6 rows — one per hour per conversion. +-- Quantity readings are converted to a rate per hour and scaled by overlap duration. +-- Flow/raw readings are already a rate, normalized to per hour using sec_in_rate, +-- and also scaled by overlap duration. +-- The slope and intercept from cik_vary are stored directly so the continuous +-- aggregate can apply them without any additional joins. +INSERT INTO hypertable_hourly_split +SELECT + r.meter_id, + CASE WHEN u.unit_represent = 'quantity'::unit_represent_type THEN + -- Convert to rate per hour and scale by overlap duration within this hour slice + (r.reading * 3600 / extract(EPOCH FROM (r.end_timestamp - r.start_timestamp))) * + extract(EPOCH FROM ( + least(r.end_timestamp, gen.interval_start + INTERVAL '1 hour') + - greatest(r.start_timestamp, gen.interval_start) + )) + WHEN (u.unit_represent = 'flow'::unit_represent_type OR u.unit_represent = 'raw'::unit_represent_type) THEN + -- Normalize flow/raw to per hour and scale by overlap duration + (r.reading * 3600 / u.sec_in_rate) * + extract(EPOCH FROM ( + least(r.end_timestamp, gen.interval_start + INTERVAL '1 hour') + - greatest(r.start_timestamp, gen.interval_start) + )) + END AS reading, + -- Clamp start/end timestamps to the hour boundary + greatest(r.start_timestamp, gen.interval_start) AS start_timestamp, + least(r.end_timestamp, gen.interval_start + INTERVAL '1 hour') AS end_timestamp, + u.unit_represent, + u.sec_in_rate, + c.slope, + c.intercept, + c.destination_id AS graphic_unit_id +FROM readings_hypertable r +INNER JOIN meters m ON r.meter_id = m.id +INNER JOIN units u ON m.unit_id = u.id +-- Join to cik_vary using a time range overlap to get all applicable conversions +-- for each reading. The exclusive bounds '()' ensure no two conversions overlap. +INNER JOIN cik_vary c ON c.source_id = m.unit_id + AND tsrange(c.start_time, c.end_time, '()') && tsrange(r.start_timestamp, r.end_timestamp, '[]') +-- Generate one row per hour that the reading spans +CROSS JOIN LATERAL generate_series( + date_trunc('hour', r.start_timestamp), + -- Subtract 1 hour because generate_series is end-inclusive + date_trunc_up('hour', r.end_timestamp) - INTERVAL '1 hour', + INTERVAL '1 hour' +) gen(interval_start); + +-- 9. Verify row count of the hourly split hypertable. +-- Will be higher than meter_hourly_readings_unit since each raw reading +-- can produce multiple rows (one per hour per cik_vary conversion) before aggregation. +SELECT COUNT(*) FROM hypertable_hourly_split; +SELECT * FROM hypertable_hourly_split LIMIT 10; + +-- 10. Check how many distinct (meter, hour, graphic_unit) combinations exist +-- in the hypertable. Should match the count from meter_hourly_readings_unit below. +SELECT COUNT(*) FROM ( + SELECT DISTINCT meter_id, date_trunc('hour', start_timestamp), graphic_unit_id + FROM hypertable_hourly_split +) s; + +-- 11. Check how many distinct (meter, hour, graphic_unit) combinations exist +-- in the materialized view. Should match the count from hypertable_hourly_split above. +SELECT COUNT(*) FROM ( + SELECT DISTINCT meter_id, lower(time_interval), graphic_unit_id + FROM meter_hourly_readings_unit +) s; + +-- 12. Create a continuous aggregate on top of hypertable_hourly_split. +-- This replicates the logic of meter_hourly_readings_unit but uses TimescaleDB's +-- continuous aggregate mechanism instead of a regular materialized view. +-- The reading in hypertable_hourly_split is already scaled by overlap duration, +-- so the weighted average is computed by dividing by duration to get the rate, +-- applying slope/intercept, then weighting by duration and normalizing. +-- This matches the two-step logic in meter_hourly_readings_unit: +-- Step 1 (base_hourly CTE): splits and computes raw reading rate +-- Step 2 (outer SELECT): applies cik_vary conversion +CREATE MATERIALIZED VIEW meter_hourly_readings_unit_cagg +WITH (timescaledb.continuous) AS +SELECT + meter_id, + graphic_unit_id, + time_bucket('1 hour', start_timestamp) AS bucket, + -- Weighted average reading rate by slice duration, with cik_vary conversion applied. + -- Dividing reading by duration recovers the rate, applying slope/intercept converts it, + -- then multiplying by duration and dividing by total duration gives the weighted average. + sum( + (reading / extract(EPOCH FROM (end_timestamp - start_timestamp)) * slope + intercept) + * extract(EPOCH FROM (end_timestamp - start_timestamp)) + ) / sum(extract(EPOCH FROM (end_timestamp - start_timestamp))) AS reading_rate, + -- Max and min rates with cik_vary conversion applied + max(reading / extract(EPOCH FROM (end_timestamp - start_timestamp)) * slope + intercept) AS max_rate, + min(reading / extract(EPOCH FROM (end_timestamp - start_timestamp)) * slope + intercept) AS min_rate, + unit_represent, + sec_in_rate +FROM hypertable_hourly_split +GROUP BY meter_id, graphic_unit_id, bucket, unit_represent, sec_in_rate; + +-- 13. Verify row count of the continuous aggregate. +-- Should match the count from meter_hourly_readings_unit. +SELECT COUNT(*) FROM meter_hourly_readings_unit_cagg; +SELECT * FROM meter_hourly_readings_unit_cagg LIMIT 10; + +-- 14. Compare reading_rate values between the original materialized view and +-- the continuous aggregate. Results are ordered by largest difference first +-- to surface any inaccuracies. Differences should be zero or extremely close +-- to zero (floating point rounding only). +SELECT + mv.meter_id, + lower(mv.time_interval) AS mv_time, + mv.reading_rate AS mv_reading_rate, + cagg.bucket AS cagg_time, + cagg.reading_rate AS cagg_reading_rate, + mv.reading_rate - cagg.reading_rate AS difference +FROM meter_hourly_readings_unit mv +INNER JOIN meter_hourly_readings_unit_cagg cagg + ON mv.meter_id = cagg.meter_id + AND lower(mv.time_interval) = cagg.bucket + AND mv.graphic_unit_id = cagg.graphic_unit_id +ORDER BY abs(mv.reading_rate - cagg.reading_rate) DESC +LIMIT 20; + +-- 15. Verify accuracy of max_rate and min_rate between the original materialized +-- view (meter_hourly_readings_unit) and the continuous aggregate +-- (meter_hourly_readings_unit_cagg). Results are ordered by largest max_rate +-- difference first to surface any inaccuracies. Differences should be zero or +-- extremely close to zero (floating point rounding only). +SELECT + mv.meter_id, + lower(mv.time_interval) AS mv_time, + mv.max_rate AS mv_max_rate, + cagg.max_rate AS cagg_max_rate, + mv.max_rate - cagg.max_rate AS max_difference, + mv.min_rate AS mv_min_rate, + cagg.min_rate AS cagg_min_rate, + mv.min_rate - cagg.min_rate AS min_difference +FROM meter_hourly_readings_unit mv +INNER JOIN meter_hourly_readings_unit_cagg cagg + ON mv.meter_id = cagg.meter_id + AND lower(mv.time_interval) = cagg.bucket + AND mv.graphic_unit_id = cagg.graphic_unit_id +ORDER BY abs(mv.max_rate - cagg.max_rate) DESC +LIMIT 20; \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 1b354cf399..3152cfde31 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,13 +16,14 @@ services: - POSTGRES_PASSWORD=pleaseChange # default postgres password that should be changed for security. volumes: - ./postgres-data:/var/lib/postgresql/data/pgdata + - ./src/server/tmp/timescaleddb:/benchmarkscripts:ro healthcheck: test: ["CMD-SHELL", "pg_isready -U postgres"] interval: 10s timeout: 10s retries: 3 - # ports: - # - "5432:5432" + ports: + - "5432:5432" # Uncomment the above lines to enable access to the PostgreSQL server # from the host machine. # Web service runs Node @@ -111,4 +112,3 @@ services: /bin/sh -c " rm -f /tmp/.X99-lock && Xvfb :99 -screen 0 1024x768x16" - \ No newline at end of file diff --git a/package.json b/package.json index 07cc7b5e6d..3402054e58 100644 --- a/package.json +++ b/package.json @@ -16,8 +16,8 @@ "check:typescript": "./src/scripts/checkTypescript.sh", "check:types": "tsc -p . --noEmit", "check:lint": "eslint --ignore-path .eslintignore --ext .ts,.tsx,.js,.jsx ./src/client/app", - "test": "mocha --timeout 15000 \"src/server/test/**/*.js\"", - "testsome": "mocha --timeout 15000", + "test": "mocha --timeout 5000 \"src/server/test/**/*.js\"", + "testsome": "mocha --timeout 5000", "createdb": "node ./src/server/services/createDB.js", "developerdb": "node -e 'require(\"./src/server/util/developer.js\").createShiftReadingsFunction()'", "migratedb": "node ./src/server/services/migrateDB.js", @@ -28,11 +28,12 @@ "editUser": "node ./src/server/services/user/editUser.js", "sendLogEmail": "node ./src/server/services/sendLogEmail.js", "refreshReadingViews": "node -e 'require(\"./src/server/services/refreshReadingViews\").refreshReadingViews()'", - "refreshAllReadingViews": "node -e 'require(\"./src/server/services/refreshAllReadingViews\").refreshAllReadingViews()'", + "refreshAllReadingViews": "node -e 'require(\"./src/server/services/refreshAllReadingViews\")()'", + "rebuildAllReadingViews": "node -e 'require(\"./src/server/services/refreshAllReadingViews\")({ rebuild: true })'", "refreshDailyReadingViews": "node -e 'require(\"./src/server/services/refreshReadingViews\").refreshReadingViews()'", "refreshHourlyReadingViews": "node -e 'require(\"./src/server/services/refreshHourlyReadingViews\").refreshHourlyReadingViews()'", "refreshGroupsDeepMeters": "node -e 'require(\"./src/server/services/refreshGroupsDeepMetersView\").refreshGroupsDeepMetersView()'", - "updateCikAndViews": "node -e 'require(\"./src/server/services/graph/redoCik.js\").updateCikAndViews()'", + "updateCikAndViews": "node -e 'require(\"./src/server/services/graph/redoCik.js\").updateCikVaryAndViews()'", "obvius:showConfigfiles": "node ./src/server/services/obvius/showConfigfiles.js", "obvius:purgeConfigfiles": "node ./src/server/services/obvius/purgeConfigfiles.js", "generateFourDayTestingData": "node -e 'require(\"./src/server/data/automatedTestingData\").generateFourDayTestingData()'", diff --git a/src/client/app/types/redux/ciks.ts b/src/client/app/types/redux/ciks.ts index f1df008714..f20f657e06 100644 --- a/src/client/app/types/redux/ciks.ts +++ b/src/client/app/types/redux/ciks.ts @@ -5,6 +5,4 @@ export interface CikData { meterUnitId: number; nonMeterUnitId: number; - slope: number; - intercept: number; } diff --git a/src/server/data/automatedTestingData.js b/src/server/data/automatedTestingData.js index 2e2efdb823..b3335aa343 100644 --- a/src/server/data/automatedTestingData.js +++ b/src/server/data/automatedTestingData.js @@ -8,11 +8,11 @@ const { generateSine, generateCosine } = require('./generateTestingData'); const Unit = require('../models/Unit'); +const Group = require('../models/Group'); const { insertUnits, insertStandardUnits, insertConversions, insertStandardConversions, insertMeters, insertGroups } = require('../util/insertData'); const { getConnection } = require('../db'); -const { redoCik } = require('../services/graph/redoCik'); -const { refreshAllReadingViews } = require('../services/refreshAllReadingViews'); -const fs = require('fs').promises; +const { redoCikVary } = require('../services/graph/redoCik'); +const refreshAllReadingViews = require('../services/refreshAllReadingViews'); // Define the start and end date for data generation. const DEFAULT_OPTIONS = { @@ -1335,7 +1335,7 @@ async function insertSpecialUnitsConversionsMetersGroups() { await insertSpecialConversions(conn); // Recreate the Cik entries since changed units/conversions. // Do now since needed to insert meters with suffix units. - await redoCik(conn); + await redoCikVary(conn); // Generate the mathematical test data needed. console.log(`Start loading each set of test data into OED meters, may take minutes):\n`); // This is very fast so wait since simpler and easier to see if this part fails. @@ -1343,10 +1343,12 @@ async function insertSpecialUnitsConversionsMetersGroups() { // Now do the large dataset generation. await testData(); // Recreate the Cik entries since changed meters. - await redoCik(conn); - // Refresh the readings since added new ones. - await refreshAllReadingViews(); + await redoCikVary(conn); await insertGroups(specialGroups, conn); + // Refresh groups deep meters view after adding new groups. + await Group.refreshGroupsDeepMetersView(conn); + // Refresh the readings since added new ones & groups. + await refreshAllReadingViews(); } /* diff --git a/src/server/data/websiteData.js b/src/server/data/websiteData.js index 36f4f3338f..ef4f1621e2 100644 --- a/src/server/data/websiteData.js +++ b/src/server/data/websiteData.js @@ -11,8 +11,8 @@ const Unit = require('../models/Unit'); const Group = require('../models/Group'); -const { redoCik } = require('../services/graph/redoCik'); -const { refreshAllReadingViews } = require('../services/refreshAllReadingViews'); +const { redoCikVary } = require('../services/graph/redoCik'); +const refreshAllReadingViews = require('../services/refreshAllReadingViews'); const { getConnection } = require('../db'); const { insertUnits, insertStandardUnits, insertConversions, insertStandardConversions, insertMeters, insertGroups } = require('../util/insertData'); const { shiftReadings } = require('../util/developer'); @@ -753,12 +753,12 @@ async function insertWebsiteData() { await insertConversions(conversions, conn); // Recreate the Cik entries since changed units/conversions. // Do now since needed to insert meters with suffix units. - await redoCik(conn); + await redoCikVary(conn); console.log(`Start loading each set of test data into OED meters (${meters.length} files of varying length, may take minutes):`); // await Meter.insertMany(meters, conn); await insertMeters(meters, conn); // Recreate the Cik entries since changed meters. - await redoCik(conn); + await redoCikVary(conn); await insertGroups(groups, conn); // Refresh groups deep meters view after adding new groups. await Group.refreshGroupsDeepMetersView(conn); diff --git a/src/server/migrations/2.0.0-3.0.0/index.js b/src/server/migrations/2.0.0-3.0.0/index.js new file mode 100644 index 0000000000..03ac116e64 --- /dev/null +++ b/src/server/migrations/2.0.0-3.0.0/index.js @@ -0,0 +1,19 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +const database = require('../../models/database'); +const sqlFile = database.sqlFile; + +module.exports = { + fromVersion: '2.0.0', + toVersion: '3.0.0', + up: async db => { + await db.none(sqlFile('../migrations/2.0.0-3.0.0/sql/cik/alter_cik_table.sql')); + await db.none(sqlFile('../migrations/2.0.0-3.0.0/sql/readings/drop_old_views.sql')); + await db.none(sqlFile('../migrations/2.0.0-3.0.0/sql/readings/create_reading_views.sql')); + await db.none(sqlFile('../migrations/2.0.0-3.0.0/sql/readings/create_function_get_3d_readings.sql')); + await db.none(sqlFile('../migrations/2.0.0-3.0.0/sql/readings/create_function_get_compare_readings.sql')); + } +}; \ No newline at end of file diff --git a/src/server/migrations/2.0.0-3.0.0/sql/cik/alter_cik_table.sql b/src/server/migrations/2.0.0-3.0.0/sql/cik/alter_cik_table.sql new file mode 100644 index 0000000000..6b7ae23591 --- /dev/null +++ b/src/server/migrations/2.0.0-3.0.0/sql/cik/alter_cik_table.sql @@ -0,0 +1,10 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +--Add the new columns with appropriate infinity defaults +ALTER TABLE cik + DROP COLUMN slope, + DROP COLUMN intercept +; diff --git a/src/server/migrations/2.0.0-3.0.0/sql/readings/create_function_get_3d_readings.sql b/src/server/migrations/2.0.0-3.0.0/sql/readings/create_function_get_3d_readings.sql new file mode 100644 index 0000000000..fd0ba1b4f7 --- /dev/null +++ b/src/server/migrations/2.0.0-3.0.0/sql/readings/create_function_get_3d_readings.sql @@ -0,0 +1,288 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +-- By indexing both columns together, the database can efficiently handle queries that involve both meter_id and time_interval. +-- Created to support the usage of the view by 3d. +-- TODO verify helps with newer views. +CREATE INDEX IF NOT EXISTS idx_meter_hourly_readings_unit_meter_time +ON meter_hourly_readings_unit (meter_id, lower(time_interval)); +-- TODO How does this relate to index for same view in create_reading_views? Are both needed? +CREATE INDEX if not exists idx_two_group_hourly_readings_unit +ON group_hourly_readings_unit (group_id, graphic_unit_id, lower(time_interval)); + +/* +This takes tsrange_to_shrink which is the requested time range to plot and makes sure it does +not exceed the start/end times for the readings in the supplied meter. This can be an issue, in particular, +because infinity is used to indicate to graph all readings. This version does it to the nearest +day by using the day reading view and is used by 3D readings which only allow days and a single meter. + */ +CREATE OR REPLACE FUNCTION shrink_tsrange_to_meter_readings_by_day(tsrange_to_shrink TSRANGE, meter_id_desired INTEGER) + RETURNS TSRANGE +AS $$ +DECLARE + readings_max_tsrange TSRANGE; +BEGIN + SELECT tsrange(min(lower(time_interval)), max(upper(time_interval))) INTO readings_max_tsrange + FROM meter_daily_readings_unit + where meter_id = meter_id_desired; + RETURN tsrange_to_shrink * readings_max_tsrange; +END; +$$ LANGUAGE 'plpgsql'; + +/* Similar to meter version but for a group */ +CREATE OR REPLACE FUNCTION shrink_tsrange_to_group_readings_by_day(tsrange_to_shrink TSRANGE, group_id_desired INTEGER) + RETURNS TSRANGE +AS $$ +DECLARE + readings_max_tsrange TSRANGE; +BEGIN + SELECT tsrange(min(lower(time_interval)), max(upper(time_interval))) INTO readings_max_tsrange + FROM group_daily_readings_unit + where group_id = group_id_desired; + RETURN tsrange_to_shrink * readings_max_tsrange; +END; +$$ LANGUAGE 'plpgsql'; + + +-- Determines the spacing between 3D points. It uses the lowest one for all meters passed +-- that is valid. +CREATE OR REPLACE FUNCTION reading_interval_3d ( + -- The desired meter ids. + IN meter_ids_requested INTEGER[], + -- The number of hours in each reading requested + IN reading_length_hours INTEGER, + -- The number of hours in each reading determined + OUT reading_length_hours_use INTEGER, + -- The number of hours in each reading determined as an interval + OUT reading_length_interval INTERVAL +) +AS $$ +DECLARE + -- The meter frequency from all meters. + meter_frequency INTERVAL; + -- The meter frequency rounded up to a whole number of hours. + meter_frequency_hour_up INTEGER; + -- The larger of the meter value and the argument sent. + max_frequency INTEGER; +BEGIN + -- Get the smallest reading frequency for all meters requested. + SELECT min(reading_frequency) INTO meter_frequency + FROM (meters m + INNER JOIN unnest(meter_ids_requested) meters(id) ON m.id = meters.id); + -- Get the seconds in the frequency from epoch, /3600 To get hours and then round up to a whole number of hours. + meter_frequency_hour_up := CEIL((SELECT * FROM EXTRACT(EPOCH FROM meter_frequency)) / 3600); + -- Use the hours that is the largest of the request and the meter values. + max_frequency := GREATEST(meter_frequency_hour_up, reading_length_hours); + -- The value used must be a divisor of 24 or greater than 12. + IF (max_frequency = 5) THEN + reading_length_hours_use := 6; + ELSIF (max_frequency = 7) THEN + reading_length_hours_use := 8; + ELSIF (max_frequency > 8 AND max_frequency < 12) THEN + reading_length_hours_use := 12; + ELSE + reading_length_hours_use := max_frequency; + END IF; + -- Hours per reading determined returned as an interval. + reading_length_interval := (reading_length_hours_use::TEXT || ' hour')::INTERVAL; +END; +$$ LANGUAGE 'plpgsql'; + + +-- Gets meters graphing data for 3D graphic by returning points that span the requested +-- length of time over the days requested. +-- New meter_3d_readings_unit function that uses new meter_hourly_readings_unit view. +CREATE OR REPLACE FUNCTION meter_3d_readings_unit ( + -- The desired meter ids. It is normally a single value for a 3D graphic. + -- TODO Should the array be changed to a single value as with group? Need to be sure client never asks for multiple. + meter_ids_requested INTEGER[], + -- The desired graphic unit of the returned data + graphic_unit_id_requested INTEGER, + -- The start and end time for the data to return + start_stamp TIMESTAMP, + end_stamp TIMESTAMP, + -- The number of hours in each reading requested + reading_length_hours INTEGER +) + RETURNS TABLE(meter_id INTEGER, reading_rate FLOAT, start_timestamp TIMESTAMP, end_timestamp TIMESTAMP) +AS $$ +DECLARE + -- Holds the range of dates for returned data that fits the actual data. + requested_range TSRANGE; + -- The number of hours in each reading determined as an interval + reading_length_interval INTERVAL; + -- Which index of the meter_id array you are currently working on. + current_meter_index INTEGER := 1; + -- The id of the meter index working on + current_meter_id INTEGER; + -- The meter frequency from all meters. + meter_frequency INTERVAL; + -- The meter frequency rounded up to a whole number of hours. + meter_frequency_hour_up INTEGER; + -- The larger of the meter value and the argument sent. + max_frequency INTEGER; + -- The number of hours in each reading determined + reading_length_hours_use INTEGER; +BEGIN + -- Find the correct number of hours per reading returned. + SELECT * from reading_interval_3d(meter_ids_requested, reading_length_hours) into reading_length_hours_use, reading_length_interval; + + -- Loop over all meters. + WHILE current_meter_index <= cardinality(meter_ids_requested) LOOP + -- ID of the current meter in loop + current_meter_id := meter_ids_requested[current_meter_index]; + + -- Get the range of days requested by calling shrink_tsrange_to_meter_readings_by_day. + -- First make requested range only be full days by dropping any partial days at start/end. + requested_range := shrink_tsrange_to_meter_readings_by_day(tsrange(date_trunc_up('day', start_stamp), date_trunc('day', end_stamp)), current_meter_id); + + IF (reading_length_hours_use <= 12) THEN + -- Need to generate_series to group the desired hours together + RETURN QUERY + -- The readings are rates in the hourly table so want to average not sum so + -- work for quantity, flow & raw. + -- The time starts at the time of the generated sequence and ends at the length + -- of each block later. This is the same as the start time of the next value + -- in the sequence (except last one). + SELECT + -- Modified to retrieve converted hourly readings from the materialized view. + mhr.meter_id as meter_id, + AVG(mhr.reading_rate) as reading_rate, + hours.hour AS start_timestamp, + hours.hour + reading_length_interval AS end_timestamp + -- This is the series that starts at the beginning of the desired days, + -- ends at the end of the desired days and steps by the desired interval. + -- You need to subtract from the last interval for the end since generate_series + -- is inclusive. + FROM ( + SELECT hour + FROM generate_series( + lower(requested_range), + upper(requested_range) - reading_length_interval, + reading_length_interval + ) hours(hour) + ) hours(hour), + -- Also need the values in the meter hourly table. + meter_hourly_readings_unit mhr + -- Only want the desired meter + WHERE mhr.meter_id = current_meter_id + -- Only want the desired graphing unit + AND mhr.graphic_unit_id = graphic_unit_id_requested + -- Only want readings that lie within this slice of the desired data + AND lower(mhr.time_interval) >= hours.hour + AND upper(mhr.time_interval) <= hours.hour + reading_length_interval + -- ensures that the start of the reading time intervals does not exceed the end of the current generated interval + AND lower(mhr.time_interval) <= hours.hour + reading_length_interval + -- Group by the start time of the generated series since all points in + -- the desired slice have the same start time for the series. + -- Also group by the meter_id since Postgres wants and desired for graphing + GROUP BY hours.hour, mhr.meter_id + -- Time sort by the meter and start time for graphing. + ORDER BY mhr.meter_id, hours.hour + ; + ELSE + -- The reading rate is more than 12 so return a single row with dummy values that easy to detect. + -- The end time differs from the start time by the meter reading frequency or min one for groups. + -- This means the meter reading frequency is too long for a 3D graphic. + RETURN QUERY + SELECT -999, -999::FLOAT, '1900-01-01 00:00:00'::TIMESTAMP, '1900-01-01 00:00:00'::TIMESTAMP + reading_length_interval + ; + END IF; + + -- Go to the next meter + current_meter_index := current_meter_index + 1; + END LOOP; +END; +$$ LANGUAGE plpgsql; + +/* Gets group graphing data for 3D graphic by returning points that span the requested + length of time over the days requested. +*/ +CREATE OR REPLACE FUNCTION group_3d_readings_unit ( + --Desire group ID + --For 3D graphics, users will only be able to select 1 group to graph. + group_id_requested INTEGER, + -- The desired graphic unit of the returned data + graphic_unit_id_requested INTEGER, + -- The start and end time for the data to return + start_stamp TIMESTAMP, + end_stamp TIMESTAMP, + -- The number of hours in each reading requested + reading_length_hours INTEGER +) + RETURNS TABLE(reading_rate FLOAT, start_timestamp TIMESTAMP, end_timestamp TIMESTAMP) +AS $$ +DECLARE + -- Holds the range of dates for returned data that fits the actual data. + requested_range TSRANGE; + --Holds the desired meter IDs in order to call meter_3d_readings_unit in the query below. + meter_ids INTEGER[]; + -- The number of hours in each reading determined + reading_length_hours_use INTEGER; + -- The number of hours in each reading determined as an interval + reading_length_interval INTERVAL; +BEGIN + --Get all the meter IDS that will be included in the group being requested. + SELECT array_agg(DISTINCT gdm.meter_id) INTO meter_ids + FROM groups_deep_meters gdm + WHERE group_id = group_id_requested; + + -- Find the correct number of hours per reading returned. + SELECT * from reading_interval_3d(meter_ids, reading_length_hours) into reading_length_hours_use, reading_length_interval; + -- Get the range of days requested by calling shrink_tsrange_to_group_readings_by_day. + -- First make requested range only be full days by dropping any partial days at start/end. + requested_range := shrink_tsrange_to_group_readings_by_day(tsrange(date_trunc_up('day', start_stamp), date_trunc('day', end_stamp)), group_id_requested); + + IF (reading_length_hours_use <= 12) THEN + -- Need to generate_series to group the desired hours together + RETURN QUERY + -- The readings are rates in the hourly table so want to average not sum so + -- work for quantity, flow & raw. + -- The time starts at the time of the generated sequence and ends at the length + -- of each block later. This is the same as the start time of the next value + -- in the sequence (except last one). + SELECT + AVG(ghr.reading_rate) as reading_rate, + hours.hour AS start_timestamp, + hours.hour + reading_length_interval AS end_timestamp + -- This is the series that starts at the beginning of the desired days, + -- ends at the end of the desired days and steps by the desired interval. + -- You need to subtract from the last interval for the end since generate_series + -- is inclusive. + FROM ( + SELECT hour + FROM generate_series( + lower(requested_range), + upper(requested_range) - reading_length_interval, + reading_length_interval + ) hours(hour) + ) hours(hour), + -- Also need the values in the group hourly table. + group_hourly_readings_unit ghr + -- Only want the desired meter + WHERE ghr.group_id = group_id_requested + -- Only want the desired graphing unit + AND ghr.graphic_unit_id = graphic_unit_id_requested + -- Only want readings that lie within this slice of the desired data + AND lower(ghr.time_interval) >= hours.hour + AND upper(ghr.time_interval) <= hours.hour + reading_length_interval + -- ensures that the start of the reading time intervals does not exceed the end of the current generated interval + AND lower(ghr.time_interval) <= hours.hour + reading_length_interval + -- Group by the start time of the generated series since all points in + -- the desired slice have the same start time for the series. + GROUP BY hours.hour + -- Time sort by the start time for graphing. + ORDER BY hours.hour + ; + ELSE + -- The reading rate is more than 12 so return a single row with dummy values that easy to detect. + -- The end time differs from the start time by the meter reading frequency or min one for groups. + -- This means the meter reading frequency is too long for a 3D graphic. + RETURN QUERY + SELECT -999::FLOAT, '1900-01-01 00:00:00'::TIMESTAMP, '1900-01-01 00:00:00'::TIMESTAMP + reading_length_interval + ; + END IF; +END; +$$ LANGUAGE 'plpgsql'; diff --git a/src/server/migrations/2.0.0-3.0.0/sql/readings/create_function_get_compare_readings.sql b/src/server/migrations/2.0.0-3.0.0/sql/readings/create_function_get_compare_readings.sql new file mode 100644 index 0000000000..457bb9ef4e --- /dev/null +++ b/src/server/migrations/2.0.0-3.0.0/sql/readings/create_function_get_compare_readings.sql @@ -0,0 +1,167 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* +This shouldn't ever be looking at more than a few weeks of data, so we don't need to deal with compression. + */ + +/* +TODO This function can probably be improved for two reasons: +1) While it was noted that you don't look at too many days, it does limit its usage to modest time lengths. +There has been thought to allowing comparisons, for example, or a whole year. It also assumes that the +frequency of readings is not too high so the number of readings looked at could be large even over +modest time frames. +2) Readings are only included if they are within the current time period. This means that if you have +a reading that crosses the timeframe they are excluded. This can be viewed as either a good thing or +a bad thing. However, if the frequency of readings is high, then large segments of time can be excluded. +For example, monthly readings would not be includes in either day, week or four week comparisons that +OED currently does. Also note that the daily and hourly views do include readings that span a time frame +so including them would be consistent. + +We need to think about how best to deal with this but one option is to use the daily and hourly tables +to get the reading values as in done with bar graphs. This needs to be a little different since partial +days can be involved. However, getting the full days from the daily table and then the hours from the +hourly table to be combined would solve this. (One could get the subhour from the raw readings but it is +unclear users would want that.) Another consider is that the current system, as is the case for bar graphs, +does not take into account missing times in readings which can lead to lower than expected values. The +daily and hourly readings would help fix this. +*/ + +/* +The following function returns data for plotting compare graphs. It works on meters. +It should not be used on raw readings. +This only works when the curr start/end times are on the hour as it uses the hourly reading view. +It will truncate to lose partial hours if they are given. The current client code will only +send full hours so this should be fine. +It is the new version of compare_readings that works with units. It takes these parameters: +meter_ids: A array of meter ids to query. +graphic_unit_id: The unit id of the unit to use for the graph. +curr_start: When the current/this time period begins for the compare. +curr_end: When the current/this time period ends for the compare. +shift: How far back in time to shift the curr_start and curr_end date/time to get the previous + times to compare. + */ +-- New version of the meter_compare_readings function that uses the meter_hourly_readings_unit view. +CREATE OR REPLACE FUNCTION meter_compare_readings_unit ( + meter_ids INTEGER[], + -- This is the graphic unit id, changed from graphic_unit_id to avoid confusion with the graphic unit id in the view. + passed_graphic_unit_id INTEGER, + curr_start TIMESTAMP, + curr_end TIMESTAMP, + shift INTERVAL +) + RETURNS TABLE(meter_id INTEGER, curr_use FLOAT, prev_use FLOAT) +AS $$ +DECLARE + curr_tsrange TSRANGE; + prev_tsrange TSRANGE; +BEGIN + curr_tsrange := tsrange(curr_start, curr_end); + prev_tsrange := tsrange(curr_start - shift, curr_end - shift); + -- Modified to retrieve converted hourly readings from the materialized view. + RETURN QUERY + WITH + curr_period AS ( + SELECT + hourly.meter_id AS meter_id, + -- It is okay to sum the flow units because hourly readings has a rate of per hour so + -- to convert to a quantity would multiply by 1 so it is the same formula. + SUM(hourly.reading_rate) AS reading + FROM meter_hourly_readings_unit hourly + -- This is getting the conversion for the meter and unit to graph. + WHERE + -- The range requested must be completely within the hour so partial hours are not included. + curr_tsrange @> hourly.time_interval AND + hourly.graphic_unit_id = passed_graphic_unit_id AND + hourly.meter_id = ANY(meter_ids) + GROUP BY hourly.meter_id + ), + prev_period AS ( + SELECT + hourly.meter_id AS meter_id, + SUM(hourly.reading_rate) AS reading + FROM meter_hourly_readings_unit hourly + -- This is getting the conversion for the meter and unit to graph. + -- The slope and intercept are used above the transform the reading to the desired unit. + WHERE + -- The range requested must be completely within the hour so partial hours are not included. + prev_tsrange @> hourly.time_interval AND + hourly.graphic_unit_id = passed_graphic_unit_id AND + hourly.meter_id = ANY(meter_ids) + GROUP BY hourly.meter_id + ) + SELECT + meters.id AS meter_id, + curr_period.reading::FLOAT AS curr_use, + prev_period.reading::FLOAT AS prev_use + FROM + unnest(meter_ids) meters(id) + -- Left joins here so we get nulls instead of missing rows if readings don't exist for some time intervals + LEFT JOIN prev_period ON meters.id = prev_period.meter_id + LEFT JOIN curr_period ON meters.id = curr_period.meter_id; +END; +$$ LANGUAGE 'plpgsql'; + + +/* +The following function returns data for plotting compare graphs. It works on groups. +It should not be used on raw readings. +See meter version for needing start/end on the hour. +It is the new version of group_compare_readings that works with units. It takes these parameters: +group_ids: A array of group ids to query. +graphic_unit_id: The unit id of the unit to use for the graph. +curr_start: When the current/this time period begins for the compare. +curr_end: When the current/this time period ends for the compare. +shift: How far back in time to shift the curr_start and curr_end date/time to get the previous + times to compare. + */ +CREATE OR REPLACE FUNCTION group_compare_readings_unit ( + group_ids INTEGER[], + requested_graphic_unit_id INTEGER, + curr_start TIMESTAMP, + curr_end TIMESTAMP, + shift INTERVAL +) + RETURNS TABLE(group_id INTEGER, curr_use FLOAT, prev_use FLOAT) +AS $$ +DECLARE + curr_tsrange TSRANGE; + prev_tsrange TSRANGE; +BEGIN + curr_tsrange := tsrange(curr_start, curr_end); + prev_tsrange := tsrange(curr_start - shift, curr_end - shift); + + RETURN QUERY + WITH + curr_period AS ( + SELECT + hourly.group_id, + SUM(hourly.reading_rate) AS reading + FROM group_hourly_readings_unit hourly + WHERE curr_tsrange @> hourly.time_interval + AND requested_graphic_unit_id = hourly.graphic_unit_id + AND hourly.group_id = ANY(group_ids) + GROUP BY hourly.group_id + ), + prev_period AS ( + SELECT + hourly.group_id, + SUM(hourly.reading_rate) AS reading + FROM group_hourly_readings_unit hourly + WHERE prev_tsrange @> hourly.time_interval + AND requested_graphic_unit_id = hourly.graphic_unit_id + AND hourly.group_id = ANY(group_ids) + GROUP BY hourly.group_id + ) + SELECT + gids.id AS group_id, + curr_period.reading::FLOAT AS curr_use, + prev_period.reading::FLOAT AS prev_use + FROM + unnest(group_ids) gids(id) + -- Left joins here so we get nulls instead of missing rows if readings don't exist for some time intervals + LEFT JOIN prev_period ON gids.id = prev_period.group_id + LEFT JOIN curr_period ON gids.id = curr_period.group_id; +END; +$$ LANGUAGE 'plpgsql'; diff --git a/src/server/migrations/2.0.0-3.0.0/sql/readings/create_reading_views.sql b/src/server/migrations/2.0.0-3.0.0/sql/readings/create_reading_views.sql new file mode 100644 index 0000000000..fa7d4abe2d --- /dev/null +++ b/src/server/migrations/2.0.0-3.0.0/sql/readings/create_reading_views.sql @@ -0,0 +1,748 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* +This takes tsrange_to_shrink which is the requested time range to plot and makes sure it does +not exceed the start/end times for all the readings. This can be an issue, in particular, +because infinity is used to indicate to graph all readings. This version does it to the nearest +day by using the day reading view since bars use to the nearest day and this should be faster. +This should be fine since bar uses the same view to get data. + */ +CREATE OR REPLACE FUNCTION shrink_tsrange_to_meters_by_day(tsrange_to_shrink TSRANGE, meter_ids INTEGER[]) + RETURNS TSRANGE +AS $$ +DECLARE + readings_max_tsrange TSRANGE; +BEGIN + SELECT tsrange(min(lower(time_interval)), max(upper(time_interval))) INTO readings_max_tsrange + FROM meter_daily_readings_unit dr + -- Get all the meter_ids in the passed array of meters. + INNER JOIN unnest(meter_ids) meters(id) ON dr.meter_id = meters.id; + -- Make the original range be to the day by dropping parts of days at start/end. + RETURN tsrange(date_trunc_up('day', lower(tsrange_to_shrink)), date_trunc('day', upper(tsrange_to_shrink))) * readings_max_tsrange; +END; +$$ LANGUAGE 'plpgsql'; + +/* + The following function takes an integer for group id and return an array of all unit ids which are compatible + to all child meters in that group. +*/ +CREATE OR REPLACE FUNCTION get_graphic_unit ( + requested_group_id INTEGER +) +RETURNS INTEGER[] AS $$ +DECLARE + src_ids INTEGER[]; + dest_ids INTEGER[]; + child_meters_unit_ids INTEGER[]; + unit_ids_compatible INTEGER[] := '{}'; + unit_id INTEGER; +BEGIN + -- get the units of all child meters in group + SELECT array_agg(DISTINCT m.unit_id) INTO child_meters_unit_ids + FROM groups_deep_meters gdm + JOIN meters m ON m.id = gdm.meter_id + WHERE gdm.group_id = requested_group_id; + + -- get all possible destination units + SELECT array_agg(u.id) INTO dest_ids + FROM units u JOIN cik c + ON u.id = c.destination_id; + + -- determine the compatible unit by checking if the array of all corresponding source unit + -- to a destination unit contains all child meters' units + FOREACH unit_id IN ARRAY dest_ids + LOOP + BEGIN + SELECT array_agg(source_id) INTO src_ids + FROM cik WHERE destination_id = unit_id; + + -- append each compatible unit id once into array + IF src_ids @> child_meters_unit_ids + THEN + IF NOT (unit_id = ANY (unit_ids_compatible)) + THEN + unit_ids_compatible := array_append(unit_ids_compatible, unit_id); + END IF; + END IF; + END; + END LOOP; + + RETURN unit_ids_compatible; +END; +$$ LANGUAGE 'plpgsql'; + +/* +This may still apply. +The following views are all generated. +This is necessary because they can't be wrapped in a function (otherwise predicates would not be pushed down). +*/ + +/** +The next two create a view/table that takes the raw/meter readings and averages them for each day or hour AND applies +the unit conversions from the cik_vary table. +This is used by the line graph function below to make them faster since the values +are already averaged and the conversions applied. There are two types of readings: quantity and flow/raw. The quantity +readings must be normalized by their time length. The flow/raw readings are already by time +so they are just averaged. The one table contains both types of readings but are now equivalent +so the line reading functions can use them both in the same way. + */ + +-- Current Working Versions, not dependent on old hourly_readings_unit view, uses a CTE instead +-- This version only handles 1 conversion per hourly reading +-- It can not handle multiple conversions per reading or conversions that overlap the time interval. +CREATE MATERIALIZED VIEW IF NOT EXISTS meter_hourly_readings_unit +AS +WITH base_hourly AS ( + SELECT + -- This gives the weighted average of the reading rates, defined as + -- sum(reading_rate * overlap_duration) / sum(overlap_duration) + r.meter_id, + CASE + WHEN u.unit_represent = 'quantity'::unit_represent_type THEN + ( + SUM( + -- Reading rate + (r.reading * 3600 / EXTRACT(EPOCH FROM (r.end_timestamp - r.start_timestamp))) * + -- The number of seconds that the reading shares with the interval + EXTRACT(EPOCH FROM LEAST(r.end_timestamp, gen.interval_start + INTERVAL '1 hour') - GREATEST(r.start_timestamp, gen.interval_start)) + ) / + SUM( + -- The number of seconds that the reading shares with the interval + EXTRACT(EPOCH FROM LEAST(r.end_timestamp, gen.interval_start + INTERVAL '1 hour') - GREATEST(r.start_timestamp, gen.interval_start)) + ) + ) + WHEN u.unit_represent IN ('flow'::unit_represent_type, 'raw'::unit_represent_type) THEN + ( + SUM( + -- Reading rate in per hour + (r.reading * 3600 / u.sec_in_rate) * + -- The number of seconds that the reading shares with the interval + EXTRACT(EPOCH FROM LEAST(r.end_timestamp, gen.interval_start + INTERVAL '1 hour') - GREATEST(r.start_timestamp, gen.interval_start)) + ) / + SUM( + -- The number of seconds that the reading shares with the interval + EXTRACT(EPOCH FROM LEAST(r.end_timestamp, gen.interval_start + INTERVAL '1 hour') - GREATEST(r.start_timestamp, gen.interval_start)) + ) + ) + END AS reading_rate, + -- The following code does the min/max for hourly readings + CASE + WHEN u.unit_represent = 'quantity'::unit_represent_type THEN + MAX( + -- Extract the maximum rate over each day + ( + -- Reading rate + (r.reading * 3600 / EXTRACT(EPOCH FROM (r.end_timestamp - r.start_timestamp))) * + -- The number of seconds that the reading shares with the interval + EXTRACT(EPOCH FROM LEAST(r.end_timestamp, gen.interval_start + INTERVAL '1 hour') - GREATEST(r.start_timestamp, gen.interval_start)) + ) / + -- The number of seconds that the reading shares with the interval + EXTRACT(EPOCH FROM LEAST(r.end_timestamp, gen.interval_start + INTERVAL '1 hour') - GREATEST(r.start_timestamp, gen.interval_start)) + ) + WHEN u.unit_represent IN ('flow'::unit_represent_type, 'raw'::unit_represent_type) THEN + -- For flow and raw data the max/min is per minute, so we multiply the max/min by 24 hrs * 60 min + MAX( + ( + -- Reading rate + (r.reading * 3600 / u.sec_in_rate) * + -- The number of seconds that the reading shares with the interval + EXTRACT(EPOCH FROM LEAST(r.end_timestamp, gen.interval_start + INTERVAL '1 hour') - GREATEST(r.start_timestamp, gen.interval_start)) + ) / + -- The number of seconds that the reading shares with the interval + EXTRACT(EPOCH FROM LEAST(r.end_timestamp, gen.interval_start + INTERVAL '1 hour') - GREATEST(r.start_timestamp, gen.interval_start)) + ) + END AS max_rate, + + CASE + WHEN u.unit_represent = 'quantity'::unit_represent_type THEN + MIN( + --Extract the minimum rate over each day + ( + -- Reading rate + (r.reading * 3600 / EXTRACT(EPOCH FROM (r.end_timestamp - r.start_timestamp))) * + -- The number of seconds that the reading shares with the interval + EXTRACT(EPOCH FROM LEAST(r.end_timestamp, gen.interval_start + INTERVAL '1 hour') - GREATEST(r.start_timestamp, gen.interval_start)) + ) / + -- The number of seconds that the reading shares with the interval + EXTRACT(EPOCH FROM LEAST(r.end_timestamp, gen.interval_start + INTERVAL '1 hour') - GREATEST(r.start_timestamp, gen.interval_start)) + ) + WHEN u.unit_represent IN ('flow'::unit_represent_type, 'raw'::unit_represent_type) THEN + MIN( + ( + -- Reading rate + (r.reading * 3600 / u.sec_in_rate) * + -- The number of seconds that the reading shares with the interval + EXTRACT(EPOCH FROM LEAST(r.end_timestamp, gen.interval_start + INTERVAL '1 hour') - GREATEST(r.start_timestamp, gen.interval_start)) + ) / + -- The number of seconds that the reading shares with the interval + EXTRACT(EPOCH FROM LEAST(r.end_timestamp, gen.interval_start + INTERVAL '1 hour') - GREATEST(r.start_timestamp, gen.interval_start)) + ) + END AS min_rate, + + tsrange(gen.interval_start, gen.interval_start + INTERVAL '1 hour', '()') AS time_interval + + FROM readings r + -- This sequence of joins takes the meter id to its unit and a unit. + INNER JOIN meters m ON r.meter_id = m.id + INNER JOIN units u ON m.unit_id = u.id + CROSS JOIN LATERAL generate_series( + date_trunc('hour', r.start_timestamp), + -- Subtract 1 interval width because generate_series is end-inclusive + date_trunc_up('hour', r.end_timestamp) - INTERVAL '1 hour', + INTERVAL '1 hour' + ) gen(interval_start) + GROUP BY r.meter_id, gen.interval_start, u.unit_represent +) +SELECT + m.id AS meter_id, + SUM(bh.reading_rate * c.slope + c.intercept) AS reading_rate, + SUM(bh.min_rate * c.slope + c.intercept) AS min_rate, + SUM(bh.max_rate * c.slope + c.intercept) AS max_rate, + bh.time_interval, + c.destination_id AS graphic_unit_id + +FROM base_hourly bh +JOIN meters m ON m.id = bh.meter_id +JOIN units u ON u.id = m.unit_id +JOIN cik_vary c ON c.source_id = m.unit_id AND tsrange(c.start_time, c.end_time, '()') && bh.time_interval +GROUP BY m.id, graphic_unit_id, bh.time_interval +-- The order by ensures that the materialized view will be clustered in this way. +ORDER BY bh.time_interval, meter_id; + +-- Used by the line/3d/compare functions. +CREATE INDEX if not exists idx_meter_hourly_ordering ON meter_hourly_readings_unit (meter_id, graphic_unit_id, lower(time_interval)); + +-- Current working version. Retrieves converted data from meter_hourly_readings_unit and averages it to the day. +CREATE MATERIALIZED VIEW IF NOT EXISTS +meter_daily_readings_unit + AS SELECT + h.meter_id AS meter_id, + AVG(h.reading_rate) AS reading_rate, + MIN(h.min_rate) AS min_rate, + MAX(h.max_rate) AS max_rate, + tsrange(gen.interval_start, gen.interval_start + INTERVAL '1 day', '()') AS time_interval, + h.graphic_unit_id AS graphic_unit_id + + FROM meter_hourly_readings_unit h + CROSS JOIN LATERAL generate_series( + date_trunc('day', lower(h.time_interval)), + date_trunc_up('day', upper(h.time_interval)) - INTERVAL '1 hour', + INTERVAL '1 day' + ) gen(interval_start) + WHERE tsrange(gen.interval_start, gen.interval_start + INTERVAL '1 day', '()') @> h.time_interval + GROUP BY h.meter_id, h.graphic_unit_id, gen.interval_start + ORDER BY h.meter_id, graphic_unit_id, gen.interval_start; + + -- Used by the line/bar/compare functions. +CREATE INDEX if not exists idx_meter_daily_ordering ON meter_daily_readings_unit (meter_id, graphic_unit_id, lower(time_interval)); +-- This index sometimes performs faster(for the bar function) than the above index but is likely not worth the additional overhead. +-- CREATE INDEX if not exists idx_mdr_meter_graphic ON meter_daily_readings_unit (meter_id, graphic_unit_id); + +--Modified to use meter_hourly_readings_unit instead of old hourly_readings_unit view. +--No longer needs to apply conversions since that is done in meter_hourly_readings_unit view. +CREATE MATERIALIZED VIEW IF NOT EXISTS +group_hourly_readings_unit + AS SELECT + gdm.group_id, + SUM(hr.reading_rate) AS reading_rate, + hr.time_interval, + hr.graphic_unit_id + + FROM meter_hourly_readings_unit hr + INNER JOIN groups_deep_meters gdm ON hr.meter_id = gdm.meter_id + INNER JOIN unnest(get_graphic_unit(gdm.group_id)) AS gu(graphic_unit_id) ON hr.graphic_unit_id = gu.graphic_unit_id + -- group meter readings of each group on the the same hour, of the same graphic unit + GROUP BY gdm.group_id, hr.graphic_unit_id, hr.time_interval + ORDER BY gdm.group_id; + +CREATE INDEX if not exists idx_group_hourly_readings_unit ON group_hourly_readings_unit USING GIST(time_interval, graphic_unit_id, group_id); + +--Modified to use meter_daily_readings_unit instead of old daily_readings_unit view. +--No longer needs to apply conversions since that is done in meter_daily_readings_unit view. +CREATE MATERIALIZED VIEW IF NOT EXISTS +group_daily_readings_unit + AS SELECT + gdm.group_id, + SUM(dr.reading_rate) AS reading_rate, + dr.time_interval, + dr.graphic_unit_id + + FROM meter_daily_readings_unit dr + INNER JOIN groups_deep_meters gdm ON dr.meter_id = gdm.meter_id + INNER JOIN unnest(get_graphic_unit(gdm.group_id)) AS gu(graphic_unit_id) ON dr.graphic_unit_id = gu.graphic_unit_id + -- group meter readings of each group on the the same day, of the same graphic unit + GROUP BY gdm.group_id, dr.graphic_unit_id, dr.time_interval + -- order by time interval instead + ORDER BY dr.time_interval, dr.graphic_unit_id, gdm.group_id; + +-- Index on interval, graphic_unit_id, group_id +CREATE INDEX if not exists idx_group_daily_readings_unit ON group_daily_readings_unit USING GIST(time_interval, graphic_unit_id, group_id); + +/* +The following function determines the correct duration view to query from, and returns averaged or raw reading from it. +It is designed to return data for plotting line graphs. It works on meters. +It is the new version of compressed_readings_2 that works with units. It takes these parameters: +meter_ids: A array of meter ids to query. +graphic_unit_id: The unit id of the unit to use for the graphic. +start_timestamp: The start timestamp of the data to return. +end_timestamp: The end timestamp of the data to return. +point_accuracy: Tells how decisions should be made on which types of points to return. 'auto' if automatic. +max_raw_points: The maximum number of data points to return if using the raw points for a meter. Only used if 'auto' for point_accuracy. +max_hour_points: The maximum number of data points to return if using the hour view. Only used if 'auto' for point_accuracy. +Details on how this function works can be found in the devDocs in the resource generalization document. + */ +-- New version of meter_line_readings_unit that uses the new views. +CREATE OR REPLACE FUNCTION meter_line_readings_unit ( + meter_ids INTEGER[], + -- This is the graphic unit id, changed from graphic_unit_id to avoid confusion with the graphic unit id in the view. + passed_graphic_unit_id INTEGER, + start_stamp TIMESTAMP, + end_stamp TIMESTAMP, + point_accuracy reading_line_accuracy, + max_raw_points INTEGER, + max_hour_points INTEGER +) + RETURNS TABLE(meter_id INTEGER, reading_rate FLOAT, min_rate FLOAT, max_rate FLOAT, start_timestamp TIMESTAMP, end_timestamp TIMESTAMP) +AS $$ +DECLARE + requested_range TSRANGE; + requested_interval INTERVAL; + requested_interval_seconds INTEGER; + frequency INTERVAL; + frequency_seconds INTEGER; + -- Which index of the meter_id array you are currently working on. + current_meter_index INTEGER := 1; + -- The id of the meter index working on + current_meter_id INTEGER; + -- Holds accuracy for current meter. + current_point_accuracy reading_line_accuracy; + BEGIN + -- For each frequency of points, verify that you will get the minimum graphing points to use for each meter. + -- Start with the raw, then hourly and then daily if others will not work. + -- Loop over all meters. + WHILE current_meter_index <= cardinality(meter_ids) LOOP + -- Reset the point accuracy for each meter so it does what is desired. + current_point_accuracy := point_accuracy; + current_meter_id := meter_ids[current_meter_index]; + -- Make sure the time range is within the reading values for this meter. + -- There may be a better way to create the array with one element as last argument. + requested_range := shrink_tsrange_to_real_readings(tsrange(start_stamp, end_stamp, '[]'), array_append(ARRAY[]::INTEGER[], current_meter_id)); + IF (current_point_accuracy = 'auto'::reading_line_accuracy) THEN + -- The request wants automatic calculation of the points returned. + + -- The request_range will still be infinity if there is no meter data. This causes the + -- auto calculation to fail because you cannot subtract them. + -- Just check the upper range since simpler. + IF (upper(requested_range) = 'infinity') THEN + -- We know there is no data but easier to just let a query happen since fast. + -- Do daily since that should be the fastest due to the least data in most cases. + current_point_accuracy := 'daily'::reading_line_accuracy; + ELSE + -- The interval of time for the requested_range. + requested_interval := upper(requested_range) - lower(requested_range); + -- Get the seconds in the interval. + -- Wanted to use the INTO syntax used above but could not get it to work so using the set syntax. + requested_interval_seconds := (SELECT * FROM EXTRACT(EPOCH FROM requested_interval)); + -- Get the frequency that this meter reads at. + SELECT reading_frequency INTO frequency FROM meters WHERE id = current_meter_id; + -- Get the seconds in the frequency. + frequency_seconds := (SELECT * FROM EXTRACT(EPOCH FROM frequency)); + + -- The first part is making sure that there are no more than maximum raw readings to graph if use raw readings. + -- Divide the time being graphed by the frequency of reading for this meter to get the number of raw readings. + -- The second part checks if the frequency of raw readings is more than a day and use raw if this is the case + -- because even daily would interpolate points. 1 day is 24 hours * 60 minute/hour * 60 seconds/minute = 86400 seconds. + -- This can lead to too many points but do this for now since that is unlikely as you would need around 4+ years of data. + -- Note this overrides the max raw points if it applies. + IF ((requested_interval_seconds / frequency_seconds <= max_raw_points) OR (frequency_seconds >= 86400)) THEN + -- Return raw meter data. + current_point_accuracy := 'raw'::reading_line_accuracy; + -- The first part is making sure that the number of hour points is no more than maximum hourly readings. + -- Thus, check if no more than interval in seconds / (60 seconds/minute * 60 minutes/hour) = # hours in interval. + -- The second part is making sure that the frequency of reading is an hour or less (3600 seconds) + -- so you don't interpolate points by using the hourly data. + ELSIF ((requested_interval_seconds / 3600 <= max_hour_points) AND (frequency_seconds <= 3600)) THEN + -- Return hourly reading data. + current_point_accuracy := 'hourly'::reading_line_accuracy; + ELSE + -- Return daily reading data. + current_point_accuracy := 'daily'::reading_line_accuracy; + END IF; + END IF; + END IF; + -- At this point current_point_accuracy should never be 'auto'. + + IF (current_point_accuracy = 'raw'::reading_line_accuracy) THEN + -- Gets raw meter data to graph. + -- Modified to allow for raw time varying conversions. + RETURN QUERY + SELECT r.meter_id as meter_id, + CASE WHEN u.unit_represent = 'quantity'::unit_represent_type THEN + -- If it is quantity readings then need to convert to rate per hour by dividing by the time length where + -- the 3600 is needed since EPOCH is in seconds. + -- Normalize to rate over reading interval + SUM( + --Wrapped in SUM to handle multiple matching cik_vary conversions + -- Weight by conversion duration(intersection of reading and conversion time ranges is necessary because the conversion may overlap the reading time range) + (EXTRACT(EPOCH FROM ( + upper(tsrange(c.start_time, c.end_time, '()') * tsrange(r.start_timestamp, r.end_timestamp, '[]')) + - + lower(tsrange(c.start_time, c.end_time, '()') * tsrange(r.start_timestamp, r.end_timestamp, '[]')) + )) / 3600) + * (c.slope * (r.reading / (EXTRACT(EPOCH FROM (r.end_timestamp - r.start_timestamp)) / 3600)) + c.intercept) + ) / (EXTRACT(EPOCH FROM (r.end_timestamp - r.start_timestamp)) / 3600) + WHEN (u.unit_represent = 'flow'::unit_represent_type OR u.unit_represent = 'raw'::unit_represent_type) THEN + -- If it is flow or raw readings then it is already a rate so just convert it but also need to normalize + -- to per hour. + SUM( + --Wrapped in SUM to handle multiple matching cik_vary conversions + -- Weight by conversion duration (intersection of reading and conversion time ranges is necessary because the conversion may overlap the reading time range) + (EXTRACT(EPOCH FROM ( + upper(tsrange(c.start_time, c.end_time, '()') * tsrange(r.start_timestamp, r.end_timestamp, '[]')) + - + lower(tsrange(c.start_time, c.end_time, '()') * tsrange(r.start_timestamp, r.end_timestamp, '[]')) + )) / 3600) + * (c.slope * (r.reading * 3600 / u.sec_in_rate) + c.intercept) + ) / (EXTRACT(EPOCH FROM (r.end_timestamp - r.start_timestamp)) / 3600) + END AS reading_rate, + -- There is no range of values on raw/meter data so return NaN to indicate that. + -- The route will return this as null when it shows up in Redux state. + cast('NaN' AS DOUBLE PRECISION) AS min_rate, + cast('NaN' AS DOUBLE PRECISION) as max_rate, + r.start_timestamp, + r.end_timestamp + + FROM (((readings r + INNER JOIN meters m ON m.id = current_meter_id) + INNER JOIN units u ON m.unit_id = u.id) + INNER JOIN cik_vary c on c.source_id = m.unit_id + AND c.destination_id = passed_graphic_unit_id + --The condition below was added for time varying conversions (allows for multiple cik_vary rows to be applied to a single reading) + --The cik_vary exclusive bounds '()' ensures no two conversions overlap. + AND tsrange(c.start_time, c.end_time, '()') && tsrange(r.start_timestamp, r.end_timestamp, '[]')) + WHERE lower(requested_range) <= r.start_timestamp AND r.end_timestamp <= upper(requested_range) AND r.meter_id = current_meter_id + -- Added GROUP BY to allow SUM to aggregate correctly across multiple rows. + -- TODO : postgreSQL doesn't understand unit_represent cannot change for a given meter, so it has to be in group by. Might be worth finding fix. + GROUP BY r.meter_id, r.start_timestamp, r.end_timestamp, u.unit_represent + -- This ensures the data is sorted + ORDER BY r.start_timestamp ASC; + -- The first part is making sure that the number of hour points is 1440 or less. + -- Thus, check if no more than 1440 hours * 60 minutes/hour * 60 seconds/hour = 5184000 seconds. + -- The second part is making sure that the frequency of reading is an hour or less (3600 seconds) + -- so you don't interpolate points by using the hourly data. + ELSIF (current_point_accuracy = 'hourly'::reading_line_accuracy) THEN + -- Get hourly points to graph. See daily for more comments. + -- Now uses materialized view for hourly meter readings. + RETURN QUERY + -- Modified to Retrieve converted hourly readings from the materialized view. + SELECT + hourly.meter_id AS meter_id, + hourly.reading_rate AS reading_rate, + hourly.min_rate AS min_rate, + hourly.max_rate AS max_rate, + lower(hourly.time_interval) AS start_timestamp, + upper(hourly.time_interval) AS end_timestamp + FROM + meter_hourly_readings_unit AS hourly + WHERE + requested_range @> hourly.time_interval + AND hourly.meter_id = current_meter_id + AND hourly.graphic_unit_id = passed_graphic_unit_id + ORDER BY + start_timestamp ASC; + ELSE + RETURN QUERY + -- Modified to retrieve converted daily readings from the materialized view. + SELECT + daily.meter_id AS meter_id, + daily.reading_rate AS reading_rate, + daily.min_rate AS min_rate, + daily.max_rate AS max_rate, + lower(daily.time_interval) AS start_timestamp, + upper(daily.time_interval) AS end_timestamp + FROM + meter_daily_readings_unit AS daily + WHERE + requested_range @> daily.time_interval + AND daily.meter_id = current_meter_id + AND daily.graphic_unit_id = passed_graphic_unit_id + ORDER BY + start_timestamp ASC; + END IF; + current_meter_index := current_meter_index + 1; + END LOOP; +END; +$$ LANGUAGE 'plpgsql'; + + +/* +The following function determines the correct duration view to query from, and returns averaged readings from it. +It is designed to return data for plotting line graphs. It works on groups. +It is the new version of compressed_group_readings_2 that works with units. It takes these parameters: +group_ids: A array of group ids to query. +graphic_unit_id: The unit id of the unit to use for the graph. +start_timestamp: The start timestamp of the data to return. +end_timestamp: The end timestamp of the data to return. +point_accuracy: Tells how decisions should be made on which types of points to return. 'auto' if automatic. +max_hour_points: The maximum number of data points to return if using the hour view. Only used if 'auto'/'raw' for point_accuracy. +Details on how this function works can be found in the devDocs in the resource generalization document and above +in the meter function that is equivalent. + */ +CREATE OR REPLACE FUNCTION group_line_readings_unit ( + group_ids INTEGER[], + requested_graphic_unit_id INTEGER, + start_stamp TIMESTAMP, + end_stamp TIMESTAMP, + point_accuracy reading_line_accuracy, + max_hour_points INTEGER +) + RETURNS TABLE(group_id INTEGER, reading_rate FLOAT, start_timestamp TIMESTAMP, end_timestamp TIMESTAMP) +AS $$ +DECLARE + meter_ids INTEGER[]; + requested_range TSRANGE; + requested_interval INTERVAL; + requested_interval_seconds INTEGER; + meters_min_frequency INTERVAL; + +BEGIN + -- First get all the meter ids that will be included in one or more groups being queried. + -- In case meter is repeated, make this distinct. + SELECT array_agg(DISTINCT gdm.meter_id) INTO meter_ids + FROM groups_deep_meters gdm + INNER JOIN unnest(group_ids) gids(id) ON gdm.group_id = gids.id; + + -- Calculate point accuracy if request (auto) or if raw since that is not allowed for groups. + IF (point_accuracy = 'auto'::reading_line_accuracy OR point_accuracy = 'raw'::reading_line_accuracy) THEN + -- The request needs automatic calculation of the points returned. + + -- Make sure the time range is within the reading values for meters in this group. + requested_range := shrink_tsrange_to_real_readings(tsrange(start_stamp, end_stamp, '[]'), meter_ids); + -- The request_range will still be infinity if there is no meter data. This causes the + -- auto calculation to fail because you cannot subtract them. + -- Just check the upper range since simpler. + IF (upper(requested_range) = 'infinity') THEN + -- We know there is no data but easier to just let a query happen since fast. + -- Do daily since that should be the fastest due to the least data in most cases. + point_accuracy := 'daily'::reading_line_accuracy; + ELSE + -- The interval of time for the requested_range. + requested_interval := upper(requested_range) - lower(requested_range); + -- Get the seconds in the interval. + -- Wanted to use the INTO syntax used above but could not get it to work so using the set syntax. + requested_interval_seconds := (SELECT * FROM EXTRACT(EPOCH FROM requested_interval)); + -- Make sure that the number of hour points is no more than maximum hourly readings. + -- Thus, check if no more than interval in seconds / (60 seconds/minute * 60 minutes/hour) = # hours in interval. + IF (requested_interval_seconds / 3600 <= max_hour_points) THEN + -- Return hourly reading data. + point_accuracy := 'hourly'::reading_line_accuracy; + ELSE + -- Return daily reading data. + point_accuracy := 'daily'::reading_line_accuracy; + END IF; + + -- Groups can require reading interpolation because of multiple meters. For example, if one meter + -- is 30 day reading frequency then it will interpolate to hourly or daily depending other + -- meters (if exist). However, to limit this effect, if hourly has been selected automatically, + -- check if shortest meter reading frequency for this group is more than an hour and then + -- choose daily instead. + IF (point_accuracy = 'hourly'::reading_line_accuracy) THEN + -- Find the min reading frequency for all meters in the group. + SELECT min(reading_frequency) INTO meters_min_frequency + FROM (meters m + INNER JOIN unnest(meter_ids) meters(id) ON m.id = meters.id); + IF (EXTRACT(EPOCH FROM meters_min_frequency) > 3600) THEN + -- The smallest meter frequency is greater than 1 hour (3600 seconds) so use daily instead. + point_accuracy = 'daily'::reading_line_accuracy; + END IF; + END IF; + END IF; + END IF; + -- point_accuracy should either be daily or hourly at this point. + + IF (point_accuracy = 'daily'::reading_line_accuracy) THEN + RETURN QUERY + SELECT + readings.group_id, + readings.reading_rate, + lower(readings.time_interval) AS start_timestamp, + upper(readings.time_interval) AS end_timestamp + FROM group_daily_readings_unit readings + INNER JOIN unnest(group_ids) gids(id) ON readings.group_id = gids.id + WHERE readings.graphic_unit_id = requested_graphic_unit_id + AND tsrange(start_stamp, end_stamp, '[]') @> readings.time_interval + -- This ensures the data is sorted + ORDER BY readings.time_interval ASC; + + ELSIF (point_accuracy = 'hourly'::reading_line_accuracy) THEN + RETURN QUERY + SELECT + readings.group_id AS group_id, + readings.reading_rate AS reading_rate, + lower(readings.time_interval) AS start_timestamp, + upper(readings.time_interval) AS end_timestamp + FROM group_hourly_readings_unit readings + INNER JOIN unnest(group_ids) gids(id) ON readings.group_id = gids.id + WHERE readings.graphic_unit_id = requested_graphic_unit_id + AND tsrange(start_stamp, end_stamp, '[]') @> readings.time_interval + -- This ensures the data is sorted + ORDER BY readings.time_interval ASC; + END IF; +END; +$$ LANGUAGE 'plpgsql'; + + +/* +The following function returns data for plotting bar graphs. It works on meters. +It should not be used on raw readings. +It is the new version of compressed_barchart_readings_2 that works with units. It takes these parameters: +meter_ids: A array of meter ids to query. +graphic_unit_id: The unit id of the unit to use for the graph. +bar_width_days: The number of days to use for the bar width. +start_timestamp: The start timestamp of the data to return. +end_timestamp: The end timestamp of the data to return. + */ +-- New version of meter_bar_readings_unit that uses the new meter_daily_readings_unit view. +CREATE OR REPLACE FUNCTION meter_bar_readings_unit ( + meter_ids INTEGER[], + -- This is the graphic unit id, changed from graphic_unit_id to avoid confusion with the graphic unit id in the view. + passed_graphic_unit_id INTEGER, + bar_width_days INTEGER, + start_stamp TIMESTAMP, + end_stamp TIMESTAMP +) + RETURNS TABLE(meter_id INTEGER, reading FLOAT, start_timestamp TIMESTAMP, end_timestamp TIMESTAMP) +AS $$ +DECLARE + bar_width INTERVAL; + real_tsrange TSRANGE; + real_start_stamp TIMESTAMP; + real_end_stamp TIMESTAMP; + num_bars INTEGER; +BEGIN + -- This is how wide (time interval) for each bar. + bar_width := INTERVAL '1 day' * bar_width_days; + /* + This rounds to the day for the start and end times requested. It then shrinks in case the actual readings span + less time than the request. This can commonly happen when you get +/-infinity for all readings available. + It uses the day reading view because that is faster than using all the readings. + This has an issue associated with it: + + 1) If the readings at the start/end have a partial day then it shows up as a day. The original code did: + real_tsrange := shrink_tsrange_to_real_readings(tsrange(date_trunc_up('day', start_stamp), date_trunc('day', end_stamp))); + and did not have this issue since it used the readings and then truncated up/down. + A more general solution would be to change the daily (and hourly) view so it does not include partial ones at start/end. + This would fix this case and also impact other uses in what seems a positive way. + Note this does not address that missing days in a bar width get no value so the bar will likely read low. + */ + real_tsrange := shrink_tsrange_to_meters_by_day(tsrange(start_stamp, end_stamp), meter_ids); + -- Get the actual start/end time rounded to the nearest day from the range. + real_start_stamp := lower(real_tsrange); + real_end_stamp := upper(real_tsrange); + -- This gives the number of whole bars that will fit within the real start/end times. For example, if the number of days + -- between start and end is 14 days and the bar width is 3 days then you get 4. + num_bars := floor(extract(EPOCH FROM real_end_stamp - real_start_stamp) / extract(EPOCH FROM bar_width)); + -- This makes the full bars go from the end time to as far back in time as possible. + -- This means that if some time was dropped to get full bars it is at the start of the interval. + -- It was felt that the most recent readings are the most important so drop older ones. + -- It also helps with maps since they use the latest bar for their value. + real_start_stamp := real_end_stamp - (num_bars * bar_width); + -- Since the inner join on the generate_series adds the bar_width, we need to back up the + -- end timestamp by that amount so it stops at the desired end timestamp. + real_end_stamp := real_end_stamp - bar_width; + + RETURN QUERY + SELECT + -- Modified to retrieve converted daily readings from the materialized view. + mdr.meter_id AS meter_id, + sum(mdr.reading_rate * 24) AS reading, + bars.interval_start AS start_timestamp, + bars.interval_start + bar_width AS end_timestamp + + FROM meter_daily_readings_unit mdr + INNER JOIN generate_series(real_start_stamp, real_end_stamp, bar_width) bars(interval_start) + ON tsrange(bars.interval_start, bars.interval_start + bar_width, '[]') @> mdr.time_interval + INNER JOIN unnest(meter_ids) meters(id) ON mdr.meter_id = meters.id + INNER JOIN meters m ON m.id = meters.id + INNER JOIN units u ON m.unit_id = u.id AND u.unit_represent != 'raw'::unit_represent_type + WHERE mdr.graphic_unit_id = passed_graphic_unit_id + GROUP BY mdr.meter_id, bars.interval_start; + +END; +$$ LANGUAGE 'plpgsql'; + + +/* +The following function returns data for plotting bar graphs. It works on groups. +It should not be used on raw readings. +It is the new version of compressed_barchart_group_readings_2 that works with units. It takes these parameters: +group_ids: A array of group ids to query. +graphic_unit_id: The unit id of the unit to use for the graph. +bar_width_days: The number of days to use for the bar width. +start_timestamp: The start timestamp of the data to return. +end_timestamp: The end timestamp of the data to return. + */ +CREATE OR REPLACE FUNCTION group_bar_readings_unit ( + group_ids INTEGER[], + requested_graphic_unit_id INTEGER, + bar_width_days INTEGER, + start_stamp TIMESTAMP, + end_stamp TIMESTAMP +) + RETURNS TABLE(group_id INTEGER, reading FLOAT, start_timestamp TIMESTAMP, end_timestamp TIMESTAMP) +AS $$ +DECLARE + bar_width INTERVAL; + real_tsrange TSRANGE; + real_start_stamp TIMESTAMP; + real_end_stamp TIMESTAMP; + num_bars INTEGER; + readings_max_tsrange TSRANGE; +BEGIN + bar_width := INTERVAL '1 day' * bar_width_days; + + SELECT tsrange(min(lower(time_interval)), max(upper(time_interval))) INTO readings_max_tsrange + FROM group_daily_readings_unit dr + -- Get all the group ids passed in. + INNER JOIN unnest(group_ids) gids(id) ON dr.group_id = gids.id; + + real_tsrange := tsrange(date_trunc_up('day', start_stamp), date_trunc('day', end_stamp)) * readings_max_tsrange; + -- Get the actual start/end time rounded to the nearest day from the range. + real_start_stamp := lower(real_tsrange); + real_end_stamp := upper(real_tsrange); + -- This gives the number of whole bars that will fit within the real start/end times. For example, if the number of days + -- between start and end is 14 days and the bar width is 3 days then you get 4. + num_bars := floor(extract(EPOCH FROM real_end_stamp - real_start_stamp) / extract(EPOCH FROM bar_width)); + -- This makes the full bars go from the end time to as far back in time as possible. + -- This means that if some time was dropped to get full bars it is at the start of the interval. + -- It was felt that the most recent readings are the most important so drop older ones. + -- It also helps with maps since they use the latest bar for their value. + real_start_stamp := real_end_stamp - (num_bars * bar_width); + -- Since the inner join on the generate_series adds the bar_width, we need to back up the + -- end timestamp by that amount so it stops at the desired end timestamp. + real_end_stamp := real_end_stamp - bar_width; + + RETURN QUERY + SELECT + -- readings.reading_rate is the weighted average reading rate per hour over the day. + -- Convert to a quantity by multiplying by the time in hours which is 24 since daily values. + -- reading is the sum of all readings within one bar. + readings.group_id AS group_id, + SUM(readings.reading_rate * 24) AS reading, + bars.interval_start AS start_timestamp, + bars.interval_start + bar_width AS end_timestamp + + FROM (((group_daily_readings_unit readings + INNER JOIN generate_series(real_start_stamp, real_end_stamp, bar_width) bars(interval_start) + ON tsrange(bars.interval_start, bars.interval_start + bar_width, '[]') @> readings.time_interval) + -- Don't return bar data if raw since cannot sum. + INNER JOIN units u ON readings.graphic_unit_id = u.id AND u.unit_represent != 'raw'::unit_represent_type) + INNER JOIN unnest(group_ids) gids(id) ON readings.group_id = gids.id) + -- Use the readings in the passed in graphic unit + WHERE readings.graphic_unit_id = requested_graphic_unit_id + + GROUP BY readings.group_id, bars.interval_start; +END; +$$ LANGUAGE 'plpgsql'; diff --git a/src/server/migrations/2.0.0-3.0.0/sql/readings/drop_old_views.sql b/src/server/migrations/2.0.0-3.0.0/sql/readings/drop_old_views.sql new file mode 100644 index 0000000000..c396a9a068 --- /dev/null +++ b/src/server/migrations/2.0.0-3.0.0/sql/readings/drop_old_views.sql @@ -0,0 +1,14 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +--Remove old materialized views and index that are no longer used. +DROP MATERIALIZED VIEW IF EXISTS daily_readings_unit; +DROP MATERIALIZED VIEW IF EXISTS hourly_readings_unit; +DROP INDEX IF EXISTS idx_daily_readings_unit; + +--Dropping group views since they will be recreated; +DROP MATERIALIZED VIEW IF EXISTS group_daily_readings_unit; +DROP MATERIALIZED VIEW IF EXISTS group_hourly_readings_unit; +DROP INDEX IF EXISTS idx_group_daily_readings_unit; +DROP INDEX IF EXISTS idx_group_hourly_readings_unit; \ No newline at end of file diff --git a/src/server/models/Cik.js b/src/server/models/Cik.js index bad52ec031..c0cbcebc28 100644 --- a/src/server/models/Cik.js +++ b/src/server/models/Cik.js @@ -8,20 +8,15 @@ const sqlFile = database.sqlFile; /** * Represents the Cik conversion model. * @see src/server/services/graph/createConversionArrays.js for details on Cik array. - * [0]: is slope, [1]: is intercept, [2]: is not used here. */ class Cik { /** * @param {*} meterUnitId The id of the meter unit. * @param {*} nonMeterUnitId The id of the non meter unit. - * @param {*} slope The slope of the conversion. - * @param {*} intercept The intercept of the conversion. */ constructor(meterUnitId, nonMeterUnitId, slope, intercept) { this.meterUnitId = meterUnitId; this.nonMeterUnitId = nonMeterUnitId; - this.slope = slope; - this.intercept = intercept; } /** @@ -39,7 +34,7 @@ class Cik { * @returns the created Cik object */ static mapRow(row) { - return new Cik(row.meter_unit_id, row.non_meter_unit_id, row.slope, row.intercept); + return new Cik(row.meter_unit_id, row.non_meter_unit_id); } /** @@ -51,29 +46,6 @@ class Cik { const rows = await conn.any(sqlFile('cik/get_cik.sql')); return rows.map(Cik.mapRow); } - - /** - * Inserts each element of the array with an actual conversion into the cik table. - * The current values in the table are removed first. - * @param {*} cik is the OED conversion array from the graph. - * @param {*} conn The database connection to use. - */ - static async insert(cik, conn) { - // TODO This should be a transaction to avoid issues for any request made to the database. - - // Remove all the current values in the table. - await conn.none(sqlFile('cik/delete_all_conversions.sql')); - - // Loop over all conversions in cik array and insert each in DB. - cik.forEach(async (conversion) => { - await conn.none(sqlFile('cik/insert_new_cik.sql'), { - sourceId: conversion.source, - destinationId: conversion.destination, - slope: conversion.slope, - intercept: conversion.intercept - }); - }); - } } module.exports = Cik; diff --git a/src/server/models/CikVary.js b/src/server/models/CikVary.js index f4ebdc9370..20f5cf678a 100644 --- a/src/server/models/CikVary.js +++ b/src/server/models/CikVary.js @@ -70,6 +70,9 @@ class CikVary { * @param {*} queryTime Timestamp to check validity. * @returns Matching CikVary objects */ + // TODO: Research whether this lookup is used by external consumers. It has + // no in-repository callers and references a misspelled/nonexistent SQL path; + // remove the method and its unused SQL file if no supported caller needs it. static async getBySourceDestinationStartEnd(conn, sourceId, destinationId, queryTime) { const rows = await conn.any(sqlFile('cik_vary/get_cik_vart_by_source_destination_start_end.sql'), { sourceId, @@ -86,21 +89,68 @@ class CikVary { * @param {*} conn The database connection to use. */ static async insert(cikVaryArr, conn) { - return conn.tx(async t => { + return conn.tx(async t => { + // cik_vary // Remove all the current values in the table. await t.none(sqlFile('cik_vary/delete_all_cik_vary.sql')); - - // Loop over all conversions in array and insert each in DB. - for (const conversion of cikVaryArr) { - await t.none(sqlFile('cik_vary/insert_new_cik_vary.sql'), { - sourceId: conversion.source, - destinationId: conversion.destination, - startTime: conversion.start_time, - endTime: conversion.end_time, + // Insert all time-varying conversions in one database statement. + // jsonb_to_recordset preserves the existing typed insert behavior while + // avoiding one application/database round trip per conversion segment. + await t.none(` + INSERT INTO cik_vary ( + source_id, + destination_id, + start_time, + end_time, + slope, + intercept + ) + SELECT + conversion.source_id, + conversion.destination_id, + conversion.start_time, + conversion.end_time, + conversion.slope, + conversion.intercept + FROM jsonb_to_recordset(\${conversions:json}::jsonb) AS conversion( + source_id INTEGER, + destination_id INTEGER, + start_time TIMESTAMP, + end_time TIMESTAMP, + slope FLOAT, + intercept FLOAT + ) + `, { + conversions: cikVaryArr.map(conversion => ({ + source_id: conversion.source, + destination_id: conversion.destination, + start_time: conversion.start_time, + end_time: conversion.end_time, slope: conversion.slope, intercept: conversion.intercept - }); - } + })) + }); + + // cik + // Remove all the current values in the table. + await t.none(sqlFile('cik/delete_all_cik.sql')); + // The following finds each unique (by source/i and destination/k) entry in cik_vary and then + // inserts and entry in cik. Done as one sql call to be more efficient. + // It is also possible to create the needed information during createCikVaryArray for each source + // and destination. That might be a little more efficient but this way is simple and guarantees + // that cik_vary and cik represent the same information. + await t.none(sqlFile('cik/insert_unique_cik_vary_in_cik.sql')); + + // Existing hourly split rows retain the conversion metadata that was + // current when they were created. Group graphic-unit compatibility also + // depends on cik. Mark both derived datasets stale in the same transaction + // as the cik_vary and cik replacement. + await t.none(` + UPDATE reading_aggregate_state + SET rebuild_revision = rebuild_revision + 1, + group_cache_revision = group_cache_revision + 1 + WHERE id = 1 + `); }); } } diff --git a/src/server/models/ConversionSegment.js b/src/server/models/ConversionSegment.js index 2c6104e82e..9ffc2ba835 100644 --- a/src/server/models/ConversionSegment.js +++ b/src/server/models/ConversionSegment.js @@ -55,6 +55,18 @@ class ConversionSegment { row.note); } + /** + * Retrieves all conversion segments in edge and chronological order. + * Used when rebuilding derived conversion tables so metadata is loaded once + * instead of queried again for every path edge. + * @param conn the connection to use + * @returns {Promise>} + */ + static async getAll(conn) { + const rows = await conn.any(sqlFile('conversionSegment/get_all.sql')); + return rows.map(ConversionSegment.mapRow); + } + /** * Returns a promise to get all conversion segments with the given source id and destination id from the database. * If the conversion segment doesn't exist then return null. @@ -64,7 +76,7 @@ class ConversionSegment { * @returns {Promise.} */ static async getBySourceDestination(sourceId, destinationId, conn) { - const rows = await conn.many(sqlFile('conversionSegment/get_by_source_destination.sql'), { + const rows = await conn.any(sqlFile('conversionSegment/get_by_source_destination.sql'), { sourceId: sourceId, destinationId: destinationId }); @@ -330,4 +342,4 @@ class ConversionSegment { } -module.exports = ConversionSegment; \ No newline at end of file +module.exports = ConversionSegment; diff --git a/src/server/models/Group.js b/src/server/models/Group.js index fffb7225c3..7db2ccd7a2 100644 --- a/src/server/models/Group.js +++ b/src/server/models/Group.js @@ -116,6 +116,19 @@ class Group { return rows.map(Group.mapRow); } + /** + * Retrieves all groups and their cached deep-meter IDs in one query. + * @param conn the connection to be used + * @returns {Promise>} + */ + static async getAllWithDeepMeters(conn) { + const rows = await conn.any(sqlFile('group/get_all_groups_with_deep_meters.sql')); + return rows.map(row => ({ + group: Group.mapRow(row), + deepMeters: row.deep_meters + })); + } + /** * Returns a promise to retrive all groups that displayable is equal to true. * @param {*} conn The connection to be used @@ -173,6 +186,9 @@ class Group { * Removes first array entry if only one and null * @param {[]} array array to remove null */ + // TODO: Research whether any external/custom query can still return [null] + // here. get_all_children.sql now returns empty arrays, so this method and its + // calls in getImmediateChildren can likely be removed. static purgeNull(array) { if (array.length === 1 && array[0] === null) { // Length 1 and only item null so remove from array. @@ -226,13 +242,20 @@ class Group { } /** - * Refreshes the groups deep meters view. + * Refreshes the groups deep meters and graphic-unit caches. * Should be called whenever a group's child meters or groups are edited. * @param conn The connection to use * @returns {Promise} */ static refreshGroupsDeepMetersView(conn) { - return conn.none('REFRESH MATERIALIZED VIEW groups_deep_meters'); + return conn.none(` + DO $$ + BEGIN + PERFORM update_groups_deep_meters_cache(); + PERFORM update_group_graphic_units_cache(); + END + $$; + `); } /** diff --git a/src/server/models/Reading.js b/src/server/models/Reading.js index 822b635da7..ce48f922ef 100644 --- a/src/server/models/Reading.js +++ b/src/server/models/Reading.js @@ -9,6 +9,10 @@ const log = require('../log'); const sqlFile = database.sqlFile; +// Keep each statement bounded so large CSV and meter imports avoid both +// per-reading round trips and excessively large PostgreSQL query parameters. +const READING_INSERT_BATCH_SIZE = 1000; + class Reading { /** * Creates a new reading @@ -34,18 +38,17 @@ class Reading { } /** - * Returns a promise to create the function and materialized views that aggregate - * readings by various time intervals. - * @param conn the database connection to use - * @returns {Promise} + * @deprecated Retained for the legacy PostgreSQL materialized-view schema. */ + // TODO: Research whether deployments or external scripts still invoke the + // legacy schema helpers below. Their in-repository setup calls are commented + // out; remove the helpers and legacy SQL once TimescaleDB is the only schema. static createReadingsMaterializedViews(conn) { return conn.none(sqlFile('reading/create_reading_views.sql')); } /** - * Returns a promise to create the compare function - * @param conn the database connection to use + * @deprecated Retained for the legacy PostgreSQL materialized-view schema. */ static createCompareReadingsFunction(conn) { return conn.none(sqlFile('reading/create_function_get_compare_readings.sql')); @@ -62,63 +65,56 @@ class Reading { } /** - * Returns a promise to create the 3D readings function - * @param conn the database connection to use + * @deprecated Retained for the legacy PostgreSQL materialized-view schema. */ static create3DReadingsFunction(conn) { return conn.none(sqlFile('reading/create_function_get_3d_readings.sql')); } /** - * Refreshes the hourly readings view. - * Should be called at least once a day but need to do hourly if the site wants zooming in - * to see hourly data as it is available. This function can take more time than refreshing - * the daily readings so be sure calling it more frequently does not impact the - * server response time. If only called once a day, then probably best to do so in the middle - * of the night as suggested for daily refresh. - * @param conn The connection to use - * @returns {Promise} + * @deprecated Use TimeScaleDBReading.refreshReadings(). */ static refreshHourlyReadings(conn) { - // This can't be a function because you can't call REFRESH inside a function - // TODO This will be removed once we completely transition to the unit version. - return conn.none('REFRESH MATERIALIZED VIEW hourly_readings_unit'); + // TODO: Remove the retained legacy implementation once the hypertable implementation is finalized: + // return conn.none('REFRESH MATERIALIZED VIEW meter_hourly_readings_unit'); + // Required lazily to avoid a circular dependency through database.js. + // eslint-disable-next-line global-require + return require('./TimeScaleDB/Reading').refreshMeterHourlyReadings(conn); } /** - * Refreshes the daily readings view. - * Should be called at least once a day, preferably in the middle of the night. - * @param conn The connection to use - * @returns {Promise} + * @deprecated Use TimeScaleDBReading.refreshReadings(). */ static refreshDailyReadings(conn) { - // This can't be a function because you can't call REFRESH inside a function - return conn.none('REFRESH MATERIALIZED VIEW daily_readings_unit'); + // TODO: Remove the retained legacy implementation once the hypertable implementation is finalized: + // return conn.none('REFRESH MATERIALIZED VIEW meter_daily_readings_unit'); + // Required lazily to avoid a circular dependency through database.js. + // eslint-disable-next-line global-require + return require('./TimeScaleDB/Reading').refreshMeterDailyReadings(conn); } - + /** - * Refreshes meter readings views. - * Should be called at least once a day, preferably in the middle of the night. - * @param conn The connection to use - * @returns {Promise} + * @deprecated Use TimeScaleDBReading.refreshReadings(). */ - static async refreshMeterReadingsViews(conn) { - await conn.none('REFRESH MATERIALIZED VIEW hourly_readings_unit'); - await conn.none('REFRESH MATERIALIZED VIEW daily_readings_unit'); + static refreshMeterReadingsViews(conn) { + // TODO: Remove the retained legacy refreshes once the hypertable implementation is finalized: + // await conn.none('REFRESH MATERIALIZED VIEW meter_hourly_readings_unit'); + // await conn.none('REFRESH MATERIALIZED VIEW meter_daily_readings_unit'); + // Required lazily to avoid a circular dependency through database.js. + // eslint-disable-next-line global-require + return require('./TimeScaleDB/Reading').refreshMeterReadings(conn); } - /** - * Refreshes group readings views. - * Should be called at least once a day, preferably in the middle of the night. - * @param conn The connection to use - * @returns {Promise} + * @deprecated Use TimeScaleDBReading.refreshReadings(). */ static refreshGroupReadingsViews(conn) { - // It is safe to refresh the hourly and daily group views in parallel since they - // do not depend one each other unlike meters. - return Promise.all([conn.none('REFRESH MATERIALIZED VIEW group_hourly_readings_unit'), - conn.none('REFRESH MATERIALIZED VIEW group_daily_readings_unit')]); + // TODO: Remove the retained legacy refreshes once the hypertable implementation is finalized: + // return Promise.all([conn.none('REFRESH MATERIALIZED VIEW group_hourly_readings_unit'), + // conn.none('REFRESH MATERIALIZED VIEW group_daily_readings_unit')]); + // Required lazily to avoid a circular dependency through database.js. + // eslint-disable-next-line global-require + return require('./TimeScaleDB/Reading').refreshGroupReadings(conn); } /** @@ -157,10 +153,7 @@ class Reading { * @returns {Promise.<>} */ static insertAll(readings, conn) { - return conn.tx(t => t.sequence(function seq(i) { - const seqT = this; - return readings[i] && readings[i].insert(seqT); - })); + return Reading.insertInBatches(readings, conn); } /** @@ -170,10 +163,32 @@ class Reading { * @returns {Promise.<>} */ static insertOrUpdateAll(readings, conn) { - return conn.tx(t => t.sequence(function seq(i) { - const seqT = this; - return readings[i] && readings[i].insertOrUpdate(seqT); - })); + /* + * Sequential upserts kept the first end timestamp but applied the last + * reading value when an input batch repeated a meter/start key. Collapse + * those duplicates before the set-based insert to preserve that behavior + * and avoid PostgreSQL updating the same row twice in one statement. + */ + const readingsByKey = new Map(); + for (const reading of readings) { + const key = `${reading.meterID}:${reading.startTimestamp.valueOf()}`; + const firstReading = readingsByKey.get(key); + if (firstReading === undefined) { + readingsByKey.set(key, reading); + } else { + readingsByKey.set(key, new Reading( + firstReading.meterID, + reading.reading, + firstReading.startTimestamp, + firstReading.endTimestamp + )); + } + } + return Reading.insertInBatches( + Array.from(readingsByKey.values()), + conn, + 'ON CONFLICT (meter_id, start_timestamp) DO UPDATE SET reading = EXCLUDED.reading' + ); } /** @@ -183,10 +198,53 @@ class Reading { * @returns {Promise} */ static insertOrIgnoreAll(readings, conn) { - return conn.tx(t => t.sequence(function seq(i) { - const seqT = this; - return readings[i] && readings[i].insertOrIgnore(seqT); - })); + return Reading.insertInBatches( + readings, + conn, + 'ON CONFLICT (meter_id, start_timestamp) DO NOTHING' + ); + } + + /** + * Inserts readings in set-based batches within one transaction. + * @param {array} readings the readings to insert + * @param conn the connection to use + * @param {string} conflictClause fixed conflict behavior for the caller + * @returns {Promise} + */ + static insertInBatches(readings, conn, conflictClause = '') { + return conn.tx(async t => { + for (let offset = 0; offset < readings.length; offset += READING_INSERT_BATCH_SIZE) { + const batch = readings.slice(offset, offset + READING_INSERT_BATCH_SIZE); + /* + * jsonb_to_recordset turns each chunk into typed rows inside + * PostgreSQL, replacing one application/database round trip per + * reading while still firing the existing maintenance trigger. + */ + await t.none(` + INSERT INTO readings (meter_id, reading, start_timestamp, end_timestamp) + SELECT + input.meter_id, + input.reading, + input.start_timestamp, + input.end_timestamp + FROM jsonb_to_recordset(\${readings:json}::jsonb) AS input( + meter_id INTEGER, + reading FLOAT, + start_timestamp TIMESTAMP, + end_timestamp TIMESTAMP + ) + ${conflictClause} + `, { + readings: batch.map(reading => ({ + meter_id: reading.meterID, + reading: reading.reading, + start_timestamp: reading.startTimestamp, + end_timestamp: reading.endTimestamp + })) + }); + } + }); } /** @@ -203,6 +261,23 @@ class Reading { return parseInt(row[0].count); } + /** + * Returns the total number of readings for all supplied meters in one query. + * @param {number[]} meterIDs meter IDs whose readings should be counted + * @param startDate inclusive reading start bound + * @param endDate inclusive reading end bound + * @param conn the connection to use + * @returns {number} + */ + static async getCountByMeterIDsAndDateRange(meterIDs, startDate, endDate, conn) { + const { count } = await conn.one(sqlFile('reading/get_count_by_meter_ids_and_date_range.sql'), { + meterIDs, + startDate, + endDate + }); + return parseInt(count); + } + /** * Returns a promise to get all of the readings for this meter from the database. * @param meterID The id of the meter to find readings for diff --git a/src/server/models/TimeScaleDB/Reading.js b/src/server/models/TimeScaleDB/Reading.js new file mode 100644 index 0000000000..a319c5f84f --- /dev/null +++ b/src/server/models/TimeScaleDB/Reading.js @@ -0,0 +1,316 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +const database = require('../database'); +const moment = require('moment'); + +const sqlFile = database.sqlFile; + +class Reading { + + /** + * Creates shared reading functions required by the TimescaleDB schema and + * graphing functions. + * + * @param conn the database connection to use + * @returns {Promise} + */ + static createReadingHelpers(conn) { + return conn.none(sqlFile('reading/TimeScaleDB/update_reading_views.sql')); + } + + /** + * Creates the TimescaleDB prerequisite objects required by the hourly and + * daily continuous aggregates. + * + * This creates: + * - hypertable_hourly_split + * - supporting indexes + * - trigger function + * - trigger + * - rebuild function + * + * @param conn the database connection to use + * @returns {Promise} + */ + static createPrerequisites(conn) { + return conn.none(sqlFile('reading/TimeScaleDB/create_prerequisites.sql')); + } + + /** + * Creates the group-level dependency objects required by the group hourly + * and daily continuous aggregates. + * + * This creates: + * - groups_deep_meters_cache table + * - group_graphic_units_cache table + * - supporting indexes + * + * The group continuous aggregates cannot depend on dynamic joins to: + * - groups_deep_meters_cache maintenance logic + * - legacy graphic-unit compatibility function + * + * Therefore, these objects are maintained as physical tables that can be + * refreshed before refreshing the group continuous aggregates. + * + * @param conn the database connection to use + * @returns {Promise} + */ + static createGroupDependencies(conn) { + return conn.none(sqlFile('reading/TimeScaleDB/create_group_dependencies.sql')); + } + + /** + * Creates the TimescaleDB continuous aggregate used for hourly meter + * readings. + * + * @param conn the database connection to use + * @returns {Promise} + */ + static createHourlyReadings(conn) { + return conn.none(sqlFile('reading/TimeScaleDB/create_hourly_readings.sql')); + } + + /** + * Creates the group hourly materialized view over the TimescaleDB meter + * aggregate. + * + * @param conn the database connection to use + * @returns {Promise} + */ + static createGroupHourlyReadings(conn) { + return conn.none(sqlFile('reading/TimeScaleDB/create_group_hourly_readings.sql')); + } + + /** + * Creates the group daily materialized view over the TimescaleDB meter + * aggregate. + * + * @param conn the database connection to use + * @returns {Promise} + */ + static createGroupDailyReadings(conn) { + return conn.none(sqlFile('reading/TimeScaleDB/create_group_daily_readings.sql')); + } + + /** + * Creates the TimescaleDB continuous aggregate used for daily meter + * readings. + * + * @param conn the database connection to use + * @returns {Promise} + */ + static createDailyReadings(conn) { + return conn.none(sqlFile('reading/TimeScaleDB/create_daily_readings.sql')); + } + + /** + * Updates meter_line_readings_unit() to use the TimescaleDB continuous + * aggregates for hourly and daily queries. + * + * @param conn the database connection to use + * @returns {Promise} + */ + static updateMeterLineReadings(conn) { + return conn.none(sqlFile('reading/TimeScaleDB/update_meter_line_readings_unit.sql')); + } + + /** + * Updates group_line_readings_unit() to use the group materialized views + * backed by TimescaleDB meter aggregates for hourly and daily queries. + * + * @param conn the database connection to use + * @returns {Promise} + */ + static updateGroupLineReadings(conn) { + return conn.none(sqlFile('reading/TimeScaleDB/update_group_line_readings_unit.sql')); + } + + /** + * Updates meter_bar_readings_unit() and group_bar_readings_unit() to use the group materialized views + * backed by TimescaleDB meter aggregates for group_daily_readings_unit_cagg. + * + * @param conn the database connection to use + * @returns {Promise} + */ + static updateMeterGroupBar(conn) { + return conn.none(sqlFile('reading/TimeScaleDB/update_meter_group_bar.sql')); + } + + /** + * Updates meter_compare_readings_unit() and group_compare_readings_unit() to use the group + * materialized views backed by TimescaleDB meter aggregates for group_daily_readings_unit_cagg. + * + * @param conn the database connection to use + * @returns {Promise} + */ + static updateCompareReadings(conn) { + return conn.none(sqlFile('reading/TimeScaleDB/update_function_get_compare_readings.sql')); + } + + /** + * Updates the 3D reading functions to use TimescaleDB continuous + * aggregates. + * + * @param conn the database connection to use + * @returns {Promise} + */ + static updateFunctionGet3DReadings(conn) { + return conn.none(sqlFile('reading/TimeScaleDB/update_function_get_3d_readings.sql')); + } + + /** + * TODO: Remove this function once the hypertable implementation is finalized. + * Removes legacy PostgreSQL materialized reading views after their + * TimescaleDB replacements and all dependent functions are installed. + * + * @param conn the database connection to use + * @returns {Promise} + */ + static dropLegacyReadingViews(conn) { + return conn.none(sqlFile('reading/TimeScaleDB/drop_legacy_reading_views.sql')); + } + + /** + * Refreshes the TimescaleDB continuous aggregates. + * + * This should be called after importing or modifying readings, especially + * when historical data may have been inserted or updated. + * + * @param conn the database connection to use + * @returns {Promise} + */ + static async rebuildReadings(conn) { + // Rebuild the hypertable in case conversion metadata (cik_vary) + // has changed or historical readings were imported. + await conn.any('SELECT rebuild_hourly_hypertable_split()'); + + await Reading.refreshReadings(conn); + } + + /** + * Validates and expands an optional refresh range to UTC bucket boundaries. + * TimescaleDB refreshes only complete buckets, so a partial final bucket + * must expand to the start of the following bucket. + * + * @param startTimestamp inclusive start of the affected range + * @param endTimestamp exclusive end of the affected range + * @param bucketPrecision moment-supported bucket precision + * @returns {{refreshStart: moment.Moment|null, refreshEnd: moment.Moment|null}} + */ + static getRefreshRange(startTimestamp = null, endTimestamp = null, bucketPrecision = 'day') { + if ((startTimestamp === null) !== (endTimestamp === null)) { + throw new Error('Both startTimestamp and endTimestamp are required for a bounded TimescaleDB refresh.'); + } + + const refreshStart = startTimestamp === null ? null : moment.utc(startTimestamp).startOf(bucketPrecision); + const refreshEnd = endTimestamp === null ? null : moment.utc(endTimestamp).startOf(bucketPrecision); + if (refreshEnd !== null && !moment.utc(endTimestamp).isSame(refreshEnd)) { + refreshEnd.add(1, bucketPrecision); + } + + return { refreshStart, refreshEnd }; + } + + /** + * Refreshes only the hourly meter continuous aggregate. + */ + static async refreshMeterHourlyReadings(conn, startTimestamp = null, endTimestamp = null) { + const { refreshStart, refreshEnd } = Reading.getRefreshRange(startTimestamp, endTimestamp, 'hour'); + await conn.none( + "CALL refresh_continuous_aggregate('meter_hourly_readings_unit_cagg', ${startTimestamp}, ${endTimestamp})", + { startTimestamp: refreshStart, endTimestamp: refreshEnd } + ); + } + + /** + * Refreshes only the daily meter continuous aggregate. + */ + static async refreshMeterDailyReadings(conn, startTimestamp = null, endTimestamp = null) { + const { refreshStart, refreshEnd } = Reading.getRefreshRange(startTimestamp, endTimestamp); + await conn.none( + "CALL refresh_continuous_aggregate('meter_daily_readings_unit_cagg', ${startTimestamp}, ${endTimestamp})", + { startTimestamp: refreshStart, endTimestamp: refreshEnd } + ); + } + + /** + * Refreshes the hourly and daily meter continuous aggregates. + */ + static async refreshMeterReadings(conn, startTimestamp = null, endTimestamp = null) { + await Reading.refreshMeterHourlyReadings(conn, startTimestamp, endTimestamp); + await Reading.refreshMeterDailyReadings(conn, startTimestamp, endTimestamp); + } + + /** + * Refreshes the group caches and group continuous aggregates. + */ + static async refreshGroupReadings(conn, startTimestamp = null, endTimestamp = null) { + // Group caches depend on relatively infrequent membership, meter-unit, + // and conversion changes. Reading-only imports can skip this work. + const groupCacheState = await conn.one(` + SELECT group_cache_revision, completed_group_cache_revision + FROM reading_aggregate_state + WHERE id = 1 + `); + if (BigInt(groupCacheState.group_cache_revision) + > BigInt(groupCacheState.completed_group_cache_revision)) { + await conn.none(` + DO $$ + BEGIN + PERFORM update_groups_deep_meters_cache(); + PERFORM update_group_graphic_units_cache(); + END + $$; + `); + // Record only the revision observed before the refresh. If another + // source change commits concurrently, its newer revision remains + // pending and the next refresh will update the caches again. + await conn.none(` + UPDATE reading_aggregate_state + SET completed_group_cache_revision = GREATEST( + completed_group_cache_revision, + \${refreshedRevision} + ) + WHERE id = 1 + `, { refreshedRevision: groupCacheState.group_cache_revision }); + } + + // Hourly and daily CAGGs use different bucket widths. Give each the + // smallest complete refresh window that covers the changed readings. + const hourlyRange = Reading.getRefreshRange(startTimestamp, endTimestamp, 'hour'); + await conn.none( + "CALL refresh_continuous_aggregate('group_hourly_readings_unit_cagg', ${startTimestamp}, ${endTimestamp})", + { startTimestamp: hourlyRange.refreshStart, endTimestamp: hourlyRange.refreshEnd } + ); + + const dailyRange = Reading.getRefreshRange(startTimestamp, endTimestamp, 'day'); + await conn.none( + "CALL refresh_continuous_aggregate('group_daily_readings_unit_cagg', ${startTimestamp}, ${endTimestamp})", + { startTimestamp: dailyRange.refreshStart, endTimestamp: dailyRange.refreshEnd } + ); + } + + /** + * Refreshes the TimescaleDB continuous aggregates for an optional time + * range. Omitting the range refreshes all materialized data. + * + * Hourly must be refreshed before daily because the daily continuous + * aggregate depends on the hourly continuous aggregate. + * + * @param conn the database connection to use + * @param startTimestamp inclusive start of the affected range + * @param endTimestamp exclusive end of the affected range + * @returns {Promise} + */ + static async refreshReadings(conn, startTimestamp = null, endTimestamp = null) { + await Reading.refreshMeterReadings(conn, startTimestamp, endTimestamp); + await Reading.refreshGroupReadings(conn, startTimestamp, endTimestamp); + } + +} + + +module.exports = Reading; diff --git a/src/server/models/database.js b/src/server/models/database.js index d46e91a188..3d7825f45c 100644 --- a/src/server/models/database.js +++ b/src/server/models/database.js @@ -31,6 +31,9 @@ function getDB(connectionParameters) { * Get the name of the database current being worked on. * @returns {string} */ +// TODO: Research whether any downstream consumers use the exported currentDB +// accessor. There are no in-repository callers and no currentDB backing value; +// if external callers do not rely on it, remove this function and its export. function getCurrentDB() { return currentDB; } @@ -90,6 +93,8 @@ async function createSchema(conn) { const Week = require('./Week'); const Cik = require('./Cik'); const CikVary = require('./CikVary'); + // TimescaleDB + const TimeScaleDBReading = require('./TimeScaleDB/Reading'); /* eslint-enable global-require */ await Unit.createUnitTypesEnum(conn); @@ -120,14 +125,29 @@ async function createSchema(conn) { await LogEmail.createTable(conn); await LogMsg.createLogMsgTypeEnum(conn); await LogMsg.createTable(conn); - await Reading.createReadingsMaterializedViews(conn); - await Reading.createCompareReadingsFunction(conn); - // For 3D reading - await Reading.create3DReadingsFunction(conn); + // TODO: Remove these retained legacy setup calls once the hypertable implementation is finalized. + // await Reading.createReadingsMaterializedViews(conn); + // await Reading.createCompareReadingsFunction(conn); + // await Reading.create3DReadingsFunction(conn); await Baseline.createTable(conn); await Map.createTable(conn); await conn.none(sqlFile('baseline/create_function_get_average_reading.sql')); await Configfile.createTable(conn); + // Create the TimescaleDB continuous aggregate view for readings + await TimeScaleDBReading.createReadingHelpers(conn); + await TimeScaleDBReading.createPrerequisites(conn); + await TimeScaleDBReading.createGroupDependencies(conn); + await TimeScaleDBReading.createHourlyReadings(conn); + await TimeScaleDBReading.createDailyReadings(conn); + await TimeScaleDBReading.createGroupHourlyReadings(conn); + await TimeScaleDBReading.createGroupDailyReadings(conn); + await TimeScaleDBReading.updateMeterLineReadings(conn); + await TimeScaleDBReading.updateGroupLineReadings(conn); + await TimeScaleDBReading.updateMeterGroupBar(conn); + await TimeScaleDBReading.updateCompareReadings(conn); + await TimeScaleDBReading.updateFunctionGet3DReadings(conn); + //TODO: Remove these retained legacy setup calls once the hypertable implementation is finalized. + // await TimeScaleDBReading.dropLegacyReadingViews(conn); } module.exports = { diff --git a/src/server/routes/ciks.js b/src/server/routes/ciks.js index 5eabee2f83..8dccf68eb5 100644 --- a/src/server/routes/ciks.js +++ b/src/server/routes/ciks.js @@ -15,8 +15,6 @@ function formatCikForResponse(item) { return { meterUnitId: item.meterUnitId, nonMeterUnitId: item.nonMeterUnitId, - slope: item.slope, - intercept: item.intercept } } @@ -31,4 +29,4 @@ router.get('/', async (req, res) => { } catch (err) { log.error(`Error while performing GET ciks details query: ${err}`); } -}); \ No newline at end of file +}); diff --git a/src/server/routes/conversionArray.js b/src/server/routes/conversionArray.js index 9b8f0eb6d0..f5ede58c05 100644 --- a/src/server/routes/conversionArray.js +++ b/src/server/routes/conversionArray.js @@ -4,8 +4,8 @@ const express = require('express'); const { getConnection } = require('../db'); -const { redoCik } = require('../services/graph/redoCik'); -const { refreshAllReadingViews } = require('../services/refreshAllReadingViews'); +const { redoCikVary } = require('../services/graph/redoCik'); +const refreshAllReadingViews = require('../services/refreshAllReadingViews'); const { adminAuthMiddleware } = require('./authenticator'); const router = express.Router(); @@ -16,7 +16,7 @@ const router = express.Router(); router.post('/refresh', adminAuthMiddleware('conversion refresh system data'), async (req, res) => { if (req.body.redoCik) { const conn = getConnection(); - await redoCik(conn); + await redoCikVary(conn); } if (req.body.refreshReadingViews) { await refreshAllReadingViews(); diff --git a/src/server/routes/csv.js b/src/server/routes/csv.js index df4701b150..c432b54b65 100644 --- a/src/server/routes/csv.js +++ b/src/server/routes/csv.js @@ -21,7 +21,7 @@ const saveCsv = require('../services/csvPipeline/saveCsv'); const uploadMeters = require('../services/csvPipeline/uploadMeters'); const uploadReadings = require('../services/csvPipeline/uploadReadings'); const zlib = require('zlib'); -const { refreshAllReadingViews } = require('../services/refreshAllReadingViews'); +const refreshAllReadingViews = require('../services/refreshAllReadingViews'); const { success, failure } = require('../services/csvPipeline/success'); /** Middleware validation */ @@ -159,6 +159,8 @@ router.post('/readings', validateReadingsCsvUploadParams, async (req, res) => { let csvFilepath; let isAllReadingsOk; let msgTotal; + let startTimestamp; + let endTimestamp; try { log.info(`The uploaded file ${uploadedFilepath} was created to upload readings csv data`); let fileBuffer = await fs.readFile(uploadedFilepath); @@ -174,10 +176,17 @@ router.post('/readings', validateReadingsCsvUploadParams, async (req, res) => { csvFilepath = uploadedFilepath; } const conn = getConnection(); - ({ isAllReadingsOk, msgTotal } = await uploadReadings(req, res, csvFilepath, conn)); + ({ isAllReadingsOk, msgTotal, startTimestamp, endTimestamp } = await uploadReadings(req, res, csvFilepath, conn)); if (isRefreshReadings) { // Refresh readings so show when daily data is used. - await refreshAllReadingViews(); + // can also not provide the startTimestamp and endTimestamp to refresh all readings. + // The idea of including the startTimestamp and endTimestamp is to refresh only the readings + // that were just inserted or updated. However, if the user does not provide these timestamps, + // then the refresh will take longer to complete for dataset that contains large number of buckets + // as all the bucket will be checked for update bit. + await refreshAllReadingViews(startTimestamp && endTimestamp + ? { startTimestamp, endTimestamp, rebuild: false } + : undefined); } } catch (error) { failure(req, res, error); diff --git a/src/server/routes/groups.js b/src/server/routes/groups.js index f1372f2159..a2ca76a85a 100644 --- a/src/server/routes/groups.js +++ b/src/server/routes/groups.js @@ -52,17 +52,12 @@ function formatToOnlyNameID(item) { router.get('/', optionalAuthMiddleware, async (req, res) => { const conn = getConnection(); try { - const rows = await Group.getAll(conn); - deepChildren = []; - promises = await rows.map(async (row) => { - const deepChildren = await Group.getDeepMetersByGroupID(row.id, conn); - return { ...row, children: deepChildren }; - }) - Promise.all(promises).then(function (values) { - res.json(values.map(formatGroupForResponse)); - }) + const rows = await Group.getAllWithDeepMeters(conn); + res.json(rows.map(({ group, deepMeters }) => + formatGroupForResponse({ ...group, children: deepMeters }))); } catch (err) { log.error(`Error while preforming GET all groups query: ${err}`, err); + res.sendStatus(500); } }); @@ -450,4 +445,3 @@ router.post('/delete', adminAuthMiddleware('delete groups'), async (req, res) => }); module.exports = router; - diff --git a/src/server/routes/readings.js b/src/server/routes/readings.js index 98b3e81177..3ec88c9034 100644 --- a/src/server/routes/readings.js +++ b/src/server/routes/readings.js @@ -43,18 +43,21 @@ router.get('/line/count/meters/:meter_ids', optionalAuthMiddleware, async (req, const meterIDs = req.params.meter_ids.split(',').map(s => parseInt(s)); const timeInterval = TimeInterval.fromString(req.query.timeInterval); try { - let count = 0; - for (var i = 0; i < meterIDs.length; i++) { - const curr = await Reading.getCountByMeterIDAndDateRange(meterIDs[i], timeInterval.startTimestamp, timeInterval.endTimestamp, conn); - count += curr - } + // Count every requested meter in one statement so response time + // does not grow by one sequential database round trip per meter. + const count = await Reading.getCountByMeterIDsAndDateRange( + meterIDs, + timeInterval.startTimestamp, + timeInterval.endTimestamp, + conn + ); res.send(JSON.stringify(count)); } catch (err) { log.error(`Error while performing GET readings COUNT for line with meters ${meterIDs} with time interval ${timeInterval}: ${err}`, err); res.sendStatus(500); } } -}) +}); // TODO This route should be limiting access to large file responses to the appropriate users. // Currently it is done in the component but also needs to be here. @@ -106,4 +109,3 @@ router.get('/line/raw/meter/:meter_id', optionalAuthMiddleware, async (req, res) module.exports = router; - diff --git a/src/server/services/createDB.js b/src/server/services/createDB.js index e18041123d..ca7a3e2274 100644 --- a/src/server/services/createDB.js +++ b/src/server/services/createDB.js @@ -6,7 +6,8 @@ const { createSchema } = require('../models/database'); const { log } = require('../log'); const { getConnection } = require('../db'); const { insertStandardUnits, insertStandardConversions } = require('../util/insertData'); -const { redoCik } = require('../services/graph/redoCik'); +const { redoCikVary } = require('../services/graph/redoCik'); +const refreshAllReadingViews = require('./refreshAllReadingViews'); (async function createSchemaWrapper() { const conn = getConnection(); @@ -14,11 +15,15 @@ const { redoCik } = require('../services/graph/redoCik'); await createSchema(conn); await insertStandardUnits(conn); await insertStandardConversions(conn); - await redoCik(conn); + await redoCikVary(conn); + // redoCikVary replaces conversion metadata atomically and marks the + // denormalized hourly split table stale. Rebuild it before the server is + // allowed to start so a restart cannot serve aggregates using old slopes. + await refreshAllReadingViews(); log.info('Schema created', null, true); process.exitCode = 0; } catch (err) { log.error(`Error creating schema: ${err}`, err, skipMail = true); process.exitCode = 1; } -}()); \ No newline at end of file +}()); diff --git a/src/server/services/eGauge/readEgaugeData.js b/src/server/services/eGauge/readEgaugeData.js index ecdb4d0d53..8f83614d62 100644 --- a/src/server/services/eGauge/readEgaugeData.js +++ b/src/server/services/eGauge/readEgaugeData.js @@ -19,8 +19,9 @@ async function readEgaugeData(meter, conn) { const meterReadings = await requestor.getMeterReadings(); await requestor.logout() - // Store the readings in the database. - await loadArrayInput(dataRows = meterReadings, + // Store the readings and return the accepted time range so the caller can + // refresh only the affected continuous-aggregate buckets. + return loadArrayInput(dataRows = meterReadings, meterID = meter.id, mapRowToModel = row => { const readRate = row[0]; diff --git a/src/server/services/eGauge/updateEgaugeMeters.js b/src/server/services/eGauge/updateEgaugeMeters.js index e1145da9ac..d25633791c 100644 --- a/src/server/services/eGauge/updateEgaugeMeters.js +++ b/src/server/services/eGauge/updateEgaugeMeters.js @@ -9,7 +9,7 @@ const updateMeters = require('../updateMeters'); const { log } = require('../../log'); const { getConnection } = require('../../db'); const readEgaugeData = require('./readEgaugeData'); -const { refreshMeterReadingsViews } = require('../../models/Reading'); +const refreshAllReadingViews = require('../refreshAllReadingViews'); /** * For every enabled eGauge meter, update the readings in the database. @@ -22,14 +22,25 @@ async function updateEgaugeMeters() { const allMeters = await Meter.getEnabled(conn); // We only want the eGauge meters. const metersToUpdate = allMeters.filter(m => m.type === Meter.type.EGAUGE); - // Ignoring that loadArrayInput is called in this sequence and returns values - // since this is only called by an automated process at this time. - // Issues from the pipeline will be logged by called functions. - await updateMeters(readEgaugeData, metersToUpdate, conn); - // We refresh the readings so they can be graphed to see the new ones. - // TODO If the system is getting other types of meters this may cause the refresh - // to happen multiple times. Might want to work on this in the future. - await refreshMeterReadingsViews(conn); + // Issues from the pipeline are logged by the called functions. Successful + // results include the accepted reading range for each meter. + const updateResults = await updateMeters(readEgaugeData, metersToUpdate, conn); + const readingRanges = updateResults.filter(result => result.startTimestamp && result.endTimestamp); + + // Refresh the readings so newly imported values can be graphed. Combining + // all meter ranges keeps this to one refresh without checking unrelated + // historical buckets. + // TODO If the system gets other meter types in the same schedule, coordinate + // their ranges so aggregate refreshes are not performed multiple times. + if (readingRanges.length > 0) { + const startTimestamp = readingRanges.reduce((earliest, result) => + result.startTimestamp.isBefore(earliest) ? result.startTimestamp : earliest, + readingRanges[0].startTimestamp); + const endTimestamp = readingRanges.reduce((latest, result) => + result.endTimestamp.isAfter(latest) ? result.endTimestamp : latest, + readingRanges[0].endTimestamp); + await refreshAllReadingViews({ startTimestamp, endTimestamp }); + } } catch (err) { log.error(`Error fetching eGauge meter data: ${err}`, err); } diff --git a/src/server/services/graph/createConversionArrays.js b/src/server/services/graph/createConversionArrays.js index d376952902..d3fdb44364 100644 --- a/src/server/services/graph/createConversionArrays.js +++ b/src/server/services/graph/createConversionArrays.js @@ -2,70 +2,12 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -const { SemicolonPreference } = require('typescript'); const Unit = require('../../models/Unit'); +const Conversion = require('../../models/Conversion'); +const ConversionSegment = require('../../models/ConversionSegment'); const { getPath } = require('./createConversionGraph'); -const { pathConversion } = require('./pathConversion'); const { timeVaryingPathConversion } = require('./timeVaryingPathConversion'); -/** - * Returns the Cik which gives the slope, intercept and suffix name between each meter and unit - * where it is NaN, Nan, '' if no conversion. - * @param {*} graph The conversion graph. - * @param {*} conn The connection to use. - * @returns - */ -async function createCikArray(graph, conn) { - // Get the vertices associated with the sources (meters) and destinations (units, suffix). - // In principle we could just get units associated with meters that have a visible meter since only - // those can be used. However, this means we would need to update Cik if an admin updates the visibility of - // a meter or adds a new meter that is visible and associated with a meter unit that was not in Cik. - // To avoid having to redo Cik for any meter update, all meter units are included. Note it is less likely - // that there is an unused meter unit. - // For units of type unit or suffix, we could exclude any that no user can see. While this might eliminate - // some units, the number that are not visible to an admin is likely to be small. As with meter units, OED - // would need to update Cik if any unit has its visible status changed. Not doing this avoids having to update - // Cik on unit changes (only on conversion change because adding a new unit that has no conversion means - // all its values in Cik would indicate no conversion). Note we do exclude if displayable is none - // since those cannot be graphed. The original unit with a suffix string is excluded by OED during - // processing of suffix units. - // The final consideration is how much including the extra items will cost. The larger array should not impact - // the speed of looking up an item in Cik. Sending Cik to the client will be larger but note that if - // there are 30 meter units and 100 unit/suffix units then Cik has 3000 items. This will - // not be large, esp. compared to the rest of the startup payload of code. Thus, including all the - // units should still be very efficient and the bytes saved by doing all the extra work above will be small. - const sources = await Unit.getTypeMeter(conn); - // This excludes units that have displayable none since cannot be graphed. - const destinations = (await Unit.getTypeUnit(conn)).concat(await Unit.getTypeSuffix(conn)); - // Size of each of these. - // Create an array to hold the values. Each entry will have integer souce it, integer destination id, double slope, - // double intercept, and string suffix. - const c = []; - for (const source of sources) { - for (const destination of destinations) { - const sourceId = source.id; - const destinationId = destination.id; - // The shortest path from source to destination. - const path = getPath(graph, sourceId, destinationId); - // Check if the path exists. - // If not, we will do nothing since the array has been initialized with [Nan, Nan, '']. - if (path !== null) { - const [slope, intercept] = await pathConversion(path, conn); - // All suffix units were dealt in src/server/services/graph/handleSuffixUnits.js - // so all units with suffix have displayable of none. - // This means this path has a suffix of "" (empty) so it does not matter. - // The name of any unit associated with a suffix was already set correctly. - // Thus, we can just use the destination identifier as the unit name. - c.push({ source: sourceId, destination: destinationId, slope: slope, intercept: intercept }); - } - } - } - // TODO: The table in the database for the logical Cik needs to be wiped and these values stored. This code - // will be added once the database table for using it to get readings is set. - // At the moment, we just return the array. - return c; -} - /** * Returns the CikVary array: all time-varying conversions between each meter and unit. * Each entry is { source, destination, start_time, end_time, slope, intercept } @@ -74,8 +16,34 @@ async function createCikArray(graph, conn) { * @returns Array of time-varying conversion segments. */ async function createCikVaryArray(graph, conn) { - const sources = await Unit.getTypeMeter(conn); - const destinations = (await Unit.getTypeUnit(conn)).concat(await Unit.getTypeSuffix(conn)); + /* + * Conversion rebuilding visits the same graph edges through many + * source/destination paths. Load the small metadata tables once so path + * traversal does not issue database queries inside the nested loops. + */ + const [sources, destinationUnits, suffixUnits, conversions, conversionSegments] = await Promise.all([ + Unit.getTypeMeter(conn), + Unit.getTypeUnit(conn), + Unit.getTypeSuffix(conn), + Conversion.getAll(conn), + ConversionSegment.getAll(conn) + ]); + const destinations = destinationUnits.concat(suffixUnits); + const conversionsByEdge = new Map(conversions.map(conversion => [ + `${conversion.sourceId}:${conversion.destinationId}`, + conversion + ])); + const segmentsByEdge = new Map(); + for (const segment of conversionSegments) { + const key = `${segment.sourceId}:${segment.destinationId}`; + const edgeSegments = segmentsByEdge.get(key); + if (edgeSegments === undefined) { + segmentsByEdge.set(key, [segment]); + } else { + edgeSegments.push(segment); + } + } + const metadata = { conversionsByEdge, segmentsByEdge }; const c = []; // Iterate over all possible meter unit sources @@ -88,7 +56,7 @@ async function createCikVaryArray(graph, conn) { const path = getPath(graph, sourceId, destinationId); // If a valid path exists, compute all time-varying conversion segments along that path if (path !== null) { - const segments = await timeVaryingPathConversion(path, conn); + const segments = await timeVaryingPathConversion(path, conn, metadata); // Add all segments to the result array segments.forEach(seg => { c.push({ @@ -107,6 +75,5 @@ async function createCikVaryArray(graph, conn) { } module.exports = { - createCikArray, createCikVaryArray -} \ No newline at end of file +}; diff --git a/src/server/services/graph/handleSuffixUnits.js b/src/server/services/graph/handleSuffixUnits.js index 7cff1922ba..208a48fdd5 100644 --- a/src/server/services/graph/handleSuffixUnits.js +++ b/src/server/services/graph/handleSuffixUnits.js @@ -82,14 +82,14 @@ async function verifyConversion(expectedSlope, expectedIntercept, source, destin ); // Insert the new conversion to the graph graph.addLink(sourceId, destinationId); - } -// TODO: This check needs to be finalized soon to reflect the updated Conversion table -// else if (currentConversion.slope !== expectedSlope || currentConversion.intercept !== expectedIntercept) { -// // While unlikely, the conversion changed so update -// currentConversion.slope = expectedSlope; -// currentConversion.intercept = expectedIntercept; -// await currentConversion.update(conn); -// } + } + // TODO: This check needs to be finalized soon to reflect the updated Conversion table + // else if (currentConversion.slope !== expectedSlope || currentConversion.intercept !== expectedIntercept) { + // // While unlikely, the conversion changed so update + // currentConversion.slope = expectedSlope; + // currentConversion.intercept = expectedIntercept; + // await currentConversion.update(conn); + // } } /** diff --git a/src/server/services/graph/pathConversion.js b/src/server/services/graph/pathConversion.js index aa2efc668b..4d0b6c90ed 100644 --- a/src/server/services/graph/pathConversion.js +++ b/src/server/services/graph/pathConversion.js @@ -3,6 +3,7 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ const Conversion = require('../../models/Conversion'); +const ConversionSegment = require('../../models/ConversionSegment'); const Unit = require('../../models/Unit'); /** @@ -14,23 +15,34 @@ const Unit = require('../../models/Unit'); * @returns */ async function conversionValues(sourceUnit, destinationUnit, conn) { - let desiredConversion = await Conversion.getBySourceDestination(sourceUnit, destinationUnit, conn); + let desiredConversion = await ConversionSegment.getBySourceDestination(sourceUnit, destinationUnit, conn); let slope; let intercept; let suffix; - if (desiredConversion === null) { + if (desiredConversion.length === 0) { // Did not find the conversion. Since conversion should exist, it must be the other way around and bidirectional. - desiredConversion = await Conversion.getBySourceDestination(destinationUnit, sourceUnit, conn); + desiredConversion = await ConversionSegment.getBySourceDestination(destinationUnit, sourceUnit, conn); if (desiredConversion === null || desiredConversion.bidirectional === false) { // This should never happen. It should have been in the table one way or the other. throw Error(`The conversions from ${sourceUnit} to ${destinationUnit} doesn't exist`); + } else if (desiredConversion.length !== 1) { + // TODO This needs to be fixed up to handle suffix units on paths with time-varying conversions. + // There are multiple conversion segments so time-varying. + throw Error(`The conversions from ${destinationUnit} to ${sourceUnit} is time-varying so cannot include`); } + desiredConversion = desiredConversion[0]; // We need to invert the conversion since it needs to go the other way from how stored. [slope, intercept] = invertConversion(desiredConversion.slope, desiredConversion.intercept); // Since we inverted the conversion, we use the suffix from the destination. suffix = (await Unit.getById(destinationUnit, conn)).suffix; } else { + // TODO This needs to be fixed up to handle suffix units on paths with time-varying conversions. + if (desiredConversion.length !== 1) { + // There are multiple conversion segments so time-varying. + throw Error(`The conversions from ${sourceUnit} to ${destinationUnit} is time-varying so cannot include`); + } // We found it in the desired order. + desiredConversion = desiredConversion[0]; slope = desiredConversion.slope; intercept = desiredConversion.intercept; suffix = (await Unit.getById(sourceUnit, conn)).suffix; @@ -102,4 +114,4 @@ async function pathConversion(path, conn) { return [slope, intercept, suffix]; } -module.exports = { pathConversion, invertConversion, updatedConversion }; \ No newline at end of file +module.exports = { pathConversion, invertConversion, updatedConversion }; diff --git a/src/server/services/graph/redoCik.js b/src/server/services/graph/redoCik.js index a55a103ad6..ddbcf3c8aa 100644 --- a/src/server/services/graph/redoCik.js +++ b/src/server/services/graph/redoCik.js @@ -3,37 +3,11 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ const { createConversionGraph } = require('./createConversionGraph'); -const { createCikArray, createCikVaryArray } = require('./createConversionArrays'); -const Cik = require('../../models/Cik'); +const { createCikVaryArray } = require('./createConversionArrays'); const CikVary = require('../../models/CikVary'); const { handleSuffixUnits } = require('./handleSuffixUnits'); const { getConnection } = require('../../db'); -const { refreshAllReadingViews } = require('../../services/refreshAllReadingViews'); - -/** - * Creates Cik based on units and conversions and then inserts these values - * in the cik table in the database. - */ -async function redoCik(conn) { - // Create graph based on units and conversions. - const graph = await createConversionGraph(conn); - // Processes suffix units to update graph and database. - await handleSuffixUnits(graph, conn); - // Uses final graph to create cik array. - const cik = await createCikArray(graph, conn); - // Inserts cik array into database where old values are deleted. - await Cik.insert(cik, conn); -} - -/** - * Needed to call from npm run. Give new name so hopefully won't use in regular code. -*/ -async function updateCikAndViews() { - const conn = getConnection(); - await redoCik(conn); - // We need to update views if Cik changes. - await refreshAllReadingViews(); -} +const refreshAllReadingViews = require('../../services/refreshAllReadingViews'); /** * Creates CikVary based on units and conversion segments and then inserts these values @@ -42,12 +16,12 @@ async function updateCikAndViews() { async function redoCikVary(conn) { // Create graph based on units and conversion segments. const graph = await createConversionGraph(conn); - + // Processes suffix units to update graph and database (not used for now). await handleSuffixUnits(graph, conn); // Uses final graph to create cik_vary array. const cikVary = await createCikVaryArray(graph, conn); - + // Inserts cik_vary array into database where old values are deleted. await CikVary.insert(cikVary, conn); } @@ -59,12 +33,10 @@ async function updateCikVaryAndViews() { const conn = getConnection(); await redoCikVary(conn); // We need to update views if CikVary changes. - await refreshAllReadingViews(); + await refreshAllReadingViews({ rebuild: true }); } module.exports = { - redoCik, - updateCikAndViews, redoCikVary, updateCikVaryAndViews }; diff --git a/src/server/services/graph/timeVaryingPathConversion.js b/src/server/services/graph/timeVaryingPathConversion.js index 8c007ed703..c11e797df8 100644 --- a/src/server/services/graph/timeVaryingPathConversion.js +++ b/src/server/services/graph/timeVaryingPathConversion.js @@ -13,9 +13,10 @@ const updatedConversion = require('./pathConversion').updatedConversion; * The algorithm aligns all segments and combines them for each time range. * @param {*} path Array of units (nodes) from source to destination. * @param {*} conn Database connection. + * @param metadata optional preloaded conversion and segment maps * @returns Array of {source, destination, startTime, endTime, slope, intercept} */ -async function timeVaryingPathConversion(path, conn) { +async function timeVaryingPathConversion(path, conn, metadata = null) { // 1. Fetch and sort segments for each edge const edgeSegments = []; @@ -23,17 +24,25 @@ async function timeVaryingPathConversion(path, conn) { const sourceId = path[i].id; const destinationId = path[i + 1].id; //segments are sorted by start_time in getBySourceDestination - let segments = await ConversionSegment.getBySourceDestination(sourceId, destinationId, conn); + const edgeKey = `${sourceId}:${destinationId}`; + let segments = metadata === null + ? await ConversionSegment.getBySourceDestination(sourceId, destinationId, conn) + : metadata.segmentsByEdge.get(edgeKey) || []; // Did not find the conversion segments. Since conversion should exist, it must be the other way around and bidirectional. if (!segments || segments.length === 0) { // Check if reverse conversion exists and is bidirectional - const reverseConversion = await Conversion.getBySourceDestination(destinationId, sourceId, conn); + const reverseKey = `${destinationId}:${sourceId}`; + const reverseConversion = metadata === null + ? await Conversion.getBySourceDestination(destinationId, sourceId, conn) + : metadata.conversionsByEdge.get(reverseKey); // This should never happen. It should have been in the table one way or the other. if (!reverseConversion || !reverseConversion.bidirectional) { throw Error(`No bidirectional conversion found between ${sourceId} and ${destinationId}`); } // Fetch reverse segments and invert them - const reverseSegments = await ConversionSegment.getBySourceDestination(destinationId, sourceId, conn); + const reverseSegments = metadata === null + ? await ConversionSegment.getBySourceDestination(destinationId, sourceId, conn) + : metadata.segmentsByEdge.get(reverseKey) || []; // This is also really weird that it exist and yet no segments found. if (!reverseSegments || reverseSegments.length === 0) { throw Error(`No conversion segments found for reverse direction between ${destinationId} and ${sourceId}`); diff --git a/src/server/services/pipeline-in-progress/loadArrayInput.js b/src/server/services/pipeline-in-progress/loadArrayInput.js index 9af0408da5..5ad6f274ab 100644 --- a/src/server/services/pipeline-in-progress/loadArrayInput.js +++ b/src/server/services/pipeline-in-progress/loadArrayInput.js @@ -59,7 +59,15 @@ async function loadArrayInput(dataRows, meterID, mapRowToModel, timeSort, readin '\n and the pipeline returned these messages: ' + msgTotal; }) } - return { isAllReadingsOk, msgTotal }; + const readingRange = readingsToInsert.length === 0 ? {} : { + startTimestamp: readingsToInsert.reduce((earliest, reading) => + reading.startTimestamp.isBefore(earliest) ? reading.startTimestamp : earliest, + readingsToInsert[0].startTimestamp), + endTimestamp: readingsToInsert.reduce((latest, reading) => + reading.endTimestamp.isAfter(latest) ? reading.endTimestamp : latest, + readingsToInsert[0].endTimestamp) + }; + return { isAllReadingsOk, msgTotal, ...readingRange }; } module.exports = loadArrayInput; diff --git a/src/server/services/refreshAllReadingViews.js b/src/server/services/refreshAllReadingViews.js index febc36718c..3fe4b3660b 100644 --- a/src/server/services/refreshAllReadingViews.js +++ b/src/server/services/refreshAllReadingViews.js @@ -6,23 +6,68 @@ const { log } = require('../log'); const { getConnection } = require('../db'); -const Reading = require('../models/Reading'); +const TimeScaleDBReading = require('../models/TimeScaleDB/Reading'); +// TODO: Remove this retained legacy import once the hypertable implementation is finalized. +// const Reading = require('../models/Reading'); + +// Arbitrary, stable application namespace key for a session-level PostgreSQL +// advisory lock. Every aggregate refresher must use this same key; the numeric +// value has no transaction ID or database-object meaning. +// Introduced because meter_hourly_readings was initially encountering deadlocks. +const REFRESH_ADVISORY_LOCK_ID = 724536221; + +async function timedRefresh(label, operation) { + const start = Date.now(); + await operation(); + log.info(`${label} completed in ${Date.now() - start} ms`); +} + +/** + * Refreshes the TimescaleDB reading aggregates while holding a shared + * advisory lock so concurrent imports cannot refresh them simultaneously. + * Refreshes are incremental by default. A full split-table rebuild runs only + * when explicitly requested or when denormalized source data is marked stale. + */ +async function refreshAllReadingViews(options = {}) { + const { startTimestamp = null, endTimestamp = null, rebuild = false } = options; + if ((startTimestamp === null) !== (endTimestamp === null)) { + throw new Error('Both startTimestamp and endTimestamp are required for a bounded reading refresh.'); + } -/** - * This function is changed from refreshing hourly and daily readings - * views in parallel using Promise.all() into one by one because - * daily readings calculation depends on hourly readings. -*/ -async function refreshAllReadingViews() { const conn = getConnection(); - // Refresh meter readings views - log.info('Refreshing Materialized Hourly and Daily Readings Views'); - await Reading.refreshMeterReadingsViews(conn); - log.info('Materialized Hourly and Daily Readings Views Refreshed'); - // Refresh group views - log.info('Refreshing Group Reading Views'); - await Reading.refreshGroupReadingsViews(conn); - log.info('refreshAllReadingViews completed'); + await conn.task(async task => { + await task.one('SELECT pg_advisory_lock(${lockId})', { lockId: REFRESH_ADVISORY_LOCK_ID }); + try { + // TODO: Remove these retained legacy refresh calls once the hypertable implementation is finalized. + // await timedRefresh('Legacy meter reading views refresh', () => Reading.refreshMeterReadingsViews(task)); + const rebuildState = await task.one(` + SELECT rebuild_revision, completed_rebuild_revision + FROM reading_aggregate_state + WHERE id = 1 + `); + const rebuildRequired = rebuild + || BigInt(rebuildState.rebuild_revision) > BigInt(rebuildState.completed_rebuild_revision); + + if (rebuildRequired) { + await timedRefresh('TimescaleDB reading aggregates rebuild', () => TimeScaleDBReading.rebuildReadings(task)); + await task.none(` + UPDATE reading_aggregate_state + SET completed_rebuild_revision = GREATEST( + completed_rebuild_revision, + \${rebuiltRevision} + ) + WHERE id = 1 + `, { rebuiltRevision: rebuildState.rebuild_revision }); + } else { + await timedRefresh('TimescaleDB reading aggregates range refresh', () => + TimeScaleDBReading.refreshReadings(task, startTimestamp, endTimestamp)); + } + // await timedRefresh('Legacy group reading views refresh', () => Reading.refreshGroupReadingsViews(task)); + } finally { + await task.one('SELECT pg_advisory_unlock(${lockId})', { lockId: REFRESH_ADVISORY_LOCK_ID }); + } + }); + log.info('All reading aggregates synchronized'); } -module.exports = { refreshAllReadingViews }; +module.exports = refreshAllReadingViews; diff --git a/src/server/services/refreshGroupsDeepMetersView.js b/src/server/services/refreshGroupsDeepMetersView.js index 6668bcb66f..0c2884d751 100644 --- a/src/server/services/refreshGroupsDeepMetersView.js +++ b/src/server/services/refreshGroupsDeepMetersView.js @@ -6,19 +6,28 @@ const { log } = require('../log'); const { getConnection } = require('../db'); -const Group = require('../models/Group'); -const Reading = require('../models/Reading'); +// TODO: Remove this redundant pre-hypertable refresh path once the hypertable implementation is finalized. +// const Group = require('../models/Group'); +const TimeScaleDBReading = require('../models/TimeScaleDB/Reading'); +// TODO: Remove this retained legacy import once the hypertable implementation is finalized. +// const Reading = require('../models/Reading'); async function refreshGroupsDeepMetersView() { const conn = getConnection(); - // Refresh groups deep meters view - log.info('Refreshing Materialized Groups Deep Meters View'); - await Group.refreshGroupsDeepMetersView(conn); - log.info('Materialized Groups Deep Meters View Refreshed'); - // Refresh group readings views - log.info('Refreshing Group Reading Views'); - await Reading.refreshGroupReadingsViews(conn); - log.info('...Group Views Refreshed!'); + // TODO: Remove this retained legacy refresh block once the hypertable implementation is finalized. + // log.info('Refreshing Materialized Groups Deep Meters View'); + // await Group.refreshGroupsDeepMetersView(conn); + // log.info('Materialized Groups Deep Meters View Refreshed'); + // log.info('Refreshing Group Reading Views'); + // await Reading.refreshGroupReadingsViews(conn); + // log.info('...Group Views Refreshed!'); + // refreshGroupReadings updates both group caches before refreshing only + // the group aggregates. Meter aggregates are maintained independently. + log.info('Refreshing TimeScaleDB Group Reading Views'); + // TODO: Remove this retained broad refresh once the hypertable implementation is finalized. + // await TimeScaleDBReading.refreshReadings(conn); + await TimeScaleDBReading.refreshGroupReadings(conn); + log.info('...TimeScaleDB Group Views Refreshed!'); } module.exports = { refreshGroupsDeepMetersView }; diff --git a/src/server/services/refreshHourlyReadingViews.js b/src/server/services/refreshHourlyReadingViews.js index d11308a7bc..e41520ca95 100644 --- a/src/server/services/refreshHourlyReadingViews.js +++ b/src/server/services/refreshHourlyReadingViews.js @@ -4,21 +4,24 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -const { log } = require('../log'); +const refreshAllReadingViews = require('./refreshAllReadingViews'); -const { getConnection } = require('../db'); -const Reading = require('../models/Reading'); +// TODO: Remove this retained legacy implementation once the hypertable +// implementation is finalized. +// const { log } = require('../log'); +// const { getConnection } = require('../db'); +// const Reading = require('../models/Reading'); /** * @deprecated OED only supports refreshing all views so please use refreshAllReadingViews. * See src/server/services/refreshAllReadingViews.js */ async function refreshHourlyReadingViews() { - const conn = getConnection(); - - log.info('Refreshing Materialized Hourly Reading Views'); - await Reading.refreshHourlyReadings(conn); - log.info('Materialized Hourly View Refreshed'); + // const conn = getConnection(); + // log.info('Refreshing Materialized Hourly Reading Views'); + // await Reading.refreshHourlyReadings(conn); + // log.info('Materialized Hourly View Refreshed'); + await refreshAllReadingViews(); } module.exports = { refreshHourlyReadingViews }; diff --git a/src/server/services/refreshReadingViews.js b/src/server/services/refreshReadingViews.js index 9868a21065..5f891aba62 100644 --- a/src/server/services/refreshReadingViews.js +++ b/src/server/services/refreshReadingViews.js @@ -4,29 +4,24 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -const { log } = require('../log'); +const refreshAllReadingViews = require('./refreshAllReadingViews'); -const { getConnection } = require('../db'); -const Reading = require('../models/Reading'); - -// While the name of this function is refreshReadingViews, the purpose -// of this function is to refresh the materialized daily views. -// To make changes in refreshing all reading views, modify -// /src/services/refreshAllReadingViews.js. -/** - * Refreshes daily view. - */ +// TODO: Remove this retained legacy implementation once the hypertable +// implementation is finalized. +// const { log } = require('../log'); +// const { getConnection } = require('../db'); +// const Reading = require('../models/Reading'); /** * @deprecated OED only supports refreshing all views so please use refreshAllReadingViews. * See src/server/services/refreshAllReadingViews.js */ async function refreshReadingViews() { - const conn = getConnection(); - - log.info('Refreshing Materialized Daily Reading Views'); - await Reading.refreshDailyReadings(conn); - log.info('Daily View Refreshed'); + // const conn = getConnection(); + // log.info('Refreshing Materialized Daily Reading Views'); + // await Reading.refreshDailyReadings(conn); + // log.info('Daily View Refreshed'); + await refreshAllReadingViews(); } module.exports = { refreshReadingViews }; diff --git a/src/server/services/updateMeters.js b/src/server/services/updateMeters.js index 773a28cf9e..aa5a2d3c65 100644 --- a/src/server/services/updateMeters.js +++ b/src/server/services/updateMeters.js @@ -12,16 +12,14 @@ const { log } = require('../log'); * @param dataReader {function} A function to fetch readings from each meter * @param metersToUpdate [Meter] An array of meters to be updated * @param conn the database connection to use - * @returns {Promise.} + * @returns {Promise} successful results returned by the data readers */ async function updateAllMeters(dataReader, metersToUpdate, conn) { log.info(`Getting meter data`); try { // Do all the network requests in parallel and log errors. - // Ignoring that loadArrayInput is called in this sequence and returns values - // since this is only called by an automated process at this time. // Issues from the pipeline will be logged by called functions. - await Promise.all( + const results = await Promise.all( metersToUpdate .map(meter => dataReader(meter, conn)) .map(p => p.catch(err => { @@ -33,8 +31,12 @@ async function updateAllMeters(dataReader, metersToUpdate, conn) { return null; }))); log.info('Update finished'); + // Preserve each successful reader's metadata, such as its imported time + // range, while retaining the existing per-meter failure isolation. + return results.filter(result => result !== null); } catch (err) { log.error(`Error updating all meters: ${err}`, err); + return []; } } diff --git a/src/server/sql/cik/create_cik_table.sql b/src/server/sql/cik/create_cik_table.sql index c36e43e31f..62d44ba567 100644 --- a/src/server/sql/cik/create_cik_table.sql +++ b/src/server/sql/cik/create_cik_table.sql @@ -6,7 +6,5 @@ CREATE TABLE IF NOT EXISTS cik ( source_id INTEGER REFERENCES units(id), destination_id INTEGER REFERENCES units(id), - slope FLOAT, - intercept FLOAT, PRIMARY KEY (source_id, destination_id) ); diff --git a/src/server/sql/cik/delete_all_conversions.sql b/src/server/sql/cik/delete_all_cik.sql similarity index 80% rename from src/server/sql/cik/delete_all_conversions.sql rename to src/server/sql/cik/delete_all_cik.sql index b5d6437cd6..8b39e43605 100644 --- a/src/server/sql/cik/delete_all_conversions.sql +++ b/src/server/sql/cik/delete_all_cik.sql @@ -4,5 +4,5 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ --- Remove all current values from the cik table. +-- Remove all current values from the cik_vary table. DELETE FROM cik; diff --git a/src/server/sql/cik/get_all_conversions.sql b/src/server/sql/cik/get_all_conversions.sql index 8bc771843e..d6fc1d421c 100644 --- a/src/server/sql/cik/get_all_conversions.sql +++ b/src/server/sql/cik/get_all_conversions.sql @@ -2,5 +2,8 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +-- TODO: Research whether external maintenance scripts use this query. No +-- in-repository JavaScript loads it; remove it if there are no external users. + -- Get all conversions in cik table. SELECT * FROM cik; diff --git a/src/server/sql/cik/get_cik.sql b/src/server/sql/cik/get_cik.sql index c1339c54ce..6fe00b9c8e 100644 --- a/src/server/sql/cik/get_cik.sql +++ b/src/server/sql/cik/get_cik.sql @@ -4,7 +4,7 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ --- Get all ciks through joining cik and units tables. -SELECT source_id AS meter_unit_id, destination_id AS non_meter_unit_id, slope, intercept +-- Get all ciks +SELECT source_id AS meter_unit_id, destination_id AS non_meter_unit_id FROM cik ; diff --git a/src/server/sql/cik/get_conversion.sql b/src/server/sql/cik/get_conversion.sql deleted file mode 100644 index 44c79aee25..0000000000 --- a/src/server/sql/cik/get_conversion.sql +++ /dev/null @@ -1,6 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - --- Get conversion for specific row and column in the cik table (same as Cik array). -SELECT * FROM cik WHERE source_id=${sourceId} and destination_id=${destinationId}; diff --git a/src/server/sql/cik/insert_new_cik.sql b/src/server/sql/cik/insert_new_cik.sql index 98ab06078e..a7e12a5976 100644 --- a/src/server/sql/cik/insert_new_cik.sql +++ b/src/server/sql/cik/insert_new_cik.sql @@ -2,6 +2,9 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +-- TODO: Research whether external maintenance scripts use this single-row +-- insert. CikVary.insert now populates cik set-wise; remove this file if unused. + -- Inserts a new conversion into the cik table. -INSERT INTO cik (source_id, destination_id, slope, intercept) -VALUES (${sourceId}, ${destinationId}, ${slope}, ${intercept}); +INSERT INTO cik (source_id, destination_id) +VALUES (${sourceId}, ${destinationId}); diff --git a/src/server/sql/cik/insert_unique_cik_vary_in_cik.sql b/src/server/sql/cik/insert_unique_cik_vary_in_cik.sql new file mode 100644 index 0000000000..645de89966 --- /dev/null +++ b/src/server/sql/cik/insert_unique_cik_vary_in_cik.sql @@ -0,0 +1,11 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +-- Put each unique source + destination in cik_vary into cik. +INSERT INTO cik (source_id, destination_id) +SELECT DISTINCT source_id, destination_id +FROM cik_vary +; diff --git a/src/server/sql/cik_vary/get_cik_vary_by_source_destination_start_end.sql b/src/server/sql/cik_vary/get_cik_vary_by_source_destination_start_end.sql index 69103e4b7d..1adb6288a4 100644 --- a/src/server/sql/cik_vary/get_cik_vary_by_source_destination_start_end.sql +++ b/src/server/sql/cik_vary/get_cik_vary_by_source_destination_start_end.sql @@ -2,5 +2,9 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +-- TODO: Research whether this lookup is part of a supported external API. Its +-- model method has no internal callers and currently references a misspelled +-- path; remove both if no external caller needs point-in-time lookup. + -- Get conversion for specific source, destination, and time range in cik_vary table. -SELECT * FROM cik_vary WHERE source_id=${sourceId} AND destination_id=${destinationId} AND start_time<=${queryTime} AND end_time>=${queryTime}; +SELECT * FROM cik_vary WHERE source_id=${sourceId} AND destination_id=${destinationId} AND start_time<=${queryTime} AND end_time>${queryTime}; diff --git a/src/server/sql/cik_vary/insert_new_cik_vary.sql b/src/server/sql/cik_vary/insert_new_cik_vary.sql index 5b3d18ed62..64db80f939 100644 --- a/src/server/sql/cik_vary/insert_new_cik_vary.sql +++ b/src/server/sql/cik_vary/insert_new_cik_vary.sql @@ -2,6 +2,9 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +-- TODO: Research whether external maintenance scripts use this single-row +-- insert. CikVary.insert now performs set-based insertion; remove if unused. + -- Inserts a new conversion into the cik_vary table. INSERT INTO cik_vary (source_id, destination_id, start_time, end_time, slope, intercept) VALUES (${sourceId}, ${destinationId}, ${startTime}, ${endTime}, ${slope}, ${intercept}); diff --git a/src/server/sql/conversion/create_conversions_table.sql b/src/server/sql/conversion/create_conversions_table.sql index 3ac0990e47..07376c3908 100644 --- a/src/server/sql/conversion/create_conversions_table.sql +++ b/src/server/sql/conversion/create_conversions_table.sql @@ -9,4 +9,4 @@ CREATE TABLE IF NOT EXISTS conversions ( note TEXT, CHECK (source_id != destination_id), PRIMARY KEY (source_id, destination_id) -); \ No newline at end of file +); diff --git a/src/server/sql/conversionSegment/get_all.sql b/src/server/sql/conversionSegment/get_all.sql new file mode 100644 index 0000000000..5782c13542 --- /dev/null +++ b/src/server/sql/conversionSegment/get_all.sql @@ -0,0 +1,15 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +SELECT + source_id, + destination_id, + week_patterns_id, + slope, + intercept, + start_time::TEXT AS start_time, + end_time::TEXT AS end_time, + note +FROM conversion_segments +ORDER BY source_id, destination_id, start_time; diff --git a/src/server/sql/conversionSegment/insert_new_conversion_segment.sql b/src/server/sql/conversionSegment/insert_new_conversion_segment.sql index 48170af1cb..64c448b21a 100644 --- a/src/server/sql/conversionSegment/insert_new_conversion_segment.sql +++ b/src/server/sql/conversionSegment/insert_new_conversion_segment.sql @@ -3,4 +3,4 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ INSERT INTO conversion_segments(source_id, destination_id, week_patterns_id, slope, intercept, start_time, end_time, note) -VALUES (${sourceId}, ${destinationId}, ${weekPatternsId}, ${slope}, ${intercept}, ${startTime}, ${endTime}, ${note}); \ No newline at end of file +VALUES (${sourceId}, ${destinationId}, ${weekPatternsId}, ${slope}, ${intercept}, ${startTime}, ${endTime}, ${note}); diff --git a/src/server/sql/group/create_groups_tables.sql b/src/server/sql/group/create_groups_tables.sql index 60bba06d90..59a2cb2da9 100644 --- a/src/server/sql/group/create_groups_tables.sql +++ b/src/server/sql/group/create_groups_tables.sql @@ -26,6 +26,11 @@ CREATE TABLE IF NOT EXISTS groups_immediate_children ( CHECK (parent_id != child_id) -- No self-references ); +-- The primary key supports traversal from parent to child. Parent lookups, +-- cycle detection, deletion, and foreign-key checks traverse in reverse. +CREATE INDEX IF NOT EXISTS groups_immediate_children_child_parent_idx +ON groups_immediate_children (child_id, parent_id); + /* The groups_deep_children view provides a logical table with a row for each (parent, deep child) relationship in the tree. */ @@ -64,6 +69,11 @@ CREATE TABLE IF NOT EXISTS meters_immediate_children ( CHECK (parent_id != child_id) ); +-- Support reverse meter-tree traversal and foreign-key maintenance without a +-- full scan of the relationship table. +CREATE INDEX IF NOT EXISTS meters_immediate_children_child_parent_idx +ON meters_immediate_children (child_id, parent_id); + /* Similarly to groups_deep_children, meters_deep_children provides all of the (parent, deep_child) relationships in the multitree of meter relationships. @@ -99,6 +109,11 @@ CREATE TABLE IF NOT EXISTS groups_immediate_meters ( PRIMARY KEY (group_id, meter_id) ); +-- The primary key covers group-to-meter access; cache refreshes and meter +-- deletion checks also need efficient meter-to-group access. +CREATE INDEX IF NOT EXISTS groups_immediate_meters_meter_group_idx +ON groups_immediate_meters (meter_id, group_id); + /* This view has a row for each (group, deep child meter) relationship represented by the groups DAG. It also includes a boolean column, is_shadowed, that is true when that group has another meter that monitors a superset @@ -108,51 +123,43 @@ CREATE TABLE IF NOT EXISTS groups_immediate_meters ( TODO: Deal with parent meters that are installed after their children. They only shadow them from a start-date onwards. The above to-do is probably going to require a significant reworking of some stuff. */ -CREATE MATERIALIZED VIEW IF NOT EXISTS groups_deep_meters AS - /* First we need to get all the deep child meters for each group. We just join groups_immediate_meters to - groups_deep_children to grab all the meters associated with a group or one of its deep children. - */ - - WITH all_deep_meters(group_id, meter_id) AS ( - SELECT DISTINCT -- Distinct because two children might include the same meter, and we only want it once. - gdc.parent_id AS group_id, - gim.meter_id AS meter_id - FROM groups_immediate_meters gim - INNER JOIN groups_deep_children gdc ON gdc.child_id = gim.group_id - UNION - SELECT - gim.group_id AS group_id, - gim.meter_id AS meter_id - from groups_immediate_meters gim - ) - SELECT - adm.group_id AS group_id, - adm.meter_id AS meter_id, - EXISTS( - /* - We want to mark meter-group relationships as shadowed if there is another relationship with the same - group that has a meter that is a deep parent of this meter. - We do this by looking for rows in the meters_deep_children (mdc) view where mdc.child_id is the id - of the current meter, and mdc.parent_id is the ID of some other row in all_deep_meters that has the same group ID as - our current group and has a meter id that is a deep parent of our current meter. - */ - SELECT 1 -- It doesn't matter what the result set has, only that it has at least 1 row, so we can just use '1'. - FROM all_deep_meters adm2 - INNER JOIN meters_deep_children mdc ON mdc.parent_id = adm2.meter_id AND mdc.child_id = adm.meter_id - WHERE adm2.group_id = adm.group_id - ) AS is_shadowed - FROM all_deep_meters adm; +-- TODO: Remove this retained legacy materialized-view implementation once the +-- hypertable implementation is finalized. groups_deep_meters_cache replaces it. +-- CREATE MATERIALIZED VIEW IF NOT EXISTS groups_deep_meters AS +-- WITH all_deep_meters(group_id, meter_id) AS ( +-- SELECT DISTINCT +-- gdc.parent_id AS group_id, +-- gim.meter_id AS meter_id +-- FROM groups_immediate_meters gim +-- INNER JOIN groups_deep_children gdc ON gdc.child_id = gim.group_id +-- UNION +-- SELECT +-- gim.group_id AS group_id, +-- gim.meter_id AS meter_id +-- FROM groups_immediate_meters gim +-- ) +-- SELECT +-- adm.group_id AS group_id, +-- adm.meter_id AS meter_id, +-- EXISTS( +-- SELECT 1 +-- FROM all_deep_meters adm2 +-- INNER JOIN meters_deep_children mdc +-- ON mdc.parent_id = adm2.meter_id +-- AND mdc.child_id = adm.meter_id +-- WHERE adm2.group_id = adm.group_id +-- ) AS is_shadowed +-- FROM all_deep_meters adm; CREATE OR REPLACE FUNCTION check_cyclic_groups() RETURNS TRIGGER AS $$ - DECLARE - num_rows INTEGER; BEGIN - SELECT COUNT(*) INTO num_rows - FROM groups_deep_children WHERE child_id = NEW.parent_id AND parent_id = NEW.child_id; - - IF num_rows > 0 THEN + IF EXISTS ( + SELECT 1 + FROM groups_deep_children + WHERE child_id = NEW.parent_id AND parent_id = NEW.child_id + ) THEN RAISE EXCEPTION 'Cyclic group detected'; END IF; RETURN NEW; diff --git a/src/server/sql/group/get_all_children.sql b/src/server/sql/group/get_all_children.sql index e367bdda34..e956082f67 100644 --- a/src/server/sql/group/get_all_children.sql +++ b/src/server/sql/group/get_all_children.sql @@ -4,12 +4,22 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ --- Returns a row for each group with the id, an array of the immediate children meters --- and and array of the immediate group children. --- Note it return an array with one entry of null if no child or group meters. -SELECT g.id as group_id, array_agg(DISTINCT gim.meter_id) as child_meters, array_agg(DISTINCT gic.child_id) as child_groups +-- Aggregate each relationship independently. Joining both relationship tables +-- before aggregating produces a child-meter x child-group intermediate result +-- for every group, which becomes expensive for groups with many children. +SELECT + g.id AS group_id, + COALESCE(meters.child_meters, ARRAY[]::INTEGER[]) AS child_meters, + COALESCE(child_groups.child_groups, ARRAY[]::INTEGER[]) AS child_groups FROM groups g --- Use LEFT OUTER JOIN so get result for all groups. -LEFT OUTER JOIN groups_immediate_meters gim ON g.id = gim.group_id -LEFT OUTER JOIN groups_immediate_children gic ON g.id = gic.parent_id -GROUP BY g.id; +LEFT JOIN ( + SELECT group_id, array_agg(meter_id ORDER BY meter_id) AS child_meters + FROM groups_immediate_meters + GROUP BY group_id +) meters ON meters.group_id = g.id +LEFT JOIN ( + SELECT parent_id, array_agg(child_id ORDER BY child_id) AS child_groups + FROM groups_immediate_children + GROUP BY parent_id +) child_groups ON child_groups.parent_id = g.id +ORDER BY g.id; diff --git a/src/server/sql/group/get_all_groups_with_deep_meters.sql b/src/server/sql/group/get_all_groups_with_deep_meters.sql new file mode 100644 index 0000000000..e3539760fc --- /dev/null +++ b/src/server/sql/group/get_all_groups_with_deep_meters.sql @@ -0,0 +1,17 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +-- Return group metadata and cached deep-meter membership in one set-based +-- query. This replaces one groups_deep_meters_cache query per group. +SELECT + g.*, + COALESCE( + array_agg(gdm.meter_id ORDER BY gdm.meter_id) + FILTER (WHERE gdm.meter_id IS NOT NULL), + ARRAY[]::INTEGER[] + ) AS deep_meters +FROM groups g +LEFT JOIN groups_deep_meters_cache gdm ON gdm.group_id = g.id +GROUP BY g.id +ORDER BY g.id; diff --git a/src/server/sql/group/get_deep_meters_by_group_id.sql b/src/server/sql/group/get_deep_meters_by_group_id.sql index 3865f214b9..ac772d6f73 100644 --- a/src/server/sql/group/get_deep_meters_by_group_id.sql +++ b/src/server/sql/group/get_deep_meters_by_group_id.sql @@ -4,5 +4,5 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ SELECT meter_id -FROM groups_deep_meters +FROM groups_deep_meters_cache WHERE group_id = ${id} diff --git a/src/server/sql/group/get_displayable.sql b/src/server/sql/group/get_displayable.sql index f5bd163156..8f0f85446c 100644 --- a/src/server/sql/group/get_displayable.sql +++ b/src/server/sql/group/get_displayable.sql @@ -1,4 +1,8 @@ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +-- TODO: Research whether external scripts load this query directly. The Group +-- model uses get_displayable_groups.sql for the same result; remove this +-- duplicate if no external consumer depends on its filename. SELECT * FROM groups WHERE displayable=true; diff --git a/src/server/sql/logmsg/create_logmsg_table.sql b/src/server/sql/logmsg/create_logmsg_table.sql index 1e6cfc5d80..1be75b48bc 100644 --- a/src/server/sql/logmsg/create_logmsg_table.sql +++ b/src/server/sql/logmsg/create_logmsg_table.sql @@ -10,4 +10,7 @@ CREATE TABLE IF NOT EXISTS logmsg ( log_time TIMESTAMP NOT NULL ); --- TODO Consider index optimization for queries \ No newline at end of file +-- Log retrieval filters by type and a time range, orders by time, and applies +-- a limit. Keep that path index-backed as the append-only log table grows. +CREATE INDEX IF NOT EXISTS logmsg_time_type_idx +ON logmsg (log_time, log_type); diff --git a/src/server/sql/meter/get_unit_id.sql b/src/server/sql/meter/get_unit_id.sql index 97d83eed1f..294055207c 100644 --- a/src/server/sql/meter/get_unit_id.sql +++ b/src/server/sql/meter/get_unit_id.sql @@ -2,4 +2,7 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +-- TODO: Research whether external scripts load this query directly. No +-- in-repository model or service references it; remove it if unused. + SELECT unit_id FROM meters WHERE id=${meterId}; diff --git a/src/server/sql/reading/TimeScaleDB/create_daily_readings.sql b/src/server/sql/reading/TimeScaleDB/create_daily_readings.sql new file mode 100644 index 0000000000..78b0cae890 --- /dev/null +++ b/src/server/sql/reading/TimeScaleDB/create_daily_readings.sql @@ -0,0 +1,74 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +/* + * Aggregating directly from hypertable_hourly_split, it will be used by by group_daily_readings_unit_cagg. + * Therefore it is necessaary to retain the bucket to allow for proper grouping in the next level of + * aggregation. + * + * Data flow: + * + * readings + * | + * trigger + * | + * v + * hypertable_hourly_split + * | + * v + * meter_daily_readings_unit_cagg + * + * + * Daily aggregation: + * + * Each row in hypertable_hourly_split represents the aggregated a single + * meter, graphic unit, and hour. + * + * The daily continuous aggregate groups those hourly rows into one-day + * buckets and computes: + * + * - Average hourly reading rate for the day. + * - Minimum hourly reading rate observed during the day. + * - Maximum hourly reading rate observed during the day. + * + * + * Time interval: + * + * Exposes the bucket timestamp directly + * + */ +CREATE MATERIALIZED VIEW IF NOT EXISTS meter_daily_readings_unit_cagg +WITH (timescaledb.continuous) +AS +SELECT + meter_id, + sum((reading / extract(EPOCH FROM (end_timestamp - start_timestamp)) * slope + intercept) * extract(EPOCH FROM (end_timestamp - start_timestamp))) / sum(extract(EPOCH FROM (end_timestamp - start_timestamp))) AS reading_rate, + max(reading / extract(EPOCH FROM (end_timestamp - start_timestamp)) * slope + intercept) AS max_rate, + min(reading / extract(EPOCH FROM (end_timestamp - start_timestamp)) * slope + intercept) AS min_rate, + tsrange( + time_bucket('1 day', start_timestamp), + time_bucket('1 day', start_timestamp) + INTERVAL '1 day', + '()' + ) AS time_interval, + graphic_unit_id, + time_bucket('1 day', start_timestamp) AS bucket +FROM hypertable_hourly_split +GROUP BY meter_id, time_bucket('1 day', start_timestamp), graphic_unit_id +-- Materialization order is not part of the view contract; query functions +-- apply their own ordering and use the meter/graphic-unit/bucket index below. +WITH NO DATA; + +-- This should improve meter continuous-aggregate refreshes +CREATE INDEX IF NOT EXISTS meter_daily_cagg_meter_graphic_bucket_idx +ON meter_daily_readings_unit_cagg + (meter_id, graphic_unit_id, bucket); + +/* + * Allow queries to include recent data that has not yet been materialized. + */ +ALTER MATERIALIZED VIEW meter_daily_readings_unit_cagg +SET ( + timescaledb.materialized_only = false +); diff --git a/src/server/sql/reading/TimeScaleDB/create_group_daily_readings.sql b/src/server/sql/reading/TimeScaleDB/create_group_daily_readings.sql new file mode 100644 index 0000000000..31bf5f42cf --- /dev/null +++ b/src/server/sql/reading/TimeScaleDB/create_group_daily_readings.sql @@ -0,0 +1,71 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +/* + * Purpose: + * + * This materialized view rolls up daily meter readings into daily group + * readings. + * + * + * Data flow: + * + * readings + * | + * trigger + * | + * v + * hypertable_hourly_split + * | + * v + * meter_hourly_readings_unit_cagg + * | + * v + * group_daily_readings_unit_cagg + * + * + * The view combines meter-level daily readings into group-level daily + * readings by summing all meters belonging to the group. + */ +CREATE MATERIALIZED VIEW IF NOT EXISTS group_daily_readings_unit_cagg +WITH (timescaledb.continuous) +AS +SELECT + gdm.group_id, + SUM(dr.reading_rate) AS reading_rate, + tsrange( + time_bucket('1 day', dr.bucket), + time_bucket('1 day', dr.bucket) + INTERVAL '1 day', + '()' + ) AS time_interval, + time_bucket('1 day', dr.bucket) AS bucket, + dr.graphic_unit_id +FROM meter_daily_readings_unit_cagg dr JOIN + groups_deep_meters_cache gdm ON dr.meter_id = gdm.meter_id JOIN + group_graphic_units_cache gu ON gu.group_id = gdm.group_id AND dr.graphic_unit_id = gu.graphic_unit_id +GROUP BY + gdm.group_id, + time_bucket('1 day', dr.bucket), + dr.graphic_unit_id +WITH NO DATA; + +-- This should improve group continuous-aggregate refreshes +CREATE INDEX IF NOT EXISTS group_daily_cagg_group_graphic_bucket_idx +ON group_daily_readings_unit_cagg + (group_id, graphic_unit_id, bucket); + +/* + * Allow queries to include recent data that has not yet been materialized. + */ +ALTER MATERIALIZED VIEW group_daily_readings_unit_cagg +SET ( + timescaledb.materialized_only = false +); + +/* + * Preserve existing database ownership. + */ +ALTER TABLE group_daily_readings_unit_cagg +OWNER TO oed; diff --git a/src/server/sql/reading/TimeScaleDB/create_group_dependencies.sql b/src/server/sql/reading/TimeScaleDB/create_group_dependencies.sql new file mode 100644 index 0000000000..722b2f147d --- /dev/null +++ b/src/server/sql/reading/TimeScaleDB/create_group_dependencies.sql @@ -0,0 +1,204 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +/* + * Purpose: + * + * Create and maintain the non-time-series cache tables required by the + * group continuous aggregates. + * + * + * Background: + * + * TimescaleDB continuous aggregates cannot depend on: + * + * - recursive views + * - PL/pgSQL functions + * - dynamic table-returning logic + * + * The previous group aggregation workflow depended on: + * + * groups_deep_meters_cache table + * legacy graphic-unit compatibility function + * + * These objects are replaced with cache tables that are refreshed before + * refreshing the group continuous aggregates. + * + * + * Data flow: + * + * groups_immediate_children + * | + * v + * groups_deep_children + * | + * v + * groups_deep_meters_cache + * | + * v + * group_graphic_units_cache + * | + * v + * group_hourly_readings_unit_cagg + * group_daily_readings_unit_cagg + * + * + * Refresh: + * + * The following functions should be called before refreshing group + * continuous aggregates: + * + * update_groups_deep_meters_cache() + * update_group_graphic_units_cache() + * + */ + + +/* + * 1. Cache all group to meter relationships. + * + * This replaces the previous groups_deep_meters_cache materialized view. + * + * Each row represents a meter that contributes readings to a group, + * including meters inherited through child groups. + */ + +CREATE TABLE IF NOT EXISTS groups_deep_meters_cache ( + group_id INTEGER NOT NULL REFERENCES groups(id), + meter_id INTEGER NOT NULL REFERENCES meters(id), + PRIMARY KEY(group_id, meter_id) +); + +-- This should improve group continuous-aggregate refreshes +CREATE INDEX IF NOT EXISTS groups_deep_meters_cache_meter_group_idx +ON groups_deep_meters_cache (meter_id, group_id); + + +/* + * Refresh groups_deep_meters_cache. + * + * Calculates the desired rows once, removes stale rows, and inserts new rows. + */ +CREATE OR REPLACE FUNCTION update_groups_deep_meters_cache() +RETURNS void +AS $$ +BEGIN + /* + * desired is referenced by both deletion and insertion. MATERIALIZED keeps + * the recursive group traversal from being calculated twice. + */ + WITH desired AS MATERIALIZED ( + WITH all_deep_meters(group_id, meter_id) AS ( + SELECT DISTINCT gdc.parent_id AS group_id, gim.meter_id AS meter_id + FROM groups_immediate_meters gim INNER JOIN + groups_deep_children gdc ON gdc.child_id = gim.group_id + UNION + SELECT gim.group_id, gim.meter_id + FROM groups_immediate_meters gim + ) + SELECT group_id, meter_id + FROM all_deep_meters + ), + /* + * Data-modifying CTEs execute even when their RETURNING rows are not used. + * Remove stale relationships while the main statement inserts newly + * desired relationships. + */ + removed AS ( + DELETE FROM groups_deep_meters_cache target + WHERE NOT EXISTS ( + SELECT 1 + FROM desired + WHERE desired.group_id = target.group_id + AND desired.meter_id = target.meter_id + ) + RETURNING target.group_id + ) + INSERT INTO groups_deep_meters_cache(group_id, meter_id) + SELECT desired.group_id, desired.meter_id + FROM desired + ON CONFLICT (group_id, meter_id) DO NOTHING; +END; +$$ LANGUAGE plpgsql; + +/* + * 2. Cache compatible graphic units for groups. + * + * This replaces the legacy graphic-unit compatibility function. + * + * Each row represents a graphic unit that can display all meters belonging + * to the group. + */ +CREATE TABLE IF NOT EXISTS group_graphic_units_cache ( + group_id INTEGER NOT NULL REFERENCES groups(id), + graphic_unit_id INTEGER NOT NULL REFERENCES units(id), + PRIMARY KEY(group_id, graphic_unit_id) +); + + +/* + * Refresh group graphic unit compatibility. + * + * A graphic unit is valid for a group only when every meter unit contained + * in that group has a conversion path to that graphic unit. + */ +CREATE OR REPLACE FUNCTION update_group_graphic_units_cache() +RETURNS void +AS $$ +BEGIN + + /* + * Calculate compatible graphic units once for both cache deletion and + * insertion. This is the expensive portion because it aggregates source + * units and checks conversion coverage for every group. + */ + WITH desired AS MATERIALIZED ( + WITH group_source_units AS ( + SELECT gdm.group_id, array_agg(DISTINCT m.unit_id) AS source_units + FROM groups_deep_meters_cache gdm INNER JOIN + meters m ON m.id = gdm.meter_id + GROUP BY gdm.group_id + ), + compatible_units AS ( + SELECT gsu.group_id, c.destination_id AS graphic_unit_id + FROM group_source_units gsu INNER JOIN + cik c ON c.source_id = ANY(gsu.source_units) + GROUP BY gsu.group_id, c.destination_id, gsu.source_units + HAVING array_agg(DISTINCT c.source_id) @> gsu.source_units + ) + SELECT group_id, graphic_unit_id + FROM compatible_units + ), + -- Remove compatibility rows that are no longer present in desired. + removed AS ( + DELETE FROM group_graphic_units_cache target + WHERE NOT EXISTS ( + SELECT 1 + FROM desired + WHERE desired.group_id = target.group_id + AND desired.graphic_unit_id = target.graphic_unit_id + ) + RETURNING target.group_id + ) + INSERT INTO group_graphic_units_cache(group_id, graphic_unit_id) + SELECT desired.group_id, desired.graphic_unit_id + FROM desired + ON CONFLICT (group_id, graphic_unit_id) DO NOTHING; + +END; +$$ LANGUAGE plpgsql; + +/* + * Initial population. + * + * This allows a fresh installation to immediately support group CAGGs. + * Subsequent updates should call these functions before refreshing CAGGs. + */ +DO $$ +BEGIN + PERFORM update_groups_deep_meters_cache(); + PERFORM update_group_graphic_units_cache(); +END +$$; diff --git a/src/server/sql/reading/TimeScaleDB/create_group_hourly_readings.sql b/src/server/sql/reading/TimeScaleDB/create_group_hourly_readings.sql new file mode 100644 index 0000000000..49428543e2 --- /dev/null +++ b/src/server/sql/reading/TimeScaleDB/create_group_hourly_readings.sql @@ -0,0 +1,92 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + + /* + * Purpose: + * + * This materialized view rolls up hourly meter readings into hourly group + * readings. + * + * + * Data flow: + * + * readings + * | + * trigger + * | + * v + * hypertable_hourly_split + * | + * v + * meter_hourly_readings_unit_cagg + * | + * v + * group_hourly_readings_unit_cagg + * + * + * The aggregate combines meter-level hourly readings into group-level hourly + * readings by summing all meters belonging to the group. + * + * + * The group hourly aggregate applies group membership and graphic unit + * filtering while preserving the hourly reporting interval format used by the + * existing reporting layer. + */ + +/* + * Create the group materialized view over the TimescaleDB meter aggregate. + * + * Source: + * + * meter_hourly_readings_unit_cagg + * + * The meter-level hourly aggregate already contains: + * + * - Hourly time bucketing + * - Unit conversion + * - Time-varying conversion handling + * + * This aggregate only performs the group-level rollup. + */ +CREATE MATERIALIZED VIEW IF NOT EXISTS group_hourly_readings_unit_cagg +WITH (timescaledb.continuous) +AS +SELECT + gdm.group_id, + SUM(hr.reading_rate) AS reading_rate, + tsrange( + time_bucket('1 hour', hr.bucket), + time_bucket('1 hour', hr.bucket) + INTERVAL '1 hour', + '()' + ) AS time_interval, + time_bucket('1 hour', hr.bucket) AS bucket, + hr.graphic_unit_id +FROM meter_hourly_readings_unit_cagg hr INNER JOIN + groups_deep_meters_cache gdm ON hr.meter_id = gdm.meter_id INNER JOIN + group_graphic_units_cache gu ON gu.group_id = gdm.group_id AND hr.graphic_unit_id = gu.graphic_unit_id +GROUP BY + gdm.group_id, + time_bucket('1 hour', hr.bucket), + hr.graphic_unit_id +WITH NO DATA; + +-- This should improve group continuous-aggregate refreshes +CREATE INDEX IF NOT EXISTS group_hourly_cagg_group_graphic_bucket_idx +ON group_hourly_readings_unit_cagg + (group_id, graphic_unit_id, bucket); + +/* + * Allow queries to include recent data that has not yet been materialized. + */ +ALTER MATERIALIZED VIEW group_hourly_readings_unit_cagg +SET ( + timescaledb.materialized_only = false +); + +/* + * Preserve existing database ownership. + */ +ALTER TABLE group_hourly_readings_unit_cagg +OWNER TO oed; \ No newline at end of file diff --git a/src/server/sql/reading/TimeScaleDB/create_hourly_readings.sql b/src/server/sql/reading/TimeScaleDB/create_hourly_readings.sql new file mode 100644 index 0000000000..e1afd2fd24 --- /dev/null +++ b/src/server/sql/reading/TimeScaleDB/create_hourly_readings.sql @@ -0,0 +1,117 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + + /* + * Prefrace: + * This script continues the work introduced in PR#1546, which established the + * benchmark for migrating hourly meter reading queries from PostgreSQL + * materialized views to TimescaleDB hypertables and continuous aggregates. + * + * Only the database objects required from PR#1546 were carried forward and + * adapted to integrate TimescaleDB continuous aggregates with the existing + * hourly meter reading workflow in the timeVary branch. + * + * Purpose: + * + * Create the TimescaleDB continuous aggregate used for hourly meter + * readings. + * + * Data flow: + * + * readings + * | + * trigger + * │ + * ▼ + * hypertable_hourly_split + * │ + * ▼ + * meter_hourly_readings_unit_cagg + * + * Notes: + * + * - The continuous aggregate is built from hypertable_hourly_split. + * - Hourly reading slices are maintained by the trigger created in + * create_prerequisites.sql. + * - Refreshing the aggregate is performed separately using + * refresh_continuous_aggregate() or a refresh policy. + */ + +/* + * 1. Create continuous aggregate for hourly meter readings. + * + * This continuous aggregate replaces the existing + * meter_hourly_readings_unit materialized view using TimescaleDB's + * incremental aggregation engine. + * + * Data flow: + * + * hypertable_hourly_split + * | + * v + * meter_hourly_readings_unit_cagg + * + * + * The hourly split table already contains: + * + * - hourly overlap calculations + * - cik_vary conversion parameters + * - unit metadata + * + * Therefore, the continuous aggregate does not need to join against + * cik_vary or other lookup tables during query execution. + * + * + * Reading calculation: + * + * hypertable_hourly_split stores reading contributions scaled by the + * duration overlap within each hourly slice. + * + * To calculate the final hourly reading rate: + * + * 1. Convert the stored contribution back into a rate. + * 2. Apply the cik_vary conversion: + * + * converted_value = rate * slope + intercept + * + * 3. Weight the converted rate by the duration of the slice. + * 4. Divide by the total duration to produce the weighted average. + * + * This reproduces the calculation performed by the original + * meter_hourly_readings_unit materialized view. + */ +CREATE MATERIALIZED VIEW IF NOT EXISTS meter_hourly_readings_unit_cagg +WITH (timescaledb.continuous) +AS +SELECT + meter_id, + graphic_unit_id, + time_bucket('1 hour', start_timestamp) AS bucket, + sum(( reading / extract( EPOCH FROM (end_timestamp - start_timestamp) ) * slope + intercept ) * extract(EPOCH FROM (end_timestamp - start_timestamp))) / sum(extract(EPOCH FROM (end_timestamp - start_timestamp))) AS reading_rate, + max(reading / extract(EPOCH FROM (end_timestamp - start_timestamp)) * slope + intercept) AS max_rate, + min(reading / extract(EPOCH FROM (end_timestamp - start_timestamp)) * slope + intercept) AS min_rate, + unit_represent, + sec_in_rate +FROM hypertable_hourly_split +GROUP BY + meter_id, + graphic_unit_id, + time_bucket('1 hour', start_timestamp), + unit_represent, + sec_in_rate +WITH NO DATA; + +-- This should improve meter continuous-aggregate refreshes +CREATE INDEX IF NOT EXISTS meter_hourly_cagg_meter_graphic_bucket_idx +ON meter_hourly_readings_unit_cagg + (meter_id, graphic_unit_id, bucket); + +/* + * Allow queries to include recent data that has not yet been materialized. + */ +ALTER MATERIALIZED VIEW meter_hourly_readings_unit_cagg +SET ( + timescaledb.materialized_only = false +); diff --git a/src/server/sql/reading/TimeScaleDB/create_prerequisites.sql b/src/server/sql/reading/TimeScaleDB/create_prerequisites.sql new file mode 100644 index 0000000000..8095e99533 --- /dev/null +++ b/src/server/sql/reading/TimeScaleDB/create_prerequisites.sql @@ -0,0 +1,416 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + + /* + * Prefrace: + * This script continues the work introduced in PR#1546, which established the + * benchmark for migrating hourly meter reading queries from PostgreSQL + * materialized views to TimescaleDB hypertables and continuous aggregates. + * + * Only the database objects required from PR#1546 were carried forward and + * adapted to integrate TimescaleDB continuous aggregates with the existing + * hourly meter reading workflow in the timeVary branch. + * + * Purpose: + * + * Create the TimescaleDB infrastructure required by the hourly and daily + * continuous aggregates. + * + * This script creates: + * + * 1. hypertable_hourly_split + * 2. TimescaleDB hypertable + * 3. Supporting indexes + * 4. Trigger function to maintain the hypertable + * 5. Trigger on readings + * 6. Rebuild function + * + * Data flow: + * + * readings + * │ + * ▼ + * hypertable_hourly_split + * + * Notes: + * + * - This script does not create any continuous aggregates. + * - The trigger maintains the hypertable as readings are inserted, + * updated, or deleted. + * - The rebuild function allows the hypertable to be regenerated from + * readings when required (for example after rebuilding cik_vary). + */ + + /* + * 1. Create hypertable_hourly_split. + * + * Stores readings after they have been split at both hourly and conversion + * boundaries. A single row in readings may produce multiple rows when it + * crosses an hour boundary, a cik_vary boundary, or both. + * + * Conversion information from cik_vary is stored with each hourly slice so + * downstream aggregation does not need to join back to cik_vary. + * + * Columns: + * + * reading: + * Reading contribution scaled by the overlap duration within this hour. + * + * slope/intercept/graphic_unit_id: + * Conversion parameters required to convert the reading into the target + * graphic unit that was valid at the time of the reading. + * + * unit_represent/sec_in_rate: + * Original unit metadata required to correctly aggregate quantity, flow, + * and raw readings. + */ +CREATE TABLE IF NOT EXISTS hypertable_hourly_split ( + meter_id INTEGER NOT NULL, + reading FLOAT NOT NULL, + start_timestamp TIMESTAMP WITHOUT TIME ZONE NOT NULL, + end_timestamp TIMESTAMP WITHOUT TIME ZONE NOT NULL, + unit_represent unit_represent_type NOT NULL, + sec_in_rate FLOAT NOT NULL, + slope FLOAT NOT NULL, + intercept FLOAT NOT NULL, + graphic_unit_id INTEGER NOT NULL +); + +-- All four aggregates use materialized_only=false, meaning queries can reach unmaterialized split rows. +CREATE INDEX IF NOT EXISTS hypertable_hourly_split_meter_graphic_time_idx +ON hypertable_hourly_split + (meter_id, graphic_unit_id, start_timestamp DESC); + +-- row trigger searches cik_vary by source_id and an overlapping time range +CREATE INDEX IF NOT EXISTS cik_vary_source_time_idx +ON cik_vary (source_id, start_time, end_time); + + +/* + * 2. Convert hypertable_hourly_split into a TimescaleDB hypertable. + * + * start_timestamp is used as the time dimension because hourly slices are + * primarily queried and aggregated by time range. + */ +SELECT create_hypertable( + 'hypertable_hourly_split', + by_range('start_timestamp'), + if_not_exists => TRUE +); + + +/* + * 3. Enforce uniqueness of hourly slices. + * + * The unique index prevents duplicate hourly slices for the same meter and + * time range and supports efficient maintenance operations. + * This index also improves lookup performance when synchronizing changed + * readings. + */ +CREATE UNIQUE INDEX IF NOT EXISTS hypertable_hourly_split_meter_time_idx +ON hypertable_hourly_split +( + meter_id, + start_timestamp, + end_timestamp, + graphic_unit_id, + slope, + intercept +); + +/* + * Track source changes that require rebuilding hypertable_hourly_split. + * + * A revision counter is used instead of a boolean so a source change that + * commits while a rebuild is running cannot be accidentally cleared. The + * refresher only records the revision that it actually rebuilt. + */ +CREATE TABLE IF NOT EXISTS reading_aggregate_state ( + id SMALLINT PRIMARY KEY CHECK (id = 1), + rebuild_revision BIGINT NOT NULL DEFAULT 0, + completed_rebuild_revision BIGINT NOT NULL DEFAULT 0, + group_cache_revision BIGINT NOT NULL DEFAULT 0, + completed_group_cache_revision BIGINT NOT NULL DEFAULT 0 +); + +-- Keep this prerequisite script rerunnable against databases that created the +-- state table before group-cache revision tracking was added. +ALTER TABLE reading_aggregate_state +ADD COLUMN IF NOT EXISTS group_cache_revision BIGINT NOT NULL DEFAULT 0; + +ALTER TABLE reading_aggregate_state +ADD COLUMN IF NOT EXISTS completed_group_cache_revision BIGINT NOT NULL DEFAULT 0; + +INSERT INTO reading_aggregate_state (id) +VALUES (1) +ON CONFLICT (id) DO NOTHING; + +/* + * Unit metadata is copied into hypertable_hourly_split. Mark existing split + * rows stale when a meter changes units or relevant unit metadata changes. + */ +CREATE OR REPLACE FUNCTION mark_reading_aggregate_rebuild_required() +RETURNS trigger +AS $$ +BEGIN + UPDATE reading_aggregate_state + SET rebuild_revision = rebuild_revision + 1 + WHERE id = 1; + + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS trigger_meter_unit_requires_reading_rebuild +ON meters; + +CREATE TRIGGER trigger_meter_unit_requires_reading_rebuild +AFTER UPDATE OF unit_id +ON meters +FOR EACH ROW +WHEN (OLD.unit_id IS DISTINCT FROM NEW.unit_id) +EXECUTE FUNCTION mark_reading_aggregate_rebuild_required(); + +/* + * Group membership, meter units, and conversion paths determine the contents + * of the two group dependency caches. Track those changes independently from + * reading imports so normal aggregate refreshes can skip cache maintenance. + */ +CREATE OR REPLACE FUNCTION mark_group_reading_cache_refresh_required() +RETURNS trigger +AS $$ +BEGIN + UPDATE reading_aggregate_state + SET group_cache_revision = group_cache_revision + 1 + WHERE id = 1; + + RETURN NULL; +END; +$$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS trigger_meter_unit_requires_group_cache_refresh +ON meters; + +CREATE TRIGGER trigger_meter_unit_requires_group_cache_refresh +AFTER UPDATE OF unit_id +ON meters +FOR EACH ROW +WHEN (OLD.unit_id IS DISTINCT FROM NEW.unit_id) +EXECUTE FUNCTION mark_group_reading_cache_refresh_required(); + +-- Group hierarchy changes affect inherited meter membership. +DROP TRIGGER IF EXISTS trigger_group_children_require_group_cache_refresh +ON groups_immediate_children; + +CREATE TRIGGER trigger_group_children_require_group_cache_refresh +AFTER INSERT OR UPDATE OR DELETE +ON groups_immediate_children +FOR EACH STATEMENT +EXECUTE FUNCTION mark_group_reading_cache_refresh_required(); + +-- Direct group-meter changes affect both group dependency caches. +DROP TRIGGER IF EXISTS trigger_group_meters_require_group_cache_refresh +ON groups_immediate_meters; + +CREATE TRIGGER trigger_group_meters_require_group_cache_refresh +AFTER INSERT OR UPDATE OR DELETE +ON groups_immediate_meters +FOR EACH STATEMENT +EXECUTE FUNCTION mark_group_reading_cache_refresh_required(); + +DROP TRIGGER IF EXISTS trigger_unit_metadata_requires_reading_rebuild +ON units; + +CREATE TRIGGER trigger_unit_metadata_requires_reading_rebuild +AFTER UPDATE OF unit_represent, sec_in_rate +ON units +FOR EACH ROW +WHEN ( + OLD.unit_represent IS DISTINCT FROM NEW.unit_represent + OR OLD.sec_in_rate IS DISTINCT FROM NEW.sec_in_rate +) +EXECUTE FUNCTION mark_reading_aggregate_rebuild_required(); + + +/* + * 4. Maintain hypertable_hourly_split from changes in readings. + * + * This trigger function maintains only the hourly slices associated with the + * reading affected by the trigger event. + * + * INSERT: + * Generate hourly slices for the new reading. + * + * UPDATE: + * Recalculate hourly slices for the updated reading. + * + * DELETE: + * Remove hourly slices generated from the deleted reading. + */ +CREATE OR REPLACE FUNCTION update_hourly_hypertable() +RETURNS trigger +AS $$ +BEGIN + + /* + * DELETE removes all hourly slices generated from the deleted reading. + */ + IF TG_OP = 'DELETE' THEN + + DELETE FROM hypertable_hourly_split + WHERE meter_id = OLD.meter_id + AND start_timestamp >= OLD.start_timestamp + AND end_timestamp <= OLD.end_timestamp; + + RETURN OLD; + + END IF; + + + /* + * UPDATE may change the reading duration or hour boundaries. + * Remove the hourly slices generated from the previous version + * of the reading before rebuilding them from NEW. + */ + IF TG_OP = 'UPDATE' THEN + + DELETE FROM hypertable_hourly_split + WHERE meter_id = OLD.meter_id + AND start_timestamp >= OLD.start_timestamp + AND end_timestamp <= OLD.end_timestamp; + + END IF; + + INSERT INTO hypertable_hourly_split(meter_id, reading, start_timestamp, end_timestamp, unit_represent, sec_in_rate, slope, intercept, graphic_unit_id) + SELECT + NEW.meter_id, + CASE + WHEN u.unit_represent = 'quantity'::unit_represent_type THEN + (NEW.reading * 3600 / extract(EPOCH FROM (NEW.end_timestamp - NEW.start_timestamp))) + * extract(EPOCH FROM (slice.end_timestamp - slice.start_timestamp)) + WHEN u.unit_represent IN ('flow'::unit_represent_type, 'raw'::unit_represent_type ) THEN + (NEW.reading * 3600 / u.sec_in_rate) + * extract(EPOCH FROM (slice.end_timestamp - slice.start_timestamp)) + END AS reading, + slice.start_timestamp, + slice.end_timestamp, + u.unit_represent, + u.sec_in_rate, + c.slope, + c.intercept, + c.destination_id AS graphic_unit_id + FROM meters m INNER JOIN + units u ON m.unit_id = u.id INNER JOIN + cik_vary c ON c.source_id = m.unit_id + AND c.start_time < NEW.end_timestamp + AND c.end_time > NEW.start_timestamp CROSS JOIN + LATERAL generate_series(date_trunc('hour', NEW.start_timestamp), date_trunc_up('hour', NEW.end_timestamp) - INTERVAL '1 hour', INTERVAL '1 hour') gen(interval_start) CROSS JOIN + LATERAL ( + SELECT + greatest(NEW.start_timestamp, gen.interval_start, c.start_time) AS start_timestamp, + least(NEW.end_timestamp, gen.interval_start + INTERVAL '1 hour', c.end_time) AS end_timestamp + ) slice + WHERE m.id = NEW.meter_id + AND slice.start_timestamp < slice.end_timestamp; + RETURN NEW; + +END; +$$ LANGUAGE plpgsql; + +/* + * 5. Recreate the readings trigger. + * + * PostgreSQL does not support CREATE TRIGGER IF NOT EXISTS, so the existing + * trigger is removed first to make this script safe to rerun during development + * and deployment. + */ +DROP TRIGGER IF EXISTS trigger_readings_update_hourly_hypertable +ON readings; + + +/* + * 6. Create trigger to maintain hypertable_hourly_split. + * + * The trigger fires after changes are committed to readings so the trigger + * function can query the updated source row when generating hourly slices. + * + * This trigger maintains the source data used by TimescaleDB continuous + * aggregates. + * + * TimescaleDB continuous aggregates do not refresh immediately from this + * trigger. They track affected time ranges through invalidation and are + * refreshed separately using: + * + * refresh_continuous_aggregate() + * + * or a continuous aggregate refresh policy. + * + * Events: + * + * INSERT: + * Creates hourly split records for the new reading. + * + * UPDATE: + * Removes old hourly split records and recreates them from the + * modified reading. + * + * DELETE: + * Removes hourly split records generated from the deleted reading. + */ +CREATE TRIGGER trigger_readings_update_hourly_hypertable +AFTER INSERT OR UPDATE OR DELETE +ON readings +FOR EACH ROW +EXECUTE FUNCTION update_hourly_hypertable(); + + +/* + * Rebuild hypertable_hourly_split from the current readings and cik_vary rows. + * + * This is needed after cik_vary is regenerated because the trigger stores the + * conversion metadata that exists at reading insert time. If readings were + * inserted before redoCikVary ran, or if conversion segments changed, the split + * rows must be rebuilt from the current conversion table before the continuous + * aggregate is refreshed. + */ +CREATE OR REPLACE FUNCTION rebuild_hourly_hypertable_split() +RETURNS void +AS $$ +BEGIN + DELETE FROM hypertable_hourly_split; + + INSERT INTO hypertable_hourly_split(meter_id, reading, start_timestamp, end_timestamp, unit_represent, sec_in_rate, slope, intercept, graphic_unit_id) + SELECT + r.meter_id, + CASE + WHEN u.unit_represent = 'quantity'::unit_represent_type THEN + (r.reading * 3600 / extract(EPOCH FROM(r.end_timestamp - r.start_timestamp))) + * extract(EPOCH FROM (slice.end_timestamp - slice.start_timestamp)) + WHEN u.unit_represent IN('flow'::unit_represent_type, 'raw'::unit_represent_type) THEN + (r.reading * 3600 / u.sec_in_rate) + * extract(EPOCH FROM (slice.end_timestamp - slice.start_timestamp)) + END AS reading, + slice.start_timestamp, + slice.end_timestamp, + u.unit_represent, + u.sec_in_rate, + c.slope, + c.intercept, + c.destination_id AS graphic_unit_id + FROM readings r INNER JOIN + meters m ON r.meter_id = m.id INNER JOIN + units u ON m.unit_id = u.id INNER JOIN + cik_vary c ON c.source_id = m.unit_id + AND c.start_time < r.end_timestamp + AND c.end_time > r.start_timestamp CROSS JOIN + LATERAL generate_series(date_trunc('hour', r.start_timestamp), date_trunc_up('hour', r.end_timestamp) - INTERVAL '1 hour', INTERVAL '1 hour') gen(interval_start) CROSS JOIN + LATERAL ( + SELECT + greatest(r.start_timestamp, gen.interval_start, c.start_time) AS start_timestamp, + least(r.end_timestamp, gen.interval_start + INTERVAL '1 hour', c.end_time) AS end_timestamp + ) slice + WHERE slice.start_timestamp < slice.end_timestamp; +END; +$$ LANGUAGE plpgsql; diff --git a/src/server/sql/reading/TimeScaleDB/drop_legacy_reading_views.sql b/src/server/sql/reading/TimeScaleDB/drop_legacy_reading_views.sql new file mode 100644 index 0000000000..a72148c181 --- /dev/null +++ b/src/server/sql/reading/TimeScaleDB/drop_legacy_reading_views.sql @@ -0,0 +1,17 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + + /* +TO DO: can be removed after the next release, when all users have upgraded to a version +that uses TimescaleDB continuous aggregates instead of PostgreSQL materialized views. +Remove PostgreSQL materialized reading views replaced by TimescaleDB +continuous aggregates. Group views must be dropped before the meter views +they depend on. + */ +DROP MATERIALIZED VIEW IF EXISTS group_daily_readings_unit; +DROP MATERIALIZED VIEW IF EXISTS group_hourly_readings_unit; +DROP MATERIALIZED VIEW IF EXISTS meter_daily_readings_unit; +DROP MATERIALIZED VIEW IF EXISTS meter_hourly_readings_unit; +DROP MATERIALIZED VIEW IF EXISTS groups_deep_meters; diff --git a/src/server/sql/reading/TimeScaleDB/update_function_get_3d_readings.sql b/src/server/sql/reading/TimeScaleDB/update_function_get_3d_readings.sql new file mode 100644 index 0000000000..6d756960ca --- /dev/null +++ b/src/server/sql/reading/TimeScaleDB/update_function_get_3d_readings.sql @@ -0,0 +1,264 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + + /* +This takes tsrange_to_shrink which is the requested time range to plot and makes sure it does +not exceed the start/end times for the readings in the supplied meter. This can be an issue, in particular, +because infinity is used to indicate to graph all readings. This version does it to the nearest +day by using the day reading view and is used by 3D readings which only allow days and a single meter. + */ +CREATE OR REPLACE FUNCTION shrink_tsrange_to_meter_readings_by_day(tsrange_to_shrink TSRANGE, meter_id_desired INTEGER) + RETURNS TSRANGE +AS $$ +DECLARE + readings_max_tsrange TSRANGE; +BEGIN + SELECT tsrange(min(bucket), max(bucket + INTERVAL '1 day')) INTO readings_max_tsrange + FROM meter_daily_readings_unit_cagg + where meter_id = meter_id_desired; + RETURN tsrange_to_shrink * readings_max_tsrange; +END; +$$ LANGUAGE 'plpgsql'; + +/* Similar to meter version but for a group */ +CREATE OR REPLACE FUNCTION shrink_tsrange_to_group_readings_by_day(tsrange_to_shrink TSRANGE, group_id_desired INTEGER) + RETURNS TSRANGE +AS $$ +DECLARE + readings_max_tsrange TSRANGE; +BEGIN + SELECT tsrange(min(bucket), max(bucket + INTERVAL '1 day')) INTO readings_max_tsrange + FROM group_daily_readings_unit_cagg + where group_id = group_id_desired; + RETURN tsrange_to_shrink * readings_max_tsrange; +END; +$$ LANGUAGE 'plpgsql'; + +-- Determines the spacing between 3D points. It uses the lowest valid spacing +-- for all requested meters. +CREATE OR REPLACE FUNCTION reading_interval_3d ( + IN meter_ids_requested INTEGER[], + IN reading_length_hours INTEGER, + OUT reading_length_hours_use INTEGER, + OUT reading_length_interval INTERVAL +) +AS $$ +DECLARE + meter_frequency INTERVAL; + meter_frequency_hour_up INTEGER; + max_frequency INTEGER; +BEGIN + SELECT min(reading_frequency) INTO meter_frequency + FROM meters m + INNER JOIN unnest(meter_ids_requested) meters(id) ON m.id = meters.id; + + meter_frequency_hour_up := CEIL(EXTRACT(EPOCH FROM meter_frequency) / 3600); + max_frequency := GREATEST(meter_frequency_hour_up, reading_length_hours); + + IF (max_frequency = 5) THEN + reading_length_hours_use := 6; + ELSIF (max_frequency = 7) THEN + reading_length_hours_use := 8; + ELSIF (max_frequency > 8 AND max_frequency < 12) THEN + reading_length_hours_use := 12; + ELSE + reading_length_hours_use := max_frequency; + END IF; + + reading_length_interval := (reading_length_hours_use::TEXT || ' hour')::INTERVAL; +END; +$$ LANGUAGE 'plpgsql'; + +-- Gets meters graphing data for 3D graphic by returning points that span the requested +-- length of time over the days requested. +-- New meter_3d_readings_unit function that uses new meter_hourly_readings_unit_cagg view. +CREATE OR REPLACE FUNCTION meter_3d_readings_unit ( + -- The desired meter ids. It is normally a single value for a 3D graphic. + -- TODO Should the array be changed to a single value as with group? Need to be sure client never asks for multiple. + meter_ids_requested INTEGER[], + -- The desired graphic unit of the returned data + graphic_unit_id_requested INTEGER, + -- The start and end time for the data to return + start_stamp TIMESTAMP, + end_stamp TIMESTAMP, + -- The number of hours in each reading requested + reading_length_hours INTEGER +) + RETURNS TABLE(meter_id INTEGER, reading_rate FLOAT, start_timestamp TIMESTAMP, end_timestamp TIMESTAMP) +AS $$ +DECLARE + -- Holds the range of dates for returned data that fits the actual data. + requested_range TSRANGE; + -- The number of hours in each reading determined as an interval + reading_length_interval INTERVAL; + -- Which index of the meter_id array you are currently working on. + current_meter_index INTEGER := 1; + -- The id of the meter index working on + current_meter_id INTEGER; + -- The meter frequency from all meters. + meter_frequency INTERVAL; + -- The meter frequency rounded up to a whole number of hours. + meter_frequency_hour_up INTEGER; + -- The larger of the meter value and the argument sent. + max_frequency INTEGER; + -- The number of hours in each reading determined + reading_length_hours_use INTEGER; +BEGIN + -- Find the correct number of hours per reading returned. + SELECT * from reading_interval_3d(meter_ids_requested, reading_length_hours) into reading_length_hours_use, reading_length_interval; + + -- Loop over all meters. + WHILE current_meter_index <= cardinality(meter_ids_requested) LOOP + -- ID of the current meter in loop + current_meter_id := meter_ids_requested[current_meter_index]; + + -- Get the range of days requested by calling shrink_tsrange_to_meter_readings_by_day. + -- First make requested range only be full days by dropping any partial days at start/end. + requested_range := shrink_tsrange_to_meter_readings_by_day(tsrange(date_trunc_up('day', start_stamp), date_trunc('day', end_stamp)), current_meter_id); + + IF (reading_length_hours_use <= 12) THEN + -- Need to generate_series to group the desired hours together + RETURN QUERY + -- The readings are rates in the hourly table so want to average not sum so + -- work for quantity, flow & raw. + -- The time starts at the time of the generated sequence and ends at the length + -- of each block later. This is the same as the start time of the next value + -- in the sequence (except last one). + SELECT + -- Modified to retrieve converted hourly readings from the materialized view. + mhr.meter_id as meter_id, + AVG(mhr.reading_rate) as reading_rate, + hours.hour AS start_timestamp, + hours.hour + reading_length_interval AS end_timestamp + -- This is the series that starts at the beginning of the desired days, + -- ends at the end of the desired days and steps by the desired interval. + -- You need to subtract from the last interval for the end since generate_series + -- is inclusive. + FROM ( + SELECT hour + FROM generate_series( + lower(requested_range), + upper(requested_range) - reading_length_interval, + reading_length_interval + ) hours(hour) + ) hours(hour), + -- Also need the values in the meter hourly table. + meter_hourly_readings_unit_cagg mhr + -- Only want the desired meter + WHERE mhr.meter_id = current_meter_id + -- Only want the desired graphing unit + AND mhr.graphic_unit_id = graphic_unit_id_requested + -- Only want readings that lie within this slice of the desired data + AND mhr.bucket >= hours.hour + AND mhr.bucket <= hours.hour + reading_length_interval - INTERVAL '1 hour' + -- Group by the start time of the generated series since all points in + -- the desired slice have the same start time for the series. + -- Also group by the meter_id since Postgres wants and desired for graphing + GROUP BY hours.hour, mhr.meter_id + -- Time sort by the meter and start time for graphing. + ORDER BY mhr.meter_id, hours.hour + ; + ELSE + -- The reading rate is more than 12 so return a single row with dummy values that easy to detect. + -- The end time differs from the start time by the meter reading frequency or min one for groups. + -- This means the meter reading frequency is too long for a 3D graphic. + RETURN QUERY + SELECT -999, -999::FLOAT, '1900-01-01 00:00:00'::TIMESTAMP, '1900-01-01 00:00:00'::TIMESTAMP + reading_length_interval + ; + END IF; + + -- Go to the next meter + current_meter_index := current_meter_index + 1; + END LOOP; +END; +$$ LANGUAGE plpgsql; + +/* Gets group graphing data for 3D graphic by returning points that span the requested + length of time over the days requested. +*/ +CREATE OR REPLACE FUNCTION group_3d_readings_unit ( + --Desire group ID + --For 3D graphics, users will only be able to select 1 group to graph. + group_id_requested INTEGER, + -- The desired graphic unit of the returned data + graphic_unit_id_requested INTEGER, + -- The start and end time for the data to return + start_stamp TIMESTAMP, + end_stamp TIMESTAMP, + -- The number of hours in each reading requested + reading_length_hours INTEGER +) + RETURNS TABLE(reading_rate FLOAT, start_timestamp TIMESTAMP, end_timestamp TIMESTAMP) +AS $$ +DECLARE + -- Holds the range of dates for returned data that fits the actual data. + requested_range TSRANGE; + --Holds the desired meter IDs in order to call meter_3d_readings_unit in the query below. + meter_ids INTEGER[]; + -- The number of hours in each reading determined + reading_length_hours_use INTEGER; + -- The number of hours in each reading determined as an interval + reading_length_interval INTERVAL; +BEGIN + --Get all the meter IDS that will be included in the group being requested. + SELECT array_agg(DISTINCT gdm.meter_id) INTO meter_ids + FROM groups_deep_meters_cache gdm + WHERE group_id = group_id_requested; + + -- Find the correct number of hours per reading returned. + SELECT * from reading_interval_3d(meter_ids, reading_length_hours) into reading_length_hours_use, reading_length_interval; + -- Get the range of days requested by calling shrink_tsrange_to_group_readings_by_day. + -- First make requested range only be full days by dropping any partial days at start/end. + requested_range := shrink_tsrange_to_group_readings_by_day(tsrange(date_trunc_up('day', start_stamp), date_trunc('day', end_stamp)), group_id_requested); + + IF (reading_length_hours_use <= 12) THEN + -- Need to generate_series to group the desired hours together + RETURN QUERY + -- The readings are rates in the hourly table so want to average not sum so + -- work for quantity, flow & raw. + -- The time starts at the time of the generated sequence and ends at the length + -- of each block later. This is the same as the start time of the next value + -- in the sequence (except last one). + SELECT + AVG(ghr.reading_rate) as reading_rate, + hours.hour AS start_timestamp, + hours.hour + reading_length_interval AS end_timestamp + -- This is the series that starts at the beginning of the desired days, + -- ends at the end of the desired days and steps by the desired interval. + -- You need to subtract from the last interval for the end since generate_series + -- is inclusive. + FROM ( + SELECT hour + FROM generate_series( + lower(requested_range), + upper(requested_range) - reading_length_interval, + reading_length_interval + ) hours(hour) + ) hours(hour), + -- Also need the values in the group hourly table. + group_hourly_readings_unit_cagg ghr + -- Only want the desired meter + WHERE ghr.group_id = group_id_requested + -- Only want the desired graphing unit + AND ghr.graphic_unit_id = graphic_unit_id_requested + -- Only want readings that lie within this slice of the desired data + AND ghr.bucket >= hours.hour + AND ghr.bucket <= hours.hour + reading_length_interval - INTERVAL '1 hour' + -- Group by the start time of the generated series since all points in + -- the desired slice have the same start time for the series. + GROUP BY hours.hour + -- Time sort by the start time for graphing. + ORDER BY hours.hour + ; + ELSE + -- The reading rate is more than 12 so return a single row with dummy values that easy to detect. + -- The end time differs from the start time by the meter reading frequency or min one for groups. + -- This means the meter reading frequency is too long for a 3D graphic. + RETURN QUERY + SELECT -999::FLOAT, '1900-01-01 00:00:00'::TIMESTAMP, '1900-01-01 00:00:00'::TIMESTAMP + reading_length_interval + ; + END IF; +END; +$$ LANGUAGE 'plpgsql'; diff --git a/src/server/sql/reading/TimeScaleDB/update_function_get_compare_readings.sql b/src/server/sql/reading/TimeScaleDB/update_function_get_compare_readings.sql new file mode 100644 index 0000000000..116d74b719 --- /dev/null +++ b/src/server/sql/reading/TimeScaleDB/update_function_get_compare_readings.sql @@ -0,0 +1,171 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* +This shouldn't ever be looking at more than a few weeks of data, so we don't need to deal with compression. + */ + +/* +TODO This function can probably be improved for two reasons: +1) While it was noted that you don't look at too many days, it does limit its usage to modest time lengths. +There has been thought to allowing comparisons, for example, or a whole year. It also assumes that the +frequency of readings is not too high so the number of readings looked at could be large even over +modest time frames. +2) Readings are only included if they are within the current time period. This means that if you have +a reading that crosses the timeframe they are excluded. This can be viewed as either a good thing or +a bad thing. However, if the frequency of readings is high, then large segments of time can be excluded. +For example, monthly readings would not be includes in either day, week or four week comparisons that +OED currently does. Also note that the daily and hourly views do include readings that span a time frame +so including them would be consistent. + +We need to think about how best to deal with this but one option is to use the daily and hourly tables +to get the reading values as in done with bar graphs. This needs to be a little different since partial +days can be involved. However, getting the full days from the daily table and then the hours from the +hourly table to be combined would solve this. (One could get the subhour from the raw readings but it is +unclear users would want that.) Another consider is that the current system, as is the case for bar graphs, +does not take into account missing times in readings which can lead to lower than expected values. The +daily and hourly readings would help fix this. +*/ + +/* +The following function returns data for plotting compare graphs. It works on meters. +It should not be used on raw readings. +This only works when the curr start/end times are on the hour as it uses the hourly reading view. +It will truncate to lose partial hours if they are given. The current client code will only +send full hours so this should be fine. +It is the new version of compare_readings that works with units. It takes these parameters: +meter_ids: A array of meter ids to query. +graphic_unit_id: The unit id of the unit to use for the graph. +curr_start: When the current/this time period begins for the compare. +curr_end: When the current/this time period ends for the compare. +shift: How far back in time to shift the curr_start and curr_end date/time to get the previous + times to compare. + */ +-- New version of the meter_compare_readings function that uses the meter_hourly_readings_unit_cagg view. +CREATE OR REPLACE FUNCTION meter_compare_readings_unit ( + meter_ids INTEGER[], + -- This is the graphic unit id, changed from graphic_unit_id to avoid confusion with the graphic unit id in the view. + passed_graphic_unit_id INTEGER, + curr_start TIMESTAMP, + curr_end TIMESTAMP, + shift INTERVAL +) + RETURNS TABLE(meter_id INTEGER, curr_use FLOAT, prev_use FLOAT) +AS $$ +DECLARE + curr_tsrange TSRANGE; + prev_tsrange TSRANGE; +BEGIN + curr_tsrange := tsrange(curr_start, curr_end); + prev_tsrange := tsrange(curr_start - shift, curr_end - shift); + -- Modified to retrieve converted hourly readings from the materialized view. + RETURN QUERY + WITH + curr_period AS ( + SELECT + hourly.meter_id AS meter_id, + -- It is okay to sum the flow units because hourly readings has a rate of per hour so + -- to convert to a quantity would multiply by 1 so it is the same formula. + SUM(hourly.reading_rate) AS reading + FROM meter_hourly_readings_unit_cagg hourly + -- This is getting the conversion for the meter and unit to graph. + WHERE + -- The range requested must be completely within the hour so partial hours are not included. + hourly.bucket >= lower(curr_tsrange) AND + hourly.bucket <= upper(curr_tsrange) - INTERVAL '1 hour' AND + hourly.graphic_unit_id = passed_graphic_unit_id AND + hourly.meter_id = ANY(meter_ids) + GROUP BY hourly.meter_id + ), + prev_period AS ( + SELECT + hourly.meter_id AS meter_id, + SUM(hourly.reading_rate) AS reading + FROM meter_hourly_readings_unit_cagg hourly + -- This is getting the conversion for the meter and unit to graph. + -- The slope and intercept are used above the transform the reading to the desired unit. + WHERE + -- The range requested must be completely within the hour so partial hours are not included. + hourly.bucket >= lower(prev_tsrange) AND + hourly.bucket <= upper(prev_tsrange) - INTERVAL '1 hour' AND + hourly.graphic_unit_id = passed_graphic_unit_id AND + hourly.meter_id = ANY(meter_ids) + GROUP BY hourly.meter_id + ) + SELECT + meters.id AS meter_id, + curr_period.reading::FLOAT AS curr_use, + prev_period.reading::FLOAT AS prev_use + FROM + unnest(meter_ids) meters(id) + -- Left joins here so we get nulls instead of missing rows if readings don't exist for some time intervals + LEFT JOIN prev_period ON meters.id = prev_period.meter_id + LEFT JOIN curr_period ON meters.id = curr_period.meter_id; +END; +$$ LANGUAGE 'plpgsql'; + + +/* +The following function returns data for plotting compare graphs. It works on groups. +It should not be used on raw readings. +See meter version for needing start/end on the hour. +It is the new version of group_compare_readings that works with units. It takes these parameters: +group_ids: A array of group ids to query. +graphic_unit_id: The unit id of the unit to use for the graph. +curr_start: When the current/this time period begins for the compare. +curr_end: When the current/this time period ends for the compare. +shift: How far back in time to shift the curr_start and curr_end date/time to get the previous + times to compare. + */ +CREATE OR REPLACE FUNCTION group_compare_readings_unit ( + group_ids INTEGER[], + requested_graphic_unit_id INTEGER, + curr_start TIMESTAMP, + curr_end TIMESTAMP, + shift INTERVAL +) + RETURNS TABLE(group_id INTEGER, curr_use FLOAT, prev_use FLOAT) +AS $$ +DECLARE + curr_tsrange TSRANGE; + prev_tsrange TSRANGE; +BEGIN + curr_tsrange := tsrange(curr_start, curr_end); + prev_tsrange := tsrange(curr_start - shift, curr_end - shift); + + RETURN QUERY + WITH + curr_period AS ( + SELECT + hourly.group_id, + SUM(hourly.reading_rate) AS reading + FROM group_hourly_readings_unit_cagg hourly + WHERE hourly.bucket >= lower(curr_tsrange) + AND hourly.bucket <= upper(curr_tsrange) - INTERVAL '1 hour' + AND requested_graphic_unit_id = hourly.graphic_unit_id + AND hourly.group_id = ANY(group_ids) + GROUP BY hourly.group_id + ), + prev_period AS ( + SELECT + hourly.group_id, + SUM(hourly.reading_rate) AS reading + FROM group_hourly_readings_unit_cagg hourly + WHERE hourly.bucket >= lower(prev_tsrange) + AND hourly.bucket <= upper(prev_tsrange) - INTERVAL '1 hour' + AND requested_graphic_unit_id = hourly.graphic_unit_id + AND hourly.group_id = ANY(group_ids) + GROUP BY hourly.group_id + ) + SELECT + gids.id AS group_id, + curr_period.reading::FLOAT AS curr_use, + prev_period.reading::FLOAT AS prev_use + FROM + unnest(group_ids) gids(id) + -- Left joins here so we get nulls instead of missing rows if readings don't exist for some time intervals + LEFT JOIN prev_period ON gids.id = prev_period.group_id + LEFT JOIN curr_period ON gids.id = curr_period.group_id; +END; +$$ LANGUAGE 'plpgsql'; diff --git a/src/server/sql/reading/TimeScaleDB/update_group_line_readings_unit.sql b/src/server/sql/reading/TimeScaleDB/update_group_line_readings_unit.sql new file mode 100644 index 0000000000..cdfa590f76 --- /dev/null +++ b/src/server/sql/reading/TimeScaleDB/update_group_line_readings_unit.sql @@ -0,0 +1,124 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +/* +The following function determines the correct duration view to query from, and returns averaged readings from it. +It is designed to return data for plotting line graphs. It works on groups. +It is the new version of compressed_group_readings_2 that works with units. It takes these parameters: +group_ids: A array of group ids to query. +graphic_unit_id: The unit id of the unit to use for the graph. +start_timestamp: The start timestamp of the data to return. +end_timestamp: The end timestamp of the data to return. +point_accuracy: Tells how decisions should be made on which types of points to return. 'auto' if automatic. +max_hour_points: The maximum number of data points to return if using the hour view. Only used if 'auto'/'raw' for point_accuracy. +Details on how this function works can be found in the devDocs in the resource generalization document and above +in the meter function that is equivalent. + */ +CREATE OR REPLACE FUNCTION group_line_readings_unit ( + group_ids INTEGER[], + requested_graphic_unit_id INTEGER, + start_stamp TIMESTAMP, + end_stamp TIMESTAMP, + point_accuracy reading_line_accuracy, + max_hour_points INTEGER +) + RETURNS TABLE(group_id INTEGER, reading_rate FLOAT, start_timestamp TIMESTAMP, end_timestamp TIMESTAMP) +AS $$ +DECLARE + meter_ids INTEGER[]; + requested_range TSRANGE; + requested_interval INTERVAL; + requested_interval_seconds INTEGER; + meters_min_frequency INTERVAL; + +BEGIN + -- First get all the meter ids that will be included in one or more groups being queried. + -- In case meter is repeated, make this distinct. + SELECT array_agg(DISTINCT gdm.meter_id) INTO meter_ids + FROM groups_deep_meters_cache gdm + INNER JOIN unnest(group_ids) gids(id) ON gdm.group_id = gids.id; + + -- Calculate point accuracy if request (auto) or if raw since that is not allowed for groups. + IF (point_accuracy = 'auto'::reading_line_accuracy OR point_accuracy = 'raw'::reading_line_accuracy) THEN + -- The request needs automatic calculation of the points returned. + + -- Make sure the time range is within the reading values for meters in this group. + requested_range := shrink_tsrange_to_real_readings(tsrange(start_stamp, end_stamp, '[]'), meter_ids); + -- The request_range will still be infinity if there is no meter data. This causes the + -- auto calculation to fail because you cannot subtract them. + -- Just check the upper range since simpler. + IF (upper(requested_range) = 'infinity') THEN + -- We know there is no data but easier to just let a query happen since fast. + -- Do daily since that should be the fastest due to the least data in most cases. + point_accuracy := 'daily'::reading_line_accuracy; + ELSE + -- The interval of time for the requested_range. + requested_interval := upper(requested_range) - lower(requested_range); + -- Get the seconds in the interval. + -- Wanted to use the INTO syntax used above but could not get it to work so using the set syntax. + requested_interval_seconds := (SELECT * FROM EXTRACT(EPOCH FROM requested_interval)); + -- Make sure that the number of hour points is no more than maximum hourly readings. + -- Thus, check if no more than interval in seconds / (60 seconds/minute * 60 minutes/hour) = # hours in interval. + IF (requested_interval_seconds / 3600 <= max_hour_points) THEN + -- Return hourly reading data. + point_accuracy := 'hourly'::reading_line_accuracy; + ELSE + -- Return daily reading data. + point_accuracy := 'daily'::reading_line_accuracy; + END IF; + + -- Groups can require reading interpolation because of multiple meters. For example, if one meter + -- is 30 day reading frequency then it will interpolate to hourly or daily depending other + -- meters (if exist). However, to limit this effect, if hourly has been selected automatically, + -- check if shortest meter reading frequency for this group is more than an hour and then + -- choose daily instead. + IF (point_accuracy = 'hourly'::reading_line_accuracy) THEN + -- Find the min reading frequency for all meters in the group. + SELECT min(reading_frequency) INTO meters_min_frequency + FROM (meters m + INNER JOIN unnest(meter_ids) meters(id) ON m.id = meters.id); + IF (EXTRACT(EPOCH FROM meters_min_frequency) > 3600) THEN + -- The smallest meter frequency is greater than 1 hour (3600 seconds) so use daily instead. + point_accuracy = 'daily'::reading_line_accuracy; + END IF; + END IF; + END IF; + END IF; + -- point_accuracy should either be daily or hourly at this point. + + IF (point_accuracy = 'daily'::reading_line_accuracy) THEN + RETURN QUERY + SELECT + readings.group_id, + readings.reading_rate, + readings.bucket AS start_timestamp, + readings.bucket + INTERVAL '1 day' AS end_timestamp + FROM group_daily_readings_unit_cagg readings + INNER JOIN unnest(group_ids) gids(id) ON readings.group_id = gids.id + WHERE readings.graphic_unit_id = requested_graphic_unit_id + -- Undefined API bounds arrive as NULL. Convert them to PostgreSQL + -- infinities while keeping bucket directly usable by its B-tree index. + AND readings.bucket >= COALESCE(start_stamp, '-infinity'::TIMESTAMP) + AND readings.bucket <= COALESCE(end_stamp, 'infinity'::TIMESTAMP) - INTERVAL '1 day' + ORDER BY readings.bucket ASC; + + ELSIF (point_accuracy = 'hourly'::reading_line_accuracy) THEN + RETURN QUERY + SELECT + readings.group_id AS group_id, + readings.reading_rate AS reading_rate, + readings.bucket AS start_timestamp, + readings.bucket + INTERVAL '1 hour' AS end_timestamp + FROM group_hourly_readings_unit_cagg readings + INNER JOIN unnest(group_ids) gids(id) ON readings.group_id = gids.id + WHERE readings.graphic_unit_id = requested_graphic_unit_id + -- Undefined API bounds arrive as NULL. Convert them to PostgreSQL + -- infinities while keeping bucket directly usable by its B-tree index. + AND readings.bucket >= COALESCE(start_stamp, '-infinity'::TIMESTAMP) + AND readings.bucket <= COALESCE(end_stamp, 'infinity'::TIMESTAMP) - INTERVAL '1 hour' + ORDER BY readings.bucket ASC; + END IF; +END; +$$ LANGUAGE 'plpgsql'; diff --git a/src/server/sql/reading/TimeScaleDB/update_meter_group_bar.sql b/src/server/sql/reading/TimeScaleDB/update_meter_group_bar.sql new file mode 100644 index 0000000000..5a9df98d49 --- /dev/null +++ b/src/server/sql/reading/TimeScaleDB/update_meter_group_bar.sql @@ -0,0 +1,161 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +/* +The following function returns data for plotting bar graphs. It works on meters. +It should not be used on raw readings. +It is the new version of compressed_barchart_readings_2 that works with units. It takes these parameters: +meter_ids: A array of meter ids to query. +graphic_unit_id: The unit id of the unit to use for the graph. +bar_width_days: The number of days to use for the bar width. +start_timestamp: The start timestamp of the data to return. +end_timestamp: The end timestamp of the data to return. + */ +-- New version of meter_bar_readings_unit that uses the new meter_daily_readings_unit_cagg view. +CREATE OR REPLACE FUNCTION meter_bar_readings_unit ( + meter_ids INTEGER[], + -- This is the graphic unit id, changed from graphic_unit_id to avoid confusion with the graphic unit id in the view. + passed_graphic_unit_id INTEGER, + bar_width_days INTEGER, + start_stamp TIMESTAMP, + end_stamp TIMESTAMP +) + RETURNS TABLE(meter_id INTEGER, reading FLOAT, start_timestamp TIMESTAMP, end_timestamp TIMESTAMP) +AS $$ +DECLARE + bar_width INTERVAL; + real_tsrange TSRANGE; + real_start_stamp TIMESTAMP; + real_end_stamp TIMESTAMP; + num_bars INTEGER; +BEGIN + -- This is how wide (time interval) for each bar. + bar_width := INTERVAL '1 day' * bar_width_days; + /* + This rounds to the day for the start and end times requested. It then shrinks in case the actual readings span + less time than the request. This can commonly happen when you get +/-infinity for all readings available. + It uses the day reading view because that is faster than using all the readings. + This has an issue associated with it: + + 1) If the readings at the start/end have a partial day then it shows up as a day. The original code did: + real_tsrange := shrink_tsrange_to_real_readings(tsrange(date_trunc_up('day', start_stamp), date_trunc('day', end_stamp))); + and did not have this issue since it used the readings and then truncated up/down. + A more general solution would be to change the daily (and hourly) view so it does not include partial ones at start/end. + This would fix this case and also impact other uses in what seems a positive way. + Note this does not address that missing days in a bar width get no value so the bar will likely read low. + */ + real_tsrange := shrink_tsrange_to_meters_by_day(tsrange(start_stamp, end_stamp), meter_ids); + -- Get the actual start/end time rounded to the nearest day from the range. + real_start_stamp := lower(real_tsrange); + real_end_stamp := upper(real_tsrange); + -- This gives the number of whole bars that will fit within the real start/end times. For example, if the number of days + -- between start and end is 14 days and the bar width is 3 days then you get 4. + num_bars := floor(extract(EPOCH FROM real_end_stamp - real_start_stamp) / extract(EPOCH FROM bar_width)); + -- This makes the full bars go from the end time to as far back in time as possible. + -- This means that if some time was dropped to get full bars it is at the start of the interval. + -- It was felt that the most recent readings are the most important so drop older ones. + -- It also helps with maps since they use the latest bar for their value. + real_start_stamp := real_end_stamp - (num_bars * bar_width); + -- Since the inner join on the generate_series adds the bar_width, we need to back up the + -- end timestamp by that amount so it stops at the desired end timestamp. + real_end_stamp := real_end_stamp - bar_width; + + RETURN QUERY + SELECT + -- Modified to retrieve converted daily readings from the materialized view. + mdr.meter_id AS meter_id, + sum(mdr.reading_rate * 24) AS reading, + bars.interval_start AS start_timestamp, + bars.interval_start + bar_width AS end_timestamp + + FROM meter_daily_readings_unit_cagg mdr + INNER JOIN generate_series(real_start_stamp, real_end_stamp, bar_width) bars(interval_start) + ON mdr.bucket >= bars.interval_start + AND mdr.bucket <= bars.interval_start + bar_width - INTERVAL '1 day' + INNER JOIN unnest(meter_ids) meters(id) ON mdr.meter_id = meters.id + INNER JOIN meters m ON m.id = meters.id + INNER JOIN units u ON m.unit_id = u.id AND u.unit_represent != 'raw'::unit_represent_type + WHERE mdr.graphic_unit_id = passed_graphic_unit_id + GROUP BY mdr.meter_id, bars.interval_start + ORDER BY mdr.meter_id, bars.interval_start; + +END; +$$ LANGUAGE 'plpgsql'; + + +/* +The following function returns data for plotting bar graphs. It works on groups. +It should not be used on raw readings. +It is the new version of compressed_barchart_group_readings_2 that works with units. It takes these parameters: +group_ids: A array of group ids to query. +graphic_unit_id: The unit id of the unit to use for the graph. +bar_width_days: The number of days to use for the bar width. +start_timestamp: The start timestamp of the data to return. +end_timestamp: The end timestamp of the data to return. + */ +CREATE OR REPLACE FUNCTION group_bar_readings_unit ( + group_ids INTEGER[], + requested_graphic_unit_id INTEGER, + bar_width_days INTEGER, + start_stamp TIMESTAMP, + end_stamp TIMESTAMP +) + RETURNS TABLE(group_id INTEGER, reading FLOAT, start_timestamp TIMESTAMP, end_timestamp TIMESTAMP) +AS $$ +DECLARE + bar_width INTERVAL; + real_tsrange TSRANGE; + real_start_stamp TIMESTAMP; + real_end_stamp TIMESTAMP; + num_bars INTEGER; + readings_max_tsrange TSRANGE; +BEGIN + bar_width := INTERVAL '1 day' * bar_width_days; + + SELECT tsrange(min(bucket), max(bucket + INTERVAL '1 day')) INTO readings_max_tsrange + FROM group_daily_readings_unit_cagg dr + -- Get all the group ids passed in. + INNER JOIN unnest(group_ids) gids(id) ON dr.group_id = gids.id; + + real_tsrange := tsrange(date_trunc_up('day', start_stamp), date_trunc('day', end_stamp)) * readings_max_tsrange; + -- Get the actual start/end time rounded to the nearest day from the range. + real_start_stamp := lower(real_tsrange); + real_end_stamp := upper(real_tsrange); + -- This gives the number of whole bars that will fit within the real start/end times. For example, if the number of days + -- between start and end is 14 days and the bar width is 3 days then you get 4. + num_bars := floor(extract(EPOCH FROM real_end_stamp - real_start_stamp) / extract(EPOCH FROM bar_width)); + -- This makes the full bars go from the end time to as far back in time as possible. + -- This means that if some time was dropped to get full bars it is at the start of the interval. + -- It was felt that the most recent readings are the most important so drop older ones. + -- It also helps with maps since they use the latest bar for their value. + real_start_stamp := real_end_stamp - (num_bars * bar_width); + -- Since the inner join on the generate_series adds the bar_width, we need to back up the + -- end timestamp by that amount so it stops at the desired end timestamp. + real_end_stamp := real_end_stamp - bar_width; + + RETURN QUERY + SELECT + -- readings.reading_rate is the weighted average reading rate per hour over the day. + -- Convert to a quantity by multiplying by the time in hours which is 24 since daily values. + -- reading is the sum of all readings within one bar. + readings.group_id AS group_id, + SUM(readings.reading_rate * 24) AS reading, + bars.interval_start AS start_timestamp, + bars.interval_start + bar_width AS end_timestamp + + FROM (((group_daily_readings_unit_cagg readings + INNER JOIN generate_series(real_start_stamp, real_end_stamp, bar_width) bars(interval_start) + ON readings.bucket >= bars.interval_start + AND readings.bucket <= bars.interval_start + bar_width - INTERVAL '1 day') + -- Don't return bar data if raw since cannot sum. + INNER JOIN units u ON readings.graphic_unit_id = u.id AND u.unit_represent != 'raw'::unit_represent_type) + INNER JOIN unnest(group_ids) gids(id) ON readings.group_id = gids.id) + -- Use the readings in the passed in graphic unit + WHERE readings.graphic_unit_id = requested_graphic_unit_id + + GROUP BY readings.group_id, bars.interval_start + ORDER BY readings.group_id, bars.interval_start; +END; +$$ LANGUAGE 'plpgsql'; diff --git a/src/server/sql/reading/TimeScaleDB/update_meter_line_readings_unit.sql b/src/server/sql/reading/TimeScaleDB/update_meter_line_readings_unit.sql new file mode 100644 index 0000000000..d2296b58ea --- /dev/null +++ b/src/server/sql/reading/TimeScaleDB/update_meter_line_readings_unit.sql @@ -0,0 +1,360 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +/* + * Function: meter_line_readings_unit + * + * Purpose: + * + * Retrieve meter readings optimized for rendering line graphs. This function + * determines the most appropriate data resolution (raw, hourly, or daily) + * based on the requested time range, meter reading frequency, and configured + * point limits. + * + * This function is the unit-aware replacement for compressed_readings_2. + * It supports graphic unit conversion and time-varying unit conversions. + * + * + * Data sources: + * + * Raw: + * Reads directly from the readings table and applies cik_vary conversion + * during query execution. + * + * Hourly: + * Uses the TimescaleDB continuous aggregate: + * + * meter_hourly_readings_unit_cagg + * + * The continuous aggregate is built on top of: + * + * hypertable_hourly_split + * + * Hourly splitting and time-varying conversion metadata are applied + * before aggregation, avoiding runtime joins to cik_vary. + * + * Daily: + * Uses the TimescaleDB continuous aggregate: + * + * meter_daily_readings_unit_cagg + * + * The daily continuous aggregate is built on top of: + * + * meter_hourly_readings_unit_cagg + * + * Daily aggregation reuses precomputed hourly aggregate values instead + * of recalculating from the hourly split hypertable. + * + * + * Resolution selection: + * + * When point_accuracy = 'auto', the function selects the highest resolution + * possible while staying within the requested point limits. + * + * Selection order: + * + * 1. Raw readings + * 2. Hourly continuous aggregate + * 3. Daily continuous aggregate + * + * + * Notes: + * + * - Each meter is processed independently because meters may have different + * reading frequencies and available data ranges. + * + * - The hourly data path previously queried the PostgreSQL materialized view: + * + * meter_hourly_readings_unit + * + * and now uses the TimescaleDB continuous aggregate: + * + * meter_hourly_readings_unit_cagg + * + * - The daily data path previously queried the PostgreSQL materialized view: + * + * meter_daily_readings_unit + * + * and now uses the TimescaleDB continuous aggregate: + * + * meter_daily_readings_unit_cagg + * + * - Design details: + * + * meter_line_readings_unit + * | + * +----------------+----------------+ + * | | | + * v v v + * readings hourly continuous daily continuous + * aggregate aggregate + * | | + * v v + * meter_hourly_readings_unit_cagg + * | + * v + * meter_daily_readings_unit_cagg + */ + + +/* + * DAILY READINGS + * + * Uses TimescaleDB daily continuous aggregate. + * + * Data source: + * + * meter_daily_readings_unit_cagg + * + * The daily continuous aggregate rolls up hourly aggregate values into + * one-day intervals. + * + * The time interval is stored as a PostgreSQL tsrange: + * + * ("2021-06-01 00:00:00","2021-06-02 00:00:00") + * + * The lower and upper bounds are converted back into timestamps so the + * result matches the function return type. + */ + + +/* +The following function determines the correct duration view to query from, and returns averaged or raw reading from it. +It is designed to return data for plotting line graphs. It works on meters. +It is the new version of compressed_readings_2 that works with units. It takes these parameters: +meter_ids: A array of meter ids to query. +graphic_unit_id: The unit id of the unit to use for the graphic. +start_timestamp: The start timestamp of the data to return. +end_timestamp: The end timestamp of the data to return. +point_accuracy: Tells how decisions should be made on which types of points to return. 'auto' if automatic. +max_raw_points: The maximum number of data points to return if using the raw points for a meter. Only used if 'auto' for point_accuracy. +max_hour_points: The maximum number of data points to return if using the hour view. Only used if 'auto' for point_accuracy. +Details on how this function works can be found in the devDocs in the resource generalization document. + */ +-- New version of meter_line_readings_unit that uses the new views. +CREATE OR REPLACE FUNCTION meter_line_readings_unit ( + meter_ids INTEGER[], + -- This is the graphic unit id, changed from graphic_unit_id to avoid confusion with the graphic unit id in the view. + passed_graphic_unit_id INTEGER, + start_stamp TIMESTAMP, + end_stamp TIMESTAMP, + point_accuracy reading_line_accuracy, + max_raw_points INTEGER, + max_hour_points INTEGER +) + RETURNS TABLE(meter_id INTEGER, reading_rate FLOAT, min_rate FLOAT, max_rate FLOAT, start_timestamp TIMESTAMP, end_timestamp TIMESTAMP) +AS $$ +BEGIN + RETURN QUERY + /* + * Process all requested meters as one set. WITH ORDINALITY preserves the + * caller's meter order in the final result. + */ + WITH requested_meters AS ( + SELECT requested.id AS meter_id, requested.request_order + FROM unnest(meter_ids) WITH ORDINALITY requested(id, request_order) + ), + /* + * Use the readings indexes to find the first start and last end for every + * requested meter without aggregating its complete reading history. + */ + reading_bounds AS ( + SELECT + requested.meter_id, + first_reading.start_timestamp AS min_start_timestamp, + last_reading.end_timestamp AS max_end_timestamp + FROM (SELECT DISTINCT rm.meter_id FROM requested_meters rm) requested + LEFT JOIN LATERAL ( + SELECT r.start_timestamp + FROM readings r + WHERE r.meter_id = requested.meter_id + ORDER BY r.start_timestamp + LIMIT 1 + ) first_reading ON TRUE + LEFT JOIN LATERAL ( + SELECT r.end_timestamp + FROM readings r + WHERE r.meter_id = requested.meter_id + ORDER BY r.end_timestamp DESC + LIMIT 1 + ) last_reading ON TRUE + ), + /* + * Restrict the requested range to the readings available for each meter. + * An absent reading bound produces an unbounded range, preserving the + * behavior of shrink_tsrange_to_real_readings(). + */ + meter_ranges AS ( + SELECT + rm.meter_id, + rm.request_order, + m.unit_id, + m.reading_frequency, + tsrange(start_stamp, end_stamp, '[]') + * tsrange(bounds.min_start_timestamp, bounds.max_end_timestamp) AS requested_range + FROM requested_meters rm + INNER JOIN meters m ON m.id = rm.meter_id + LEFT JOIN reading_bounds bounds ON bounds.meter_id = rm.meter_id + ), + /* + * Select raw, hourly, or daily resolution independently for each meter. + * + * Raw is selected when the estimated number of readings does not exceed + * max_raw_points. A frequency of at least one day also stays raw because + * hourly or daily aggregation would interpolate additional points. + * + * Hourly is selected when the requested number of hours does not exceed + * max_hour_points and the meter frequency is no greater than one hour. + * All remaining meters use daily data. + */ + meter_resolutions AS ( + SELECT + mr.*, + CASE + WHEN point_accuracy <> 'auto'::reading_line_accuracy THEN point_accuracy + WHEN upper(mr.requested_range) = 'infinity'::TIMESTAMP THEN 'daily'::reading_line_accuracy + WHEN ( + EXTRACT(EPOCH FROM (upper(mr.requested_range) - lower(mr.requested_range))) + / EXTRACT(EPOCH FROM mr.reading_frequency) <= max_raw_points + OR EXTRACT(EPOCH FROM mr.reading_frequency) >= 86400 + ) THEN 'raw'::reading_line_accuracy + WHEN ( + EXTRACT(EPOCH FROM (upper(mr.requested_range) - lower(mr.requested_range))) / 3600 + <= max_hour_points + AND EXTRACT(EPOCH FROM mr.reading_frequency) <= 3600 + ) THEN 'hourly'::reading_line_accuracy + ELSE 'daily'::reading_line_accuracy + END AS selected_accuracy + FROM meter_ranges mr + ), + /* + * RAW READINGS + * + * Apply time-varying conversions directly to raw readings while preserving + * the original meter-reading interval. When multiple cik_vary segments + * overlap one reading, duration-weight their converted rates into the one + * raw point represented by that reading. + */ + raw_results AS ( + SELECT + selected.request_order, + r.meter_id, + CASE + WHEN u.unit_represent = 'quantity'::unit_represent_type THEN + SUM( + (EXTRACT(EPOCH FROM ( + upper(tsrange(c.start_time, c.end_time, '()') * tsrange(r.start_timestamp, r.end_timestamp, '[]')) + - lower(tsrange(c.start_time, c.end_time, '()') * tsrange(r.start_timestamp, r.end_timestamp, '[]')) + )) / 3600) + * (c.slope * (r.reading / (EXTRACT(EPOCH FROM (r.end_timestamp - r.start_timestamp)) / 3600)) + c.intercept) + ) / (EXTRACT(EPOCH FROM (r.end_timestamp - r.start_timestamp)) / 3600) + WHEN u.unit_represent IN ('flow'::unit_represent_type, 'raw'::unit_represent_type) THEN + SUM( + (EXTRACT(EPOCH FROM ( + upper(tsrange(c.start_time, c.end_time, '()') * tsrange(r.start_timestamp, r.end_timestamp, '[]')) + - lower(tsrange(c.start_time, c.end_time, '()') * tsrange(r.start_timestamp, r.end_timestamp, '[]')) + )) / 3600) + * (c.slope * (r.reading * 3600 / u.sec_in_rate) + c.intercept) + ) / (EXTRACT(EPOCH FROM (r.end_timestamp - r.start_timestamp)) / 3600) + END AS reading_rate, + /* + * Raw meter data has no min/max range. NaN is converted to null by + * the route when the result is stored in Redux state. + */ + 'NaN'::DOUBLE PRECISION AS min_rate, + 'NaN'::DOUBLE PRECISION AS max_rate, + r.start_timestamp, + r.end_timestamp + FROM meter_resolutions selected + INNER JOIN readings r ON r.meter_id = selected.meter_id + INNER JOIN units u ON u.id = selected.unit_id + INNER JOIN cik_vary c + ON c.source_id = selected.unit_id + AND c.destination_id = passed_graphic_unit_id + /* + * Allow multiple time-varying conversion segments to contribute + * when they overlap a reading. + */ + AND c.start_time < r.end_timestamp + AND c.end_time > r.start_timestamp + WHERE selected.selected_accuracy = 'raw'::reading_line_accuracy + AND r.start_timestamp >= lower(selected.requested_range) + AND r.end_timestamp <= upper(selected.requested_range) + GROUP BY + selected.request_order, + r.meter_id, + r.start_timestamp, + r.end_timestamp, + u.unit_represent + ), + /* + * HOURLY READINGS + * + * Use the TimescaleDB hourly continuous aggregate. Direct bucket bounds + * allow the meter/graphic-unit/bucket index to constrain the time range. + */ + hourly_results AS ( + SELECT + selected.request_order, + hourly.meter_id, + hourly.reading_rate, + hourly.min_rate, + hourly.max_rate, + hourly.bucket AS start_timestamp, + hourly.bucket + INTERVAL '1 hour' AS end_timestamp + FROM meter_resolutions selected + INNER JOIN meter_hourly_readings_unit_cagg hourly + ON hourly.meter_id = selected.meter_id + AND hourly.graphic_unit_id = passed_graphic_unit_id + AND hourly.bucket >= lower(selected.requested_range) + AND hourly.bucket <= upper(selected.requested_range) - INTERVAL '1 hour' + WHERE selected.selected_accuracy = 'hourly'::reading_line_accuracy + ), + /* + * DAILY READINGS + * + * Use the TimescaleDB daily continuous aggregate with the same indexable + * complete-bucket bounds. + */ + daily_results AS ( + SELECT + selected.request_order, + daily.meter_id, + daily.reading_rate, + daily.min_rate, + daily.max_rate, + daily.bucket AS start_timestamp, + daily.bucket + INTERVAL '1 day' AS end_timestamp + FROM meter_resolutions selected + INNER JOIN meter_daily_readings_unit_cagg daily + ON daily.meter_id = selected.meter_id + AND daily.graphic_unit_id = passed_graphic_unit_id + AND daily.bucket >= lower(selected.requested_range) + AND daily.bucket <= upper(selected.requested_range) - INTERVAL '1 day' + WHERE selected.selected_accuracy = 'daily'::reading_line_accuracy + ), + /* + * Each meter appears in exactly one resolution branch. UNION ALL avoids + * unnecessary duplicate elimination. + */ + results AS ( + SELECT * FROM raw_results + UNION ALL + SELECT * FROM hourly_results + UNION ALL + SELECT * FROM daily_results + ) + SELECT + results.meter_id, + results.reading_rate, + results.min_rate, + results.max_rate, + results.start_timestamp, + results.end_timestamp + FROM results + -- Preserve the original per-meter chronological result ordering. + ORDER BY results.request_order, results.start_timestamp; +END; +$$ LANGUAGE 'plpgsql'; diff --git a/src/server/sql/reading/TimeScaleDB/update_reading_views.sql b/src/server/sql/reading/TimeScaleDB/update_reading_views.sql new file mode 100644 index 0000000000..aa027445ea --- /dev/null +++ b/src/server/sql/reading/TimeScaleDB/update_reading_views.sql @@ -0,0 +1,112 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +/* +Rounds a timestamp up to the next interval. + */ +CREATE OR REPLACE FUNCTION date_trunc_up(interval_precision TEXT, ts TIMESTAMP) + RETURNS TIMESTAMP LANGUAGE SQL +IMMUTABLE +AS $$ +SELECT CASE + WHEN ts = date_trunc(interval_precision, ts) THEN ts + ELSE date_trunc(interval_precision, ts + ('1 ' || interval_precision)::INTERVAL) + END +$$; + +/* +Restricts a requested range to the available raw readings for the supplied +meters. + */ +CREATE OR REPLACE FUNCTION shrink_tsrange_to_real_readings(tsrange_to_shrink TSRANGE, meter_ids INTEGER[]) + RETURNS TSRANGE +AS $$ +DECLARE + readings_max_tsrange TSRANGE; +BEGIN + /* + * Find each meter's first start and last end with the readings indexes. + * Aggregating only those boundary rows avoids scanning complete histories + * whenever a group line request automatically chooses its resolution. + */ + SELECT tsrange(min(first_reading.start_timestamp), max(last_reading.end_timestamp)) + INTO readings_max_tsrange + FROM (SELECT DISTINCT id FROM unnest(meter_ids) requested(id)) meters + LEFT JOIN LATERAL ( + SELECT r.start_timestamp + FROM readings r + WHERE r.meter_id = meters.id + ORDER BY r.start_timestamp + LIMIT 1 + ) first_reading ON TRUE + LEFT JOIN LATERAL ( + SELECT r.end_timestamp + FROM readings r + WHERE r.meter_id = meters.id + ORDER BY r.end_timestamp DESC + LIMIT 1 + ) last_reading ON TRUE; + RETURN tsrange_to_shrink * readings_max_tsrange; +END; +$$ LANGUAGE 'plpgsql'; + +-- TODO: Remove this retained legacy function once the hypertable +-- implementation is finalized. group_graphic_units_cache replaces it. +-- CREATE OR REPLACE FUNCTION get_graphic_unit(requested_group_id INTEGER) +-- RETURNS INTEGER[] AS $$ +-- DECLARE +-- src_ids INTEGER[]; +-- dest_ids INTEGER[]; +-- child_meters_unit_ids INTEGER[]; +-- unit_ids_compatible INTEGER[] := '{}'; +-- unit_id INTEGER; +-- BEGIN +-- SELECT array_agg(DISTINCT m.unit_id) INTO child_meters_unit_ids +-- FROM groups_deep_meters_cache gdm +-- JOIN meters m ON m.id = gdm.meter_id +-- WHERE gdm.group_id = requested_group_id; +-- +-- SELECT array_agg(u.id) INTO dest_ids +-- FROM units u +-- JOIN cik c ON u.id = c.destination_id; +-- +-- FOREACH unit_id IN ARRAY dest_ids +-- LOOP +-- SELECT array_agg(source_id) INTO src_ids +-- FROM cik +-- WHERE destination_id = unit_id; +-- +-- IF src_ids @> child_meters_unit_ids +-- AND NOT (unit_id = ANY (unit_ids_compatible)) +-- THEN +-- unit_ids_compatible := array_append(unit_ids_compatible, unit_id); +-- END IF; +-- END LOOP; +-- +-- RETURN unit_ids_compatible; +-- END; +-- $$ LANGUAGE 'plpgsql'; + +/* +This takes tsrange_to_shrink which is the requested time range to plot and makes sure it does +not exceed the start/end times for all the readings. This can be an issue, in particular, +because infinity is used to indicate to graph all readings. This version does it to the nearest +day by using the day reading view since bars use to the nearest day and this should be faster. +This should be fine since bar uses the same view to get data. + */ +CREATE OR REPLACE FUNCTION shrink_tsrange_to_meters_by_day(tsrange_to_shrink TSRANGE, meter_ids INTEGER[]) + RETURNS TSRANGE +AS $$ +DECLARE + readings_max_tsrange TSRANGE; +BEGIN + SELECT tsrange(min(bucket), max(bucket + INTERVAL '1 day')) INTO readings_max_tsrange + FROM meter_daily_readings_unit_cagg dr + -- Get all the meter_ids in the passed array of meters. + INNER JOIN unnest(meter_ids) meters(id) ON dr.meter_id = meters.id; + -- Make the original range be to the day by dropping parts of days at start/end. + RETURN tsrange(date_trunc_up('day', lower(tsrange_to_shrink)), date_trunc('day', upper(tsrange_to_shrink))) * readings_max_tsrange; +END; +$$ LANGUAGE 'plpgsql'; diff --git a/src/server/sql/reading/create_function_get_3d_readings.sql b/src/server/sql/reading/create_function_get_3d_readings.sql index 37d6572963..822ebde9f9 100644 --- a/src/server/sql/reading/create_function_get_3d_readings.sql +++ b/src/server/sql/reading/create_function_get_3d_readings.sql @@ -1,12 +1,13 @@ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -- By indexing both columns together, the database can efficiently handle queries that involve both meter_id and time_interval. -- Created to support the usage of the view by 3d. -CREATE INDEX IF NOT EXISTS idx_hourly_readings_unit_meter_time -ON hourly_readings_unit (meter_id, lower(time_interval)); +-- TODO verify helps with newer views. +CREATE INDEX IF NOT EXISTS idx_meter_hourly_readings_unit_meter_time +ON meter_hourly_readings_unit (meter_id, lower(time_interval)); -- TODO How does this relate to index for same view in create_reading_views? Are both needed? CREATE INDEX if not exists idx_two_group_hourly_readings_unit ON group_hourly_readings_unit (group_id, graphic_unit_id, lower(time_interval)); @@ -24,7 +25,7 @@ DECLARE readings_max_tsrange TSRANGE; BEGIN SELECT tsrange(min(lower(time_interval)), max(upper(time_interval))) INTO readings_max_tsrange - FROM daily_readings_unit + FROM meter_daily_readings_unit where meter_id = meter_id_desired; RETURN tsrange_to_shrink * readings_max_tsrange; END; @@ -48,10 +49,10 @@ $$ LANGUAGE 'plpgsql'; -- Determines the spacing between 3D points. It uses the lowest one for all meters passed -- that is valid. CREATE OR REPLACE FUNCTION reading_interval_3d ( - -- The desired meter ids. - IN meter_ids_requested INTEGER[], + -- The desired meter ids. + IN meter_ids_requested INTEGER[], -- The number of hours in each reading requested - IN reading_length_hours INTEGER, + IN reading_length_hours INTEGER, -- The number of hours in each reading determined OUT reading_length_hours_use INTEGER, -- The number of hours in each reading determined as an interval @@ -59,157 +60,149 @@ CREATE OR REPLACE FUNCTION reading_interval_3d ( ) AS $$ DECLARE - -- The meter frequency from all meters. + -- The meter frequency from all meters. meter_frequency INTERVAL; - -- The meter frequency rounded up to a whole number of hours. + -- The meter frequency rounded up to a whole number of hours. meter_frequency_hour_up INTEGER; - -- The larger of the meter value and the argument sent. - max_frequency INTEGER; + -- The larger of the meter value and the argument sent. + max_frequency INTEGER; BEGIN - -- Get the smallest reading frequency for all meters requested. - SELECT min(reading_frequency) INTO meter_frequency + -- Get the smallest reading frequency for all meters requested. + SELECT min(reading_frequency) INTO meter_frequency FROM (meters m INNER JOIN unnest(meter_ids_requested) meters(id) ON m.id = meters.id); -- Get the seconds in the frequency from epoch, /3600 To get hours and then round up to a whole number of hours. - meter_frequency_hour_up := CEIL((SELECT * FROM EXTRACT(EPOCH FROM meter_frequency)) / 3600); - -- Use the hours that is the largest of the request and the meter values. - max_frequency := GREATEST(meter_frequency_hour_up, reading_length_hours); - -- The value used must be a divisor of 24 or greater than 12. - IF (max_frequency = 5) THEN - reading_length_hours_use := 6; - ELSIF (max_frequency = 7) THEN - reading_length_hours_use := 8; - ELSIF (max_frequency > 8 AND max_frequency < 12) THEN - reading_length_hours_use := 12; - ELSE - reading_length_hours_use := max_frequency; - END IF; - -- Hours per reading determined returned as an interval. - reading_length_interval := (reading_length_hours_use::TEXT || ' hour')::INTERVAL; + meter_frequency_hour_up := CEIL((SELECT * FROM EXTRACT(EPOCH FROM meter_frequency)) / 3600); + -- Use the hours that is the largest of the request and the meter values. + max_frequency := GREATEST(meter_frequency_hour_up, reading_length_hours); + -- The value used must be a divisor of 24 or greater than 12. + IF (max_frequency = 5) THEN + reading_length_hours_use := 6; + ELSIF (max_frequency = 7) THEN + reading_length_hours_use := 8; + ELSIF (max_frequency > 8 AND max_frequency < 12) THEN + reading_length_hours_use := 12; + ELSE + reading_length_hours_use := max_frequency; + END IF; + -- Hours per reading determined returned as an interval. + reading_length_interval := (reading_length_hours_use::TEXT || ' hour')::INTERVAL; END; $$ LANGUAGE 'plpgsql'; -- Gets meters graphing data for 3D graphic by returning points that span the requested --- length of time over the days requested. This function can be slower than line readings --- so is designed to be called for one year or less of data. +-- length of time over the days requested. +-- New meter_3d_readings_unit function that uses new meter_hourly_readings_unit view. CREATE OR REPLACE FUNCTION meter_3d_readings_unit ( - -- The desired meter ids. It is normally a single value for a 3D graphic but groups - -- may need multiple meters. - meter_ids_requested INTEGER[], - -- The desired graphic unit of the returned data - graphic_unit_id INTEGER, - -- The start/end time for the data to return - start_stamp TIMESTAMP, - end_stamp TIMESTAMP, - -- The number of hours in each reading returned - reading_length_hours INTEGER + -- The desired meter ids. It is normally a single value for a 3D graphic. + -- TODO Should the array be changed to a single value as with group? Need to be sure client never asks for multiple. + meter_ids_requested INTEGER[], + -- The desired graphic unit of the returned data + graphic_unit_id_requested INTEGER, + -- The start and end time for the data to return + start_stamp TIMESTAMP, + end_stamp TIMESTAMP, + -- The number of hours in each reading requested + reading_length_hours INTEGER ) - RETURNS TABLE(meter_id INTEGER, reading_rate FLOAT, start_timestamp TIMESTAMP, end_timestamp TIMESTAMP) + RETURNS TABLE(meter_id INTEGER, reading_rate FLOAT, start_timestamp TIMESTAMP, end_timestamp TIMESTAMP) AS $$ DECLARE - -- Holds the range of dates for returned data that fits the actual data. - requested_range TSRANGE; - -- The slope of the conversion from meter to graphing units - slope FLOAT; - -- The intercept of the conversion from meter to graphing units - intercept FLOAT; - -- The length of each reading returned as an interval - reading_length_interval INTERVAL; + -- Holds the range of dates for returned data that fits the actual data. + requested_range TSRANGE; + -- The number of hours in each reading determined as an interval + reading_length_interval INTERVAL; -- Which index of the meter_id array you are currently working on. current_meter_index INTEGER := 1; -- The id of the meter index working on current_meter_id INTEGER; - -- The meter frequency from all meters. + -- The meter frequency from all meters. meter_frequency INTERVAL; - -- The meter frequency rounded up to a whole number of hours. + -- The meter frequency rounded up to a whole number of hours. meter_frequency_hour_up INTEGER; - -- The larger of the meter value and the argument sent. - max_frequency INTEGER; - -- The actual number of hours in a reading to use. - reading_length_hours_use INTEGER; + -- The larger of the meter value and the argument sent. + max_frequency INTEGER; + -- The number of hours in each reading determined + reading_length_hours_use INTEGER; BEGIN -- Find the correct number of hours per reading returned. SELECT * from reading_interval_3d(meter_ids_requested, reading_length_hours) into reading_length_hours_use, reading_length_interval; -- Loop over all meters. WHILE current_meter_index <= cardinality(meter_ids_requested) LOOP - -- ID of the current meter in loop - current_meter_id := meter_ids_requested[current_meter_index]; + -- ID of the current meter in loop + current_meter_id := meter_ids_requested[current_meter_index]; - -- Get the conversion from the current meter's unit to the desired graphing unit. - SELECT c.slope, c.intercept into slope, intercept - FROM meters m - INNER JOIN cik c on c.source_id = m.unit_id AND c.destination_id = graphic_unit_id - WHERE m.id = current_meter_id - ; - -- Get the range of days requested by calling shrink_tsrange_to_meter_readings_by_day. - -- First make requested range only be full days by dropping any partial days at start/end. - requested_range := shrink_tsrange_to_meter_readings_by_day(tsrange(date_trunc_up('day', start_stamp), date_trunc('day', end_stamp)), current_meter_id); + -- Get the range of days requested by calling shrink_tsrange_to_meter_readings_by_day. + -- First make requested range only be full days by dropping any partial days at start/end. + requested_range := shrink_tsrange_to_meter_readings_by_day(tsrange(date_trunc_up('day', start_stamp), date_trunc('day', end_stamp)), current_meter_id); - IF (reading_length_hours_use <= 12) THEN - -- Need to generate_series to group the desired hours together - RETURN QUERY - -- The readings are rates in the hourly table so want to average not sum so - -- work for quantity, flow & raw. - -- The time starts at the time of the generated sequence and ends at the length - -- of each block later. This is the same as the start time of the next value - -- in the sequence (except last one). - SELECT - hr.meter_id as meter_id, - AVG(hr.reading_rate) * slope + intercept as reading_rate, - hours.hour AS start_timestamp, - hours.hour + reading_length_interval AS end_timestamp - -- This is the series that starts at the beginning of the desired days, - -- ends at the end of the desired days and steps by the desired interval. - -- You need to subtract from the last interval for the end since generate_series - -- is inclusive. - FROM ( - SELECT hour - FROM generate_series( - lower(requested_range), - upper(requested_range) - reading_length_interval, - reading_length_interval - ) hours(hour) - ) hours(hour), - -- Also need the values in the hourly table. - hourly_readings_unit hr - -- Only want the desired meter - WHERE hr.meter_id = current_meter_id - -- Only want readings that lie within this slice of the desired data - AND lower(hr.time_interval) >= hours.hour - AND upper(hr.time_interval) <= hours.hour + reading_length_interval - -- ensures that the start of the reading time intervals does not exceed the end of the current generated interval - AND lower(hr.time_interval) <= hours.hour + reading_length_interval - -- Group by the start time of the generated series since all points in - -- the desired slice have the same start time for the series. - -- Also group by the meter_id since Postgres wants and desired for graphing - GROUP BY hours.hour, hr.meter_id - -- Time sort by the meter and start time for graphing. - ORDER BY hr.meter_id, hours.hour - ; - ELSE - -- The reading rate is more than 12 so return a single row with dummy values that easy to detect. - -- The end time differs from the start time by the meter reading frequency or min one for groups. - -- This means the meter reading frequency is too long for a 3D graphic. - RETURN QUERY - SELECT -999, -999::FLOAT, '1900-01-01 00:00:00'::TIMESTAMP, '1900-01-01 00:00:00'::TIMESTAMP + reading_length_interval - ; - END IF; + IF (reading_length_hours_use <= 12) THEN + -- Need to generate_series to group the desired hours together + RETURN QUERY + -- The readings are rates in the hourly table so want to average not sum so + -- work for quantity, flow & raw. + -- The time starts at the time of the generated sequence and ends at the length + -- of each block later. This is the same as the start time of the next value + -- in the sequence (except last one). + SELECT + -- Modified to retrieve converted hourly readings from the materialized view. + mhr.meter_id as meter_id, + AVG(mhr.reading_rate) as reading_rate, + hours.hour AS start_timestamp, + hours.hour + reading_length_interval AS end_timestamp + -- This is the series that starts at the beginning of the desired days, + -- ends at the end of the desired days and steps by the desired interval. + -- You need to subtract from the last interval for the end since generate_series + -- is inclusive. + FROM ( + SELECT hour + FROM generate_series( + lower(requested_range), + upper(requested_range) - reading_length_interval, + reading_length_interval + ) hours(hour) + ) hours(hour), + -- Also need the values in the meter hourly table. + meter_hourly_readings_unit mhr + -- Only want the desired meter + WHERE mhr.meter_id = current_meter_id + -- Only want the desired graphing unit + AND mhr.graphic_unit_id = graphic_unit_id_requested + -- Only want readings that lie within this slice of the desired data + AND lower(mhr.time_interval) >= hours.hour + AND upper(mhr.time_interval) <= hours.hour + reading_length_interval + -- ensures that the start of the reading time intervals does not exceed the end of the current generated interval + AND lower(mhr.time_interval) <= hours.hour + reading_length_interval + -- Group by the start time of the generated series since all points in + -- the desired slice have the same start time for the series. + -- Also group by the meter_id since Postgres wants and desired for graphing + GROUP BY hours.hour, mhr.meter_id + -- Time sort by the meter and start time for graphing. + ORDER BY mhr.meter_id, hours.hour + ; + ELSE + -- The reading rate is more than 12 so return a single row with dummy values that easy to detect. + -- The end time differs from the start time by the meter reading frequency or min one for groups. + -- This means the meter reading frequency is too long for a 3D graphic. + RETURN QUERY + SELECT -999, -999::FLOAT, '1900-01-01 00:00:00'::TIMESTAMP, '1900-01-01 00:00:00'::TIMESTAMP + reading_length_interval + ; + END IF; - -- Go to the next meter + -- Go to the next meter current_meter_index := current_meter_index + 1; END LOOP; END; $$ LANGUAGE plpgsql; - -/* Gets group meters graphing data for 3D graphic by returning points that span the requested +/* Gets group graphing data for 3D graphic by returning points that span the requested length of time over the days requested. */ CREATE OR REPLACE FUNCTION group_3d_readings_unit ( --Desire group ID - --For 3D graphics, users will only be able to select 1 group to graph. + --For 3D graphics, users will only be able to select 1 group to graph. group_id_requested INTEGER, -- The desired graphic unit of the returned data graphic_unit_id_requested INTEGER, @@ -217,13 +210,13 @@ CREATE OR REPLACE FUNCTION group_3d_readings_unit ( start_stamp TIMESTAMP, end_stamp TIMESTAMP, -- The number of hours in each reading requested - reading_length_hours INTEGER + reading_length_hours INTEGER ) RETURNS TABLE(reading_rate FLOAT, start_timestamp TIMESTAMP, end_timestamp TIMESTAMP) AS $$ DECLARE - -- Holds the range of dates for returned data that fits the actual data. - requested_range TSRANGE; + -- Holds the range of dates for returned data that fits the actual data. + requested_range TSRANGE; --Holds the desired meter IDs in order to call meter_3d_readings_unit in the query below. meter_ids INTEGER[]; -- The number of hours in each reading determined @@ -231,65 +224,65 @@ DECLARE -- The number of hours in each reading determined as an interval reading_length_interval INTERVAL; BEGIN - --Get all the meter IDS that will be included in the group being requested. + --Get all the meter IDS that will be included in the group being requested. SELECT array_agg(DISTINCT gdm.meter_id) INTO meter_ids - FROM groups_deep_meters gdm - WHERE group_id = group_id_requested; + FROM groups_deep_meters_cache gdm + WHERE group_id = group_id_requested; -- Find the correct number of hours per reading returned. SELECT * from reading_interval_3d(meter_ids, reading_length_hours) into reading_length_hours_use, reading_length_interval; - -- Get the range of days requested by calling shrink_tsrange_to_group_readings_by_day. - -- First make requested range only be full days by dropping any partial days at start/end. - requested_range := shrink_tsrange_to_group_readings_by_day(tsrange(date_trunc_up('day', start_stamp), date_trunc('day', end_stamp)), group_id_requested); + -- Get the range of days requested by calling shrink_tsrange_to_group_readings_by_day. + -- First make requested range only be full days by dropping any partial days at start/end. + requested_range := shrink_tsrange_to_group_readings_by_day(tsrange(date_trunc_up('day', start_stamp), date_trunc('day', end_stamp)), group_id_requested); IF (reading_length_hours_use <= 12) THEN - -- Need to generate_series to group the desired hours together - RETURN QUERY - -- The readings are rates in the hourly table so want to average not sum so - -- work for quantity, flow & raw. - -- The time starts at the time of the generated sequence and ends at the length - -- of each block later. This is the same as the start time of the next value - -- in the sequence (except last one). - SELECT - AVG(ghr.reading_rate) as reading_rate, - hours.hour AS start_timestamp, - hours.hour + reading_length_interval AS end_timestamp - -- This is the series that starts at the beginning of the desired days, - -- ends at the end of the desired days and steps by the desired interval. - -- You need to subtract from the last interval for the end since generate_series - -- is inclusive. - FROM ( - SELECT hour - FROM generate_series( - lower(requested_range), - upper(requested_range) - reading_length_interval, - reading_length_interval - ) hours(hour) - ) hours(hour), - -- Also need the values in the group hourly table. - group_hourly_readings_unit ghr - -- Only want the desired meter - WHERE ghr.group_id = group_id_requested + -- Need to generate_series to group the desired hours together + RETURN QUERY + -- The readings are rates in the hourly table so want to average not sum so + -- work for quantity, flow & raw. + -- The time starts at the time of the generated sequence and ends at the length + -- of each block later. This is the same as the start time of the next value + -- in the sequence (except last one). + SELECT + AVG(ghr.reading_rate) as reading_rate, + hours.hour AS start_timestamp, + hours.hour + reading_length_interval AS end_timestamp + -- This is the series that starts at the beginning of the desired days, + -- ends at the end of the desired days and steps by the desired interval. + -- You need to subtract from the last interval for the end since generate_series + -- is inclusive. + FROM ( + SELECT hour + FROM generate_series( + lower(requested_range), + upper(requested_range) - reading_length_interval, + reading_length_interval + ) hours(hour) + ) hours(hour), + -- Also need the values in the group hourly table. + group_hourly_readings_unit ghr + -- Only want the desired meter + WHERE ghr.group_id = group_id_requested -- Only want the desired graphing unit AND ghr.graphic_unit_id = graphic_unit_id_requested - -- Only want readings that lie within this slice of the desired data - AND lower(ghr.time_interval) >= hours.hour - AND upper(ghr.time_interval) <= hours.hour + reading_length_interval - -- ensures that the start of the reading time intervals does not exceed the end of the current generated interval - AND lower(ghr.time_interval) <= hours.hour + reading_length_interval - -- Group by the start time of the generated series since all points in - -- the desired slice have the same start time for the series. - GROUP BY hours.hour - -- Time sort by the start time for graphing. - ORDER BY hours.hour - ; - ELSE - -- The reading rate is more than 12 so return a single row with dummy values that easy to detect. - -- The end time differs from the start time by the meter reading frequency or min one for groups. - -- This means the meter reading frequency is too long for a 3D graphic. - RETURN QUERY - SELECT -999::FLOAT, '1900-01-01 00:00:00'::TIMESTAMP, '1900-01-01 00:00:00'::TIMESTAMP + reading_length_interval - ; - END IF; + -- Only want readings that lie within this slice of the desired data + AND lower(ghr.time_interval) >= hours.hour + AND upper(ghr.time_interval) <= hours.hour + reading_length_interval + -- ensures that the start of the reading time intervals does not exceed the end of the current generated interval + AND lower(ghr.time_interval) <= hours.hour + reading_length_interval + -- Group by the start time of the generated series since all points in + -- the desired slice have the same start time for the series. + GROUP BY hours.hour + -- Time sort by the start time for graphing. + ORDER BY hours.hour + ; + ELSE + -- The reading rate is more than 12 so return a single row with dummy values that easy to detect. + -- The end time differs from the start time by the meter reading frequency or min one for groups. + -- This means the meter reading frequency is too long for a 3D graphic. + RETURN QUERY + SELECT -999::FLOAT, '1900-01-01 00:00:00'::TIMESTAMP, '1900-01-01 00:00:00'::TIMESTAMP + reading_length_interval + ; + END IF; END; $$ LANGUAGE 'plpgsql'; diff --git a/src/server/sql/reading/create_function_get_compare_readings.sql b/src/server/sql/reading/create_function_get_compare_readings.sql index e605236c74..457bb9ef4e 100644 --- a/src/server/sql/reading/create_function_get_compare_readings.sql +++ b/src/server/sql/reading/create_function_get_compare_readings.sql @@ -42,9 +42,11 @@ curr_end: When the current/this time period ends for the compare. shift: How far back in time to shift the curr_start and curr_end date/time to get the previous times to compare. */ +-- New version of the meter_compare_readings function that uses the meter_hourly_readings_unit view. CREATE OR REPLACE FUNCTION meter_compare_readings_unit ( meter_ids INTEGER[], - graphic_unit_id INTEGER, + -- This is the graphic unit id, changed from graphic_unit_id to avoid confusion with the graphic unit id in the view. + passed_graphic_unit_id INTEGER, curr_start TIMESTAMP, curr_end TIMESTAMP, shift INTERVAL @@ -57,40 +59,37 @@ DECLARE BEGIN curr_tsrange := tsrange(curr_start, curr_end); prev_tsrange := tsrange(curr_start - shift, curr_end - shift); - + -- Modified to retrieve converted hourly readings from the materialized view. RETURN QUERY WITH curr_period AS ( SELECT - meters.id AS meter_id, - -- Convert the reading based on the conversion found below. + hourly.meter_id AS meter_id, -- It is okay to sum the flow units because hourly readings has a rate of per hour so -- to convert to a quantity would multiply by 1 so it is the same formula. - SUM(hourly.reading_rate) * c.slope + c.intercept AS reading - FROM (((hourly_readings_unit hourly - INNER JOIN unnest(meter_ids) meters(id) ON hourly.meter_id = meters.id) - INNER JOIN meters m ON m.id = meters.id) + SUM(hourly.reading_rate) AS reading + FROM meter_hourly_readings_unit hourly -- This is getting the conversion for the meter and unit to graph. - -- The slope and intercept are used above the transform the reading to the desired unit. - INNER JOIN cik c on c.source_id = m.unit_id AND c.destination_id = graphic_unit_id) - WHERE curr_tsrange @> hourly.time_interval - GROUP BY meters.id, c.slope, c.intercept + WHERE + -- The range requested must be completely within the hour so partial hours are not included. + curr_tsrange @> hourly.time_interval AND + hourly.graphic_unit_id = passed_graphic_unit_id AND + hourly.meter_id = ANY(meter_ids) + GROUP BY hourly.meter_id ), prev_period AS ( SELECT - meters.id AS meter_id, - -- Convert the reading based on the conversion found below. - -- It is okay to sum the flow units because hourly readings has a rate of per hour so - -- to convert to a quantity would multiply by 1 so it is the same formula. - SUM(hourly.reading_rate) * c.slope + c.intercept AS reading - FROM (((hourly_readings_unit hourly - INNER JOIN unnest(meter_ids) meters(id) ON hourly.meter_id = meters.id) - INNER JOIN meters m ON m.id = meters.id) + hourly.meter_id AS meter_id, + SUM(hourly.reading_rate) AS reading + FROM meter_hourly_readings_unit hourly -- This is getting the conversion for the meter and unit to graph. -- The slope and intercept are used above the transform the reading to the desired unit. - INNER JOIN cik c on c.source_id = m.unit_id AND c.destination_id = graphic_unit_id) - WHERE prev_tsrange @> hourly.time_interval - GROUP BY meters.id, c.slope, c.intercept + WHERE + -- The range requested must be completely within the hour so partial hours are not included. + prev_tsrange @> hourly.time_interval AND + hourly.graphic_unit_id = passed_graphic_unit_id AND + hourly.meter_id = ANY(meter_ids) + GROUP BY hourly.meter_id ) SELECT meters.id AS meter_id, @@ -142,6 +141,7 @@ BEGIN FROM group_hourly_readings_unit hourly WHERE curr_tsrange @> hourly.time_interval AND requested_graphic_unit_id = hourly.graphic_unit_id + AND hourly.group_id = ANY(group_ids) GROUP BY hourly.group_id ), prev_period AS ( @@ -151,6 +151,7 @@ BEGIN FROM group_hourly_readings_unit hourly WHERE prev_tsrange @> hourly.time_interval AND requested_graphic_unit_id = hourly.graphic_unit_id + AND hourly.group_id = ANY(group_ids) GROUP BY hourly.group_id ) SELECT diff --git a/src/server/sql/reading/create_reading_views.sql b/src/server/sql/reading/create_reading_views.sql index de521a4721..81aa00cbf4 100644 --- a/src/server/sql/reading/create_reading_views.sql +++ b/src/server/sql/reading/create_reading_views.sql @@ -9,6 +9,9 @@ Trying to only use case statements led to issues so the following functions mix case and if statements. */ +-- We need a gist index to support the @> operation. +CREATE EXTENSION IF NOT EXISTS btree_gist; + /* Rounds a timestamp up to the next interval */ @@ -54,7 +57,7 @@ DECLARE readings_max_tsrange TSRANGE; BEGIN SELECT tsrange(min(lower(time_interval)), max(upper(time_interval))) INTO readings_max_tsrange - FROM daily_readings_unit dr + FROM meter_daily_readings_unit dr -- Get all the meter_ids in the passed array of meters. INNER JOIN unnest(meter_ids) meters(id) ON dr.meter_id = meters.id; -- Make the original range be to the day by dropping parts of days at start/end. @@ -62,281 +65,257 @@ BEGIN END; $$ LANGUAGE 'plpgsql'; -/* - The following views are all generated in src/server/models/Reading.js in createReadingsMaterializedViews. - This is necessary because they can't be wrapped in a function (otherwise predicates would not be pushed down). -*/ +-- TODO: Remove this retained legacy function once the hypertable +-- implementation is finalized. group_graphic_units_cache replaces it. +-- CREATE OR REPLACE FUNCTION get_graphic_unit ( +-- requested_group_id INTEGER +-- ) +-- RETURNS INTEGER[] AS $$ +-- DECLARE +-- src_ids INTEGER[]; +-- dest_ids INTEGER[]; +-- child_meters_unit_ids INTEGER[]; +-- unit_ids_compatible INTEGER[] := '{}'; +-- unit_id INTEGER; +-- BEGIN +-- SELECT array_agg(DISTINCT m.unit_id) INTO child_meters_unit_ids +-- FROM groups_deep_meters_cache gdm +-- JOIN meters m ON m.id = gdm.meter_id +-- WHERE gdm.group_id = requested_group_id; +-- +-- SELECT array_agg(u.id) INTO dest_ids +-- FROM units u JOIN cik c +-- ON u.id = c.destination_id; +-- +-- FOREACH unit_id IN ARRAY dest_ids +-- LOOP +-- BEGIN +-- SELECT array_agg(source_id) INTO src_ids +-- FROM cik WHERE destination_id = unit_id; +-- +-- IF src_ids @> child_meters_unit_ids +-- THEN +-- IF NOT (unit_id = ANY (unit_ids_compatible)) +-- THEN +-- unit_ids_compatible := array_append(unit_ids_compatible, unit_id); +-- END IF; +-- END IF; +-- END; +-- END LOOP; +-- +-- RETURN unit_ids_compatible; +-- END; +-- $$ LANGUAGE 'plpgsql'; /* -The query shared by all of these views gets slow when one of two things happen: - 1) It has to scan a large percentage of the readings table - 2) It has to generate a large number of rows (by compressing to a small interval) -We pick the best of both worlds by only materializing the large duration tables (day+ and then hour+). -These produce fewer rows, making them acceptable to store, -but they benefit from materialization because they require a scan of a large percentage of -the readings table (to aggregate data over a large time range). The hourly table may not be that much smaller than -the meter data but it can make it much faster for meters that read at sub-hour intervals so it's worth the -extra disk space. - -The daily and hourly views are used when they give a minimum number of points as specified by the supplied -parameter. It first tries daily since this is fastest, then hourly and finally uses raw/meter data if necessary. -The goal is that the number of readings touched is never that large and when doing raw/meter readings the -time range should be small so the number of readings retrieved is not large. It is assumed that the indices/optimizations -allow for getting a subset of the raw/meter readings quickly. - */ +This may still apply. +The following views are all generated. +This is necessary because they can't be wrapped in a function (otherwise predicates would not be pushed down). +*/ /** -The next two create a view/table that takes the raw/meter readings and averages them for each day or hour. +The next two create a view/table that takes the raw/meter readings and averages them for each day or hour AND applies +the unit conversions from the cik_vary table. This is used by the line graph function below to make them faster since the values -are already averaged. There are two types of readings: quantity and flow/raw. The quantity +are already averaged and the conversions applied. There are two types of readings: quantity and flow/raw. The quantity readings must be normalized by their time length. The flow/raw readings are already by time so they are just averaged. The one table contains both types of readings but are now equivalent so the line reading functions can use them both in the same way. */ -CREATE MATERIALIZED VIEW IF NOT EXISTS -hourly_readings_unit - AS SELECT +-- Current Working Versions, not dependent on old hourly_readings_unit view, uses a CTE instead +-- This version only handles 1 conversion per hourly reading +-- It can not handle multiple conversions per reading or conversions that overlap the time interval. +CREATE MATERIALIZED VIEW IF NOT EXISTS meter_hourly_readings_unit +AS +WITH base_hourly AS ( + SELECT -- This gives the weighted average of the reading rates, defined as -- sum(reading_rate * overlap_duration) / sum(overlap_duration) - r.meter_id AS meter_id, - CASE WHEN u.unit_represent = 'quantity'::unit_represent_type THEN - (sum( - (r.reading * 3600 / (extract(EPOCH FROM (r.end_timestamp - r.start_timestamp)))) -- Reading rate in kw - * - extract(EPOCH FROM -- The number of seconds that the reading shares with the interval - least(r.end_timestamp, gen.interval_start + '1 hour'::INTERVAL) - - - greatest(r.start_timestamp, gen.interval_start) - ) - ) / sum( - extract(EPOCH FROM -- The number of seconds that the reading shares with the interval - least(r.end_timestamp, gen.interval_start + '1 hour'::INTERVAL) - - - greatest(r.start_timestamp, gen.interval_start) - ) - )) - WHEN (u.unit_represent = 'flow'::unit_represent_type OR u.unit_represent = 'raw'::unit_represent_type) THEN - (sum( - (r.reading * 3600 / u.sec_in_rate) -- Reading rate in per hour - * - extract(EPOCH FROM -- The number of seconds that the reading shares with the interval - least(r.end_timestamp, gen.interval_start + '1 hour'::INTERVAL) - - - greatest(r.start_timestamp, gen.interval_start) - ) - ) / sum( - extract(EPOCH FROM -- The number of seconds that the reading shares with the interval - least(r.end_timestamp, gen.interval_start + '1 hour'::INTERVAL) - - - greatest(r.start_timestamp, gen.interval_start) - ) - )) + r.meter_id, + CASE + WHEN u.unit_represent = 'quantity'::unit_represent_type THEN + ( + SUM( + -- Reading rate + (r.reading * 3600 / EXTRACT(EPOCH FROM (r.end_timestamp - r.start_timestamp))) * + -- The number of seconds that the reading shares with the interval + EXTRACT(EPOCH FROM LEAST(r.end_timestamp, gen.interval_start + INTERVAL '1 hour') - GREATEST(r.start_timestamp, gen.interval_start)) + ) / + SUM( + -- The number of seconds that the reading shares with the interval + EXTRACT(EPOCH FROM LEAST(r.end_timestamp, gen.interval_start + INTERVAL '1 hour') - GREATEST(r.start_timestamp, gen.interval_start)) + ) + ) + WHEN u.unit_represent IN ('flow'::unit_represent_type, 'raw'::unit_represent_type) THEN + ( + SUM( + -- Reading rate in per hour + (r.reading * 3600 / u.sec_in_rate) * + -- The number of seconds that the reading shares with the interval + EXTRACT(EPOCH FROM LEAST(r.end_timestamp, gen.interval_start + INTERVAL '1 hour') - GREATEST(r.start_timestamp, gen.interval_start)) + ) / + SUM( + -- The number of seconds that the reading shares with the interval + EXTRACT(EPOCH FROM LEAST(r.end_timestamp, gen.interval_start + INTERVAL '1 hour') - GREATEST(r.start_timestamp, gen.interval_start)) + ) + ) END AS reading_rate, - -- The following code does the min/max for hourly readings - CASE WHEN u.unit_represent = 'quantity'::unit_represent_type THEN - (max(( -- Extract the maximum rate over each day - (r.reading * 3600 / (extract(EPOCH FROM (r.end_timestamp - r.start_timestamp)))) -- Reading rate in kw - * - extract(EPOCH FROM -- The number of seconds that the reading shares with the interval - least(r.end_timestamp, gen.interval_start + '1 hour'::INTERVAL) - - - greatest(r.start_timestamp, gen.interval_start) - ) - ) / ( - extract(EPOCH FROM -- The number of seconds that the reading shares with the interval - least(r.end_timestamp, gen.interval_start + '1 hour'::INTERVAL) - - - greatest(r.start_timestamp, gen.interval_start) - ) - ))) - WHEN (u.unit_represent = 'flow'::unit_represent_type OR u.unit_represent = 'raw'::unit_represent_type) THEN - (max(( -- For flow and raw data the max/min is per minute, so we multiply the max/min by 24 hrs * 60 min - (r.reading * 3600 / u.sec_in_rate) -- Reading rate in kw - * - extract(EPOCH FROM -- The number of seconds that the reading shares with the interval - least(r.end_timestamp, gen.interval_start + '1 hour'::INTERVAL) - - - greatest(r.start_timestamp, gen.interval_start) - ) - ) / ( - extract(EPOCH FROM -- The number of seconds that the reading shares with the interval - least(r.end_timestamp, gen.interval_start + '1 hour'::INTERVAL) - - - greatest(r.start_timestamp, gen.interval_start) - ) - ))) - END as max_rate, - - CASE WHEN u.unit_represent = 'quantity'::unit_represent_type THEN - (min(( --Extract the minimum rate over each day - (r.reading * 3600 / (extract(EPOCH FROM (r.end_timestamp - r.start_timestamp)))) -- Reading rate in kw - * - extract(EPOCH FROM -- The number of seconds that the reading shares with the interval - least(r.end_timestamp, gen.interval_start + '1 hour'::INTERVAL) - - - greatest(r.start_timestamp, gen.interval_start) - ) - ) / ( - extract(EPOCH FROM -- The number of seconds that the reading shares with the interval - least(r.end_timestamp, gen.interval_start + '1 hour'::INTERVAL) - - - greatest(r.start_timestamp, gen.interval_start) - ) - ))) - WHEN (u.unit_represent = 'flow'::unit_represent_type OR u.unit_represent = 'raw'::unit_represent_type) THEN - (min(( - (r.reading * 3600 / u.sec_in_rate) -- Reading rate in kw - * - extract(EPOCH FROM -- The number of seconds that the reading shares with the interval - least(r.end_timestamp, gen.interval_start + '1 hour'::INTERVAL) - - - greatest(r.start_timestamp, gen.interval_start) - ) - ) / ( - extract(EPOCH FROM -- The number of seconds that the reading shares with the interval - least(r.end_timestamp, gen.interval_start + '1 hour'::INTERVAL) - - - greatest(r.start_timestamp, gen.interval_start) - ) - ))) - END as min_rate, - - tsrange(gen.interval_start, gen.interval_start + '1 hour'::INTERVAL, '()') AS time_interval - FROM ((readings r + CASE + WHEN u.unit_represent = 'quantity'::unit_represent_type THEN + MAX( + -- Extract the maximum rate over each day + ( + -- Reading rate + (r.reading * 3600 / EXTRACT(EPOCH FROM (r.end_timestamp - r.start_timestamp))) * + -- The number of seconds that the reading shares with the interval + EXTRACT(EPOCH FROM LEAST(r.end_timestamp, gen.interval_start + INTERVAL '1 hour') - GREATEST(r.start_timestamp, gen.interval_start)) + ) / + -- The number of seconds that the reading shares with the interval + EXTRACT(EPOCH FROM LEAST(r.end_timestamp, gen.interval_start + INTERVAL '1 hour') - GREATEST(r.start_timestamp, gen.interval_start)) + ) + WHEN u.unit_represent IN ('flow'::unit_represent_type, 'raw'::unit_represent_type) THEN + -- For flow and raw data the max/min is per minute, so we multiply the max/min by 24 hrs * 60 min + MAX( + ( + -- Reading rate + (r.reading * 3600 / u.sec_in_rate) * + -- The number of seconds that the reading shares with the interval + EXTRACT(EPOCH FROM LEAST(r.end_timestamp, gen.interval_start + INTERVAL '1 hour') - GREATEST(r.start_timestamp, gen.interval_start)) + ) / + -- The number of seconds that the reading shares with the interval + EXTRACT(EPOCH FROM LEAST(r.end_timestamp, gen.interval_start + INTERVAL '1 hour') - GREATEST(r.start_timestamp, gen.interval_start)) + ) + END AS max_rate, + + CASE + WHEN u.unit_represent = 'quantity'::unit_represent_type THEN + MIN( + --Extract the minimum rate over each day + ( + -- Reading rate + (r.reading * 3600 / EXTRACT(EPOCH FROM (r.end_timestamp - r.start_timestamp))) * + -- The number of seconds that the reading shares with the interval + EXTRACT(EPOCH FROM LEAST(r.end_timestamp, gen.interval_start + INTERVAL '1 hour') - GREATEST(r.start_timestamp, gen.interval_start)) + ) / + -- The number of seconds that the reading shares with the interval + EXTRACT(EPOCH FROM LEAST(r.end_timestamp, gen.interval_start + INTERVAL '1 hour') - GREATEST(r.start_timestamp, gen.interval_start)) + ) + WHEN u.unit_represent IN ('flow'::unit_represent_type, 'raw'::unit_represent_type) THEN + MIN( + ( + -- Reading rate + (r.reading * 3600 / u.sec_in_rate) * + -- The number of seconds that the reading shares with the interval + EXTRACT(EPOCH FROM LEAST(r.end_timestamp, gen.interval_start + INTERVAL '1 hour') - GREATEST(r.start_timestamp, gen.interval_start)) + ) / + -- The number of seconds that the reading shares with the interval + EXTRACT(EPOCH FROM LEAST(r.end_timestamp, gen.interval_start + INTERVAL '1 hour') - GREATEST(r.start_timestamp, gen.interval_start)) + ) + END AS min_rate, + + tsrange(gen.interval_start, gen.interval_start + INTERVAL '1 hour', '()') AS time_interval + + FROM readings r -- This sequence of joins takes the meter id to its unit and a unit. - INNER JOIN meters m ON r.meter_id = m.id) - INNER JOIN units u ON m.unit_id = u.id) - CROSS JOIN LATERAL generate_series( - date_trunc('hour', r.start_timestamp), - -- Subtract 1 interval width because generate_series is end-inclusive - date_trunc_up('hour', r.end_timestamp) - '1 hour'::INTERVAL, - '1 hour'::INTERVAL - ) gen(interval_start) + INNER JOIN meters m ON r.meter_id = m.id + INNER JOIN units u ON m.unit_id = u.id + CROSS JOIN LATERAL generate_series( + date_trunc('hour', r.start_timestamp), + -- Subtract 1 interval width because generate_series is end-inclusive + date_trunc_up('hour', r.end_timestamp) - INTERVAL '1 hour', + INTERVAL '1 hour' + ) gen(interval_start) GROUP BY r.meter_id, gen.interval_start, u.unit_represent - -- The order by ensures that the materialized view will be clustered in this way. - ORDER BY gen.interval_start, r.meter_id; - +) +SELECT + m.id AS meter_id, + SUM(bh.reading_rate * c.slope + c.intercept) AS reading_rate, + SUM(bh.min_rate * c.slope + c.intercept) AS min_rate, + SUM(bh.max_rate * c.slope + c.intercept) AS max_rate, + bh.time_interval, + c.destination_id AS graphic_unit_id + +FROM base_hourly bh +JOIN meters m ON m.id = bh.meter_id +JOIN units u ON u.id = m.unit_id +JOIN cik_vary c ON c.source_id = m.unit_id AND tsrange(c.start_time, c.end_time, '()') && bh.time_interval +GROUP BY m.id, graphic_unit_id, bh.time_interval +-- The order by ensures that the materialized view will be clustered in this way. +ORDER BY bh.time_interval, meter_id; + +-- Used by the line/3d/compare functions. +CREATE INDEX if not exists idx_meter_hourly_ordering ON meter_hourly_readings_unit (meter_id, graphic_unit_id, lower(time_interval)); + +-- Current working version. Retrieves converted data from meter_hourly_readings_unit and averages it to the day. CREATE MATERIALIZED VIEW IF NOT EXISTS -daily_readings_unit +meter_daily_readings_unit AS SELECT - h.meter_id AS meter_id, - avg(h.reading_rate) AS reading_rate, - max(h.max_rate) AS max_rate, - min(h.min_rate) AS min_rate, - - tsrange(gen.interval_start, gen.interval_start + '1 day'::INTERVAL, '()') AS time_interval - FROM ((hourly_readings_unit h - INNER JOIN meters m ON h.meter_id = m.id) - INNER JOIN units u ON m.unit_id = u.id) - CROSS JOIN LATERAL generate_series( - date_trunc('day', lower(h.time_interval)), - date_trunc_up('day', upper(h.time_interval)) - '1 hour'::INTERVAL, - '1 day'::INTERVAL - ) gen(interval_start) - GROUP BY h.meter_id, gen.interval_start, u.unit_represent - ORDER BY gen.interval_start, h.meter_id; - --- TODO Check if needed and when to use as not done for hourly. --- With the index added in 3D readings, this should be consider as part of the decision --- on if this is needed. -CREATE EXTENSION IF NOT EXISTS btree_gist; --- We need a gist index to support the @> operation. -CREATE INDEX if not exists idx_daily_readings_unit ON daily_readings_unit USING GIST(time_interval, meter_id); - -/* - The following function takes an integer for group id and return an array of all unit ids which are compatible - to all child meters in that group. -*/ -CREATE OR REPLACE FUNCTION get_graphic_unit ( - requested_group_id INTEGER -) -RETURNS INTEGER[] AS $$ -DECLARE - src_ids INTEGER[]; - dest_ids INTEGER[]; - child_meters_unit_ids INTEGER[]; - unit_ids_compatible INTEGER[] := '{}'; - unit_id INTEGER; - -BEGIN - -- get the units of all child meters in group - SELECT array_agg(DISTINCT m.unit_id) INTO child_meters_unit_ids - FROM groups_deep_meters gdm - JOIN meters m ON m.id = gdm.meter_id - WHERE gdm.group_id = requested_group_id; - - -- get all possible destination units - SELECT array_agg(u.id) INTO dest_ids - FROM units u JOIN cik c - ON u.id = c.destination_id; - - -- determine the compatible unit by checking if the array of all corresponding source unit - -- to a destination unit contains all child meters' units - FOREACH unit_id IN ARRAY dest_ids - LOOP - BEGIN - SELECT array_agg(source_id) INTO src_ids - FROM cik WHERE destination_id = unit_id; - - -- append each compatible unit id once into array - IF src_ids @> child_meters_unit_ids - THEN - IF NOT (unit_id = ANY (unit_ids_compatible)) - THEN - unit_ids_compatible := array_append(unit_ids_compatible, unit_id); - END IF; - END IF; - END; - END LOOP; + h.meter_id AS meter_id, + AVG(h.reading_rate) AS reading_rate, + MIN(h.min_rate) AS min_rate, + MAX(h.max_rate) AS max_rate, + tsrange(gen.interval_start, gen.interval_start + INTERVAL '1 day', '()') AS time_interval, + h.graphic_unit_id AS graphic_unit_id + + FROM meter_hourly_readings_unit h + CROSS JOIN LATERAL generate_series( + date_trunc('day', lower(h.time_interval)), + date_trunc_up('day', upper(h.time_interval)) - INTERVAL '1 hour', + INTERVAL '1 day' + ) gen(interval_start) + WHERE tsrange(gen.interval_start, gen.interval_start + INTERVAL '1 day', '()') @> h.time_interval + GROUP BY h.meter_id, h.graphic_unit_id, gen.interval_start + ORDER BY h.meter_id, graphic_unit_id, gen.interval_start; + + -- Used by the line/bar/compare functions. +CREATE INDEX if not exists idx_meter_daily_ordering ON meter_daily_readings_unit (meter_id, graphic_unit_id, lower(time_interval)); +-- This index sometimes performs faster(for the bar function) than the above index but is likely not worth the additional overhead. +-- CREATE INDEX if not exists idx_mdr_meter_graphic ON meter_daily_readings_unit (meter_id, graphic_unit_id); + +--Modified to use meter_hourly_readings_unit instead of old hourly_readings_unit view. +--No longer needs to apply conversions since that is done in meter_hourly_readings_unit view. +CREATE MATERIALIZED VIEW IF NOT EXISTS +group_hourly_readings_unit + AS SELECT + gdm.group_id, + SUM(hr.reading_rate) AS reading_rate, + hr.time_interval, + hr.graphic_unit_id - RETURN unit_ids_compatible; -END; -$$ LANGUAGE 'plpgsql'; + FROM meter_hourly_readings_unit hr + INNER JOIN groups_deep_meters_cache gdm ON hr.meter_id = gdm.meter_id + INNER JOIN group_graphic_units_cache gu + ON gu.group_id = gdm.group_id AND hr.graphic_unit_id = gu.graphic_unit_id + -- group meter readings of each group on the the same hour, of the same graphic unit + GROUP BY gdm.group_id, hr.graphic_unit_id, hr.time_interval + ORDER BY gdm.group_id; +CREATE INDEX if not exists idx_group_hourly_readings_unit ON group_hourly_readings_unit USING GIST(time_interval, graphic_unit_id, group_id); +--Modified to use meter_daily_readings_unit instead of old daily_readings_unit view. +--No longer needs to apply conversions since that is done in meter_daily_readings_unit view. CREATE MATERIALIZED VIEW IF NOT EXISTS group_daily_readings_unit AS SELECT gdm.group_id, - sum(dr.reading_rate * c.slope + c.intercept) AS reading_rate, + SUM(dr.reading_rate) AS reading_rate, dr.time_interval, - gu.graphic_unit_id AS graphic_unit_id - - FROM (((((daily_readings_unit dr - INNER JOIN groups_deep_meters gdm ON dr.meter_id = gdm.meter_id) - INNER JOIN meters m ON m.id = dr.meter_id) - INNER JOIN units u ON m.unit_id = u.id) - INNER JOIN cik c on c.source_id = m.unit_id) - INNER JOIN unnest(get_graphic_unit(gdm.group_id)) AS gu(graphic_unit_id) ON c.destination_id = gu.graphic_unit_id) + dr.graphic_unit_id + + FROM meter_daily_readings_unit dr + INNER JOIN groups_deep_meters_cache gdm ON dr.meter_id = gdm.meter_id + INNER JOIN group_graphic_units_cache gu + ON gu.group_id = gdm.group_id AND dr.graphic_unit_id = gu.graphic_unit_id -- group meter readings of each group on the the same day, of the same graphic unit - GROUP BY gdm.group_id, gu.graphic_unit_id, dr.time_interval -- order by time interval instead - ORDER BY dr.time_interval, gu.graphic_unit_id, gdm.group_id; + GROUP BY gdm.group_id, dr.graphic_unit_id, dr.time_interval + -- order by time interval instead + ORDER BY dr.time_interval, dr.graphic_unit_id, gdm.group_id; -- Index on interval, graphic_unit_id, group_id CREATE INDEX if not exists idx_group_daily_readings_unit ON group_daily_readings_unit USING GIST(time_interval, graphic_unit_id, group_id); -CREATE MATERIALIZED VIEW IF NOT EXISTS -group_hourly_readings_unit - AS SELECT - gdm.group_id, - sum(hr.reading_rate * c.slope + c.intercept) AS reading_rate, - hr.time_interval, - gu.graphic_unit_id AS graphic_unit_id - - FROM (((((hourly_readings_unit hr - INNER JOIN groups_deep_meters gdm ON hr.meter_id = gdm.meter_id) - INNER JOIN meters m ON m.id = hr.meter_id) - INNER JOIN units u ON m.unit_id = u.id) - INNER JOIN cik c on c.source_id = m.unit_id) - INNER JOIN unnest(get_graphic_unit(gdm.group_id)) AS gu(graphic_unit_id) ON c.destination_id = gu.graphic_unit_id) - -- group meter readings of each group on the the same hour, of the same graphic unit - GROUP BY gdm.group_id, gu.graphic_unit_id, hr.time_interval - ORDER BY gdm.group_id; - -CREATE INDEX if not exists idx_group_hourly_readings_unit ON group_hourly_readings_unit USING GIST(time_interval, graphic_unit_id, group_id); - /* The following function determines the correct duration view to query from, and returns averaged or raw reading from it. It is designed to return data for plotting line graphs. It works on meters. @@ -350,9 +329,11 @@ max_raw_points: The maximum number of data points to return if using the raw poi max_hour_points: The maximum number of data points to return if using the hour view. Only used if 'auto' for point_accuracy. Details on how this function works can be found in the devDocs in the resource generalization document. */ +-- New version of meter_line_readings_unit that uses the new views. CREATE OR REPLACE FUNCTION meter_line_readings_unit ( meter_ids INTEGER[], - graphic_unit_id INTEGER, + -- This is the graphic unit id, changed from graphic_unit_id to avoid confusion with the graphic unit id in the view. + passed_graphic_unit_id INTEGER, start_stamp TIMESTAMP, end_stamp TIMESTAMP, point_accuracy reading_line_accuracy, @@ -431,16 +412,36 @@ DECLARE IF (current_point_accuracy = 'raw'::reading_line_accuracy) THEN -- Gets raw meter data to graph. + -- Modified to allow for raw time varying conversions. RETURN QUERY SELECT r.meter_id as meter_id, CASE WHEN u.unit_represent = 'quantity'::unit_represent_type THEN -- If it is quantity readings then need to convert to rate per hour by dividing by the time length where -- the 3600 is needed since EPOCH is in seconds. - ((r.reading / (extract(EPOCH FROM (r.end_timestamp - r.start_timestamp)) / 3600)) * c.slope + c.intercept) + -- Normalize to rate over reading interval + SUM( + --Wrapped in SUM to handle multiple matching cik_vary conversions + -- Weight by conversion duration(intersection of reading and conversion time ranges is necessary because the conversion may overlap the reading time range) + (EXTRACT(EPOCH FROM ( + upper(tsrange(c.start_time, c.end_time, '()') * tsrange(r.start_timestamp, r.end_timestamp, '[]')) + - + lower(tsrange(c.start_time, c.end_time, '()') * tsrange(r.start_timestamp, r.end_timestamp, '[]')) + )) / 3600) + * (c.slope * (r.reading / (EXTRACT(EPOCH FROM (r.end_timestamp - r.start_timestamp)) / 3600)) + c.intercept) + ) / (EXTRACT(EPOCH FROM (r.end_timestamp - r.start_timestamp)) / 3600) WHEN (u.unit_represent = 'flow'::unit_represent_type OR u.unit_represent = 'raw'::unit_represent_type) THEN -- If it is flow or raw readings then it is already a rate so just convert it but also need to normalize -- to per hour. - ((r.reading * 3600 / u.sec_in_rate) * c.slope + c.intercept) + SUM( + --Wrapped in SUM to handle multiple matching cik_vary conversions + -- Weight by conversion duration (intersection of reading and conversion time ranges is necessary because the conversion may overlap the reading time range) + (EXTRACT(EPOCH FROM ( + upper(tsrange(c.start_time, c.end_time, '()') * tsrange(r.start_timestamp, r.end_timestamp, '[]')) + - + lower(tsrange(c.start_time, c.end_time, '()') * tsrange(r.start_timestamp, r.end_timestamp, '[]')) + )) / 3600) + * (c.slope * (r.reading * 3600 / u.sec_in_rate) + c.intercept) + ) / (EXTRACT(EPOCH FROM (r.end_timestamp - r.start_timestamp)) / 3600) END AS reading_rate, -- There is no range of values on raw/meter data so return NaN to indicate that. -- The route will return this as null when it shows up in Redux state. @@ -448,11 +449,19 @@ DECLARE cast('NaN' AS DOUBLE PRECISION) as max_rate, r.start_timestamp, r.end_timestamp + FROM (((readings r INNER JOIN meters m ON m.id = current_meter_id) INNER JOIN units u ON m.unit_id = u.id) - INNER JOIN cik c on c.source_id = m.unit_id AND c.destination_id = graphic_unit_id) + INNER JOIN cik_vary c on c.source_id = m.unit_id + AND c.destination_id = passed_graphic_unit_id + --The condition below was added for time varying conversions (allows for multiple cik_vary rows to be applied to a single reading) + --The cik_vary exclusive bounds '()' ensures no two conversions overlap. + AND tsrange(c.start_time, c.end_time, '()') && tsrange(r.start_timestamp, r.end_timestamp, '[]')) WHERE lower(requested_range) <= r.start_timestamp AND r.end_timestamp <= upper(requested_range) AND r.meter_id = current_meter_id + -- Added GROUP BY to allow SUM to aggregate correctly across multiple rows. + -- TODO : postgreSQL doesn't understand unit_represent cannot change for a given meter, so it has to be in group by. Might be worth finding fix. + GROUP BY r.meter_id, r.start_timestamp, r.end_timestamp, u.unit_represent -- This ensures the data is sorted ORDER BY r.start_timestamp ASC; -- The first part is making sure that the number of hour points is 1440 or less. @@ -461,45 +470,42 @@ DECLARE -- so you don't interpolate points by using the hourly data. ELSIF (current_point_accuracy = 'hourly'::reading_line_accuracy) THEN -- Get hourly points to graph. See daily for more comments. + -- Now uses materialized view for hourly meter readings. RETURN QUERY - SELECT hourly.meter_id AS meter_id, - -- Convert the reading based on the conversion found below. - -- Hourly readings are already averaged correctly into a rate. - hourly.reading_rate * c.slope + c.intercept as reading_rate, - hourly.min_rate * c.slope + c.intercept AS min_rate, - hourly.max_rate * c.slope + c.intercept AS max_rate, + -- Modified to Retrieve converted hourly readings from the materialized view. + SELECT + hourly.meter_id AS meter_id, + hourly.reading_rate AS reading_rate, + hourly.min_rate AS min_rate, + hourly.max_rate AS max_rate, lower(hourly.time_interval) AS start_timestamp, upper(hourly.time_interval) AS end_timestamp - FROM ((hourly_readings_unit hourly - INNER JOIN meters m ON m.id = current_meter_id) - INNER JOIN cik c on c.source_id = m.unit_id AND c.destination_id = graphic_unit_id) - WHERE requested_range @> time_interval AND hourly.meter_id = current_meter_id - -- This ensures the data is sorted - ORDER BY start_timestamp ASC; + FROM + meter_hourly_readings_unit AS hourly + WHERE + requested_range @> hourly.time_interval + AND hourly.meter_id = current_meter_id + AND hourly.graphic_unit_id = passed_graphic_unit_id + ORDER BY + start_timestamp ASC; ELSE - -- Get daily points to graph. This should be an okay number but can be too many - -- if there are a lot of days of readings. - -- TODO Someday consider averaging days if too many. RETURN QUERY + -- Modified to retrieve converted daily readings from the materialized view. SELECT daily.meter_id AS meter_id, - -- Convert the reading based on the conversion found below. - -- Daily readings are already averaged correctly into a rate. - daily.reading_rate * c.slope + c.intercept as reading_rate, - daily.min_rate * c.slope + c.intercept AS min_rate, - daily.max_rate * c.slope + c.intercept AS max_rate, + daily.reading_rate AS reading_rate, + daily.min_rate AS min_rate, + daily.max_rate AS max_rate, lower(daily.time_interval) AS start_timestamp, upper(daily.time_interval) AS end_timestamp - FROM ((daily_readings_unit daily - -- Get all the meter_ids in the passed array of meters. - -- This sequence of joins takes the meter id to its unit and a unit. - INNER JOIN meters m ON m.id = current_meter_id) - -- This is getting the conversion for the meter and unit to graph. - -- The slope and intercept are used above the transform the reading to the desired unit. - INNER JOIN cik c on c.source_id = m.unit_id AND c.destination_id = graphic_unit_id) - WHERE requested_range @> time_interval AND daily.meter_id = current_meter_id - -- This ensures the data is sorted - ORDER BY start_timestamp ASC; + FROM + meter_daily_readings_unit AS daily + WHERE + requested_range @> daily.time_interval + AND daily.meter_id = current_meter_id + AND daily.graphic_unit_id = passed_graphic_unit_id + ORDER BY + start_timestamp ASC; END IF; current_meter_index := current_meter_index + 1; END LOOP; @@ -541,7 +547,7 @@ BEGIN -- First get all the meter ids that will be included in one or more groups being queried. -- In case meter is repeated, make this distinct. SELECT array_agg(DISTINCT gdm.meter_id) INTO meter_ids - FROM groups_deep_meters gdm + FROM groups_deep_meters_cache gdm INNER JOIN unnest(group_ids) gids(id) ON gdm.group_id = gids.id; -- Calculate point accuracy if request (auto) or if raw since that is not allowed for groups. @@ -599,7 +605,6 @@ BEGIN readings.reading_rate, lower(readings.time_interval) AS start_timestamp, upper(readings.time_interval) AS end_timestamp - FROM group_daily_readings_unit readings INNER JOIN unnest(group_ids) gids(id) ON readings.group_id = gids.id WHERE readings.graphic_unit_id = requested_graphic_unit_id @@ -635,9 +640,11 @@ bar_width_days: The number of days to use for the bar width. start_timestamp: The start timestamp of the data to return. end_timestamp: The end timestamp of the data to return. */ +-- New version of meter_bar_readings_unit that uses the new meter_daily_readings_unit view. CREATE OR REPLACE FUNCTION meter_bar_readings_unit ( meter_ids INTEGER[], - graphic_unit_id INTEGER, + -- This is the graphic unit id, changed from graphic_unit_id to avoid confusion with the graphic unit id in the view. + passed_graphic_unit_id INTEGER, bar_width_days INTEGER, start_stamp TIMESTAMP, end_stamp TIMESTAMP @@ -683,27 +690,22 @@ BEGIN real_end_stamp := real_end_stamp - bar_width; RETURN QUERY - SELECT dr.meter_id AS meter_id, - -- dr.reading_rate is the weighted average reading rate per hour over the day. - -- Convert to a quantity by multiplying by the time in hours which is 24 since daily values. - -- Then convert the reading based on the conversion found below. - sum(dr.reading_rate * 24) * c.slope + c.intercept AS reading, + SELECT + -- Modified to retrieve converted daily readings from the materialized view. + mdr.meter_id AS meter_id, + sum(mdr.reading_rate * 24) AS reading, bars.interval_start AS start_timestamp, bars.interval_start + bar_width AS end_timestamp - FROM (((((daily_readings_unit dr + + FROM meter_daily_readings_unit mdr INNER JOIN generate_series(real_start_stamp, real_end_stamp, bar_width) bars(interval_start) - ON tsrange(bars.interval_start, bars.interval_start + bar_width, '[]') @> dr.time_interval) - -- Get all the meter_ids in the passed array of meters. - INNER JOIN unnest(meter_ids) meters(id) ON dr.meter_id = meters.id) - -- This sequence of joins takes the meter id to its unit and in the final join - -- it then get the desired conversion. - INNER JOIN meters m ON m.id = meters.id) - -- Don't return bar data if raw since cannot sum. - INNER JOIN units u ON m.unit_id = u.id AND u.unit_represent != 'raw'::unit_represent_type) - -- This is getting the conversion for the meter (source_id) and unit to graph (destination_id). - -- The slope and intercept are used above the transform the reading to the desired unit. - INNER JOIN cik c on c.source_id = m.unit_id AND c.destination_id = graphic_unit_id) - GROUP BY dr.meter_id, bars.interval_start, c.slope, c.intercept; + ON tsrange(bars.interval_start, bars.interval_start + bar_width, '[]') @> mdr.time_interval + INNER JOIN unnest(meter_ids) meters(id) ON mdr.meter_id = meters.id + INNER JOIN meters m ON m.id = meters.id + INNER JOIN units u ON m.unit_id = u.id AND u.unit_represent != 'raw'::unit_represent_type + WHERE mdr.graphic_unit_id = passed_graphic_unit_id + GROUP BY mdr.meter_id, bars.interval_start; + END; $$ LANGUAGE 'plpgsql'; @@ -775,7 +777,7 @@ BEGIN INNER JOIN units u ON readings.graphic_unit_id = u.id AND u.unit_represent != 'raw'::unit_represent_type) INNER JOIN unnest(group_ids) gids(id) ON readings.group_id = gids.id) -- Use the readings in the passed in graphic unit - WHERE readings.graphic_unit_id = requested_graphic_unit_id + WHERE readings.graphic_unit_id = requested_graphic_unit_id GROUP BY readings.group_id, bars.interval_start; END; diff --git a/src/server/sql/reading/create_readings_table.sql b/src/server/sql/reading/create_readings_table.sql index 316ba43561..1da854ec8d 100644 --- a/src/server/sql/reading/create_readings_table.sql +++ b/src/server/sql/reading/create_readings_table.sql @@ -11,3 +11,7 @@ CREATE TABLE IF NOT EXISTS readings ( CHECK (start_timestamp < readings.end_timestamp), PRIMARY KEY (meter_id, start_timestamp) ); + +-- Supports indexed lookup of the latest reading end for each meter. +CREATE INDEX IF NOT EXISTS readings_meter_end_timestamp_idx +ON readings (meter_id, end_timestamp DESC); diff --git a/src/server/sql/reading/get_count_by_meter_ids_and_date_range.sql b/src/server/sql/reading/get_count_by_meter_ids_and_date_range.sql new file mode 100644 index 0000000000..60a54113ed --- /dev/null +++ b/src/server/sql/reading/get_count_by_meter_ids_and_date_range.sql @@ -0,0 +1,10 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +-- Count all requested meters together to avoid one database query per meter. +SELECT COUNT(*) +FROM readings +WHERE meter_id = ANY(${meterIDs}::INTEGER[]) + AND start_timestamp >= COALESCE(${startDate}, '-infinity'::TIMESTAMP) + AND end_timestamp <= COALESCE(${endDate}, 'infinity'::TIMESTAMP); diff --git a/src/server/test/common.js b/src/server/test/common.js index c0d201fe4c..66a57a4d41 100644 --- a/src/server/test/common.js +++ b/src/server/test/common.js @@ -19,6 +19,7 @@ const { log, LogLevel } = require('../log'); // TODO: Move logging disabling to a better place. log.level = LogLevel.SILENT; log.emailLevel = LogLevel.SILENT; +log.logToFile = false; const User = require('../models/User'); const { getDB, createSchema, stopDB } = require('../models/database'); diff --git a/src/server/test/db/cikVaryTests.js b/src/server/test/db/cikVaryTests.js index 210c0559a3..16b09bc2ac 100644 --- a/src/server/test/db/cikVaryTests.js +++ b/src/server/test/db/cikVaryTests.js @@ -10,6 +10,9 @@ const Conversion = require('../../models/Conversion'); const Unit = require('../../models/Unit'); const ConversionSegment = require('../../models/ConversionSegment'); const CikVary = require('../../models/CikVary'); +const TimeScaleDBReading = require('../../models/TimeScaleDB/Reading'); +const refreshAllReadingViews = require('../../services/refreshAllReadingViews'); +const sinon = require('sinon'); async function setupTestData(conn) { await new Unit(undefined, 'Unit 10', 'Unit 10', Unit.unitRepresentType.QUANTITY, 1000, Unit.unitType.METER, '', Unit.displayableType.ADMIN, true, 'Note 10').insert(conn); @@ -115,4 +118,76 @@ mocha.describe('redoCikVary integration', function () { expect(grouped[key]).to.deep.equal(slopes); }); }); + + mocha.it('should require a reading aggregate rebuild after replacing cik_vary', async function () { + await redoCikVary(conn); + + const state = await conn.one(` + SELECT rebuild_revision, completed_rebuild_revision + FROM reading_aggregate_state + WHERE id = 1 + `); + expect(Number(state.rebuild_revision)).to.be.greaterThan(Number(state.completed_rebuild_revision)); + }); + + mocha.it('should complete a pending rebuild during a default refresh', async function () { + await redoCikVary(conn); + await refreshAllReadingViews(); + + const state = await conn.one(` + SELECT rebuild_revision, completed_rebuild_revision + FROM reading_aggregate_state + WHERE id = 1 + `); + expect(state.completed_rebuild_revision).to.equal(state.rebuild_revision); + }); + + mocha.it('should complete pending group cache maintenance during a default refresh', async function () { + await redoCikVary(conn); + await refreshAllReadingViews(); + + const state = await conn.one(` + SELECT group_cache_revision, completed_group_cache_revision + FROM reading_aggregate_state + WHERE id = 1 + `); + expect(state.completed_group_cache_revision).to.equal(state.group_cache_revision); + }); + + mocha.it('should align hourly and daily refreshes to their own bucket boundaries', function () { + const startTimestamp = '2022-08-18 10:15:00'; + const endTimestamp = '2022-08-18 11:45:00'; + const hourlyRange = TimeScaleDBReading.getRefreshRange(startTimestamp, endTimestamp, 'hour'); + const dailyRange = TimeScaleDBReading.getRefreshRange(startTimestamp, endTimestamp, 'day'); + + expect(hourlyRange.refreshStart.toISOString()).to.equal('2022-08-18T10:00:00.000Z'); + expect(hourlyRange.refreshEnd.toISOString()).to.equal('2022-08-18T12:00:00.000Z'); + expect(dailyRange.refreshStart.toISOString()).to.equal('2022-08-18T00:00:00.000Z'); + expect(dailyRange.refreshEnd.toISOString()).to.equal('2022-08-19T00:00:00.000Z'); + }); + + mocha.it('should require a rebuild after changing split-row unit metadata', async function () { + await conn.none(` + UPDATE units + SET sec_in_rate = sec_in_rate + 1 + WHERE id = \${unitId} + `, { unitId: unit10Id }); + + const state = await conn.one(` + SELECT rebuild_revision, completed_rebuild_revision + FROM reading_aggregate_state + WHERE id = 1 + `); + expect(Number(state.rebuild_revision)).to.be.greaterThan(Number(state.completed_rebuild_revision)); + }); + + mocha.it('should not rebuild by default when split-row sources are current', async function () { + const rebuildSpy = sinon.spy(TimeScaleDBReading, 'rebuildReadings'); + try { + await refreshAllReadingViews(); + expect(rebuildSpy.called).to.equal(false); + } finally { + rebuildSpy.restore(); + } + }); }); diff --git a/src/server/test/db/compareTests.js b/src/server/test/db/compareTests.js index 70df939e97..398be51f4f 100644 --- a/src/server/test/db/compareTests.js +++ b/src/server/test/db/compareTests.js @@ -12,9 +12,9 @@ const Group = require('../../models/Group'); const Unit = require('../../models/Unit'); const { insertStandardUnits, insertStandardConversions } = require('../../util/insertData'); const { insertSpecialUnits, insertSpecialConversions } = require('../../data/automatedTestingData'); -const { redoCik } = require('../../services/graph/redoCik'); -const { refreshAllReadingViews } = require('../../services/refreshAllReadingViews'); +const refreshAllReadingViews = require('../../services/refreshAllReadingViews'); const { refreshGroupsDeepMetersView } = require('../../services/refreshGroupsDeepMetersView'); +const { redoCikVary } = require('../../services/graph/redoCik'); mocha.describe('Compare readings', () => { let meter, graphicUnitId, conversionSlope, conn; @@ -32,7 +32,6 @@ mocha.describe('Compare readings', () => { await insertStandardConversions(conn); await insertSpecialUnits(conn); await insertSpecialConversions(conn); - await redoCik(conn); // Make the meter be a kWh meter. const meterUnitId = (await Unit.getByName('Electric_Utility', conn)).id; await new Meter( @@ -66,6 +65,7 @@ mocha.describe('Compare readings', () => { undefined // reading frequency ).insert(conn); meter = await Meter.getByName('Meter', conn); + await redoCikVary(conn); await Reading.insertAll([ new Reading(meter.id, 1, prevStart, prevEnd), new Reading(meter.id, 10, currStart, currEnd) diff --git a/src/server/test/db/conversionTests.js b/src/server/test/db/conversionTests.js index 62cd23edf3..c4edfd31e7 100644 --- a/src/server/test/db/conversionTests.js +++ b/src/server/test/db/conversionTests.js @@ -2,8 +2,11 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +const database = require('../../models/database'); +const sqlFile = database.sqlFile; const { mocha, expect, testDB } = require('../common'); const Conversion = require('../../models/Conversion'); +const ConversionSegment = require('../../models/ConversionSegment'); const Unit = require('../../models/Unit'); /** @@ -15,12 +18,29 @@ function expectConversionToBeEquivalent(expected, actual) { expect(actual).to.have.property('sourceId', expected.sourceId); expect(actual).to.have.property('destinationId', expected.destinationId); expect(actual).to.have.property('bidirectional', expected.bidirectional); + expect(actual).to.have.property('note', expected.note); +} + +/** + * Compares the expected and actual conversion segments. + * @param {*} expected The expected conversion. + * @param {*} actual The actual conversion. + */ +function expectConversionSegmentToBeEquivalent(expected, actual) { + // console.log('expected: ', expected); + // console.log('actual: ', actual); + expect(actual).to.have.property('sourceId', expected.sourceId); + expect(actual).to.have.property('destinationId', expected.destinationId); + expect(actual).to.have.property('weekPatternsId', expected.weekPatternsId); expect(actual).to.have.property('slope', expected.slope); expect(actual).to.have.property('intercept', expected.intercept); + expect(actual).to.have.property('startTime', expected.startTime); + expect(actual).to.have.property('endTime', expected.endTime); expect(actual).to.have.property('note', expected.note); } mocha.describe('Conversions', () => { + let unitAId, unitBId, conversionSegmentPreInsert; mocha.beforeEach(async () => { conn = testDB.getConnection(); const unitA = new Unit(undefined, 'Unit A', 'Unit A', Unit.unitRepresentType.QUANTITY, 1000, @@ -29,44 +49,54 @@ mocha.describe('Conversions', () => { Unit.unitType.METER, 'Suffix B', Unit.displayableType.ALL, true, 'Note B'); await unitA.insert(conn); await unitB.insert(conn); + unitAId = (await Unit.getByName('Unit A', conn)).id; + unitBId = (await Unit.getByName('Unit B', conn)).id; + conversionSegmentPreInsert = new ConversionSegment(unitAId, unitBId, null, 1.23, 3.14, '-infinity', 'infinity', 'Segment note'); }); - mocha.it('can be saved and retrived', async () => { + mocha.it('can be saved and retrieved', async () => { const conn = testDB.getConnection(); - const unitAId = (await Unit.getByName('Unit A', conn)).id; - const unitBId = (await Unit.getByName('Unit B', conn)).id; - const conversionPreInsert = new Conversion(unitAId, unitBId, false, 1.23, 4.56, 'Note'); - await conversionPreInsert.insert(conn); + const conversionPreInsert = new Conversion(unitAId, unitBId, false, 'Note'); + await conversionPreInsert.insert(conversionSegmentPreInsert.weekPatternsId, conversionSegmentPreInsert.slope, conversionSegmentPreInsert.intercept, conversionSegmentPreInsert.note, conn); // Gets conversion by source and destination. - const conversionPostInsertBySourceDestination = await Conversion.getBySourceDestination(unitAId, unitBId, conn); - expectConversionToBeEquivalent(conversionPreInsert, conversionPostInsertBySourceDestination); + const conversionPostInsert = await Conversion.getBySourceDestination(unitAId, unitBId, conn); + expectConversionToBeEquivalent(conversionPreInsert, conversionPostInsert); + // Gets conversion segment by source and destination. + const conversionSegmentPostInsert = await ConversionSegment.getBySourceDestination(unitAId, unitBId, conn); + expect(conversionSegmentPostInsert.length).to.equal(1); + expectConversionSegmentToBeEquivalent(conversionSegmentPreInsert, conversionSegmentPostInsert[0]); }); - mocha.it('can be updated and retrived', async () => { + mocha.it('can be updated and retrieved', async () => { const conn = testDB.getConnection(); - const unitAId = (await Unit.getByName('Unit A', conn)).id; - const unitBId = (await Unit.getByName('Unit B', conn)).id; - const conversionPreInsert = new Conversion(unitAId, unitBId, true, 1.23, 4.56, 'Note'); - await conversionPreInsert.insert(conn); - + const conversionPreInsert = new Conversion(unitAId, unitBId, true, 'Note'); + await conversionPreInsert.insert(conversionSegmentPreInsert.weekPatternsId, conversionSegmentPreInsert.slope, conversionSegmentPreInsert.intercept, conversionSegmentPreInsert.note, conn); // Updates the conversion. Note that the sourceId and destinationId can't be changed. conversionPreInsert.bidirectional = false; - conversionPreInsert.intercept = 3.14; conversionPreInsert.note = 'New note'; await conversionPreInsert.update(conn); - - const covnersionPostInsert = await Conversion.getBySourceDestination(unitAId, unitBId, conn); - expectConversionToBeEquivalent(conversionPreInsert, covnersionPostInsert); + // Checks conversion and segment. + const conversionPostInsert = await Conversion.getBySourceDestination(unitAId, unitBId, conn); + expectConversionToBeEquivalent(conversionPreInsert, conversionPostInsert); + const conversionSegmentPostInsert = await ConversionSegment.getBySourceDestination(unitAId, unitBId, conn); + // Conversion segment should not have been changed. + expect(conversionSegmentPostInsert.length).to.equal(1); + expectConversionSegmentToBeEquivalent(conversionSegmentPreInsert, conversionSegmentPostInsert[0]); }); mocha.it('can be deleted', async () => { const conn = testDB.getConnection(); - const unitAId = (await Unit.getByName('Unit A', conn)).id; - const unitBId = (await Unit.getByName('Unit B', conn)).id; - const conversionPreInsert = new Conversion(unitAId, unitBId, true, 1.23, 4.56, 'Note'); - await conversionPreInsert.insert(conn); + const conversionPreInsert = new Conversion(unitAId, unitBId, true, 'Note'); + await conversionPreInsert.insert(conversionSegmentPreInsert.weekPatternsId, conversionSegmentPreInsert.slope, conversionSegmentPreInsert.intercept, conversionSegmentPreInsert.segmentNote, conn); + // Remove the conversion segment created. + await conn.none(sqlFile('conversionSegment/delete_conversion_segment.sql'), { + sourceId: conversionSegmentPreInsert.sourceId, + destinationId: conversionSegmentPreInsert.destinationId, + startTime: conversionSegmentPreInsert.startTime, + endTime: conversionSegmentPreInsert.endTime + }); await Conversion.delete(unitAId, unitBId, conn); - + // Check that gone. const conversionPostInsert = await Conversion.getBySourceDestination(unitAId, unitBId, conn); expect(conversionPostInsert).to.be.equal(null); }); diff --git a/src/server/test/db/groupTests.js b/src/server/test/db/groupTests.js index 1eda7bae53..98159f6ce8 100644 --- a/src/server/test/db/groupTests.js +++ b/src/server/test/db/groupTests.js @@ -197,6 +197,22 @@ mocha.describe('Groups', () => { expect(meters).to.deep.equal([lovedMeter.id]); }); + mocha.it('removes disowned meters from the deep meter cache', async () => { + conn = testDB.getConnection(); + const parent = await Group.getByName('GA', conn); + const lovedMeter = await Meter.getByName('MA', conn); + const impendingOrphan = await Meter.getByName('MB', conn); + + await parent.adoptMeter(lovedMeter.id, conn); + await parent.adoptMeter(impendingOrphan.id, conn); + await refreshGroupsDeepMetersView(); + await parent.disownMeter(impendingOrphan.id, conn); + await refreshGroupsDeepMetersView(); + + const deepMeters = await Group.getDeepMetersByGroupID(parent.id, conn); + expect(deepMeters).to.deep.equal([lovedMeter.id]); + }); + mocha.it('can be deleted', async () => { conn = testDB.getConnection(); const unwanted = await Group.getByName('GA', conn); diff --git a/src/server/test/db/readingTests.js b/src/server/test/db/readingTests.js index 41f2b3dbfd..b7b7c6ea92 100644 --- a/src/server/test/db/readingTests.js +++ b/src/server/test/db/readingTests.js @@ -5,7 +5,7 @@ /** * This class is for testing meter readings. */ -const { mocha, expect, testDB } = require('../common'); +const { mocha, expect, testDB, chai, app } = require('../common'); const moment = require('moment'); const Meter = require('../../models/Meter'); const Reading = require('../../models/Reading'); @@ -75,6 +75,24 @@ mocha.describe('Readings', () => { await Reading.insertOrUpdateAll([reading1Updated, reading2], conn); const retrievedReadings = await Reading.getAllByMeterID(meter.id, conn); expect(retrievedReadings).to.have.length(2); + expect(retrievedReadings[0].reading).to.equal(2); + expect(retrievedReadings[0].endTimestamp.isSame(endTimestamp1)).to.equal(true); + }); + mocha.it('preserves sequential upsert behavior for repeated keys in one bulk input', async () => { + const conn = testDB.getConnection(); + const startTimestamp = moment.utc('2017-01-01'); + const firstEndTimestamp = moment.utc(startTimestamp).add(1, 'hour'); + const laterEndTimestamp = moment.utc(startTimestamp).add(2, 'hours'); + await Reading.insertOrUpdateAll([ + new Reading(meter.id, 1, startTimestamp, firstEndTimestamp), + new Reading(meter.id, 2, startTimestamp, laterEndTimestamp) + ], conn); + + const [retrievedReading] = await Reading.getAllByMeterID(meter.id, conn); + expect(retrievedReading.reading).to.equal(2); + // The original row-by-row upsert changed only reading, so the first end + // timestamp remains part of the compatibility contract. + expect(retrievedReading.endTimestamp.isSame(firstEndTimestamp)).to.equal(true); }); mocha.it('can keep any data already in the DB', async () => { const conn = testDB.getConnection(); @@ -91,4 +109,47 @@ mocha.describe('Readings', () => { expect(retrievedReading.endTimestamp.isSame(endTimestamp)).to.equal(true); expect(retrievedReading.reading).to.equal(1); }); + mocha.it('counts readings for multiple meters in one query', async () => { + const conn = testDB.getConnection(); + await new Meter(undefined, 'Second meter', null, false, true, Meter.type.MAMAC, null, gps).insert(conn); + const secondMeter = await Meter.getByName('Second meter', conn); + const firstStart = moment.utc('2018-01-01'); + const firstEnd = moment.utc(firstStart).add(1, 'hour'); + const secondStart = moment.utc(firstEnd); + const secondEnd = moment.utc(secondStart).add(1, 'hour'); + + await Reading.insertAll([ + new Reading(meter.id, 1, firstStart, firstEnd), + new Reading(meter.id, 2, secondStart, secondEnd), + new Reading(secondMeter.id, 3, firstStart, firstEnd) + ], conn); + + const count = await Reading.getCountByMeterIDsAndDateRange( + [meter.id, secondMeter.id], + firstStart, + firstEnd, + conn + ); + expect(count).to.equal(2); + }); + mocha.it('returns the combined multi-meter count from the readings API', async () => { + const conn = testDB.getConnection(); + await new Meter(undefined, 'Second meter', null, false, true, Meter.type.MAMAC, null, gps).insert(conn); + const secondMeter = await Meter.getByName('Second meter', conn); + const startTimestamp = moment.utc('2018-01-01'); + const endTimestamp = moment.utc(startTimestamp).add(1, 'hour'); + await Reading.insertAll([ + new Reading(meter.id, 1, startTimestamp, endTimestamp), + new Reading(secondMeter.id, 2, startTimestamp, endTimestamp) + ], conn); + + const response = await chai.request(app) + .get(`/api/readings/line/count/meters/${meter.id},${secondMeter.id}`) + .query({ timeInterval: `${startTimestamp.format()}_${endTimestamp.format()}` }); + + expect(response).to.have.status(200); + // The existing route serializes the number explicitly, so Chai exposes + // the response through text rather than parsing it into response.body. + expect(response.text).to.equal('2'); + }); }); diff --git a/src/server/test/db/timeVaryingReadingsTests.js b/src/server/test/db/timeVaryingReadingsTests.js new file mode 100644 index 0000000000..d55101080b --- /dev/null +++ b/src/server/test/db/timeVaryingReadingsTests.js @@ -0,0 +1,284 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const { mocha, expect, testDB } = require('../common'); +const refreshAllReadingViews = require('../../services/refreshAllReadingViews'); + +const START = '2021-06-01 00:00:00'; +const END = '2021-06-06 00:00:00'; +const DELTA = 0.0000001; + +const quantityReadings = [ + [24, '2021-06-01 00:00:00', '2021-06-02 00:00:00'], + [21, '2021-06-02 00:00:00', '2021-06-02 12:00:00'], + [27, '2021-06-02 12:00:00', '2021-06-03 00:00:00'], + [72, '2021-06-03 00:00:00', '2021-06-04 00:00:00'], + [96, '2021-06-04 00:00:00', '2021-06-05 00:00:00'], + [120, '2021-06-05 00:00:00', '2021-06-06 00:00:00'] +]; + +const flowReadings = [ + [1, '2021-06-01 00:00:00', '2021-06-02 00:00:00'], + [2, '2021-06-02 00:00:00', '2021-06-03 00:00:00'], + [2.6875, '2021-06-03 00:00:00', '2021-06-03 09:00:00'], + [3.125, '2021-06-03 09:00:00', '2021-06-03 21:00:00'], + [3.4375, '2021-06-03 21:00:00', '2021-06-04 00:00:00'], + [4, '2021-06-04 00:00:00', '2021-06-05 00:00:00'], + [5, '2021-06-05 00:00:00', '2021-06-06 00:00:00'] +]; + +const conversionSegments = [ + ['-infinity', '2021-06-02 04:00:00', 3], + ['2021-06-02 04:00:00', '2021-06-03 18:00:00', 5], + ['2021-06-03 18:00:00', '2021-06-05 00:00:00', 7], + ['2021-06-05 00:00:00', 'infinity', 9] +]; + +function expectRate(row, expected) { + expect(row.reading_rate).to.be.closeTo(expected, DELTA); +} + +mocha.describe('Time-varying reading conversion boundaries', function () { + let conn; + let quantityMeterId; + let quantityGraphicUnitId; + let flowMeterId; + let flowGraphicUnitId; + + mocha.beforeEach(async function () { + conn = testDB.getConnection(); + + const units = await conn.many(` + INSERT INTO units ( + name, identifier, unit_represent, sec_in_rate, type_of_unit, + suffix, displayable, preferred_display, note + ) + VALUES + ('TV Quantity Source', 'tv_quantity_source', 'quantity', 3600, 'meter', '', 'none', false, ''), + ('TV Quantity Graphic', 'tv_quantity_graphic', 'quantity', 3600, 'unit', '', 'all', false, ''), + ('TV Flow Source', 'tv_flow_source', 'flow', 60, 'meter', '', 'none', false, ''), + ('TV Flow Graphic', 'tv_flow_graphic', 'flow', 60, 'unit', '', 'all', false, '') + RETURNING id, name + `); + const unitIds = Object.fromEntries(units.map(unit => [unit.name, unit.id])); + quantityGraphicUnitId = unitIds['TV Quantity Graphic']; + flowGraphicUnitId = unitIds['TV Flow Graphic']; + + const meters = await conn.many(` + INSERT INTO meters ( + name, enabled, displayable, meter_type, default_timezone_meter, + identifier, unit_id, default_graphic_unit, reading_frequency + ) + VALUES + ('TV Quantity Meter', false, true, 'other', 'UTC', 'tv_quantity_meter', + \${quantitySourceId}, \${quantityGraphicUnitId}, INTERVAL '1 hour'), + ('TV Flow Meter', false, true, 'other', 'UTC', 'tv_flow_meter', + \${flowSourceId}, \${flowGraphicUnitId}, INTERVAL '1 hour') + RETURNING id, name + `, { + quantitySourceId: unitIds['TV Quantity Source'], + quantityGraphicUnitId, + flowSourceId: unitIds['TV Flow Source'], + flowGraphicUnitId + }); + const meterIds = Object.fromEntries(meters.map(meter => [meter.name, meter.id])); + quantityMeterId = meterIds['TV Quantity Meter']; + flowMeterId = meterIds['TV Flow Meter']; + + for (const [sourceId, destinationId] of [ + [unitIds['TV Quantity Source'], quantityGraphicUnitId], + [unitIds['TV Flow Source'], flowGraphicUnitId] + ]) { + await conn.none('INSERT INTO cik (source_id, destination_id) VALUES (\${sourceId}, \${destinationId})', { + sourceId, + destinationId + }); + for (const [startTime, endTime, slope] of conversionSegments) { + await conn.none(` + INSERT INTO cik_vary (source_id, destination_id, start_time, end_time, slope, intercept) + VALUES (\${sourceId}, \${destinationId}, \${startTime}, \${endTime}, \${slope}, 0) + `, { sourceId, destinationId, startTime, endTime, slope }); + } + } + + const readingRows = [ + ...quantityReadings.map(reading => [quantityMeterId, ...reading]), + ...flowReadings.map(reading => [flowMeterId, ...reading]) + ]; + await conn.none(` + INSERT INTO readings (meter_id, reading, start_timestamp, end_timestamp) + SELECT meter_id, reading, start_timestamp, end_timestamp + FROM jsonb_to_recordset(\${readings:json}::jsonb) AS input( + meter_id INTEGER, + reading FLOAT, + start_timestamp TIMESTAMP, + end_timestamp TIMESTAMP + ) + `, { + readings: readingRows.map(([meterId, reading, startTimestamp, endTimestamp]) => ({ + meter_id: meterId, + reading, + start_timestamp: startTimestamp, + end_timestamp: endTimestamp + })) + }); + + await refreshAllReadingViews({ rebuild: true }); + }); + + mocha.it('splits quantity and flow hourly rows at each conversion boundary', async function () { + const quantity = await conn.many(` + SELECT bucket, reading_rate, min_rate, max_rate + FROM meter_hourly_readings_unit_cagg + WHERE meter_id = \${meterId} AND graphic_unit_id = \${graphicUnitId} + ORDER BY bucket + `, { meterId: quantityMeterId, graphicUnitId: quantityGraphicUnitId }); + const flow = await conn.many(` + SELECT bucket, reading_rate, min_rate, max_rate + FROM meter_hourly_readings_unit_cagg + WHERE meter_id = \${meterId} AND graphic_unit_id = \${graphicUnitId} + ORDER BY bucket + `, { meterId: flowMeterId, graphicUnitId: flowGraphicUnitId }); + + expect(quantity).to.have.lengthOf(120); + expect(flow).to.have.lengthOf(120); + + for (const [hour, expected] of [[24, 5.25], [28, 8.75], [36, 11.25], [66, 21], [96, 45]]) { + expectRate(quantity[hour], expected); + expect(quantity[hour].min_rate).to.be.closeTo(expected, DELTA); + expect(quantity[hour].max_rate).to.be.closeTo(expected, DELTA); + } + for (const [hour, expected] of [[24, 360], [28, 600], [57, 937.5], [66, 1312.5], [69, 1443.75], [96, 2700]]) { + expectRate(flow[hour], expected); + expect(flow[hour].min_rate).to.be.closeTo(expected, DELTA); + expect(flow[hour].max_rate).to.be.closeTo(expected, DELTA); + } + }); + + mocha.it('duration-weights a conversion boundary inside an hour', async function () { + await conn.any(` + UPDATE cik_vary + SET end_time = '2021-06-02 04:30:00' + WHERE end_time = '2021-06-02 04:00:00'; + + UPDATE cik_vary + SET start_time = '2021-06-02 04:30:00' + WHERE start_time = '2021-06-02 04:00:00'; + + SELECT rebuild_hourly_hypertable_split(); + `); + await refreshAllReadingViews(); + + const splitRows = await conn.many(` + SELECT meter_id, start_timestamp, end_timestamp, slope + FROM hypertable_hourly_split + WHERE meter_id IN (\${quantityMeterId}, \${flowMeterId}) + AND start_timestamp >= '2021-06-02 04:00:00' + AND end_timestamp <= '2021-06-02 05:00:00' + ORDER BY meter_id, start_timestamp + `, { quantityMeterId, flowMeterId }); + expect(splitRows).to.have.lengthOf(4); + expect(splitRows.map(row => row.end_timestamp.diff(row.start_timestamp, 'seconds'))) + .to.deep.equal([1800, 1800, 1800, 1800]); + + const quantity = await conn.one(` + SELECT reading_rate, min_rate, max_rate + FROM meter_hourly_readings_unit_cagg + WHERE meter_id = \${meterId} AND graphic_unit_id = \${graphicUnitId} + AND bucket = '2021-06-02 04:00:00' + `, { meterId: quantityMeterId, graphicUnitId: quantityGraphicUnitId }); + const flow = await conn.one(` + SELECT reading_rate, min_rate, max_rate + FROM meter_hourly_readings_unit_cagg + WHERE meter_id = \${meterId} AND graphic_unit_id = \${graphicUnitId} + AND bucket = '2021-06-02 04:00:00' + `, { meterId: flowMeterId, graphicUnitId: flowGraphicUnitId }); + + expectRate(quantity, 7); + expect(quantity.min_rate).to.be.closeTo(5.25, DELTA); + expect(quantity.max_rate).to.be.closeTo(8.75, DELTA); + expectRate(flow, 480); + expect(flow.min_rate).to.be.closeTo(360, DELTA); + expect(flow.max_rate).to.be.closeTo(600, DELTA); + }); + + mocha.it('uses duration-weighted daily averages', async function () { + const quantity = await conn.many(` + SELECT bucket, reading_rate, min_rate, max_rate + FROM meter_daily_readings_unit_cagg + WHERE meter_id = \${meterId} AND graphic_unit_id = \${graphicUnitId} + ORDER BY bucket + `, { meterId: quantityMeterId, graphicUnitId: quantityGraphicUnitId }); + const flow = await conn.many(` + SELECT bucket, reading_rate, min_rate, max_rate + FROM meter_daily_readings_unit_cagg + WHERE meter_id = \${meterId} AND graphic_unit_id = \${graphicUnitId} + ORDER BY bucket + `, { meterId: flowMeterId, graphicUnitId: flowGraphicUnitId }); + + [3, 9.41666666666667, 16.5, 28, 45].forEach((expected, index) => expectRate(quantity[index], expected)); + [180, 560, 998.4375, 1680, 2700].forEach((expected, index) => expectRate(flow[index], expected)); + expect(quantity[1].min_rate).to.be.closeTo(5.25, DELTA); + expect(quantity[1].max_rate).to.be.closeTo(11.25, DELTA); + expect(flow[2].min_rate).to.be.closeTo(806.25, DELTA); + expect(flow[2].max_rate).to.be.closeTo(1443.75, DELTA); + }); + + mocha.it('preserves raw meter-reading intervals with duration-weighted conversions', async function () { + const quantity = await conn.func('meter_line_readings_unit', [ + [quantityMeterId], quantityGraphicUnitId, START, END, 'raw', 1440, 1440 + ]); + const flow = await conn.func('meter_line_readings_unit', [ + [flowMeterId], flowGraphicUnitId, START, END, 'raw', 1440, 1440 + ]); + + expect(quantity).to.have.lengthOf(quantityReadings.length); + expect(flow).to.have.lengthOf(flowReadings.length); + [3, 7.58333333333333, 11.25, 16.5, 28, 45] + .forEach((expected, index) => expectRate(quantity[index], expected)); + [180, 560, 806.25, 1031.25, 1443.75, 1680, 2700] + .forEach((expected, index) => expectRate(flow[index], expected)); + + quantity.forEach((row, index) => { + expect(row.start_timestamp.format('YYYY-MM-DD HH:mm:ss')).to.equal(quantityReadings[index][1]); + expect(row.end_timestamp.format('YYYY-MM-DD HH:mm:ss')).to.equal(quantityReadings[index][2]); + }); + flow.forEach((row, index) => { + expect(row.start_timestamp.format('YYYY-MM-DD HH:mm:ss')).to.equal(flowReadings[index][1]); + expect(row.end_timestamp.format('YYYY-MM-DD HH:mm:ss')).to.equal(flowReadings[index][2]); + }); + }); + + mocha.it('keeps one-hour 3D results consistent with the hourly line', async function () { + const quantity = await conn.func('meter_3d_readings_unit', [ + [quantityMeterId], quantityGraphicUnitId, START, END, 1 + ]); + const flow = await conn.func('meter_3d_readings_unit', [ + [flowMeterId], flowGraphicUnitId, START, END, 1 + ]); + + expect(quantity).to.have.lengthOf(120); + expect(flow).to.have.lengthOf(120); + expectRate(quantity[24], 5.25); + expectRate(quantity[28], 8.75); + expectRate(flow[57], 937.5); + expectRate(flow[66], 1312.5); + }); + + mocha.it('produces one-day bars from the corrected daily rates', async function () { + const quantity = await conn.func('meter_bar_readings_unit', [ + [quantityMeterId], quantityGraphicUnitId, 1, START, END + ]); + const flow = await conn.func('meter_bar_readings_unit', [ + [flowMeterId], flowGraphicUnitId, 1, START, END + ]); + + [72, 226, 396, 672, 1080].forEach((expected, index) => { + expect(quantity[index].reading).to.be.closeTo(expected, DELTA); + }); + [4320, 13440, 23962.5, 40320, 64800].forEach((expected, index) => { + expect(flow[index].reading).to.be.closeTo(expected, DELTA); + }); + }); +}); diff --git a/src/server/test/db/unitReadingsTests.js b/src/server/test/db/unitReadingsTests.js index d8577fdaa1..dd698a78d6 100644 --- a/src/server/test/db/unitReadingsTests.js +++ b/src/server/test/db/unitReadingsTests.js @@ -11,18 +11,21 @@ const Reading = require('../../models/Reading'); const Group = require('../../models/Group'); const { generateSineData } = require('../../data/generateTestingData'); const Unit = require('../../models/Unit'); -const Conversion = require('../../models/Conversion'); const { insertStandardUnits, insertStandardConversions } = require('../../util/insertData'); const { insertSpecialUnits, insertSpecialConversions } = require('../../data/automatedTestingData'); -const { redoCik } = require('../../services/graph/redoCik'); +const { insertUnits, insertConversions } = require('../../util/insertData'); +const { redoCikVary } = require('../../services/graph/redoCik'); const { refreshGroupsDeepMetersView } = require('../../services/refreshGroupsDeepMetersView'); +const { getUnitId, unitDatakWh, conversionDatakWh } = require('../../util/readingsUtils'); +const refreshAllReadingViews = require('../../services/refreshAllReadingViews'); +// Readings should be accurate to many decimal places, but allow some wiggle room for database and javascript conversions const { DELTA } = require('../../util/readingsUtils.js'); // TODO add tests that check flow readings. mocha.describe('Line & bar Readings', () => { mocha.describe('Check daily and hourly reading views', () => { - let meter, conn; + let meter, unitId, conn; // Need to work in UTC time since that is what the database returns and comparing // to database values. Done in all moment objects in this test. const timestamp1 = moment.utc('2017-01-01'); @@ -34,13 +37,42 @@ mocha.describe('Line & bar Readings', () => { mocha.beforeEach(async () => { conn = testDB.getConnection(); // Insert the special units. Really only need 1-2 but this is easy. - await insertSpecialUnits(conn); + await insertUnits(unitDatakWh, false, conn); + await insertConversions(conversionDatakWh, conn); // Make the meter be a kWh meter. const meterUnitId = (await Unit.getByName('Electric_Utility', conn)).id; await new Meter(undefined, 'Meter', null, false, true, Meter.type.OTHER, 'CST', undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, - undefined, undefined, undefined, undefined, undefined, meterUnitId, meterUnitId, undefined, '01:00:00').insert(conn); + undefined, undefined, undefined, undefined, undefined, meterUnitId, unitId, undefined, '01:00:00').insert(conn); + await redoCikVary(conn); meter = await Meter.getByName('Meter', conn); + // kwh = await Unit.getByName('kWh', conn); + unitId = await getUnitId('kWh'); + }); + + mocha.it('Does not create legacy materialized reading views or graphic-unit function', async () => { + const { count } = await conn.one(` + SELECT count(*)::INTEGER AS count + FROM pg_matviews + WHERE matviewname = ANY(\${legacy_view_names}) + `, { + legacy_view_names: [ + 'meter_hourly_readings_unit', + 'meter_daily_readings_unit', + 'group_hourly_readings_unit', + 'group_daily_readings_unit', + 'groups_deep_meters' + ] + }); + expect(count).to.equal(0); + + const { function_count } = await conn.one(` + SELECT count(*)::INTEGER AS function_count + FROM pg_proc + WHERE pronamespace = current_schema()::regnamespace + AND proname = 'get_graphic_unit' + `); + expect(function_count).to.equal(0); }); mocha.it('Hourly readings when readings line up with hour', async () => { @@ -50,10 +82,10 @@ mocha.describe('Line & bar Readings', () => { new Reading(meter.id, 300, timestamp3, timestamp4), new Reading(meter.id, 400, timestamp4, timestamp5) ], conn); - // Refresh meter views but only hourly view needs the change - await Reading.refreshMeterReadingsViews(conn); - const { meter_id, reading_rate } = await conn.one('SELECT * FROM hourly_readings_unit WHERE lower(time_interval)=${start_timestamp};', - { start_timestamp: timestamp1 }); + // Refresh the hourly readings view because it is materialized. + await Reading.refreshHourlyReadings(conn); + const { meter_id, reading_rate } = await conn.one('SELECT * FROM meter_hourly_readings_unit_cagg WHERE bucket=${start_timestamp} and graphic_unit_id = ${graphic_unit};', + { start_timestamp: timestamp1, graphic_unit: unitId }); expect(meter_id).to.equal(meter.id); expect(reading_rate).to.equal(100); }); @@ -69,8 +101,8 @@ mocha.describe('Line & bar Readings', () => { await Reading.refreshMeterReadingsViews(conn); const { meter_id, reading_rate } = await conn.one( - 'SELECT * FROM daily_readings_unit WHERE time_interval && tsrange(${start_timestamp}, ${end_timestamp});', - { start_timestamp: timestamp1, end_timestamp: timestamp2 }); + 'SELECT * FROM meter_daily_readings_unit_cagg WHERE time_interval && tsrange(${start_timestamp}, ${end_timestamp}) and graphic_unit_id = ${graphic_unit};', + { start_timestamp: timestamp1, end_timestamp: timestamp2, graphic_unit: unitId }); expect(meter_id).to.equal(meter.id); expect(reading_rate).to.equal((100 + 200 + 300 + 400) / 4); }); @@ -85,7 +117,7 @@ mocha.describe('Line & bar Readings', () => { await Reading.refreshMeterReadingsViews(conn); - const rows = await conn.many('SELECT * FROM daily_readings_unit;'); + const rows = await conn.many('SELECT * FROM meter_daily_readings_unit_cagg;'); expect(rows).to.have.length(2); expect(rows[0].meter_id).to.equal(meter.id); expect(rows[1].meter_id).to.equal(meter.id); @@ -111,54 +143,55 @@ mocha.describe('Line & bar Readings', () => { await Reading.refreshMeterReadingsViews(conn); - const { meter_id, reading_rate } = await conn.one('SELECT * FROM daily_readings_unit WHERE lower(time_interval) = ${start_timestamp};', - { start_timestamp: day1Start }); + const { meter_id, reading_rate } = await conn.one('SELECT * FROM meter_daily_readings_unit_cagg WHERE bucket = ${start_timestamp} and graphic_unit_id = ${graphic_unit};', + { start_timestamp: day1Start, graphic_unit: unitId }); expect(meter_id).to.equal(meter.id); expect(reading_rate).to.be.closeTo(((50 * 1) + (100 * 2)) / (1 + 2), 0.0001); }); }); - // mocha.describe('Check intervals', () => { - // let meter, graphicUnitId, conn; - - // mocha.beforeEach(async function () { - // conn = testDB.getConnection(); - // // Insert the standard and special units and conversions. Really only need 1-2 but this is easy. - // await insertStandardUnits(conn); - // await insertStandardConversions(conn); - // await insertSpecialUnits(conn); - // await insertSpecialConversions(conn); - // await redoCik(conn); - // // Make the meter be a kWh meter. - // const meterUnitId = (await Unit.getByName('Electric_Utility', conn)).id; - // await new Meter(undefined, 'Meter', null, false, true, Meter.type.OTHER, 'CST', undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, - // undefined, undefined, undefined, undefined, undefined, meterUnitId, meterUnitId, undefined, '1 year').insert(conn); - // meter = await Meter.getByName('Meter', conn); - // // Make the graphic unit be MegaJoules. - // graphicUnitId = (await Unit.getByName('MJ', conn)).id; - // }); - - // // TODO This test no longer does what is desired because so few readings will return the 1 raw point. - // // Until we have an interface to allow setting the frequency desired this test is commented out. - // mocha.it('Correctly shrinks infinite intervals', async () => { - // // TODO: Test infinite range with bounded timestamp to ensure proper shrink - // const yearStart = moment.utc('2018-01-01'); - // const yearEnd = yearStart.clone().add(1, 'year'); - - // await Reading.insertAll([ - // new Reading(meter.id, 100, yearStart, yearEnd) - // ], conn); - // // Refresh daily reading views. - // await Reading.refreshDailyReadings(conn); - // const allReadings = await Reading.getMeterLineReadings([meter.id], graphicUnitId, null, null, conn); - // const meterReadings = allReadings[meter.id]; - // expect(meterReadings.length).to.equal(365); // 365 days in a year - // const aRow = meterReadings[0]; - // const rowWidth = moment.duration(aRow.end_timestamp.diff(aRow.start_timestamp)); - // expect(rowWidth.asDays()).to.equal(1); - // }); - // }); + mocha.describe('Check intervals', () => { + let meter, graphicUnitId, conn; + + mocha.beforeEach(async function () { + conn = testDB.getConnection(); + // Insert the standard and special units and conversions. Really only need 1-2 but this is easy. + await insertStandardUnits(conn); + await insertStandardConversions(conn); + await insertSpecialUnits(conn); + await insertSpecialConversions(conn); + await redoCikVary(conn); + // Make the meter be a kWh meter. + const meterUnitId = (await Unit.getByName('Electric_Utility', conn)).id; + // Put a shorter meter reading frequency so it will return daily rather than raw since only one reading. + graphicUnitId = (await Unit.getByName('kWh', conn)).id; + await new Meter(undefined, 'Meter', null, false, true, Meter.type.OTHER, 'CST', undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, + undefined, undefined, undefined, undefined, undefined, meterUnitId, graphicUnitId, undefined, '15 minutes').insert(conn); + meter = await Meter.getByName('Meter', conn); + // Make the graphic unit be MegaJoules. + }); + + // TODO This test no longer does what is desired because so few readings will return the 1 raw point. + // Until we have an interface to allow setting the frequency desired this test is commented out. + mocha.it('Correctly shrinks infinite intervals', async function () { + // TODO: Test infinite range with bounded timestamp to ensure proper shrink + const yearStart = moment.utc('2018-01-01'); + const yearEnd = yearStart.clone().add(1, 'year'); + + await Reading.insertAll([ + new Reading(meter.id, 100, yearStart, yearEnd) + ], conn); + // Refresh both meter aggregates once. Group aggregates are not used by this test. + await Reading.refreshMeterReadingsViews(conn); + const allReadings = await Reading.getMeterLineReadings([meter.id], graphicUnitId, null, null, conn); + const meterReadings = allReadings[meter.id]; + expect(meterReadings.length).to.equal(365); // 365 days in a year + const aRow = meterReadings[0]; + const rowWidth = moment.duration(aRow.end_timestamp.diff(aRow.start_timestamp)); + expect(rowWidth.asDays()).to.equal(1); + }); + }); // TODO modify so checks values too. @@ -170,16 +203,13 @@ mocha.describe('Line & bar Readings', () => { const endDate = '2020-03-02 00:00:00'; mocha.beforeEach(async function () { - // Extend timeout because a longer time with more data being created. The value is somewhat - // arbitrary and can be made larger if you get timeouts. - this.timeout(20000); conn = testDB.getConnection(); // Insert the standard and special units and conversions. Really only need 1-2 but this is easy. await insertStandardUnits(conn); await insertStandardConversions(conn); await insertSpecialUnits(conn); await insertSpecialConversions(conn); - await redoCik(conn); + await redoCikVary(conn); // Make the meter be a kWh meter. const meterUnitId = (await Unit.getByName('Electric_Utility', conn)).id; await new Meter(undefined, 'Meter', null, false, true, Meter.type.OTHER, 'CST', undefined, undefined, undefined, undefined, @@ -238,16 +268,13 @@ mocha.describe('Line & bar Readings', () => { const endDate = '2020-03-02 00:00:00'; mocha.beforeEach(async function () { - // Extend timeout because a longer time with more data being created. The value is somewhat - // arbitrary and can be made larger if you get timeouts. - this.timeout(20000); conn = testDB.getConnection(); // Insert the standard and special units and conversions. Really only need 1-2 but this is easy. await insertStandardUnits(conn); await insertStandardConversions(conn); await insertSpecialUnits(conn); await insertSpecialConversions(conn); - await redoCik(conn); + await redoCikVary(conn); // Make the meter be a kWh meter. const meterUnitId = (await Unit.getByName('Electric_Utility', conn)).id; // The default frequency of reading is 15-min so set to 23 for this meter. @@ -314,7 +341,7 @@ mocha.describe('Line & bar Readings', () => { await insertStandardConversions(conn); await insertSpecialUnits(conn); await insertSpecialConversions(conn); - await redoCik(conn); + await redoCikVary(conn); // Make the meter be a kWh meter. meterUnitId = (await Unit.getByName('Electric_Utility', conn)).id; await new Meter(undefined, 'Meter', null, false, true, Meter.type.OTHER, 'CST', undefined, undefined, undefined, undefined, @@ -418,6 +445,42 @@ mocha.describe('Line & bar Readings', () => { expect(meterReadings.length).to.equal(1); expect(meterReadings[0].reading_rate).to.be.closeTo(100 / (15 / 60) * conversionSlope, 0.00001); }); + + mocha.it('Selects different resolutions for multiple meters in one query', async () => { + const meterUnitId = (await Unit.getByName('Electric_Utility', conn)).id; + await new Meter(undefined, 'Slow Meter', null, false, true, Meter.type.OTHER, 'CST', undefined, undefined, undefined, undefined, + undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, + undefined, undefined, undefined, undefined, undefined, meterUnitId, meterUnitId, undefined, '1 day').insert(conn); + const slowMeter = await Meter.getByName('Slow Meter', conn); + const start = moment.utc('2018-01-01'); + const middle = start.clone().add(1, 'hour'); + const end = middle.clone().add(1, 'hour'); + + await Reading.insertAll([ + new Reading(meter.id, 100, start, middle), + new Reading(meter.id, 200, middle, end), + new Reading(slowMeter.id, 300, start, middle), + new Reading(slowMeter.id, 400, middle, end) + ], conn); + await Reading.refreshMeterReadingsViews(conn); + + const rows = await conn.func('meter_line_readings_unit', [ + [meter.id, slowMeter.id], + graphicUnitId, + start, + end, + 'auto', + 1, + 10 + ]); + const fastMeterRows = rows.filter(row => row.meter_id === meter.id); + const slowMeterRows = rows.filter(row => row.meter_id === slowMeter.id); + + expect(fastMeterRows).to.have.length(2); + expect(slowMeterRows).to.have.length(2); + expect(Number.isNaN(fastMeterRows[0].min_rate)).to.equal(false); + expect(Number.isNaN(slowMeterRows[0].min_rate)).to.equal(true); + }); }); mocha.describe('Group line readings', () => { @@ -432,7 +495,7 @@ mocha.describe('Line & bar Readings', () => { await insertStandardConversions(conn); await insertSpecialUnits(conn); await insertSpecialConversions(conn); - await redoCik(conn); + await redoCikVary(conn); // Make the meter be a kWh meter. meterUnitId = (await Unit.getByName('Electric_Utility', conn)).id; await new Meter(undefined, 'Meter1', null, false, true, Meter.type.OTHER, 'CST', undefined, undefined, undefined, undefined, @@ -534,7 +597,7 @@ mocha.describe('Line & bar Readings', () => { await insertStandardConversions(conn); await insertSpecialUnits(conn); await insertSpecialConversions(conn); - await redoCik(conn); + await redoCikVary(conn); // Make the meter be a kWh meter. meterUnitId = (await Unit.getByName('Electric_Utility', conn)).id; await new Meter(undefined, 'Meter', null, false, true, Meter.type.OTHER, 'CST', undefined, undefined, undefined, undefined, diff --git a/src/server/test/updateMetersTests.js b/src/server/test/updateMetersTests.js new file mode 100644 index 0000000000..58d38e7c6f --- /dev/null +++ b/src/server/test/updateMetersTests.js @@ -0,0 +1,36 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +const chai = require('chai'); +const mocha = require('mocha'); +const sinon = require('sinon'); +const { log } = require('../log'); +const updateMeters = require('../services/updateMeters'); + +const expect = chai.expect; + +mocha.describe('updateMeters', () => { + mocha.it('returns successful data-reader results and omits failures', async () => { + const infoStub = sinon.stub(log, 'info'); + const errorStub = sinon.stub(log, 'error'); + const meters = [{ id: 1 }, { id: 2 }, { id: 3 }]; + const dataReader = async meter => { + if (meter.id === 2) { + throw new Error('expected test failure'); + } + return { meterId: meter.id }; + }; + + try { + const results = await updateMeters(dataReader, meters, {}); + expect(results).to.deep.equal([{ meterId: 1 }, { meterId: 3 }]); + expect(errorStub.calledOnce).to.equal(true); + } finally { + infoStub.restore(); + errorStub.restore(); + } + }); +}); diff --git a/src/server/test/web/csvPipelineTest.js b/src/server/test/web/csvPipelineTest.js index 7abc7fceaf..53c644ec8e 100644 --- a/src/server/test/web/csvPipelineTest.js +++ b/src/server/test/web/csvPipelineTest.js @@ -8,7 +8,7 @@ const Reading = require('../../models/Reading'); const Point = require('../../models/Point'); const Unit = require('../../models/Unit'); const { insertStandardUnits, insertStandardConversions, insertUnits, insertConversions } = require('../../util/insertData') -const { redoCik } = require('../../services/graph/redoCik'); +const { redoCikVary } = require('../../services/graph/redoCik'); const util = require('util'); const fs = require('fs'); const csv = require('csv'); @@ -560,7 +560,7 @@ for (let fileKey in testCases) { ]; await insertConversions(conversions, conn); // Recreate the Cik entries since changed units/conversions. - await redoCik(conn); + await redoCikVary(conn); // We don't need to refresh views since we get readings directly from DB readings table. }); const numUploads = testCases[fileKey].chaiRequest.length; @@ -584,7 +584,7 @@ for (let fileKey in testCases) { Meter.type.OTHER, // type null, // timezone ) - meter.insert(conn); + await meter.insert(conn); } let inputFile = testCases[fileKey]['fileName'][index]; let inputPath = `${__dirname}/csvPipeline/${inputFile}`; diff --git a/src/server/test/web/readingsBarGroupBasic.js b/src/server/test/web/readingsBarGroupBasic.js index 51f345d718..dc890aa1af 100644 --- a/src/server/test/web/readingsBarGroupBasic.js +++ b/src/server/test/web/readingsBarGroupBasic.js @@ -17,7 +17,9 @@ const { prepareTest, unitDatakWh, conversionDatakWh } = require('../../util/readingsUtils'); -mocha.describe('readings API', () => { +mocha.describe('readings API', function () { + // Group bar setup rebuilds and refreshes the TimescaleDB meter and group + // aggregates mocha.describe('readings test, test if data returned by API is as expected', () => { mocha.describe('for bar charts', () => { mocha.describe('basic for groups', () => { diff --git a/src/server/test/web/readingsBarGroupFlow.js b/src/server/test/web/readingsBarGroupFlow.js index 55c42aa572..bd8a858128 100644 --- a/src/server/test/web/readingsBarGroupFlow.js +++ b/src/server/test/web/readingsBarGroupFlow.js @@ -17,7 +17,9 @@ const { prepareTest, METER_ID, GROUP_ID } = require('../../util/readingsUtils'); -mocha.describe('readings API', () => { +mocha.describe('readings API', function () { + // Group bar setup rebuilds and refreshes the TimescaleDB meter and group + // aggregates mocha.describe('readings test, test if data returned by API is as expected', () => { mocha.describe('for bar charts', () => { mocha.describe('for flow groups', () => { diff --git a/src/server/test/web/readingsBarGroupQuantity.js b/src/server/test/web/readingsBarGroupQuantity.js index 70fb9e6fa5..efccbc3f43 100644 --- a/src/server/test/web/readingsBarGroupQuantity.js +++ b/src/server/test/web/readingsBarGroupQuantity.js @@ -23,7 +23,9 @@ const { prepareTest, meterDatakWhGroups, groupDatakWh } = require('../../util/readingsUtils'); -mocha.describe('readings API', () => { +mocha.describe('readings API', function () { + // Group bar setup rebuilds and refreshes the TimescaleDB meter and group + // aggregates mocha.describe('readings test, test if data returned by API is as expected', () => { mocha.describe('for bar charts', () => { mocha.describe('for quantity groups', () => { diff --git a/src/server/test/web/readingsCompareMeterFlow.js b/src/server/test/web/readingsCompareMeterFlow.js index 354a7bca54..a09b89e176 100644 --- a/src/server/test/web/readingsCompareMeterFlow.js +++ b/src/server/test/web/readingsCompareMeterFlow.js @@ -11,7 +11,7 @@ const Unit = require('../../models/Unit'); const { prepareTest, expectCompareToEqualExpected, getUnitId, - METER_ID} = require('../../util/readingsUtils'); + METER_ID } = require('../../util/readingsUtils'); mocha.describe('readings API', () => { mocha.describe('readings test, test if data returned by API is as expected', () => { @@ -72,7 +72,6 @@ mocha.describe('readings API', () => { await prepareTest(unitData, conversionData, meterData); const unitId = await getUnitId('kW'); const expected = [1990.55774277443, 2057.611897078]; - const res = await chai.request(app) .get(`/api/compareReadings/meters/${METER_ID}`) .query({ @@ -84,7 +83,6 @@ mocha.describe('readings API', () => { expectCompareToEqualExpected(res, expected); }); - mocha.it('C16: 7 day shift end 2022-10-31 17:00:00 for 15 minute reading intervals and flow units & thing as thing where rate is 36', async () => { // These are the 2D arrays for units and conversions to feed into the database // For Thing units. @@ -114,7 +112,6 @@ mocha.describe('readings API', () => { note: 'special unit' } ]; - const conversionDataThing_36 = [ { // c15 @@ -126,7 +123,6 @@ mocha.describe('readings API', () => { note: 'Thing_36 → thing unit' } ]; - const meterDataThing_36 = [ { name: 'Thing_36 thing unit', @@ -141,15 +137,12 @@ mocha.describe('readings API', () => { id: METER_ID } ] - // Initialize test database with "thing" data await prepareTest(unitDataThing, conversionDataThing_36, meterDataThing_36); - // Get the unit ID since the DB could use any value const unitId = await getUnitId('thing unit'); // Expected was taken from the `curr use, prev use` column for this test case, since this is a compare readings test const expected = [199055.77427744, 205761.1897078]; - // Create a request to the API and save the response // Note: the api paths are located in app.js, but this specific one points to compareReadings.js const res = await chai.request(app).get(`/api/compareReadings/meters/${METER_ID}`) @@ -159,11 +152,9 @@ mocha.describe('readings API', () => { shift: 'P7D', graphicUnitId: unitId }); - // Check that the API reading is equal to what it is expected to equal expectCompareToEqualExpected(res, expected, METER_ID); }) - mocha.it('C17: 1 full day shift for 15 minute reading intervals and flow units & kW as kW', async () => { // Units and Conversions const unitData = [ @@ -192,7 +183,6 @@ mocha.describe('readings API', () => { note: 'special unit' } ]; - const conversionDataElectric = [ { // c4 @@ -204,7 +194,6 @@ mocha.describe('readings API', () => { note: 'Electric → kW' } ]; - const meterDataElectric = [ { name: 'Electric kW', @@ -219,11 +208,9 @@ mocha.describe('readings API', () => { id: METER_ID } ]; - await prepareTest(unitData, conversionDataElectric, meterDataElectric); const unitId = await getUnitId('kW'); const expected = [1210.55315436926, 1349.13987250313]; - const res = await chai.request(app).get(`/api/compareReadings/meters/${METER_ID}`) .query({ curr_start: '2022-10-30 00:00:00', @@ -231,13 +218,11 @@ mocha.describe('readings API', () => { shift: "P1D", graphicUnitId: unitId }); - expectCompareToEqualExpected(res, expected); - }); - mocha.it('C18: 28 day shift for 26 days for 15 minute reading intervals and flow units & kW as kW', async () =>{ - unitData = [ + mocha.it('C18: 28 day shift for 26 days for 15 minute reading intervals and flow units & kW as kW', async () => { + unitData = [ { //u4 name: 'kW', @@ -274,7 +259,6 @@ mocha.describe('readings API', () => { note: 'Electric → kW' } ]; - const meterData = [ { name: 'Electric kW', @@ -289,10 +273,9 @@ mocha.describe('readings API', () => { id: METER_ID } ]; - await prepareTest(unitData, conversionData, meterData); const unitId = await getUnitId('kW'); - const expected = [30830.9420431404, 31064.5397007187]; + const expected = [30830.9420431404, 31064.5397007187]; const res = await chai.request(app).get(`/api/compareReadings/meters/${METER_ID}`) .query({ @@ -301,11 +284,10 @@ mocha.describe('readings API', () => { shift: 'P28D', graphicUnitId: unitId }); - - expectCompareToEqualExpected(res, expected, METER_ID); - }); - - + expectCompareToEqualExpected(res, expected, METER_ID); + }); + + mocha.it('C19: 7 day shift end 2022-11-01 15:00:00 (beyond data) for 15 minute reading intervals and quantity units & kW as kW', async () => { const unitData = [ { @@ -361,7 +343,6 @@ mocha.describe('readings API', () => { await prepareTest(unitData, conversionData, meterData); const unitId = await getUnitId('kW'); const expected = [2283.20315493009, 3286.9345597083]; - const res = await chai.request(app) .get(`/api/compareReadings/meters/${METER_ID}`) .query({ @@ -370,13 +351,12 @@ mocha.describe('readings API', () => { shift: 'P7D', graphicUnitId: unitId }); - expectCompareToEqualExpected(res, expected); }); - mocha.it('C20: 28 day shift end 2022-10-31 17:12:34 (partial hour) for 15 minute reading intervals and quantity units & kW as kW', async () =>{ + mocha.it('C20: 28 day shift end 2022-10-31 17:12:34 (partial hour) for 15 minute reading intervals and quantity units & kW as kW', async () => { // Units and Conversions - unitData = [ + unitData = [ { //u4 name: 'kW', @@ -413,7 +393,6 @@ mocha.describe('readings API', () => { note: 'Electric → kW' } ]; - const meterData = [ { name: 'Electric kW', @@ -430,7 +409,7 @@ mocha.describe('readings API', () => { ]; await prepareTest(unitData, conversionData, meterData); const unitId = await getUnitId('kW'); - const expected = [27067.4812056454, 27222.4619148768]; + const expected = [27067.4812056454, 27222.4619148768]; const res = await chai.request(app).get(`/api/compareReadings/meters/${METER_ID}`) .query({ @@ -439,10 +418,8 @@ mocha.describe('readings API', () => { shift: 'P28D', graphicUnitId: unitId }); - expectCompareToEqualExpected(res, expected, METER_ID); }); - }); }); }); diff --git a/src/server/test/web/readingsCompareMeterQuantity.js b/src/server/test/web/readingsCompareMeterQuantity.js index d5652eb207..7608acf480 100644 --- a/src/server/test/web/readingsCompareMeterQuantity.js +++ b/src/server/test/web/readingsCompareMeterQuantity.js @@ -183,7 +183,7 @@ mocha.describe('readings API', () => { await prepareTest(unitData, conversionData, meterDatakWh); // Get the unit ID since the DB could use any value. const unitId = await getUnitId('MJ'); - const expected = [11232.0660730344, 12123.0051081528]; + const expected = [11232.0660730344, 12123.0051081528]; // for compare, need the unitID, currentStart, currentEnd, shift const res = await chai.request(app).get(`/api/compareReadings/meters/${METER_ID}`) .query({ @@ -200,7 +200,7 @@ mocha.describe('readings API', () => { // Use predefined unit and conversion data const unitData = unitDatakWh.concat([ //adding u3, u16 - { + { // u3 name: 'MJ', identifier: 'megaJoules', @@ -212,7 +212,7 @@ mocha.describe('readings API', () => { preferredDisplay: false, note: 'MJ' }, - { + { // u16 name: 'BTU', identifier: '', @@ -227,7 +227,7 @@ mocha.describe('readings API', () => { ]); const conversionData = conversionDatakWh.concat([ // adding c2, c3 - { + { // c2 sourceName: 'kWh', destinationName: 'MJ', @@ -236,7 +236,7 @@ mocha.describe('readings API', () => { intercept: 0, note: 'MJ → BTU' }, - { + { // c3 sourceName: 'MJ', destinationName: 'BTU', @@ -246,7 +246,7 @@ mocha.describe('readings API', () => { note: 'MJ → BTU' } ]); - + // load data into database await prepareTest(unitData, conversionData, meterDatakWh); @@ -260,31 +260,31 @@ mocha.describe('readings API', () => { curr_end: '2022-10-31 17:00:00', shift: 'P1D', graphicUnitId: unitId - }); - expectCompareToEqualExpected(res, expected); - }); + }); + expectCompareToEqualExpected(res, expected); + }); mocha.it('C11: 1 day shift end 2022-10-31 17:00:00 for 15 minute reading intervals and quantity units & kWh as BTU reverse conversion', async () => { const unitData = unitDatakWh.concat([ // adding units u3, u16 { // u3 - name: 'MJ', - identifier: 'megaJoules', - unitRepresent: Unit.unitRepresentType.QUANTITY, - secInRate: 3600, typeOfUnit: Unit.unitType.UNIT, - suffix: '', displayable: Unit.displayableType.ALL, - preferredDisplay: false, + name: 'MJ', + identifier: 'megaJoules', + unitRepresent: Unit.unitRepresentType.QUANTITY, + secInRate: 3600, typeOfUnit: Unit.unitType.UNIT, + suffix: '', displayable: Unit.displayableType.ALL, + preferredDisplay: false, note: 'MJ' }, { // u16 - name: 'BTU', identifier: '', - unitRepresent: Unit.unitRepresentType.QUANTITY, - secInRate: 3600, - typeOfUnit: Unit.unitType.UNIT, - suffix: '', displayable: Unit.displayableType.ALL, - preferredDisplay: true, + name: 'BTU', identifier: '', + unitRepresent: Unit.unitRepresentType.QUANTITY, + secInRate: 3600, + typeOfUnit: Unit.unitType.UNIT, + suffix: '', displayable: Unit.displayableType.ALL, + preferredDisplay: true, note: 'OED created standard unit' }, ]); @@ -301,14 +301,14 @@ mocha.describe('readings API', () => { }, { // c3 - sourceName: 'MJ', - destinationName: 'BTU', - bidirectional: true, - slope: 947.8, - intercept: 0, - note: 'MJ → BTU' + sourceName: 'MJ', + destinationName: 'BTU', + bidirectional: true, + slope: 947.8, + intercept: 0, + note: 'MJ → BTU' }, - + ]); // redefining the meterData as the unit is different const meterData = [ @@ -421,11 +421,11 @@ mocha.describe('readings API', () => { // for compare, need the unitID, currentStart, currentEnd, shift const res = await chai.request(app).get(`/api/compareReadings/meters/${METER_ID}`) .query({ - curr_start: '2022-10-31 00:00:00', - curr_end: '2022-10-31 17:00:00', - shift: 'P1D', - graphicUnitId: unitId - }); + curr_start: '2022-10-31 00:00:00', + curr_end: '2022-10-31 17:00:00', + shift: 'P1D', + graphicUnitId: unitId + }); expectCompareToEqualExpected(res, expected); }); diff --git a/src/server/test/web/readingsLineMeterRaw.js b/src/server/test/web/readingsLineMeterRaw.js index af0d0baab4..d41c72af92 100644 --- a/src/server/test/web/readingsLineMeterRaw.js +++ b/src/server/test/web/readingsLineMeterRaw.js @@ -3,638 +3,638 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ /* - This file tests the readings retrieval API for line chart raw meters. - See: https://github.com/OpenEnergyDashboard/DesignDocs/blob/main/testing/testing.md for information. + This file tests the readings retrieval API for line chart raw meters. + See: https://github.com/OpenEnergyDashboard/DesignDocs/blob/main/testing/testing.md for information. */ const { chai, mocha, app } = require('../common'); const Unit = require('../../models/Unit'); const { prepareTest, - parseExpectedCsv, - expectReadingToEqualExpected, - createTimeString, - getUnitId, - ETERNITY, - METER_ID, - unitDatakWh, - conversionDatakWh, - meterDatakWh } = require('../../util/readingsUtils'); + parseExpectedCsv, + expectReadingToEqualExpected, + createTimeString, + getUnitId, + ETERNITY, + METER_ID, + unitDatakWh, + conversionDatakWh, + meterDatakWh } = require('../../util/readingsUtils'); mocha.describe('readings API', () => { - mocha.describe('readings test, test if data returned by API is as expected', () => { - mocha.describe('for line charts', () => { - mocha.describe('for raw meters', () => { - // Test 15 minutes over all time for raw unit. - mocha.it('L9: should have daily points for 15 minute reading intervals and raw units with +-inf start/end time & Celsius as Celsius', async () => { - const unitData = [ - { - // u6 - name: 'C', - identifier: '', - unitRepresent: Unit.unitRepresentType.RAW, - secInRate: 3600, - typeOfUnit: Unit.unitType.UNIT, - suffix: '', - displayable: Unit.displayableType.ALL, - preferredDisplay: true, - note: 'Celsius' - }, - { - // u7 - name: 'Degrees', - identifier: '', - unitRepresent: Unit.unitRepresentType.RAW, - secInRate: 3600, - typeOfUnit: Unit.unitType.METER, - suffix: '', - displayable: Unit.displayableType.NONE, - preferredDisplay: false, - note: 'special unit' - } - ]; - const conversionData = [ - { - // c5 - sourceName: 'Degrees', - destinationName: 'C', - bidirectional: false, - slope: 1, - intercept: 0, - note: 'Degrees → C' - } - ]; - const meterData = [ - { - name: 'Degrees Celsius', - unit: 'Degrees', - defaultGraphicUnit: 'C', - displayable: true, - gps: undefined, - note: 'special meter', - file: 'test/web/readingsData/readings_ri_15_days_75.csv', - deleteFile: false, - readingFrequency: '15 minutes', - id: METER_ID - } - ]; + mocha.describe('readings test, test if data returned by API is as expected', () => { + mocha.describe('for line charts', () => { + mocha.describe('for raw meters', () => { + // Test 15 minutes over all time for raw unit. + mocha.it('L9: should have daily points for 15 minute reading intervals and raw units with +-inf start/end time & Celsius as Celsius', async () => { + const unitData = [ + { + // u6 + name: 'C', + identifier: '', + unitRepresent: Unit.unitRepresentType.RAW, + secInRate: 3600, + typeOfUnit: Unit.unitType.UNIT, + suffix: '', + displayable: Unit.displayableType.ALL, + preferredDisplay: true, + note: 'Celsius' + }, + { + // u7 + name: 'Degrees', + identifier: '', + unitRepresent: Unit.unitRepresentType.RAW, + secInRate: 3600, + typeOfUnit: Unit.unitType.METER, + suffix: '', + displayable: Unit.displayableType.NONE, + preferredDisplay: false, + note: 'special unit' + } + ]; + const conversionData = [ + { + // c5 + sourceName: 'Degrees', + destinationName: 'C', + bidirectional: false, + slope: 1, + intercept: 0, + note: 'Degrees → C' + } + ]; + const meterData = [ + { + name: 'Degrees Celsius', + unit: 'Degrees', + defaultGraphicUnit: 'C', + displayable: true, + gps: undefined, + note: 'special meter', + file: 'test/web/readingsData/readings_ri_15_days_75.csv', + deleteFile: false, + readingFrequency: '15 minutes', + id: METER_ID + } + ]; - await prepareTest(unitData, conversionData, meterData); - // Get the unit ID since the DB could use any value. - const unitId = await getUnitId('C'); - // Reuse same file as flow since value should be the same values. - const expected = await parseExpectedCsv('src/server/test/web/readingsData/expected_line_ri_15_mu_kW_gu_kW_st_-inf_et_inf.csv'); + await prepareTest(unitData, conversionData, meterData); + // Get the unit ID since the DB could use any value. + const unitId = await getUnitId('C'); + // Reuse same file as flow since value should be the same values. + const expected = await parseExpectedCsv('src/server/test/web/readingsData/expected_line_ri_15_mu_kW_gu_kW_st_-inf_et_inf.csv'); - const res = await chai.request(app).get(`/api/unitReadings/line/meters/${METER_ID}`) - .query({ timeInterval: ETERNITY.toString(), graphicUnitId: unitId }); - expectReadingToEqualExpected(res, expected) - }); - mocha.it('L14: should have daily points for 15 minute reading intervals and raw units with +-inf start/end time & C as F with intercept', async () => { - const unitData = [ - { - // u6 - name: 'C', - identifier: '', - unitRepresent: Unit.unitRepresentType.RAW, - secInRate: 3600, - typeOfUnit: Unit.unitType.UNIT, - suffix: '', - displayable: Unit.displayableType.ALL, - preferredDisplay: true, - note: 'Celsius' - }, - { - // u7 - name: 'Degrees', - identifier: '', - unitRepresent: Unit.unitRepresentType.RAW, - secInRate: 3600, - typeOfUnit: Unit.unitType.METER, - suffix: '', - displayable: Unit.displayableType.NONE, - preferredDisplay: false, - note: 'special unit' - }, - { - // u8 - name: 'F', - identifier: '', - unitRepresent: Unit.unitRepresentType.RAW, - secInRate: 3600, - typeOfUnit: Unit.unitType.UNIT, - suffix: '', - displayable: Unit.displayableType.ALL, - preferredDisplay: false, - note: 'OED created standard unit' - } - ]; - const conversionData = [ - { - // c5 - sourceName: 'Degrees', - destinationName: 'C', - bidirectional: false, - slope: 1, - intercept: 0, - note: 'Degrees → C' - }, - { - // c7 - sourceName: 'C', - destinationName: 'F', - bidirectional: true, - slope: 1.8, - intercept: 32, - note: 'Celsius → Fahrenheit' - } - ]; - const meterData = [ - { - name: 'Degrees F', - unit: 'Degrees', - defaultGraphicUnit: 'F', - displayable: true, - gps: undefined, - note: 'special meter', - file: 'test/web/readingsData/readings_ri_15_days_75.csv', - deleteFile: false, - readingFrequency: '15 minutes', - id: METER_ID - } - ]; + const res = await chai.request(app).get(`/api/unitReadings/line/meters/${METER_ID}`) + .query({ timeInterval: ETERNITY.toString(), graphicUnitId: unitId }); + expectReadingToEqualExpected(res, expected) + }); + mocha.it('L14: should have daily points for 15 minute reading intervals and raw units with +-inf start/end time & C as F with intercept', async () => { + const unitData = [ + { + // u6 + name: 'C', + identifier: '', + unitRepresent: Unit.unitRepresentType.RAW, + secInRate: 3600, + typeOfUnit: Unit.unitType.UNIT, + suffix: '', + displayable: Unit.displayableType.ALL, + preferredDisplay: true, + note: 'Celsius' + }, + { + // u7 + name: 'Degrees', + identifier: '', + unitRepresent: Unit.unitRepresentType.RAW, + secInRate: 3600, + typeOfUnit: Unit.unitType.METER, + suffix: '', + displayable: Unit.displayableType.NONE, + preferredDisplay: false, + note: 'special unit' + }, + { + // u8 + name: 'F', + identifier: '', + unitRepresent: Unit.unitRepresentType.RAW, + secInRate: 3600, + typeOfUnit: Unit.unitType.UNIT, + suffix: '', + displayable: Unit.displayableType.ALL, + preferredDisplay: false, + note: 'OED created standard unit' + } + ]; + const conversionData = [ + { + // c5 + sourceName: 'Degrees', + destinationName: 'C', + bidirectional: false, + slope: 1, + intercept: 0, + note: 'Degrees → C' + }, + { + // c7 + sourceName: 'C', + destinationName: 'F', + bidirectional: true, + slope: 1.8, + intercept: 32, + note: 'Celsius → Fahrenheit' + } + ]; + const meterData = [ + { + name: 'Degrees F', + unit: 'Degrees', + defaultGraphicUnit: 'F', + displayable: true, + gps: undefined, + note: 'special meter', + file: 'test/web/readingsData/readings_ri_15_days_75.csv', + deleteFile: false, + readingFrequency: '15 minutes', + id: METER_ID + } + ]; - await prepareTest(unitData, conversionData, meterData); - // Get the unit ID since the DB could use any value. - const unitId = await getUnitId('F'); - // Reuse same file as flow since value should be the same values. - const expected = await parseExpectedCsv('src/server/test/web/readingsData/expected_line_ri_15_mu_C_gu_F_st_-inf_et_inf.csv'); + await prepareTest(unitData, conversionData, meterData); + // Get the unit ID since the DB could use any value. + const unitId = await getUnitId('F'); + // Reuse same file as flow since value should be the same values. + const expected = await parseExpectedCsv('src/server/test/web/readingsData/expected_line_ri_15_mu_C_gu_F_st_-inf_et_inf.csv'); - const res = await chai.request(app).get(`/api/unitReadings/line/meters/${METER_ID}`) - .query({ timeInterval: ETERNITY.toString(), graphicUnitId: unitId }); - expectReadingToEqualExpected(res, expected) - }); - mocha.it('L15: should have daily points for 15 minute reading intervals and raw units with +-inf start/end time & C as F with intercept reverse conversion', async () => { - const unitData = [ - { - // u6 - name: 'C', - identifier: '', - unitRepresent: Unit.unitRepresentType.RAW, - secInRate: 3600, - typeOfUnit: Unit.unitType.UNIT, - suffix: '', - displayable: Unit.displayableType.ALL, - preferredDisplay: true, - note: 'Celsius' - }, - { - // u7 - name: 'Degrees', - identifier: '', - unitRepresent: Unit.unitRepresentType.RAW, - secInRate: 3600, - typeOfUnit: Unit.unitType.METER, - suffix: '', - displayable: Unit.displayableType.NONE, - preferredDisplay: false, - note: 'special unit' - }, - { - // u8 - name: 'F', - identifier: '', - unitRepresent: Unit.unitRepresentType.RAW, - secInRate: 3600, - typeOfUnit: Unit.unitType.UNIT, - suffix: '', - displayable: Unit.displayableType.ALL, - preferredDisplay: false, - note: 'OED created standard unit' - } - ]; - const conversionData = [ - { - // c5 - sourceName: 'Degrees', - destinationName: 'C', - bidirectional: false, - slope: 1, - intercept: 0, - note: 'Degrees → C' - }, - { - // c8 - sourceName: 'F', - destinationName: 'C', - bidirectional: true, - slope: 1 / 1.8, - intercept: -32 / 1.8, - note: 'Fahrenheit → Celsius' - } - ]; - const meterData = [ - { - name: 'Degrees F', - unit: 'Degrees', - defaultGraphicUnit: 'F', - displayable: true, - gps: undefined, - note: 'special meter', - file: 'test/web/readingsData/readings_ri_15_days_75.csv', - deleteFile: false, - readingFrequency: '15 minutes', - id: METER_ID - } - ]; + const res = await chai.request(app).get(`/api/unitReadings/line/meters/${METER_ID}`) + .query({ timeInterval: ETERNITY.toString(), graphicUnitId: unitId }); + expectReadingToEqualExpected(res, expected) + }); + mocha.it('L15: should have daily points for 15 minute reading intervals and raw units with +-inf start/end time & C as F with intercept reverse conversion', async () => { + const unitData = [ + { + // u6 + name: 'C', + identifier: '', + unitRepresent: Unit.unitRepresentType.RAW, + secInRate: 3600, + typeOfUnit: Unit.unitType.UNIT, + suffix: '', + displayable: Unit.displayableType.ALL, + preferredDisplay: true, + note: 'Celsius' + }, + { + // u7 + name: 'Degrees', + identifier: '', + unitRepresent: Unit.unitRepresentType.RAW, + secInRate: 3600, + typeOfUnit: Unit.unitType.METER, + suffix: '', + displayable: Unit.displayableType.NONE, + preferredDisplay: false, + note: 'special unit' + }, + { + // u8 + name: 'F', + identifier: '', + unitRepresent: Unit.unitRepresentType.RAW, + secInRate: 3600, + typeOfUnit: Unit.unitType.UNIT, + suffix: '', + displayable: Unit.displayableType.ALL, + preferredDisplay: false, + note: 'OED created standard unit' + } + ]; + const conversionData = [ + { + // c5 + sourceName: 'Degrees', + destinationName: 'C', + bidirectional: false, + slope: 1, + intercept: 0, + note: 'Degrees → C' + }, + { + // c8 + sourceName: 'F', + destinationName: 'C', + bidirectional: true, + slope: 1 / 1.8, + intercept: -32 / 1.8, + note: 'Fahrenheit → Celsius' + } + ]; + const meterData = [ + { + name: 'Degrees F', + unit: 'Degrees', + defaultGraphicUnit: 'F', + displayable: true, + gps: undefined, + note: 'special meter', + file: 'test/web/readingsData/readings_ri_15_days_75.csv', + deleteFile: false, + readingFrequency: '15 minutes', + id: METER_ID + } + ]; - await prepareTest(unitData, conversionData, meterData); - // Get the unit ID since the DB could use any value. - const unitId = await getUnitId('F'); - // Reuse same file as flow since value should be the same values. - const expected = await parseExpectedCsv('src/server/test/web/readingsData/expected_line_ri_15_mu_C_gu_F_st_-inf_et_inf.csv'); + await prepareTest(unitData, conversionData, meterData); + // Get the unit ID since the DB could use any value. + const unitId = await getUnitId('F'); + // Reuse same file as flow since value should be the same values. + const expected = await parseExpectedCsv('src/server/test/web/readingsData/expected_line_ri_15_mu_C_gu_F_st_-inf_et_inf.csv'); - const res = await chai.request(app).get(`/api/unitReadings/line/meters/${METER_ID}`) - .query({ timeInterval: ETERNITY.toString(), graphicUnitId: unitId }); - expectReadingToEqualExpected(res, expected) - }); - mocha.it('L16: should have daily points for 15 minute reading intervals and raw units with +-inf start/end time & C as Widget with intercept & chained', async () => { - const unitData = [ - { - // u6 - name: 'C', - identifier: '', - unitRepresent: Unit.unitRepresentType.RAW, - secInRate: 3600, - typeOfUnit: Unit.unitType.UNIT, - suffix: '', - displayable: Unit.displayableType.ALL, - preferredDisplay: true, - note: 'Celsius' - }, - { - // u7 - name: 'Degrees', - identifier: '', - unitRepresent: Unit.unitRepresentType.RAW, - secInRate: 3600, - typeOfUnit: Unit.unitType.METER, - suffix: '', - displayable: Unit.displayableType.NONE, - preferredDisplay: false, - note: 'special unit' - }, - { - // u8 - name: 'F', - identifier: '', - unitRepresent: Unit.unitRepresentType.RAW, - secInRate: 3600, - typeOfUnit: Unit.unitType.UNIT, - suffix: '', - displayable: Unit.displayableType.ALL, - preferredDisplay: false, - note: 'OED created standard unit' - }, - { - // u9 - name: 'Widget', - identifier: '', - unitRepresent: Unit.unitRepresentType.RAW, - secInRate: 3600, - typeOfUnit: Unit.unitType.UNIT, - suffix: '', - displayable: Unit.displayableType.ALL, - preferredDisplay: false, - note: 'fake unit' - } - ]; - const conversionData = [ - { - // c5 - sourceName: 'Degrees', - destinationName: 'C', - bidirectional: false, - slope: 1, - intercept: 0, - note: 'Degrees → C' - }, - { - // c7 - sourceName: 'C', - destinationName: 'F', - bidirectional: true, - slope: 1.8, - intercept: 32, - note: 'Celsius → Fahrenheit' - }, - { - // c9 - sourceName: 'F', - destinationName: 'Widget', - bidirectional: true, - slope: 5, - intercept: 3, - note: 'Fahrenheit → Widget' - } - ]; - const meterData = [ - { - name: 'Degrees Widget', - unit: 'Degrees', - defaultGraphicUnit: 'Widget', - displayable: true, - gps: undefined, - note: 'special meter', - file: 'test/web/readingsData/readings_ri_15_days_75.csv', - deleteFile: false, - readingFrequency: '15 minutes', - id: METER_ID - } - ]; + const res = await chai.request(app).get(`/api/unitReadings/line/meters/${METER_ID}`) + .query({ timeInterval: ETERNITY.toString(), graphicUnitId: unitId }); + expectReadingToEqualExpected(res, expected) + }); + mocha.it('L16: should have daily points for 15 minute reading intervals and raw units with +-inf start/end time & C as Widget with intercept & chained', async () => { + const unitData = [ + { + // u6 + name: 'C', + identifier: '', + unitRepresent: Unit.unitRepresentType.RAW, + secInRate: 3600, + typeOfUnit: Unit.unitType.UNIT, + suffix: '', + displayable: Unit.displayableType.ALL, + preferredDisplay: true, + note: 'Celsius' + }, + { + // u7 + name: 'Degrees', + identifier: '', + unitRepresent: Unit.unitRepresentType.RAW, + secInRate: 3600, + typeOfUnit: Unit.unitType.METER, + suffix: '', + displayable: Unit.displayableType.NONE, + preferredDisplay: false, + note: 'special unit' + }, + { + // u8 + name: 'F', + identifier: '', + unitRepresent: Unit.unitRepresentType.RAW, + secInRate: 3600, + typeOfUnit: Unit.unitType.UNIT, + suffix: '', + displayable: Unit.displayableType.ALL, + preferredDisplay: false, + note: 'OED created standard unit' + }, + { + // u9 + name: 'Widget', + identifier: '', + unitRepresent: Unit.unitRepresentType.RAW, + secInRate: 3600, + typeOfUnit: Unit.unitType.UNIT, + suffix: '', + displayable: Unit.displayableType.ALL, + preferredDisplay: false, + note: 'fake unit' + } + ]; + const conversionData = [ + { + // c5 + sourceName: 'Degrees', + destinationName: 'C', + bidirectional: false, + slope: 1, + intercept: 0, + note: 'Degrees → C' + }, + { + // c7 + sourceName: 'C', + destinationName: 'F', + bidirectional: true, + slope: 1.8, + intercept: 32, + note: 'Celsius → Fahrenheit' + }, + { + // c9 + sourceName: 'F', + destinationName: 'Widget', + bidirectional: true, + slope: 5, + intercept: 3, + note: 'Fahrenheit → Widget' + } + ]; + const meterData = [ + { + name: 'Degrees Widget', + unit: 'Degrees', + defaultGraphicUnit: 'Widget', + displayable: true, + gps: undefined, + note: 'special meter', + file: 'test/web/readingsData/readings_ri_15_days_75.csv', + deleteFile: false, + readingFrequency: '15 minutes', + id: METER_ID + } + ]; - await prepareTest(unitData, conversionData, meterData); - // Get the unit ID since the DB could use any value. - const unitId = await getUnitId('Widget'); - // Reuse same file as flow since value should be the same values. - const expected = await parseExpectedCsv('src/server/test/web/readingsData/expected_line_ri_15_mu_C_gu_Widget_st_-inf_et_inf.csv'); + await prepareTest(unitData, conversionData, meterData); + // Get the unit ID since the DB could use any value. + const unitId = await getUnitId('Widget'); + // Reuse same file as flow since value should be the same values. + const expected = await parseExpectedCsv('src/server/test/web/readingsData/expected_line_ri_15_mu_C_gu_Widget_st_-inf_et_inf.csv'); - const res = await chai.request(app).get(`/api/unitReadings/line/meters/${METER_ID}`) - .query({ timeInterval: ETERNITY.toString(), graphicUnitId: unitId }); - expectReadingToEqualExpected(res, expected) - }); - mocha.it('L17: should have daily points for 15 minute reading intervals and raw units with +-inf start/end time & C as Widget with intercept & chained & reverse conversions', async () => { - const unitData = [ - { - // u6 - name: 'C', - identifier: '', - unitRepresent: Unit.unitRepresentType.RAW, - secInRate: 3600, - typeOfUnit: Unit.unitType.UNIT, - suffix: '', - displayable: Unit.displayableType.ALL, - preferredDisplay: true, - note: 'Celsius' - }, - { - // u7 - name: 'Degrees', - identifier: '', - unitRepresent: Unit.unitRepresentType.RAW, - secInRate: 3600, - typeOfUnit: Unit.unitType.METER, - suffix: '', - displayable: Unit.displayableType.NONE, - preferredDisplay: false, - note: 'special unit' - }, - { - // u8 - name: 'F', - identifier: '', - unitRepresent: Unit.unitRepresentType.RAW, - secInRate: 3600, - typeOfUnit: Unit.unitType.UNIT, - suffix: '', - displayable: Unit.displayableType.ALL, - preferredDisplay: false, - note: 'OED created standard unit' - }, - { - // u9 - name: 'Widget', - identifier: '', - unitRepresent: Unit.unitRepresentType.RAW, - secInRate: 3600, - typeOfUnit: Unit.unitType.UNIT, - suffix: '', - displayable: Unit.displayableType.ALL, - preferredDisplay: false, - note: 'fake unit' - } - ]; - const conversionData = [ - { - // c5 - sourceName: 'Degrees', - destinationName: 'C', - bidirectional: false, - slope: 1, - intercept: 0, - note: 'Degrees → C' - }, - { - // c8 - sourceName: 'F', - destinationName: 'C', - bidirectional: true, - slope: 1 / 1.8, - intercept: -32 / 1.8, - note: 'Fahrenheit → Celsius' - }, - { - // c10 - sourceName: 'Widget', - destinationName: 'F', - bidirectional: true, - slope: 0.2, - intercept: -3 / 5, - note: 'Fahrenheit → Widget' - } - ]; - const meterData = [ - { - name: 'Degrees Widget', - unit: 'Degrees', - defaultGraphicUnit: 'Widget', - displayable: true, - gps: undefined, - note: 'special meter', - file: 'test/web/readingsData/readings_ri_15_days_75.csv', - deleteFile: false, - readingFrequency: '15 minutes', - id: METER_ID - } - ]; + const res = await chai.request(app).get(`/api/unitReadings/line/meters/${METER_ID}`) + .query({ timeInterval: ETERNITY.toString(), graphicUnitId: unitId }); + expectReadingToEqualExpected(res, expected) + }); + mocha.it('L17: should have daily points for 15 minute reading intervals and raw units with +-inf start/end time & C as Widget with intercept & chained & reverse conversions', async () => { + const unitData = [ + { + // u6 + name: 'C', + identifier: '', + unitRepresent: Unit.unitRepresentType.RAW, + secInRate: 3600, + typeOfUnit: Unit.unitType.UNIT, + suffix: '', + displayable: Unit.displayableType.ALL, + preferredDisplay: true, + note: 'Celsius' + }, + { + // u7 + name: 'Degrees', + identifier: '', + unitRepresent: Unit.unitRepresentType.RAW, + secInRate: 3600, + typeOfUnit: Unit.unitType.METER, + suffix: '', + displayable: Unit.displayableType.NONE, + preferredDisplay: false, + note: 'special unit' + }, + { + // u8 + name: 'F', + identifier: '', + unitRepresent: Unit.unitRepresentType.RAW, + secInRate: 3600, + typeOfUnit: Unit.unitType.UNIT, + suffix: '', + displayable: Unit.displayableType.ALL, + preferredDisplay: false, + note: 'OED created standard unit' + }, + { + // u9 + name: 'Widget', + identifier: '', + unitRepresent: Unit.unitRepresentType.RAW, + secInRate: 3600, + typeOfUnit: Unit.unitType.UNIT, + suffix: '', + displayable: Unit.displayableType.ALL, + preferredDisplay: false, + note: 'fake unit' + } + ]; + const conversionData = [ + { + // c5 + sourceName: 'Degrees', + destinationName: 'C', + bidirectional: false, + slope: 1, + intercept: 0, + note: 'Degrees → C' + }, + { + // c8 + sourceName: 'F', + destinationName: 'C', + bidirectional: true, + slope: 1 / 1.8, + intercept: -32 / 1.8, + note: 'Fahrenheit → Celsius' + }, + { + // c10 + sourceName: 'Widget', + destinationName: 'F', + bidirectional: true, + slope: 0.2, + intercept: -3 / 5, + note: 'Fahrenheit → Widget' + } + ]; + const meterData = [ + { + name: 'Degrees Widget', + unit: 'Degrees', + defaultGraphicUnit: 'Widget', + displayable: true, + gps: undefined, + note: 'special meter', + file: 'test/web/readingsData/readings_ri_15_days_75.csv', + deleteFile: false, + readingFrequency: '15 minutes', + id: METER_ID + } + ]; - await prepareTest(unitData, conversionData, meterData); - // Get the unit ID since the DB could use any value. - const unitId = await getUnitId('Widget'); - // Reuse same file as flow since value should be the same values. - const expected = await parseExpectedCsv('src/server/test/web/readingsData/expected_line_ri_15_mu_C_gu_Widget_st_-inf_et_inf.csv'); + await prepareTest(unitData, conversionData, meterData); + // Get the unit ID since the DB could use any value. + const unitId = await getUnitId('Widget'); + // Reuse same file as flow since value should be the same values. + const expected = await parseExpectedCsv('src/server/test/web/readingsData/expected_line_ri_15_mu_C_gu_Widget_st_-inf_et_inf.csv'); - const res = await chai.request(app).get(`/api/unitReadings/line/meters/${METER_ID}`) - .query({ timeInterval: ETERNITY.toString(), graphicUnitId: unitId }); - expectReadingToEqualExpected(res, expected) - }); - mocha.it('L22: should have hourly points for middle readings of 15 minute for a 60 day period and raw units & C as F with intercept', async () => { - const unitData = [ - { - // u6 - name: 'C', - identifier: '', - unitRepresent: Unit.unitRepresentType.RAW, - secInRate: 3600, - typeOfUnit: Unit.unitType.UNIT, - suffix: '', - displayable: Unit.displayableType.ALL, - preferredDisplay: true, - note: 'Celsius' - }, - { - // u7 - name: 'Degrees', - identifier: '', - unitRepresent: Unit.unitRepresentType.RAW, - secInRate: 3600, - typeOfUnit: Unit.unitType.METER, - suffix: '', - displayable: Unit.displayableType.NONE, - preferredDisplay: false, - note: 'special unit' - }, - { - // u8 - name: 'F', - identifier: '', - unitRepresent: Unit.unitRepresentType.RAW, - secInRate: 3600, - typeOfUnit: Unit.unitType.UNIT, - suffix: '', - displayable: Unit.displayableType.ALL, - preferredDisplay: false, - note: 'OED created standard unit' - } - ]; - const conversionData = [ - { - // c5 - sourceName: 'Degrees', - destinationName: 'C', - bidirectional: false, - slope: 1, - intercept: 0, - note: 'Degrees → C' - }, - { - // c7 - sourceName: 'C', - destinationName: 'F', - bidirectional: true, - slope: 1.8, - intercept: 32, - note: 'Celsius → Fahrenheit' - } - ]; - const meterData = [ - { - name: 'Degrees F', - unit: 'Degrees', - defaultGraphicUnit: 'F', - displayable: true, - gps: undefined, - note: 'special meter', - file: 'test/web/readingsData/readings_ri_15_days_75.csv', - deleteFile: false, - readingFrequency: '15 minutes', - id: METER_ID - } - ]; + const res = await chai.request(app).get(`/api/unitReadings/line/meters/${METER_ID}`) + .query({ timeInterval: ETERNITY.toString(), graphicUnitId: unitId }); + expectReadingToEqualExpected(res, expected) + }); + mocha.it('L22: should have hourly points for middle readings of 15 minute for a 60 day period and raw units & C as F with intercept', async () => { + const unitData = [ + { + // u6 + name: 'C', + identifier: '', + unitRepresent: Unit.unitRepresentType.RAW, + secInRate: 3600, + typeOfUnit: Unit.unitType.UNIT, + suffix: '', + displayable: Unit.displayableType.ALL, + preferredDisplay: true, + note: 'Celsius' + }, + { + // u7 + name: 'Degrees', + identifier: '', + unitRepresent: Unit.unitRepresentType.RAW, + secInRate: 3600, + typeOfUnit: Unit.unitType.METER, + suffix: '', + displayable: Unit.displayableType.NONE, + preferredDisplay: false, + note: 'special unit' + }, + { + // u8 + name: 'F', + identifier: '', + unitRepresent: Unit.unitRepresentType.RAW, + secInRate: 3600, + typeOfUnit: Unit.unitType.UNIT, + suffix: '', + displayable: Unit.displayableType.ALL, + preferredDisplay: false, + note: 'OED created standard unit' + } + ]; + const conversionData = [ + { + // c5 + sourceName: 'Degrees', + destinationName: 'C', + bidirectional: false, + slope: 1, + intercept: 0, + note: 'Degrees → C' + }, + { + // c7 + sourceName: 'C', + destinationName: 'F', + bidirectional: true, + slope: 1.8, + intercept: 32, + note: 'Celsius → Fahrenheit' + } + ]; + const meterData = [ + { + name: 'Degrees F', + unit: 'Degrees', + defaultGraphicUnit: 'F', + displayable: true, + gps: undefined, + note: 'special meter', + file: 'test/web/readingsData/readings_ri_15_days_75.csv', + deleteFile: false, + readingFrequency: '15 minutes', + id: METER_ID + } + ]; - await prepareTest(unitData, conversionData, meterData); - // Get the unit ID since the DB could use any value. - const unitId = await getUnitId('F'); - // Reuse same file as flow since value should be the same values. - const expected = await parseExpectedCsv('src/server/test/web/readingsData/expected_line_ri_15_mu_C_gu_F_st_2022-08-25%00#00#00_et_2022-10-24%00#00#00.csv'); + await prepareTest(unitData, conversionData, meterData); + // Get the unit ID since the DB could use any value. + const unitId = await getUnitId('F'); + // Reuse same file as flow since value should be the same values. + const expected = await parseExpectedCsv('src/server/test/web/readingsData/expected_line_ri_15_mu_C_gu_F_st_2022-08-25%00#00#00_et_2022-10-24%00#00#00.csv'); - const res = await chai.request(app).get(`/api/unitReadings/line/meters/${METER_ID}`) - .query({ timeInterval: createTimeString('2022-08-25', '00:00:00', '2022-10-24', '00:00:00'), graphicUnitId: unitId }); - expectReadingToEqualExpected(res, expected) - }); - mocha.it('L24: should have raw points for middle readings of 15 minute for a 14 day period and raw units & C as F with intercept', async () => { - const unitData = [ - { - // u6 - name: 'C', - identifier: '', - unitRepresent: Unit.unitRepresentType.RAW, - secInRate: 3600, - typeOfUnit: Unit.unitType.UNIT, - suffix: '', - displayable: Unit.displayableType.ALL, - preferredDisplay: true, - note: 'Celsius' - }, - { - // u7 - name: 'Degrees', - identifier: '', - unitRepresent: Unit.unitRepresentType.RAW, - secInRate: 3600, - typeOfUnit: Unit.unitType.METER, - suffix: '', - displayable: Unit.displayableType.NONE, - preferredDisplay: false, - note: 'special unit' - }, - { - // u8 - name: 'F', - identifier: '', - unitRepresent: Unit.unitRepresentType.RAW, - secInRate: 3600, - typeOfUnit: Unit.unitType.UNIT, - suffix: '', - displayable: Unit.displayableType.ALL, - preferredDisplay: false, - note: 'OED created standard unit' - } - ]; - const conversionData = [ - { - // c5 - sourceName: 'Degrees', - destinationName: 'C', - bidirectional: false, - slope: 1, - intercept: 0, - note: 'Degrees → C' - }, - { - // c7 - sourceName: 'C', - destinationName: 'F', - bidirectional: true, - slope: 1.8, - intercept: 32, - note: 'Celsius → Fahrenheit' - } - ]; - const meterData = [ - { - name: 'Degrees F', - unit: 'Degrees', - defaultGraphicUnit: 'F', - displayable: true, - gps: undefined, - note: 'special meter', - file: 'test/web/readingsData/readings_ri_15_days_75.csv', - deleteFile: false, - readingFrequency: '15 minutes', - id: METER_ID - } - ]; + const res = await chai.request(app).get(`/api/unitReadings/line/meters/${METER_ID}`) + .query({ timeInterval: createTimeString('2022-08-25', '00:00:00', '2022-10-24', '00:00:00'), graphicUnitId: unitId }); + expectReadingToEqualExpected(res, expected) + }); + mocha.it('L24: should have raw points for middle readings of 15 minute for a 14 day period and raw units & C as F with intercept', async () => { + const unitData = [ + { + // u6 + name: 'C', + identifier: '', + unitRepresent: Unit.unitRepresentType.RAW, + secInRate: 3600, + typeOfUnit: Unit.unitType.UNIT, + suffix: '', + displayable: Unit.displayableType.ALL, + preferredDisplay: true, + note: 'Celsius' + }, + { + // u7 + name: 'Degrees', + identifier: '', + unitRepresent: Unit.unitRepresentType.RAW, + secInRate: 3600, + typeOfUnit: Unit.unitType.METER, + suffix: '', + displayable: Unit.displayableType.NONE, + preferredDisplay: false, + note: 'special unit' + }, + { + // u8 + name: 'F', + identifier: '', + unitRepresent: Unit.unitRepresentType.RAW, + secInRate: 3600, + typeOfUnit: Unit.unitType.UNIT, + suffix: '', + displayable: Unit.displayableType.ALL, + preferredDisplay: false, + note: 'OED created standard unit' + } + ]; + const conversionData = [ + { + // c5 + sourceName: 'Degrees', + destinationName: 'C', + bidirectional: false, + slope: 1, + intercept: 0, + note: 'Degrees → C' + }, + { + // c7 + sourceName: 'C', + destinationName: 'F', + bidirectional: true, + slope: 1.8, + intercept: 32, + note: 'Celsius → Fahrenheit' + } + ]; + const meterData = [ + { + name: 'Degrees F', + unit: 'Degrees', + defaultGraphicUnit: 'F', + displayable: true, + gps: undefined, + note: 'special meter', + file: 'test/web/readingsData/readings_ri_15_days_75.csv', + deleteFile: false, + readingFrequency: '15 minutes', + id: METER_ID + } + ]; - await prepareTest(unitData, conversionData, meterData); - // Get the unit ID since the DB could use any value. - const unitId = await getUnitId('F'); - // Reuse same file as flow since value should be the same values. - const expected = await parseExpectedCsv('src/server/test/web/readingsData/expected_line_ri_15_mu_C_gu_F_st_2022-09-21%00#00#00_et_2022-10-05%00#00#00.csv'); + await prepareTest(unitData, conversionData, meterData); + // Get the unit ID since the DB could use any value. + const unitId = await getUnitId('F'); + // Reuse same file as flow since value should be the same values. + const expected = await parseExpectedCsv('src/server/test/web/readingsData/expected_line_ri_15_mu_C_gu_F_st_2022-09-21%00#00#00_et_2022-10-05%00#00#00.csv'); - const res = await chai.request(app).get(`/api/unitReadings/line/meters/${METER_ID}`) - .query({ timeInterval: createTimeString('2022-09-21', '00:00:00', '2022-10-05', '00:00:00'), graphicUnitId: unitId }); - expectReadingToEqualExpected(res, expected) - }); - }); - }); - }); + const res = await chai.request(app).get(`/api/unitReadings/line/meters/${METER_ID}`) + .query({ timeInterval: createTimeString('2022-09-21', '00:00:00', '2022-10-05', '00:00:00'), graphicUnitId: unitId }); + expectReadingToEqualExpected(res, expected) + }); + }); + }); + }); }); diff --git a/src/server/util/developer.js b/src/server/util/developer.js index 9b80c54170..2f3dc54a31 100644 --- a/src/server/util/developer.js +++ b/src/server/util/developer.js @@ -5,7 +5,7 @@ const database = require('../models/database'); const sqlFile = database.sqlFile; const { getConnection } = require('../db'); -const { refreshAllReadingViews } = require('../services/refreshAllReadingViews'); +const refreshAllReadingViews = require('../services/refreshAllReadingViews'); // These functions are designed for OED developers. diff --git a/src/server/util/insertData.js b/src/server/util/insertData.js index 72145fad9c..ed32e4efb3 100644 --- a/src/server/util/insertData.js +++ b/src/server/util/insertData.js @@ -204,9 +204,9 @@ async function insertConversions(conversionsToInsert, conn) { const destinationId = (await Unit.getByName(conversionData.destinationName, conn)).id; if (await Conversion.getBySourceDestination(sourceId, destinationId, conn) === null) { await new Conversion( - sourceId, - destinationId, - conversionData.bidirectional, + sourceId, + destinationId, + conversionData.bidirectional, conversionData.note ).insert( null, diff --git a/src/server/util/readingsUtils.js b/src/server/util/readingsUtils.js index d733d65161..eff4acd2f9 100644 --- a/src/server/util/readingsUtils.js +++ b/src/server/util/readingsUtils.js @@ -6,12 +6,11 @@ const { expect, testDB } = require('../test/common'); const { TimeInterval } = require('../../common/TimeInterval'); const { insertUnits, insertConversions, insertMeters, insertGroups } = require('./insertData'); const Unit = require('../models/Unit'); -const { redoCik } = require('../services/graph/redoCik'); -const { refreshAllReadingViews } = require('../services/refreshAllReadingViews'); +const { redoCikVary } = require('../services/graph/redoCik'); +const refreshAllReadingViews = require('../services/refreshAllReadingViews'); const readCsv = require('../services/pipeline-in-progress/readCsv'); const moment = require('moment'); const Group = require('../models/Group'); -const Reading = require('../models/Reading'); const ETERNITY = TimeInterval.unbounded(); // Readings should be accurate to many decimal places, but allow some wiggle room for database and javascript conversions @@ -21,36 +20,35 @@ const METER_ID = 100; const GROUP_ID = 200; // Some common HTTP status response codes const HTTP_CODE = { - OK: 200, - FOUND: 302, - BAD_REQUEST: 400, - NOT_FOUND: 404 + OK: 200, + FOUND: 302, + BAD_REQUEST: 400, + NOT_FOUND: 404 }; /** * Initialize test database, call the functions to insert data into the database, - * then redoCik and refresh views to ensure everything works. + * then redoCikVary and refresh views to ensure everything works. * @param {array} unitData parameters for insertUnits * @param {array} conversionData parameters for insertConversions * @param {array} meterData parameters for insertMeters * @param {array} groupData parameters for insertGroups (optional) */ async function prepareTest(unitData, conversionData, meterData, groupData = []) { - const conn = testDB.getConnection(); - await insertUnits(unitData, false, conn); - await insertConversions(conversionData, conn); - const result = await insertMeters(meterData, conn); - await insertGroups(groupData, conn); - await redoCik(conn); + const conn = testDB.getConnection(); + await insertUnits(unitData, false, conn); + await insertConversions(conversionData, conn); + const result = await insertMeters(meterData, conn); + await insertGroups(groupData, conn); + await redoCikVary(conn); - // Only refresh meter views if there is no group changes. - if (groupData.length == 0) { - await Reading.refreshMeterReadingsViews(conn); - } - else { + if (groupData.length != 0) { await Group.refreshGroupsDeepMetersView(conn); - await refreshAllReadingViews(); } + + // Refresh both the legacy materialized views and the TimescaleDB + // continuous aggregates through the centralized refresh service. + await refreshAllReadingViews(); } /** @@ -60,9 +58,9 @@ async function prepareTest(unitData, conversionData, meterData, groupData = []) * @returns {array} array of arrays similar in format to the expected JSON output of the readings api */ async function parseExpectedCsv(fileName) { - let expectedCsv = await readCsv(fileName); - expectedCsv.shift(); - return expectedCsv; + let expectedCsv = await readCsv(fileName); + expectedCsv.shift(); + return expectedCsv; }; /** @@ -71,18 +69,18 @@ async function parseExpectedCsv(fileName) { * @param {array} expected the returned array from parseExpectedCsv */ function expectReadingToEqualExpected(res, expected, id = METER_ID) { - expect(res).to.be.json; - expect(res).to.have.status(HTTP_CODE.OK); - // Did the response have the correct number of readings. - expect(res.body).to.have.property(`${id}`).to.have.lengthOf(expected.length); - // Loop over each reading - for (let i = 0; i < expected.length; i++) { - // Check that the reading's value is within the expected tolerance (DELTA). - expect(res.body).to.have.property(`${id}`).to.have.property(`${i}`).to.have.property('reading').to.be.closeTo(Number(expected[i][0]), DELTA); - // Reading has correct start/end date and time. - expect(res.body).to.have.property(`${id}`).to.have.property(`${i}`).to.have.property('startTimestamp').to.equal(Date.parse(expected[i][1])); - expect(res.body).to.have.property(`${id}`).to.have.property(`${i}`).to.have.property('endTimestamp').to.equal(Date.parse(expected[i][2])); - } + expect(res).to.be.json; + expect(res).to.have.status(HTTP_CODE.OK); + // Did the response have the correct number of readings. + expect(res.body).to.have.property(`${id}`).to.have.lengthOf(expected.length); + // Loop over each reading + for (let i = 0; i < expected.length; i++) { + // Check that the reading's value is within the expected tolerance (DELTA). + expect(res.body).to.have.property(`${id}`).to.have.property(`${i}`).to.have.property('reading').to.be.closeTo(Number(expected[i][0]), DELTA); + // Reading has correct start/end date and time. + expect(res.body).to.have.property(`${id}`).to.have.property(`${i}`).to.have.property('startTimestamp').to.equal(Date.parse(expected[i][1])); + expect(res.body).to.have.property(`${id}`).to.have.property(`${i}`).to.have.property('endTimestamp').to.equal(Date.parse(expected[i][2])); + } } /** @@ -91,19 +89,19 @@ function expectReadingToEqualExpected(res, expected, id = METER_ID) { * @param {array} expected the returned array from parseExpectedCsv */ function expectRangeToEqualExpected(res, expected, id = METER_ID) { - expect(res).to.be.json; - expect(res).to.have.status(HTTP_CODE.OK); - // Did the response have the correct number of readings. - expect(res.body).to.have.property(`${id}`).to.have.lengthOf(expected.length); - // Loop over each reading - for (let i = 0; i < expected.length; i++) { - // Check that the reading's min/max is within the expected tolerance (DELTA). - expect(res.body).to.have.property(`${id}`).to.have.property(`${i}`).to.have.property('min').to.be.closeTo(Number(expected[i][0]), DELTA); - expect(res.body).to.have.property(`${id}`).to.have.property(`${i}`).to.have.property('max').to.be.closeTo(Number(expected[i][1]), DELTA); - // Reading has correct start/end date and time. - expect(res.body).to.have.property(`${id}`).to.have.property(`${i}`).to.have.property('startTimestamp').to.equal(Date.parse(expected[i][2])); - expect(res.body).to.have.property(`${id}`).to.have.property(`${i}`).to.have.property('endTimestamp').to.equal(Date.parse(expected[i][3])); - } + expect(res).to.be.json; + expect(res).to.have.status(HTTP_CODE.OK); + // Did the response have the correct number of readings. + expect(res.body).to.have.property(`${id}`).to.have.lengthOf(expected.length); + // Loop over each reading + for (let i = 0; i < expected.length; i++) { + // Check that the reading's min/max is within the expected tolerance (DELTA). + expect(res.body).to.have.property(`${id}`).to.have.property(`${i}`).to.have.property('min').to.be.closeTo(Number(expected[i][0]), DELTA); + expect(res.body).to.have.property(`${id}`).to.have.property(`${i}`).to.have.property('max').to.be.closeTo(Number(expected[i][1]), DELTA); + // Reading has correct start/end date and time. + expect(res.body).to.have.property(`${id}`).to.have.property(`${i}`).to.have.property('startTimestamp').to.equal(Date.parse(expected[i][2])); + expect(res.body).to.have.property(`${id}`).to.have.property(`${i}`).to.have.property('endTimestamp').to.equal(Date.parse(expected[i][3])); + } } /** @@ -112,13 +110,13 @@ function expectRangeToEqualExpected(res, expected, id = METER_ID) { * @param {array} expected the returned array from parseExpectedCsv */ function expectCompareToEqualExpected(res, expected, id = METER_ID) { - expect(res).to.be.json; - expect(res).to.have.status(HTTP_CODE.OK); - // Did the response have the correct meter - expect(res.body).to.have.property(`${id}`); - // Check that the reading's values (previous value and current value) is within the expected tolerance (DELTA). - expect(res.body).to.have.property(`${id}`).to.have.property('curr_use').to.be.closeTo(Number(expected[0]), DELTA); - expect(res.body).to.have.property(`${id}`).to.have.property('prev_use').to.be.closeTo(Number(expected[1]), DELTA); + expect(res).to.be.json; + expect(res).to.have.status(HTTP_CODE.OK); + // Did the response have the correct meter + expect(res.body).to.have.property(`${id}`); + // Check that the reading's values (previous value and current value) is within the expected tolerance (DELTA). + expect(res.body).to.have.property(`${id}`).to.have.property('curr_use').to.be.closeTo(Number(expected[0]), DELTA); + expect(res.body).to.have.property(`${id}`).to.have.property('prev_use').to.be.closeTo(Number(expected[1]), DELTA); } /** @@ -129,51 +127,51 @@ function expectCompareToEqualExpected(res, expected, id = METER_ID) { * @param {boolean} noData true if 3D request cannot return data so special values, false by default */ function expectThreeDReadingToEqualExpected(res, expected, timePerReading, noData = false) { - let readingsPerDay = 24 / timePerReading; - // Number of days expected to be returned. Special of only 1 value if 3D cannot return data so special value. - let days = noData ? 1 : expected.length / readingsPerDay; - expect(res).to.be.json; - expect(res).to.have.status(HTTP_CODE.OK); - // Did the response have the correct type of properties. - expect(res.body).to.have.property('xData'); - expect(res.body).to.have.property('yData'); - expect(res.body).to.have.property('zData').to.have.lengthOf(days); - // The lengths should be correct. - expect(res.body, 'xData length').to.have.property(`xData`).to.have.lengthOf(readingsPerDay); - expect(res.body, 'yData length').to.have.property(`yData`).to.have.lengthOf(days); - expect(res.body, 'zData length').to.have.property(`zData`).to.have.lengthOf(days); - // Only check the first one but the others are checked in the loop for value. - expect(res.body.zData[0], 'zData[0] length').to.have.lengthOf(readingsPerDay); + let readingsPerDay = 24 / timePerReading; + // Number of days expected to be returned. Special of only 1 value if 3D cannot return data so special value. + let days = noData ? 1 : expected.length / readingsPerDay; + expect(res).to.be.json; + expect(res).to.have.status(HTTP_CODE.OK); + // Did the response have the correct type of properties. + expect(res.body).to.have.property('xData'); + expect(res.body).to.have.property('yData'); + expect(res.body).to.have.property('zData').to.have.lengthOf(days); + // The lengths should be correct. + expect(res.body, 'xData length').to.have.property(`xData`).to.have.lengthOf(readingsPerDay); + expect(res.body, 'yData length').to.have.property(`yData`).to.have.lengthOf(days); + expect(res.body, 'zData length').to.have.property(`zData`).to.have.lengthOf(days); + // Only check the first one but the others are checked in the loop for value. + expect(res.body.zData[0], 'zData[0] length').to.have.lengthOf(readingsPerDay); - // xData should have readingsPerDay values with the start/end time of each point in the day. - for (let hourIndex = 0; hourIndex < readingsPerDay; hourIndex++) { - expect(res.body.xData[hourIndex]).to.have.property('startTimestamp').to.be.equal(Date.parse(expected[hourIndex][1])); - expect(res.body.xData[hourIndex]).to.have.property('endTimestamp').to.be.equal(Date.parse(expected[hourIndex][2])); - } + // xData should have readingsPerDay values with the start/end time of each point in the day. + for (let hourIndex = 0; hourIndex < readingsPerDay; hourIndex++) { + expect(res.body.xData[hourIndex]).to.have.property('startTimestamp').to.be.equal(Date.parse(expected[hourIndex][1])); + expect(res.body.xData[hourIndex]).to.have.property('endTimestamp').to.be.equal(Date.parse(expected[hourIndex][2])); + } - // yData should have days values with each day start time. - // The index in expected which is first reading of each day. - let expectedIndex = 0; - for (let dayIndex = 0; dayIndex < days; dayIndex++) { - expect(res.body.yData[dayIndex]).to.be.equal(Date.parse(expected[expectedIndex][1])); - expectedIndex += readingsPerDay; - } + // yData should have days values with each day start time. + // The index in expected which is first reading of each day. + let expectedIndex = 0; + for (let dayIndex = 0; dayIndex < days; dayIndex++) { + expect(res.body.yData[dayIndex]).to.be.equal(Date.parse(expected[expectedIndex][1])); + expectedIndex += readingsPerDay; + } - // zData should be a 2D array where the first index has days values and the second has readingsPerDay - // and each value is the reading at that day and time. - // The index in expected which increases by 1. - expectedIndex = 0; - for (let dayIndex = 0; dayIndex < days; dayIndex++) { - for (let hourIndex = 0; hourIndex < readingsPerDay; hourIndex++) { - // When there are holes in the data that are filled the expected value is 'null' and requires a special check. - if (expected[expectedIndex][0] === 'null') { - expect(res.body.zData[dayIndex][hourIndex]).to.equal(null); - } else { - expect(res.body.zData[dayIndex][hourIndex]).to.be.closeTo(Number(expected[expectedIndex][0]), DELTA); - } - expectedIndex++; - } - } + // zData should be a 2D array where the first index has days values and the second has readingsPerDay + // and each value is the reading at that day and time. + // The index in expected which increases by 1. + expectedIndex = 0; + for (let dayIndex = 0; dayIndex < days; dayIndex++) { + for (let hourIndex = 0; hourIndex < readingsPerDay; hourIndex++) { + // When there are holes in the data that are filled the expected value is 'null' and requires a special check. + if (expected[expectedIndex][0] === 'null') { + expect(res.body.zData[dayIndex][hourIndex]).to.equal(null); + } else { + expect(res.body.zData[dayIndex][hourIndex]).to.be.closeTo(Number(expected[expectedIndex][0]), DELTA); + } + expectedIndex++; + } + } } /** @@ -185,8 +183,8 @@ function expectThreeDReadingToEqualExpected(res, expected, timePerReading, noDat * @returns {string} a string with the format '20XX-XX-XXT00:00:00Z_20XX-XX-XXT00:00:00Z' */ function createTimeString(startDay, startTime, endDay, endTime) { - const dateString = new TimeInterval(moment(startDay + ' ' + startTime), moment(endDay + ' ' + endTime)); - return dateString.toString(); + const dateString = new TimeInterval(moment(startDay + ' ' + startTime), moment(endDay + ' ' + endTime)); + return dateString.toString(); } /** @@ -195,113 +193,113 @@ function createTimeString(startDay, startTime, endDay, endTime) { * @returns {number} id of unitName */ async function getUnitId(unitName) { - conn = testDB.getConnection(); - const unit = await Unit.getByName(unitName, conn); - if (!unit) { - // This is not a valid unit name so return -99. - return -99; - } else { - return (await Unit.getByName(unitName, conn)).id; - } + conn = testDB.getConnection(); + const unit = await Unit.getByName(unitName, conn); + if (!unit) { + // This is not a valid unit name so return -99. + return -99; + } else { + return (await Unit.getByName(unitName, conn)).id; + } } // These units and conversions are used in many tests. // These are the 2D arrays for units, conversions to feed into the database // For kWh units. const unitDatakWh = [ - { - name: 'kWh', - identifier: '', - unitRepresent: Unit.unitRepresentType.QUANTITY, - secInRate: 3600, - typeOfUnit: Unit.unitType.UNIT, - suffix: '', - displayable: Unit.displayableType.ALL, - preferredDisplay: true, - note: 'OED created standard unit' - }, - { - name: 'Electric_Utility', - identifier: '', - unitRepresent: Unit.unitRepresentType.QUANTITY, - secInRate: 3600, - typeOfUnit: Unit.unitType.METER, - suffix: '', - displayable: Unit.displayableType.NONE, - preferredDisplay: false, - note: 'special unit' - } + { + name: 'kWh', + identifier: '', + unitRepresent: Unit.unitRepresentType.QUANTITY, + secInRate: 3600, + typeOfUnit: Unit.unitType.UNIT, + suffix: '', + displayable: Unit.displayableType.ALL, + preferredDisplay: true, + note: 'OED created standard unit' + }, + { + name: 'Electric_Utility', + identifier: '', + unitRepresent: Unit.unitRepresentType.QUANTITY, + secInRate: 3600, + typeOfUnit: Unit.unitType.METER, + suffix: '', + displayable: Unit.displayableType.NONE, + preferredDisplay: false, + note: 'special unit' + } ]; const conversionDatakWh = [ - { - sourceName: 'Electric_Utility', - destinationName: 'kWh', - bidirectional: false, - slope: 1, - intercept: 0, - note: 'Electric_Utility → kWh' - } + { + sourceName: 'Electric_Utility', + destinationName: 'kWh', + bidirectional: false, + slope: 1, + intercept: 0, + note: 'Electric_Utility → kWh' + } ]; const meterDatakWh = [ - { - name: 'Electric Utility kWh', - unit: 'Electric_Utility', - defaultGraphicUnit: 'kWh', - displayable: true, - gps: undefined, - note: 'special meter', - file: 'test/web/readingsData/readings_ri_15_days_75.csv', - deleteFile: false, - readingFrequency: '15 minutes', - // Note the meter ID is set so we know what to expect when a query is made. - id: METER_ID - } + { + name: 'Electric Utility kWh', + unit: 'Electric_Utility', + defaultGraphicUnit: 'kWh', + displayable: true, + gps: undefined, + note: 'special meter', + file: 'test/web/readingsData/readings_ri_15_days_75.csv', + deleteFile: false, + readingFrequency: '15 minutes', + // Note the meter ID is set so we know what to expect when a query is made. + id: METER_ID + } ]; const meterDatakWhOther = [ - { - name: 'Electric Utility Other', - unit: 'Electric_Utility', - defaultGraphicUnit: 'kWh', - displayable: true, - gps: undefined, - note: 'special meter', - file: 'test/web/readingsData/readings_ri_20_days_75.csv', - deleteFile: false, - readingFrequency: '20 minutes', - id: (METER_ID + 1) - } + { + name: 'Electric Utility Other', + unit: 'Electric_Utility', + defaultGraphicUnit: 'kWh', + displayable: true, + gps: undefined, + note: 'special meter', + file: 'test/web/readingsData/readings_ri_20_days_75.csv', + deleteFile: false, + readingFrequency: '20 minutes', + id: (METER_ID + 1) + } ]; const meterDatakWhGroups = meterDatakWh.concat(meterDatakWhOther); const groupDatakWh = [ - { - id: GROUP_ID, - name: 'Electric Utility kWh + Other', - displayable: true, - note: 'special group', - defaultGraphicUnit: 'kWh', - childMeters: ['Electric Utility kWh', 'Electric Utility Other'], - childGroups: [], - } + { + id: GROUP_ID, + name: 'Electric Utility kWh + Other', + displayable: true, + note: 'special group', + defaultGraphicUnit: 'kWh', + childMeters: ['Electric Utility kWh', 'Electric Utility Other'], + childGroups: [], + } ]; module.exports = { - prepareTest, - parseExpectedCsv, - expectReadingToEqualExpected, - expectRangeToEqualExpected, - expectCompareToEqualExpected, - expectThreeDReadingToEqualExpected, - createTimeString, - getUnitId, - ETERNITY, - DELTA, - METER_ID, - GROUP_ID, - HTTP_CODE, - unitDatakWh, - conversionDatakWh, - meterDatakWh, - meterDatakWhGroups, - groupDatakWh + prepareTest, + parseExpectedCsv, + expectReadingToEqualExpected, + expectRangeToEqualExpected, + expectCompareToEqualExpected, + expectThreeDReadingToEqualExpected, + createTimeString, + getUnitId, + ETERNITY, + DELTA, + METER_ID, + GROUP_ID, + HTTP_CODE, + unitDatakWh, + conversionDatakWh, + meterDatakWh, + meterDatakWhGroups, + groupDatakWh };