Add TimescaleDB meter reading aggregation design handover documentation

README.md

@@ -23,7 +23,7 @@ Note OED is transitioning to mostly public design documents. The private DevDocs
 - [MQTT.md](./MQTT.md): Looking into integrating for meter data acquisition.
 - [baseline/baseline.md](./baseline/baseline.md): Add the ability for an admin to add baselines to meters and for users to display on a graphic.
 - [enhancementToGithubAction.md](./githubAction/enhancementToGithubAction.md): securing GitHub action information.
-- [timescaleDB/timescaleDB.md](./timescaleDB/timescaleDB.md): Information on efforts to investigate TimescaleDB usage in OED.
+- [timescaleDB/Handover.md](./timescaleDB/Handover.md): Information on migration to TimescaleDB in OED and possible improvement opportunities.
 - [infisicalIntegration/infisicalIntegration.md](./infisicalIntegration/infisicalIntegration.md): Instructions for how to utilize Infisical secrets management to store database passwords.
 
 ## Information

timescaleDB/sqlScripts/compare/CompareDailyReadings.sql

@@ -0,0 +1,91 @@
+/*
+ * CompareDailyReadings.sql
+ *
+ * Purpose:
+ *
+ *   Compare results between the existing PostgreSQL materialized view
+ *
+ *       meter_daily_readings_unit
+ *
+ *   and the TimescaleDB continuous aggregate
+ *
+ *       meter_daily_readings_unit_cagg
+ *
+ *
+ * The purpose of this script is to validate that the TimescaleDB
+ * implementation produces equivalent results to the existing materialized
+ * view before considering migration.
+ *
+ * Expected result:
+ *
+ *   - reading_rate differences should be zero or within floating-point
+ *     rounding tolerance.
+ *   - max_rate and min_rate differences should be zero or within
+ *     floating-point rounding tolerance.
+ *
+ * Results are ordered by the largest differences first to make discrepancies
+ * easier to identify.
+ */
+
+
+/*
+ * 1. Compare reading_rate values.
+ *
+ * This compares the primary hourly aggregation result between:
+ *
+ *   Original:
+ *       meter_daily_readings_unit
+ *
+ *   TimescaleDB:
+ *       meter_daily_readings_unit_cagg
+ *
+ * Any non-zero differences should be investigated.
+ */
+SELECT
+    mv.meter_id,
+    LOWER(mv.time_interval) AS mv_time,
+    mv.reading_rate AS mv_reading_rate,
+    LOWER(cagg.time_interval) AS cagg_time,
+    cagg.reading_rate AS cagg_reading_rate,
+    (COALESCE(mv.reading_rate, 0) - COALESCE(cagg.reading_rate, 0)) AS difference
+FROM meter_daily_readings_unit AS mv LEFT JOIN 
+	 meter_daily_readings_unit_cagg AS cagg
+     	 ON mv.meter_id = cagg.meter_id
+    		 AND mv.time_interval = cagg.time_interval
+    		 AND mv.graphic_unit_id = cagg.graphic_unit_id
+ORDER BY
+    ABS(COALESCE(mv.reading_rate, 0) - COALESCE(cagg.reading_rate, 0)) DESC
+LIMIT 20;
+
+
+/*
+ * 2. Compare max_rate and min_rate values.
+ *
+ * This verifies that the continuous aggregate preserves the same minimum and
+ * maximum hourly rates as the original materialized view.
+ *
+ * Any non-zero differences should be investigated.
+ */
+SELECT
+    mv.meter_id,
+    LOWER(mv.time_interval) AS mv_time_start,
+	LOWER(cagg.time_interval) AS cagg_time_start,
+	UPPER(mv.time_interval) AS mv_time_end,
+	UPPER(cagg.time_interval) AS cagg_time_end,
+    mv.max_rate AS mv_max_rate,
+    cagg.max_rate AS cagg_max_rate,
+    (COALESCE(mv.max_rate, 0) - COALESCE(cagg.max_rate, 0)) AS max_difference,
+    mv.min_rate AS mv_min_rate,
+	cagg.min_rate AS cagg_min_rate,
+	(COALESCE(mv.min_rate, 0) - COALESCE(cagg.min_rate, 0)) AS min_difference
+FROM meter_daily_readings_unit AS mv LEFT JOIN 
+	 meter_daily_readings_unit_cagg AS cagg
+    	 ON mv.meter_id = cagg.meter_id
+    	 	 AND mv.time_interval = cagg.time_interval
+    	 	 AND mv.graphic_unit_id = cagg.graphic_unit_id
+ORDER BY
+    GREATEST(
+        ABS(COALESCE(mv.max_rate, 0) - COALESCE(cagg.max_rate, 0)),
+        ABS(COALESCE(mv.min_rate, 0) - COALESCE(cagg.min_rate, 0))
+    ) DESC
+LIMIT 20;

timescaleDB/sqlScripts/compare/CompareGroupDailyReadings.sql

@@ -0,0 +1,158 @@
+/*
+ * CompareGroupDailyReadings.sql
+ *
+ * Purpose:
+ *
+ *   Compare results between the existing PostgreSQL materialized view
+ *
+ *       group_daily_readings_unit
+ *
+ *   and the TimescaleDB continuous aggregate
+ *
+ *       group_daily_readings_unit_cagg
+ *
+ *
+ * The purpose of this script is to validate that the TimescaleDB
+ * implementation produces equivalent results to the existing materialized
+ * view before considering migration.
+ *
+ *
+ * Expected result:
+ *
+ *   - reading_rate differences should be zero or within floating-point
+ *     rounding tolerance.
+ *
+ *   - Missing rows should be investigated.
+ *
+ * Results are ordered by the largest differences first to make discrepancies
+ * easier to identify.
+ */
+
+
+/*
+ * 1. Compare reading_rate values.
+ *
+ * This compares the primary daily aggregation result between:
+ *
+ *   Original:
+ *       group_daily_readings_unit
+ *
+ *   TimescaleDB:
+ *       group_daily_readings_unit_cagg
+ *
+ * Any non-zero differences should be investigated.
+ */
+SELECT
+
+    mv.group_id,
+
+    LOWER(mv.time_interval) AS mv_time,
+
+    LOWER(cagg.time_interval) AS cagg_time,
+
+    mv.reading_rate AS mv_reading_rate,
+
+    cagg.reading_rate AS cagg_reading_rate,
+
+    (
+        COALESCE(mv.reading_rate, 0)
+        -
+        COALESCE(cagg.reading_rate, 0)
+    ) AS difference
+
+
+FROM group_daily_readings_unit AS mv
+
+
+LEFT JOIN group_daily_readings_unit_cagg AS cagg
+
+    ON mv.group_id = cagg.group_id
+
+    AND mv.time_interval = cagg.time_interval
+
+    AND mv.graphic_unit_id = cagg.graphic_unit_id
+
+
+ORDER BY
+
+    ABS(
+        COALESCE(mv.reading_rate, 0)
+        -
+        COALESCE(cagg.reading_rate, 0)
+    ) DESC
+
+
+LIMIT 20;
+
+
+
+/*
+ * 2. Compare row counts.
+ *
+ * This identifies missing or additional daily buckets between the two
+ * implementations.
+ */
+SELECT
+
+    COUNT(*) AS total_rows,
+
+    COUNT(cagg.group_id) AS matching_cagg_rows,
+
+    COUNT(*) - COUNT(cagg.group_id) AS missing_cagg_rows
+
+
+FROM group_daily_readings_unit AS mv
+
+
+LEFT JOIN group_daily_readings_unit_cagg AS cagg
+
+    ON mv.group_id = cagg.group_id
+
+    AND mv.time_interval = cagg.time_interval
+
+    AND mv.graphic_unit_id = cagg.graphic_unit_id;
+
+
+
+/*
+ * 3. Display missing TimescaleDB rows.
+ *
+ * These rows exist in the original materialized view but not in the
+ * continuous aggregate.
+ */
+SELECT
+
+    mv.group_id,
+
+    LOWER(mv.time_interval) AS mv_time,
+
+    UPPER(mv.time_interval) AS mv_time_end,
+
+    mv.graphic_unit_id,
+
+    mv.reading_rate
+
+
+FROM group_daily_readings_unit AS mv
+
+
+LEFT JOIN group_daily_readings_unit_cagg AS cagg
+
+    ON mv.group_id = cagg.group_id
+
+    AND mv.time_interval = cagg.time_interval
+
+    AND mv.graphic_unit_id = cagg.graphic_unit_id
+
+
+WHERE cagg.group_id IS NULL
+
+
+ORDER BY
+
+    mv.group_id,
+
+    mv.time_interval
+
+
+LIMIT 20;

timescaleDB/sqlScripts/compare/CompareGroupHourlyReadings.sql

@@ -0,0 +1,123 @@
+/*
+ * CompareGroupHourlyReadings.sql
+ *
+ * Purpose:
+ *
+ *   Compare results between the existing PostgreSQL materialized view
+ *
+ *       group_hourly_readings_unit
+ *
+ *   and the TimescaleDB continuous aggregate
+ *
+ *       group_hourly_readings_unit_cagg
+ *
+ *
+ * The purpose of this script is to validate that the TimescaleDB
+ * implementation produces equivalent results to the existing materialized
+ * view before considering migration.
+ *
+ *
+ * Expected result:
+ *
+ *   - reading_rate differences should be zero or within floating-point
+ *     rounding tolerance.
+ *
+ * Results are ordered by the largest differences first to make discrepancies
+ * easier to identify.
+ */
+
+
+/*
+ * 1. Compare reading_rate values.
+ *
+ * This compares the primary hourly aggregation result between:
+ *
+ *   Original:
+ *       group_hourly_readings_unit
+ *
+ *   TimescaleDB:
+ *       group_hourly_readings_unit_cagg
+ *
+ * Any non-zero differences should be investigated.
+ */
+SELECT
+
+    mv.group_id,
+
+    LOWER(mv.time_interval) AS mv_time,
+
+    mv.reading_rate AS mv_reading_rate,
+
+    LOWER(cagg.time_interval) AS cagg_time,
+
+    cagg.reading_rate AS cagg_reading_rate,
+
+    (
+        COALESCE(mv.reading_rate, 0)
+        -
+        COALESCE(cagg.reading_rate, 0)
+    ) AS difference
+
+
+FROM group_hourly_readings_unit AS mv
+
+INNER JOIN group_hourly_readings_unit_cagg AS cagg
+
+    ON mv.group_id = cagg.group_id
+
+    AND mv.time_interval = cagg.time_interval
+
+    AND mv.graphic_unit_id = cagg.graphic_unit_id
+
+
+ORDER BY
+
+    ABS(
+        COALESCE(mv.reading_rate, 0)
+        -
+        COALESCE(cagg.reading_rate, 0)
+    ) DESC
+
+
+LIMIT 20;
+
+
+
+/*
+ * 2. Identify rows missing from the TimescaleDB continuous aggregate.
+ *
+ * This catches cases where the continuous aggregate does not contain a
+ * corresponding group/hour/unit combination.
+ */
+SELECT
+
+    mv.group_id,
+
+    mv.time_interval,
+
+    mv.graphic_unit_id,
+
+    mv.reading_rate
+
+FROM group_hourly_readings_unit AS mv
+
+LEFT JOIN group_hourly_readings_unit_cagg AS cagg
+
+    ON mv.group_id = cagg.group_id
+
+    AND mv.time_interval = cagg.time_interval
+
+    AND mv.graphic_unit_id = cagg.graphic_unit_id
+
+
+WHERE cagg.group_id IS NULL
+
+
+ORDER BY
+
+    mv.group_id,
+
+    mv.time_interval
+
+
+LIMIT 20;