From 24c1983e530f624a5734883d9608578d386b36ac Mon Sep 17 00:00:00 2001 From: djurcola-agent Date: Sun, 30 Aug 2026 14:59:56 +1000 Subject: [PATCH 01/30] Preserve deployed BatteryWatch database release --- app/.gitignore | 6 + app/README.md | 75 ++ app/__init__.py | 1 + app/backend/__init__.py | 1 + app/backend/batterywatch_api/__init__.py | 3 + app/backend/batterywatch_api/aemo.py | 354 ++++++ .../batterywatch_api/battery_assets.py | 129 +++ app/backend/batterywatch_api/collector.py | 73 ++ .../batterywatch_api/collector_service.py | 361 ++++++ app/backend/batterywatch_api/database_main.py | 40 + .../dispatch_price_ingestion.py | 225 ++++ .../batterywatch_api/dispatch_scada.py | 200 ++++ .../dispatch_scada_ingestion.py | 343 ++++++ app/backend/batterywatch_api/fixtures.py | 56 + app/backend/batterywatch_api/main.py | 433 +++++++ app/backend/batterywatch_api/models.py | 109 ++ .../nemweb_dispatch_prices.py | 284 +++++ .../batterywatch_api/nemweb_dispatch_scada.py | 227 ++++ app/backend/batterywatch_api/nemweb_http.py | 174 +++ app/backend/batterywatch_api/runtime.py | 91 ++ app/backend/batterywatch_api/storage.py | 1024 +++++++++++++++++ app/backend/requirements.txt | 5 + app/backend/tests/__init__.py | 1 + app/backend/tests/test_aemo.py | 275 +++++ app/backend/tests/test_api.py | 47 + .../tests/test_api_database_generators.py | 427 +++++++ app/backend/tests/test_api_database_series.py | 761 ++++++++++++ app/backend/tests/test_battery_assets.py | 78 ++ app/backend/tests/test_collector.py | 151 +++ app/backend/tests/test_collector_service.py | 328 ++++++ .../tests/test_dispatch_price_ingestion.py | 234 ++++ app/backend/tests/test_dispatch_scada.py | 435 +++++++ .../tests/test_dispatch_scada_ingestion.py | 104 ++ ...est_dispatch_scada_ingestion_repository.py | 285 +++++ .../tests/test_nemweb_dispatch_prices.py | 159 +++ .../tests/test_nemweb_dispatch_scada.py | 547 +++++++++ app/backend/tests/test_nemweb_http.py | 319 +++++ app/backend/tests/test_postgresql_storage.py | 383 ++++++ app/backend/tests/test_runtime_health.py | 314 +++++ app/backend/tests/test_storage.py | 179 +++ app/config/battery_assets.json | 603 ++++++++++ app/deploy/README.md | 36 + app/deploy/backup-verify.sh | 23 + app/deploy/batterywatch-api.service | 25 + app/deploy/batterywatch-collector.service | 26 + app/deploy/migrate.sh | 41 + app/deploy/runtime.env.example | 3 + app/deploy/verify-live.sql | 10 + app/frontend/index.html | 13 + app/frontend/package-lock.json | 975 ++++++++++++++++ app/frontend/package.json | 22 + app/frontend/src/main.tsx | 68 ++ app/frontend/src/styles.css | 27 + app/frontend/tsconfig.json | 20 + app/frontend/vite.config.ts | 10 + app/migrations/001_initial_schema.sql | 107 ++ .../002_dispatch_scada_raw_ingestion.sql | 33 + .../003_dispatch_price_artifacts.sql | 26 + 58 files changed, 11309 insertions(+) create mode 100644 app/.gitignore create mode 100644 app/README.md create mode 100644 app/__init__.py create mode 100644 app/backend/__init__.py create mode 100644 app/backend/batterywatch_api/__init__.py create mode 100644 app/backend/batterywatch_api/aemo.py create mode 100644 app/backend/batterywatch_api/battery_assets.py create mode 100644 app/backend/batterywatch_api/collector.py create mode 100644 app/backend/batterywatch_api/collector_service.py create mode 100644 app/backend/batterywatch_api/database_main.py create mode 100644 app/backend/batterywatch_api/dispatch_price_ingestion.py create mode 100644 app/backend/batterywatch_api/dispatch_scada.py create mode 100644 app/backend/batterywatch_api/dispatch_scada_ingestion.py create mode 100644 app/backend/batterywatch_api/fixtures.py create mode 100644 app/backend/batterywatch_api/main.py create mode 100644 app/backend/batterywatch_api/models.py create mode 100644 app/backend/batterywatch_api/nemweb_dispatch_prices.py create mode 100644 app/backend/batterywatch_api/nemweb_dispatch_scada.py create mode 100644 app/backend/batterywatch_api/nemweb_http.py create mode 100644 app/backend/batterywatch_api/runtime.py create mode 100644 app/backend/batterywatch_api/storage.py create mode 100644 app/backend/requirements.txt create mode 100644 app/backend/tests/__init__.py create mode 100644 app/backend/tests/test_aemo.py create mode 100644 app/backend/tests/test_api.py create mode 100644 app/backend/tests/test_api_database_generators.py create mode 100644 app/backend/tests/test_api_database_series.py create mode 100644 app/backend/tests/test_battery_assets.py create mode 100644 app/backend/tests/test_collector.py create mode 100644 app/backend/tests/test_collector_service.py create mode 100644 app/backend/tests/test_dispatch_price_ingestion.py create mode 100644 app/backend/tests/test_dispatch_scada.py create mode 100644 app/backend/tests/test_dispatch_scada_ingestion.py create mode 100644 app/backend/tests/test_dispatch_scada_ingestion_repository.py create mode 100644 app/backend/tests/test_nemweb_dispatch_prices.py create mode 100644 app/backend/tests/test_nemweb_dispatch_scada.py create mode 100644 app/backend/tests/test_nemweb_http.py create mode 100644 app/backend/tests/test_postgresql_storage.py create mode 100644 app/backend/tests/test_runtime_health.py create mode 100644 app/backend/tests/test_storage.py create mode 100644 app/config/battery_assets.json create mode 100644 app/deploy/README.md create mode 100755 app/deploy/backup-verify.sh create mode 100644 app/deploy/batterywatch-api.service create mode 100644 app/deploy/batterywatch-collector.service create mode 100755 app/deploy/migrate.sh create mode 100644 app/deploy/runtime.env.example create mode 100644 app/deploy/verify-live.sql create mode 100644 app/frontend/index.html create mode 100644 app/frontend/package-lock.json create mode 100644 app/frontend/package.json create mode 100644 app/frontend/src/main.tsx create mode 100644 app/frontend/src/styles.css create mode 100644 app/frontend/tsconfig.json create mode 100644 app/frontend/vite.config.ts create mode 100644 app/migrations/001_initial_schema.sql create mode 100644 app/migrations/002_dispatch_scada_raw_ingestion.sql create mode 100644 app/migrations/003_dispatch_price_artifacts.sql diff --git a/app/.gitignore b/app/.gitignore new file mode 100644 index 0000000..99d03c7 --- /dev/null +++ b/app/.gitignore @@ -0,0 +1,6 @@ +__pycache__/ +*.py[cod] +.pytest_cache/ + +frontend/node_modules/ +frontend/dist/ diff --git a/app/README.md b/app/README.md new file mode 100644 index 0000000..5ccfa05 --- /dev/null +++ b/app/README.md @@ -0,0 +1,75 @@ +# BatteryWatch successor + +This is the standalone successor to the original BatteryWatch static site. The first vertical slice proves a new local API-to-chart path using deterministic five-minute fixtures. + +## Local development + +Backend: + +```bash +python3 -m venv .venv +. .venv/bin/activate +python -m pip install -r app/backend/requirements.txt +uvicorn batterywatch_api.main:app --app-dir app/backend --host 0.0.0.0 --port 8080 +``` + +Frontend, in a second shell: + +```bash +npm --prefix app/frontend install +npm --prefix app/frontend run dev +``` + +Open `http://127.0.0.1:5173`. The Vite development server proxies `/api` to the local API on `http://127.0.0.1:8080`. + +After a production frontend build, the API serves `app/frontend/dist` when present. The deployment contract is plain HTTP on `0.0.0.0:8080`, so the user-managed reverse proxy can terminate HTTPS and forward to `http://192.168.20.42:8080`. Database and ingestion/admin endpoints are not exposed by this slice. + +## Data semantics + +- Positive battery power means discharge/export; negative means charge/import. +- Every point is a five-minute interval and uses `energy_mwh = power_mw × 5/60`. +- Price is represented as AEMO regional RRP-shaped data in AUD/MWh. +- Missing prices remain unavailable (`null`) and are reflected in coverage. +- Gross discharge value, charging cost, and net energy value are estimates, not actual profit. They exclude efficiency losses, auxiliary consumption, degradation, FCAS, network charges, contracts, taxes, and fees. + +The fixture includes discharge, charging, zero output, a negative price, missing price, and nullable SOC cases. TimescaleDB/PostgreSQL persistence, real AEMO ingestion, historical backfill, and four-second telemetry are later slices; four-second telemetry is intentionally outside v1. + +## S2a storage contract + +The S2a persistence seam is defined in `backend/batterywatch_api/storage.py` and +`migrations/001_initial_schema.sql`. Records use UTC-aware timestamps, require +five-minute interval alignment, retain source identity and timestamps, and +carry ingestion and correction versions. Power uses positive discharge/export +and negative charge/import. Zero is a real measurement; missing price and SOC +remain `null`/unavailable and are never inferred from power. + +The logical key is generator plus interval for power/SOC and region plus +interval for NEM price. The deterministic repository keeps one effective +record per logical key: an exact replay is a no-op, a newer correction replaces +the effective record, and a stale record cannot regress it. The SQL migration +provides PostgreSQL/Timescale-ready tables and uniqueness/check constraints but +does not create a database, role, credential, extension, or service. + +The S2b code slice adds a production-replaceable `PostgreSQLRepository` using a +caller-supplied DB-API connection and parameterized SQL. It preserves this +storage boundary, keeps only the winning effective record (revision history is +not retained), and does not connect to or activate a database. The existing +fixture repository and API remain the local/deployed baseline. + +Real database provisioning and activation are a separate supervisor-only gate: +against a private PostgreSQL/TimescaleDB instance, the supervisor must apply +the migration and verify write/read-back, duplicate replay, correction, +backup, and isolated restore before any live AEMO data is written. This slice +does not perform that provisioning or validation. + +## S2c AEMO dispatch-price parser + +`backend/batterywatch_api/aemo.py` parses canonical dispatch-price CSV rows into +`RegionalPrice5m` records. It requires `SETTLEMENTDATE`, `REGIONID`, and `RRP`, +rejects malformed or duplicate logical intervals, requires an explicit timezone +for offset-free source timestamps, normalizes records to UTC/five-minute +boundaries, and preserves blank, zero, and negative RRP semantics. `RUNNO`, +`INTERVENTION`, and `APCFLAG` are retained as quality metadata; ingestion and +correction revisions are supplied by the coordinator rather than guessed from +the source row. Live AEMO fetching, scheduling, persistence, and backfill remain +separate gates. diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..9b0fbd9 --- /dev/null +++ b/app/__init__.py @@ -0,0 +1 @@ +"""BatteryWatch successor application packages.""" diff --git a/app/backend/__init__.py b/app/backend/__init__.py new file mode 100644 index 0000000..bf00399 --- /dev/null +++ b/app/backend/__init__.py @@ -0,0 +1 @@ +"""Backend package root for local test discovery.""" diff --git a/app/backend/batterywatch_api/__init__.py b/app/backend/batterywatch_api/__init__.py new file mode 100644 index 0000000..d9ed613 --- /dev/null +++ b/app/backend/batterywatch_api/__init__.py @@ -0,0 +1,3 @@ +"""Standalone BatteryWatch successor API.""" + +__version__ = "0.1.0" diff --git a/app/backend/batterywatch_api/aemo.py b/app/backend/batterywatch_api/aemo.py new file mode 100644 index 0000000..284f892 --- /dev/null +++ b/app/backend/batterywatch_api/aemo.py @@ -0,0 +1,354 @@ +"""Strict parser for canonical AEMO dispatch-price CSV rows.""" + +from __future__ import annotations + +import csv +from datetime import datetime, timedelta, timezone, tzinfo +from io import StringIO +from math import isfinite +import re + +from .storage import RegionalPrice5m + + +_REQUIRED_COLUMNS = {"SETTLEMENTDATE", "REGIONID", "RRP"} +_OPTIONAL_FLAG_COLUMNS = ("INTERVENTION", "APCFLAG", "RUNNO") + + +class AemoParseError(ValueError): + """Raised when a dispatch-price payload cannot be safely normalized.""" + + +def _parse_timestamp(value: str, *, naive_timezone: tzinfo | None) -> datetime: + normalized = value.strip().replace("/", "-").replace(" ", "T", 1) + try: + parsed = datetime.fromisoformat(normalized) + except ValueError as exc: + raise AemoParseError(f"invalid SETTLEMENTDATE: {value!r}") from exc + if parsed.tzinfo is None or parsed.utcoffset() is None: + if naive_timezone is None: + raise AemoParseError("naive SETTLEMENTDATE requires naive_timezone") + parsed = parsed.replace(tzinfo=naive_timezone) + return parsed + + +def _parse_nonnegative_flag(value: str, name: str, row_number: int) -> int: + if not value.strip(): + return 0 + try: + parsed = int(value) + except ValueError as exc: + raise AemoParseError(f"row {row_number}: {name} must be an integer") from exc + if parsed < 0: + raise AemoParseError(f"row {row_number}: {name} must be non-negative") + return parsed + + +def _parse_market_suspended(value: str, row_number: int) -> bool: + parsed = _parse_nonnegative_flag(value, "MARKETSUSPENDEDFLAG", row_number) + if parsed not in (0, 1): + raise AemoParseError( + f"row {row_number}: MARKETSUSPENDEDFLAG must be 0 or 1" + ) + return bool(parsed) + + +def _quality_flags(row: dict[str, str], row_number: int) -> tuple[str, ...]: + flags: list[str] = [] + for name in _OPTIONAL_FLAG_COLUMNS: + value = row.get(name, "") + if name == "RUNNO": + runno = _parse_nonnegative_flag(value, name, row_number) + flags.append(f"runno={runno}") + else: + flag = _parse_nonnegative_flag(value, name, row_number) + if flag: + flags.append(f"{name.lower()}={flag}") + return tuple(flags) + + +def _parse_price(value: str, row_number: int) -> float | None: + if not value.strip(): + return None + try: + return float(value) + except ValueError as exc: + raise AemoParseError(f"row {row_number}: RRP must be numeric or blank") from exc + + +def parse_dispatch_price_csv( + payload: str, + *, + source_id: str, + source_timestamp: datetime, + ingestion_version: int, + correction_version: int = 0, + naive_timezone: tzinfo | None = None, +) -> tuple[RegionalPrice5m, ...]: + """Parse strict dispatch-price rows into UTC-normalized price records. + + AEMO CSV timestamps without an offset are accepted only when the caller + supplies the source timezone explicitly. Revision values are supplied by + the ingestion coordinator; ``RUNNO`` is retained as provenance metadata. + """ + + reader = csv.DictReader(StringIO(payload)) + fieldnames = reader.fieldnames + if not fieldnames or len(fieldnames) != len(set(fieldnames)): + raise AemoParseError("CSV must have a unique header row") + missing = _REQUIRED_COLUMNS - set(fieldnames) + if missing: + raise AemoParseError(f"CSV is missing required columns: {sorted(missing)}") + + records: list[RegionalPrice5m] = [] + seen: set[tuple[str, datetime]] = set() + for row_number, row in enumerate(reader, start=2): + if None in row or any(value is None for value in row.values()): + raise AemoParseError(f"row {row_number}: malformed CSV fields") + region = row["REGIONID"].strip() + if not region: + raise AemoParseError(f"row {row_number}: REGIONID is required") + try: + interval_start = _parse_timestamp( + row["SETTLEMENTDATE"], naive_timezone=naive_timezone + ) + price = _parse_price(row["RRP"], row_number) + record = RegionalPrice5m( + region=region, + interval_start=interval_start, + price_aud_per_mwh=price, + price_status="missing" if price is None else ("negative" if price < 0 else "available"), + source_id=source_id, + source_timestamp=source_timestamp, + ingestion_version=ingestion_version, + correction_version=correction_version, + quality_flags=_quality_flags(row, row_number), + intervention=_parse_nonnegative_flag( + row.get("INTERVENTION", ""), "INTERVENTION", row_number + ), + apc_flag=_parse_nonnegative_flag( + row.get("APCFLAG", ""), "APCFLAG", row_number + ), + market_suspended=_parse_market_suspended( + row.get("MARKETSUSPENDEDFLAG", ""), row_number + ), + ) + except (TypeError, ValueError) as exc: + if isinstance(exc, AemoParseError): + raise + raise AemoParseError(f"row {row_number}: {exc}") from exc + if record.logical_key in seen: + raise AemoParseError(f"row {row_number}: duplicate region and interval") + seen.add(record.logical_key) + records.append(record) + + return tuple(sorted(records, key=lambda record: record.logical_key)) + + +_NEM_TIMEZONE = timezone(timedelta(hours=10)) +_MMS_METADATA_PREFIX = ("C", "NEMP.WORLD", "DISPATCHIS", "AEMO", "PUBLIC") +_MMS_PRICE_HEADER = ( + "I", "DISPATCH", "PRICE", "5", "SETTLEMENTDATE", "RUNNO", "REGIONID", + "DISPATCHINTERVAL", "INTERVENTION", "RRP", "EEP", "ROP", "APCFLAG", + "MARKETSUSPENDEDFLAG", "LASTCHANGED", "RAISE6SECRRP", "RAISE6SECROP", + "RAISE6SECAPCFLAG", "RAISE60SECRRP", "RAISE60SECROP", "RAISE60SECAPCFLAG", + "RAISE5MINRRP", "RAISE5MINROP", "RAISE5MINAPCFLAG", "RAISEREGRRP", + "RAISEREGROP", "RAISEREGAPCFLAG", "LOWER6SECRRP", "LOWER6SECROP", + "LOWER6SECAPCFLAG", "LOWER60SECRRP", "LOWER60SECROP", "LOWER60SECAPCFLAG", + "LOWER5MINRRP", "LOWER5MINROP", "LOWER5MINAPCFLAG", "LOWERREGRRP", + "LOWERREGROP", "LOWERREGAPCFLAG", "PRICE_STATUS", "PRE_AP_ENERGY_PRICE", + "PRE_AP_RAISE6_PRICE", "PRE_AP_RAISE60_PRICE", "PRE_AP_RAISE5MIN_PRICE", + "PRE_AP_RAISEREG_PRICE", "PRE_AP_LOWER6_PRICE", "PRE_AP_LOWER60_PRICE", + "PRE_AP_LOWER5MIN_PRICE", "PRE_AP_LOWERREG_PRICE", "RAISE1SECRRP", + "RAISE1SECROP", "RAISE1SECAPCFLAG", "LOWER1SECRRP", "LOWER1SECROP", + "LOWER1SECAPCFLAG", "PRE_AP_RAISE1_PRICE", "PRE_AP_LOWER1_PRICE", + "CUMUL_PRE_AP_ENERGY_PRICE", "CUMUL_PRE_AP_RAISE6_PRICE", + "CUMUL_PRE_AP_RAISE60_PRICE", "CUMUL_PRE_AP_RAISE5MIN_PRICE", + "CUMUL_PRE_AP_RAISEREG_PRICE", "CUMUL_PRE_AP_LOWER6_PRICE", + "CUMUL_PRE_AP_LOWER60_PRICE", "CUMUL_PRE_AP_LOWER5MIN_PRICE", + "CUMUL_PRE_AP_LOWERREG_PRICE", "CUMUL_PRE_AP_RAISE1_PRICE", + "CUMUL_PRE_AP_LOWER1_PRICE", "OCD_STATUS", "MII_STATUS", +) +_MMS_REGIONS = frozenset(("NSW1", "QLD1", "SA1", "TAS1", "VIC1")) +_MMS_STATUS_RE = re.compile(r"[A-Za-z0-9_.-]{1,32}") + + +def _mms_timestamp(value: str, row_number: int, *, aligned: bool) -> datetime: + normalized = value.strip().replace("/", "-").replace(" ", "T", 1) + try: + parsed = datetime.fromisoformat(normalized) + except (AttributeError, TypeError, ValueError) as exc: + raise AemoParseError(f"row {row_number}: invalid timestamp") from exc + if parsed.tzinfo is None or parsed.utcoffset() is None: + parsed = parsed.replace(tzinfo=_NEM_TIMEZONE) + result = parsed.astimezone(timezone.utc) + if aligned and (result.minute % 5 or result.second or result.microsecond): + raise AemoParseError(f"row {row_number}: timestamp is not five-minute aligned") + return result + + +def _mms_control(value: str, name: str, row_number: int) -> int: + if not value or not value.isascii() or not value.isdecimal(): + raise AemoParseError(f"row {row_number}: invalid {name}") + parsed = int(value) + if parsed > 2_147_483_647: + raise AemoParseError(f"row {row_number}: invalid {name}") + if name == "MARKETSUSPENDEDFLAG" and parsed not in (0, 1): + raise AemoParseError(f"row {row_number}: invalid {name}") + return parsed + + +def _mms_rrp(value: str, row_number: int) -> float | None: + if not value.strip(): + return None + try: + parsed = float(value) + except (TypeError, ValueError) as exc: + raise AemoParseError(f"row {row_number}: invalid RRP") from exc + if not isfinite(parsed): + raise AemoParseError(f"row {row_number}: invalid RRP") + return parsed + + +def parse_dispatch_price_mms_csv( + payload: str, + *, + source_id: str, + ingestion_version: int, + correction_version: int = 0, + source_timestamp: datetime | None = None, +) -> tuple[RegionalPrice5m, ...]: + """Parse one complete version-5 MMS DispatchIS regional-price report.""" + + if not isinstance(payload, str) or not payload or "\x00" in payload: + raise AemoParseError("invalid MMS CSV") + if not isinstance(source_id, str) or not source_id: + raise AemoParseError("invalid source id") + try: + rows = list(csv.reader(StringIO(payload), strict=True)) + except csv.Error as exc: + raise AemoParseError("invalid MMS CSV") from exc + if len(rows) < 4 or any(not row for row in rows): + raise AemoParseError("invalid MMS envelope") + + metadata = rows[0] + if ( + len(metadata) != 10 + or tuple(metadata[:5]) != _MMS_METADATA_PREFIX + or metadata[7] != source_id + or metadata[8] != "DISPATCHIS" + or not metadata[9].isascii() + or not metadata[9].isdecimal() + ): + raise AemoParseError("invalid MMS envelope") + try: + datetime.strptime(f"{metadata[5]} {metadata[6]}", "%Y/%m/%d %H:%M:%S") + except ValueError as exc: + raise AemoParseError("invalid MMS envelope") from exc + + footer = rows[-1] + if len(footer) != 3 or tuple(footer[:2]) != ("C", "END OF REPORT"): + raise AemoParseError("invalid MMS envelope") + try: + footer_count = int(footer[2]) + except ValueError as exc: + raise AemoParseError("invalid MMS envelope") from exc + if str(footer_count) != footer[2] or footer_count != len(rows): + raise AemoParseError("invalid MMS envelope") + + header_positions: dict[str, int] | None = None + data_rows: list[tuple[int, list[str]]] = [] + for row_number, row in enumerate(rows[1:-1], start=2): + if row[0] == "I" and len(row) >= 3 and tuple(row[1:3]) == ("DISPATCH", "PRICE"): + if tuple(row) != _MMS_PRICE_HEADER or header_positions is not None: + raise AemoParseError(f"row {row_number}: invalid PRICE header") + header_positions = {name: index for index, name in enumerate(row)} + elif row[0] == "D" and len(row) >= 3 and tuple(row[1:3]) == ("DISPATCH", "PRICE"): + if len(row) < 4 or row[3] != "5": + raise AemoParseError(f"row {row_number}: unsupported PRICE version") + if header_positions is None: + raise AemoParseError(f"row {row_number}: PRICE data precedes header") + data_rows.append((row_number, row)) + elif row[0] not in ("C", "I", "D"): + raise AemoParseError(f"row {row_number}: invalid MMS row type") + + if header_positions is None or len(data_rows) != len(_MMS_REGIONS): + raise AemoParseError("PRICE report must contain exactly five NEM regions") + + records: list[RegionalPrice5m] = [] + seen: set[tuple[str, datetime]] = set() + interval: datetime | None = None + for row_number, row in data_rows: + if len(row) != len(_MMS_PRICE_HEADER) or tuple(row[:4]) != ("D", "DISPATCH", "PRICE", "5"): + raise AemoParseError(f"row {row_number}: invalid PRICE row") + region = row[header_positions["REGIONID"]] + if region not in _MMS_REGIONS: + raise AemoParseError(f"row {row_number}: unknown NEM region") + settlement = _mms_timestamp( + row[header_positions["SETTLEMENTDATE"]], row_number, aligned=True + ) + if interval is None: + interval = settlement + elif settlement != interval: + raise AemoParseError(f"row {row_number}: mixed settlement intervals") + key = (region, settlement) + if key in seen: + raise AemoParseError(f"row {row_number}: duplicate region and interval") + seen.add(key) + + runno = _mms_control(row[header_positions["RUNNO"]], "RUNNO", row_number) + intervention = _mms_control( + row[header_positions["INTERVENTION"]], "INTERVENTION", row_number + ) + apc_flag = _mms_control(row[header_positions["APCFLAG"]], "APCFLAG", row_number) + suspended = _mms_control( + row[header_positions["MARKETSUSPENDEDFLAG"]], + "MARKETSUSPENDEDFLAG", + row_number, + ) + status = row[header_positions["PRICE_STATUS"]] + if not _MMS_STATUS_RE.fullmatch(status) or not status.isascii(): + raise AemoParseError(f"row {row_number}: invalid PRICE_STATUS") + price = _mms_rrp(row[header_positions["RRP"]], row_number) + observed_at = source_timestamp or _mms_timestamp( + row[header_positions["LASTCHANGED"]], row_number, aligned=False + ) + flags = [f"runno={runno}"] + if intervention: + flags.append(f"intervention={intervention}") + if apc_flag: + flags.append(f"apcflag={apc_flag}") + if suspended: + flags.append("market_suspended=1") + flags.append(f"aemo_price_status={status}") + try: + records.append(RegionalPrice5m( + region=region, + interval_start=settlement, + price_aud_per_mwh=price, + price_status=( + "missing" if price is None else + ("negative" if price < 0 else "available") + ), + source_id=source_id, + source_timestamp=observed_at, + ingestion_version=ingestion_version, + correction_version=correction_version, + quality_flags=tuple(flags), + intervention=intervention, + apc_flag=apc_flag, + market_suspended=bool(suspended), + )) + except (TypeError, ValueError) as exc: + raise AemoParseError(f"row {row_number}: invalid PRICE fields") from exc + + if {record.region for record in records} != _MMS_REGIONS: + raise AemoParseError("PRICE report must contain exactly five NEM regions") + return tuple(sorted(records, key=lambda record: record.logical_key)) + + +__all__ = [ + "AemoParseError", + "parse_dispatch_price_csv", + "parse_dispatch_price_mms_csv", +] diff --git a/app/backend/batterywatch_api/battery_assets.py b/app/backend/batterywatch_api/battery_assets.py new file mode 100644 index 0000000..85d7ce1 --- /dev/null +++ b/app/backend/batterywatch_api/battery_assets.py @@ -0,0 +1,129 @@ + +"""Strict loader for reviewed BatteryWatch DUID metadata.""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timezone +import json +from math import isfinite +from pathlib import Path +import re + + +_TOP_LEVEL_KEYS = {"schema_version", "assets", "excluded"} +_ASSET_KEYS = { + "duid", + "site_name", + "region", + "capacity_mw", + "storage_capacity_mwh", + "source_id", + "source_timestamp", +} +_EXCLUDED_KEYS = {"duid", "reason"} +_REGIONS = {"NSW1", "QLD1", "SA1", "TAS1", "VIC1"} +_DUID_RE = re.compile(r"^[A-Z0-9_]{1,32}$") +_SOURCE_ID_RE = re.compile(r"^[A-Za-z0-9._-]{1,128}$") + + +@dataclass(frozen=True, slots=True) +class BatteryAsset: + duid: str + site_name: str + region: str + capacity_mw: float + storage_capacity_mwh: float + source_id: str + source_timestamp: datetime + + +def _text(value: object, field: str) -> str: + if type(value) is not str or not value or value != value.strip(): + raise ValueError(f"invalid {field}") + return value + + +def _positive_number(value: object, field: str) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"invalid {field}") + normalized = float(value) + if not isfinite(normalized) or normalized <= 0: + raise ValueError(f"invalid {field}") + return normalized + + +def _timestamp(value: object) -> datetime: + text = _text(value, "source_timestamp") + if not text.endswith("Z"): + raise ValueError("invalid source_timestamp") + try: + parsed = datetime.fromisoformat(text[:-1] + "+00:00") + except ValueError: + raise ValueError("invalid source_timestamp") from None + if parsed.tzinfo is None or parsed.utcoffset() is None: + raise ValueError("invalid source_timestamp") + return parsed.astimezone(timezone.utc) + + +def load_battery_assets(path: str | Path) -> tuple[BatteryAsset, ...]: + """Load one exact reviewed config and reject ambiguity fail closed.""" + + try: + payload = json.loads(Path(path).read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise ValueError("invalid battery asset config") from exc + if type(payload) is not dict or set(payload) != _TOP_LEVEL_KEYS: + raise ValueError("invalid battery asset config") + if payload["schema_version"] != 1: + raise ValueError("invalid battery asset schema version") + raw_assets = payload["assets"] + raw_excluded = payload["excluded"] + if type(raw_assets) is not list or not raw_assets or type(raw_excluded) is not list: + raise ValueError("invalid battery asset config") + + excluded: set[str] = set() + for item in raw_excluded: + if type(item) is not dict or set(item) != _EXCLUDED_KEYS: + raise ValueError("invalid excluded battery asset") + duid = _text(item["duid"], "excluded duid") + _text(item["reason"], "excluded reason") + if not _DUID_RE.fullmatch(duid) or duid in excluded: + raise ValueError("invalid excluded battery asset") + excluded.add(duid) + + assets: list[BatteryAsset] = [] + seen: set[str] = set() + for item in raw_assets: + if type(item) is not dict or set(item) != _ASSET_KEYS: + raise ValueError("invalid battery asset") + duid = _text(item["duid"], "duid") + site_name = _text(item["site_name"], "site_name") + region = _text(item["region"], "region") + source_id = _text(item["source_id"], "source_id") + if ( + not _DUID_RE.fullmatch(duid) + or region not in _REGIONS + or not _SOURCE_ID_RE.fullmatch(source_id) + or duid in seen + or duid in excluded + ): + raise ValueError("invalid battery asset") + seen.add(duid) + assets.append( + BatteryAsset( + duid=duid, + site_name=site_name, + region=region, + capacity_mw=_positive_number(item["capacity_mw"], "capacity_mw"), + storage_capacity_mwh=_positive_number( + item["storage_capacity_mwh"], "storage_capacity_mwh" + ), + source_id=source_id, + source_timestamp=_timestamp(item["source_timestamp"]), + ) + ) + return tuple(sorted(assets, key=lambda asset: asset.duid)) + + +__all__ = ["BatteryAsset", "load_battery_assets"] diff --git a/app/backend/batterywatch_api/collector.py b/app/backend/batterywatch_api/collector.py new file mode 100644 index 0000000..20471fd --- /dev/null +++ b/app/backend/batterywatch_api/collector.py @@ -0,0 +1,73 @@ +"""One latest Dispatch SCADA collection cycle.""" + +from collections.abc import Callable +from dataclasses import dataclass +from datetime import timedelta, timezone + +from .dispatch_scada import parse_dispatch_scada_csv +from .nemweb_dispatch_scada import ( + DispatchScadaArtifact, + discover_dispatch_scada_artifacts, + extract_dispatch_scada_zip, +) +from .nemweb_http import NemwebHttpResource, fetch_nemweb_resource +from .storage import GeneratorPower5m + + +DISPATCH_SCADA_INDEX_URL = ( + "https://www.nemweb.com.au/REPORTS/CURRENT/Dispatch_SCADA/" +) +DISPATCH_SCADA_INDEX_MAX_BYTES = 2 * 1024 * 1024 +DISPATCH_SCADA_ARTIFACT_MAX_BYTES = 16 * 1024 * 1024 + + +@dataclass(frozen=True, slots=True) +class DispatchScadaCollection: + """The verified latest artifact and its parsed power records.""" + + artifact: DispatchScadaArtifact + records: tuple[GeneratorPower5m, ...] + + +def collect_latest_dispatch_scada( + ingestion_version: int, + correction_version: int = 0, + *, + fetch: Callable[..., NemwebHttpResource] = fetch_nemweb_resource, +) -> DispatchScadaCollection: + """Collect the latest canonical Dispatch SCADA artifact.""" + + index_resource = fetch( + DISPATCH_SCADA_INDEX_URL, + max_bytes=DISPATCH_SCADA_INDEX_MAX_BYTES, + ) + references = discover_dispatch_scada_artifacts( + index_resource.body.decode("utf-8"), + index_url=DISPATCH_SCADA_INDEX_URL, + ) + latest_reference = references[-1] + artifact_resource = fetch( + latest_reference.url, + max_bytes=DISPATCH_SCADA_ARTIFACT_MAX_BYTES, + ) + artifact = extract_dispatch_scada_zip( + latest_reference, + artifact_resource.body, + ) + records = parse_dispatch_scada_csv( + artifact.csv_payload, + source_artifact_id=artifact.reference.source_artifact_id, + ingestion_version=ingestion_version, + correction_version=correction_version, + naive_timezone=timezone(timedelta(hours=10)), + ) + return DispatchScadaCollection(artifact=artifact, records=records) + + +__all__ = [ + "DISPATCH_SCADA_ARTIFACT_MAX_BYTES", + "DISPATCH_SCADA_INDEX_MAX_BYTES", + "DISPATCH_SCADA_INDEX_URL", + "DispatchScadaCollection", + "collect_latest_dispatch_scada", +] diff --git a/app/backend/batterywatch_api/collector_service.py b/app/backend/batterywatch_api/collector_service.py new file mode 100644 index 0000000..52e2a08 --- /dev/null +++ b/app/backend/batterywatch_api/collector_service.py @@ -0,0 +1,361 @@ + +"""Separate Dispatch SCADA collection and persistence runtime.""" + +from __future__ import annotations + +import argparse +from collections.abc import Callable, Iterable +from dataclasses import dataclass, replace +from importlib import import_module +import json +import os +from pathlib import Path +import signal +import sys +from threading import Event +from typing import Any, Protocol + +from .battery_assets import BatteryAsset, load_battery_assets +from .collector import DispatchScadaCollection, collect_latest_dispatch_scada +from .dispatch_price_ingestion import ( + DispatchPriceArtifactReceipt, + DispatchPriceIngestionResult, + PostgreSQLDispatchPriceIngestor, +) +from .dispatch_scada_ingestion import ( + DispatchScadaArtifactReceipt, + DispatchScadaIngestionResult, + PostgreSQLDispatchScadaIngestor, + RawDispatchScadaObservation, +) +from .nemweb_dispatch_prices import DispatchPriceCollection, collect_latest_dispatch_prices +from .storage import GeneratorMetadata, GeneratorPower5m + + +_MAX_BIGINT = (1 << 63) - 1 + + +class _Ingestor(Protocol): + def ingest( + self, + receipt: DispatchScadaArtifactReceipt, + observations: Iterable[RawDispatchScadaObservation], + generators: Iterable[GeneratorMetadata] = (), + power_records: Iterable[GeneratorPower5m] = (), + ) -> DispatchScadaIngestionResult: ... + + +class _PriceIngestor(Protocol): + def ingest( + self, + receipt: DispatchPriceArtifactReceipt, + records: Iterable[Any], + ) -> DispatchPriceIngestionResult: ... + + +@dataclass(frozen=True, slots=True) +class CollectorCycleResult: + scada: DispatchScadaIngestionResult + prices: DispatchPriceIngestionResult + + @property + def raw_observation_count(self) -> int: + return self.scada.raw_observation_count + + @property + def mapped_power_count(self) -> int: + return self.scada.mapped_power_count + + @property + def replayed(self) -> bool: + return self.scada.replayed + + @property + def price_count(self) -> int: + return self.prices.price_count + + @property + def price_replayed(self) -> bool: + return self.prices.replayed + + +def _connect(database_url: str, *, connect_timeout: int) -> Any: + return import_module("psycopg").connect( + database_url, connect_timeout=connect_timeout + ) + + +def _artifact_version(collection: DispatchScadaCollection) -> int: + source_artifact_id = collection.artifact.reference.source_artifact_id + try: + version = int(source_artifact_id) + except (TypeError, ValueError, OverflowError): + raise ValueError("invalid Dispatch SCADA artifact version") from None + if not 0 <= version <= _MAX_BIGINT: + raise ValueError("invalid Dispatch SCADA artifact version") + return version + + +def run_collection_cycle( + connection: Any, + assets: Iterable[BatteryAsset], + *, + collect: Callable[..., DispatchScadaCollection] = collect_latest_dispatch_scada, + ingestor_factory: Callable[[Any], _Ingestor] = PostgreSQLDispatchScadaIngestor, +) -> DispatchScadaIngestionResult: + """Fetch, validate, map, and atomically persist one latest artifact.""" + + asset_by_duid: dict[str, BatteryAsset] = {} + for asset in assets: + if asset.duid in asset_by_duid: + raise ValueError("duplicate battery asset DUID") + asset_by_duid[asset.duid] = asset + if not asset_by_duid: + raise ValueError("battery assets must not be empty") + + collection = collect(ingestion_version=0, correction_version=0) + version = _artifact_version(collection) + artifact = collection.artifact + reference = artifact.reference + receipt = DispatchScadaArtifactReceipt( + source_artifact_id=reference.source_artifact_id, + source_url=reference.url, + zip_filename=reference.zip_filename, + csv_member_name=artifact.csv_member_name, + report_timestamp=reference.report_timestamp, + zip_sha256=artifact.zip_sha256, + raw_zip=artifact.raw_zip, + ) + + raw_rows: list[RawDispatchScadaObservation] = [] + mapped_power: list[GeneratorPower5m] = [] + mapped_duids: set[str] = set() + for record in collection.records: + if record.source_id != reference.source_artifact_id: + raise ValueError("record source artifact does not match collection") + canonical_record = GeneratorPower5m( + generator_id=record.generator_id, + interval_start=record.interval_start, + power_mw=record.power_mw, + source_id=record.source_id, + source_timestamp=record.source_timestamp, + ingestion_version=version, + correction_version=record.correction_version, + ) + raw_rows.append( + RawDispatchScadaObservation( + source_artifact_id=reference.source_artifact_id, + duid=canonical_record.generator_id, + interval_start=canonical_record.interval_start, + power_mw=canonical_record.power_mw, + source_timestamp=canonical_record.source_timestamp, + ingestion_version=canonical_record.ingestion_version, + correction_version=canonical_record.correction_version, + ) + ) + if canonical_record.generator_id in asset_by_duid: + mapped_power.append(canonical_record) + mapped_duids.add(canonical_record.generator_id) + + generators = tuple( + GeneratorMetadata( + generator_id=asset.duid, + site_name=asset.site_name, + region=asset.region, + capacity_mw=asset.capacity_mw, + storage_capacity_mwh=asset.storage_capacity_mwh, + source_id=asset.source_id, + source_timestamp=asset.source_timestamp, + ingestion_version=1, + ) + for duid, asset in sorted(asset_by_duid.items()) + if duid in mapped_duids + ) + return ingestor_factory(connection).ingest( + receipt, + tuple(raw_rows), + generators=generators, + power_records=tuple(mapped_power), + ) + + +def run_price_collection_cycle( + connection: Any, + *, + collect: Callable[..., DispatchPriceCollection] = collect_latest_dispatch_prices, + ingestor_factory: Callable[[Any], _PriceIngestor] = PostgreSQLDispatchPriceIngestor, +) -> DispatchPriceIngestionResult: + """Fetch, validate, and atomically persist one official DispatchIS artifact.""" + + collection = collect(ingestion_version=0, correction_version=0) + artifact = collection.artifact + reference = artifact.reference + try: + version = int(reference.source_artifact_id) + except (TypeError, ValueError, OverflowError): + raise ValueError("invalid DispatchIS artifact version") from None + if not 0 <= version <= _MAX_BIGINT: + raise ValueError("invalid DispatchIS artifact version") + receipt = DispatchPriceArtifactReceipt( + source_artifact_id=reference.source_artifact_id, + source_url=reference.url, + zip_filename=reference.zip_filename, + csv_member_name=artifact.csv_member_name, + report_timestamp=reference.report_timestamp, + zip_sha256=artifact.zip_sha256, + raw_zip=artifact.raw_zip, + ) + records = tuple( + replace(record, ingestion_version=version) + for record in collection.records + ) + return ingestor_factory(connection).ingest(receipt, records) + + +def run_database_cycle( + database_url: str, + assets: Iterable[BatteryAsset], + *, + connect: Callable[..., Any] = _connect, + collect: Callable[..., DispatchScadaCollection] = collect_latest_dispatch_scada, + collect_prices: Callable[..., DispatchPriceCollection] = collect_latest_dispatch_prices, + ingestor_factory: Callable[[Any], _Ingestor] = PostgreSQLDispatchScadaIngestor, + price_ingestor_factory: Callable[[Any], _PriceIngestor] = PostgreSQLDispatchPriceIngestor, +) -> CollectorCycleResult: + """Open one bounded database connection, run one cycle, and always close it.""" + + if not isinstance(database_url, str) or not database_url: + raise ValueError("database URL is required") + connection = connect(database_url, connect_timeout=10) + try: + scada = run_collection_cycle( + connection, + assets, + collect=collect, + ingestor_factory=ingestor_factory, + ) + prices = run_price_collection_cycle( + connection, + collect=collect_prices, + ingestor_factory=price_ingestor_factory, + ) + return CollectorCycleResult(scada, prices) + finally: + connection.close() + + +def run_polling_loop( + cycle: Callable[[], CollectorCycleResult], + *, + interval_seconds: int, + wait: Callable[[float], bool], +) -> CollectorCycleResult: + """Run immediately and stop cleanly when the interruptible wait is signalled.""" + + if type(interval_seconds) is not int or not 30 <= interval_seconds <= 3600: + raise ValueError("poll interval must be between 30 and 3600 seconds") + while True: + result = cycle() + if wait(float(interval_seconds)): + return result + + +def main( + argv: list[str] | None = None, + *, + environ: dict[str, str] | None = None, + connect: Callable[..., Any] = _connect, + collect: Callable[..., DispatchScadaCollection] = collect_latest_dispatch_scada, + collect_prices: Callable[..., DispatchPriceCollection] = collect_latest_dispatch_prices, + ingestor_factory: Callable[[Any], _Ingestor] = PostgreSQLDispatchScadaIngestor, + price_ingestor_factory: Callable[[Any], _PriceIngestor] = PostgreSQLDispatchPriceIngestor, +) -> int: + """Run one cycle or the supervised long-lived collector process.""" + + parser = argparse.ArgumentParser(prog="batterywatch-collector") + parser.add_argument("--once", action="store_true") + parser.add_argument("--assets-path", type=Path) + parser.add_argument("--interval-seconds", type=int) + arguments = parser.parse_args(argv) + environment = os.environ if environ is None else environ + try: + database_url = environment.get("BATTERYWATCH_DATABASE_URL", "") + assets_path = arguments.assets_path or Path( + environment.get( + "BATTERYWATCH_ASSETS_PATH", + str(Path(__file__).resolve().parents[2] / "config/battery_assets.json"), + ) + ) + interval_seconds = ( + arguments.interval_seconds + if arguments.interval_seconds is not None + else int(environment.get("BATTERYWATCH_COLLECT_INTERVAL_SECONDS", "300")) + ) + assets = load_battery_assets(assets_path) + + def cycle() -> CollectorCycleResult: + return run_database_cycle( + database_url, + assets, + connect=connect, + collect=collect, + collect_prices=collect_prices, + ingestor_factory=ingestor_factory, + price_ingestor_factory=price_ingestor_factory, + ) + + if arguments.once: + result = cycle() + else: + stop = Event() + + def request_stop(signum: int, frame: Any) -> None: + del signum, frame + stop.set() + + signal.signal(signal.SIGTERM, request_stop) + signal.signal(signal.SIGINT, request_stop) + result = run_polling_loop( + cycle, + interval_seconds=interval_seconds, + wait=stop.wait, + ) + print( + json.dumps( + { + "status": "ok", + "raw_observation_count": result.raw_observation_count, + "mapped_power_count": result.mapped_power_count, + "replayed": result.replayed, + "price_count": result.price_count, + "price_replayed": result.price_replayed, + }, + sort_keys=True, + ) + ) + return 0 + except KeyboardInterrupt: + return 0 + except Exception as error: + print( + json.dumps( + {"status": "error", "error_type": type(error).__name__}, + sort_keys=True, + ), + file=sys.stderr, + ) + return 1 + + +__all__ = [ + "CollectorCycleResult", + "main", + "run_collection_cycle", + "run_database_cycle", + "run_polling_loop", + "run_price_collection_cycle", +] + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/app/backend/batterywatch_api/database_main.py b/app/backend/batterywatch_api/database_main.py new file mode 100644 index 0000000..f92ef61 --- /dev/null +++ b/app/backend/batterywatch_api/database_main.py @@ -0,0 +1,40 @@ +"""Database-only ASGI entrypoint for the activated BatteryWatch service.""" + +from __future__ import annotations + +from collections.abc import Mapping +import os +from typing import Any + +from fastapi import FastAPI + +from .main import create_app + + +def create_database_application( + environ: Mapping[str, str] | None = None, + *, + connection_factory: Any = None, + health_tracer: Any = None, + repository_provider: Any = None, +) -> FastAPI: + """Create an application whose database mode cannot be changed by env.""" + + environment = os.environ if environ is None else environ + options: dict[str, Any] = {} + if connection_factory is not None: + options["connection_factory"] = connection_factory + if health_tracer is not None: + options["health_tracer"] = health_tracer + if repository_provider is not None: + options["repository_provider"] = repository_provider + return create_app( + data_mode="database", + database_url=environment.get("BATTERYWATCH_DATABASE_URL"), + **options, + ) + + +app = create_database_application() + +__all__ = ["app", "create_database_application"] diff --git a/app/backend/batterywatch_api/dispatch_price_ingestion.py b/app/backend/batterywatch_api/dispatch_price_ingestion.py new file mode 100644 index 0000000..d24a95e --- /dev/null +++ b/app/backend/batterywatch_api/dispatch_price_ingestion.py @@ -0,0 +1,225 @@ +"""Atomic persistence for official NEMWeb DispatchIS regional prices.""" + +from __future__ import annotations + +from contextlib import contextmanager +from dataclasses import dataclass +from datetime import datetime +from typing import Any, Iterable, Iterator, Protocol + +from .storage import RegionalPrice5m + + +_REGIONS = {"NSW1", "QLD1", "SA1", "TAS1", "VIC1"} +_MAX_BIGINT = (1 << 63) - 1 + + +@dataclass(frozen=True, slots=True) +class DispatchPriceArtifactReceipt: + source_artifact_id: str + source_url: str + zip_filename: str + csv_member_name: str + report_timestamp: datetime + zip_sha256: str + raw_zip: bytes + + +@dataclass(frozen=True, slots=True) +class DispatchPriceIngestionResult: + price_count: int + replayed: bool + + +class DispatchPriceArtifactConflictError(Exception): + """Raised when an artifact identity is already bound to different evidence.""" + + +class _Cursor(Protocol): + def execute(self, statement: str, parameters: tuple[Any, ...]) -> None: ... + def fetchone(self) -> tuple[Any, ...] | None: ... + def close(self) -> None: ... + + +class _Connection(Protocol): + def cursor(self) -> _Cursor: ... + def commit(self) -> None: ... + def rollback(self) -> None: ... + + +@contextmanager +def _managed_cursor(connection: _Connection) -> Iterator[_Cursor]: + cursor = connection.cursor() + try: + yield cursor + finally: + cursor.close() + + +_ARTIFACT_INSERT_SQL = """ +INSERT INTO dispatch_price_artifacts ( + source_artifact_id, source_url, zip_filename, csv_member_name, + report_timestamp, zip_sha256, raw_zip +) +VALUES (%s, %s, %s, %s, %s, %s, %s) +ON CONFLICT DO NOTHING RETURNING 1 +""" + +_ARTIFACT_SELECT_SQL = """ +SELECT source_artifact_id, source_url, zip_filename, csv_member_name, + report_timestamp, zip_sha256, raw_zip +FROM dispatch_price_artifacts +WHERE source_artifact_id = %s +""" + +_PRICE_UPSERT_SQL = """ +INSERT INTO nem_price_5m ( + region, + interval_start, + price_aud_per_mwh, + price_status, + intervention, + apc_flag, + market_suspended, + source_id, + source_timestamp, + ingestion_version, + correction_version, + quality_flags +) +VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) +ON CONFLICT (region, interval_start) DO UPDATE +SET price_aud_per_mwh = EXCLUDED.price_aud_per_mwh, + price_status = EXCLUDED.price_status, + intervention = EXCLUDED.intervention, + apc_flag = EXCLUDED.apc_flag, + market_suspended = EXCLUDED.market_suspended, + source_id = EXCLUDED.source_id, + source_timestamp = EXCLUDED.source_timestamp, + ingestion_version = EXCLUDED.ingestion_version, + correction_version = EXCLUDED.correction_version, + quality_flags = EXCLUDED.quality_flags +WHERE (EXCLUDED.correction_version, EXCLUDED.ingestion_version, EXCLUDED.source_timestamp) + > (nem_price_5m.correction_version, + nem_price_5m.ingestion_version, nem_price_5m.source_timestamp) +RETURNING 1 +""" + + +def _receipt_parameters(receipt: DispatchPriceArtifactReceipt) -> tuple[Any, ...]: + return ( + receipt.source_artifact_id, + receipt.source_url, + receipt.zip_filename, + receipt.csv_member_name, + receipt.report_timestamp, + receipt.zip_sha256, + receipt.raw_zip, + ) + + +def _normalized_bytes(value: Any) -> bytes: + if isinstance(value, memoryview): + return value.tobytes() + return bytes(value) + + +def _receipt_matches( + receipt: DispatchPriceArtifactReceipt, + stored: tuple[Any, ...], +) -> bool: + if len(stored) != 7: + return False + expected = _receipt_parameters(receipt) + return expected[:6] == stored[:6] and _normalized_bytes(expected[6]) == _normalized_bytes( + stored[6] + ) + + +def _price_parameters(record: RegionalPrice5m) -> tuple[Any, ...]: + return ( + record.region, + record.interval_start, + record.price_aud_per_mwh, + record.price_status, + record.intervention, + record.apc_flag, + record.market_suspended, + record.source_id, + record.source_timestamp, + record.ingestion_version, + record.correction_version, + list(record.quality_flags), + ) + + +class PostgreSQLDispatchPriceIngestor: + """Persist one canonical five-region DispatchIS artifact transactionally.""" + + def __init__(self, connection: _Connection): + self._connection = connection + + def ingest( + self, + receipt: DispatchPriceArtifactReceipt, + records: Iterable[RegionalPrice5m], + ) -> DispatchPriceIngestionResult: + try: + materialized = tuple(records) + regions = {record.region for record in materialized} + intervals = {record.interval_start for record in materialized} + if len(materialized) != 5 or regions != _REGIONS or len(intervals) != 1: + raise ValueError( + "price artifact must contain exactly the five canonical regions for one interval" + ) + if any(record.source_id != receipt.source_artifact_id for record in materialized): + raise ValueError("price source artifact does not match receipt") + if intervals != {receipt.report_timestamp}: + raise ValueError("price interval does not match artifact report timestamp") + try: + artifact_version = int(receipt.source_artifact_id) + except (TypeError, ValueError, OverflowError): + raise ValueError("invalid price artifact version") from None + if not 0 <= artifact_version <= _MAX_BIGINT or any( + record.ingestion_version != artifact_version + for record in materialized + ): + raise ValueError("price ingestion version does not match artifact version") + + applied = 0 + replayed = False + with _managed_cursor(self._connection) as cursor: + cursor.execute(_ARTIFACT_INSERT_SQL, _receipt_parameters(receipt)) + if cursor.fetchone() is None: + cursor.execute( + _ARTIFACT_SELECT_SQL, + (receipt.source_artifact_id,), + ) + stored = cursor.fetchone() + if stored is None or not _receipt_matches(receipt, stored): + raise DispatchPriceArtifactConflictError( + "DispatchIS price artifact conflicts with stored evidence" + ) + replayed = True + else: + for record in materialized: + cursor.execute(_PRICE_UPSERT_SQL, _price_parameters(record)) + if cursor.fetchone() is not None: + applied += 1 + + self._connection.commit() + return DispatchPriceIngestionResult(0 if replayed else applied, replayed) + except Exception: + try: + self._connection.rollback() + except Exception: + pass + raise + + +__all__ = [ + "DispatchPriceArtifactConflictError", + "DispatchPriceArtifactReceipt", + "DispatchPriceIngestionResult", + "PostgreSQLDispatchPriceIngestor", +] diff --git a/app/backend/batterywatch_api/dispatch_scada.py b/app/backend/batterywatch_api/dispatch_scada.py new file mode 100644 index 0000000..7d6cf83 --- /dev/null +++ b/app/backend/batterywatch_api/dispatch_scada.py @@ -0,0 +1,200 @@ +"""Parser for a canonical AEMO Dispatch SCADA artifact.""" + +from __future__ import annotations + +import csv +from datetime import datetime, timedelta, timezone, tzinfo +from io import StringIO + +from .storage import GeneratorPower5m + + +class DispatchScadaParseError(ValueError): + """Raised when a Dispatch SCADA CSV member is not canonical.""" + + +__all__ = ["DispatchScadaParseError", "parse_dispatch_scada_csv"] + +_METADATA_PREFIX = ("C", "NEMP.WORLD", "DISPATCHSCADA", "AEMO", "PUBLIC") +_HEADER = ( + "I", + "DISPATCH", + "UNIT_SCADA", + "1", + "SETTLEMENTDATE", + "DUID", + "SCADAVALUE", + "LASTCHANGED", +) +_DATA_PREFIX = ("D", "DISPATCH", "UNIT_SCADA", "1") +_FOOTER_PREFIX = ("C", "END OF REPORT") + + +def _read_rows(payload: str) -> list[list[str]]: + if not isinstance(payload, str): + raise DispatchScadaParseError("row 1: payload must be CSV text") + try: + reader = csv.reader(StringIO(payload), strict=True) + except (TypeError, ValueError, UnicodeError): + raise DispatchScadaParseError("row 1: payload must be CSV text") from None + try: + return list(reader) + except (csv.Error, TypeError, ValueError, UnicodeError): + raise DispatchScadaParseError( + f"row {max(reader.line_num, 1)}: malformed CSV" + ) from None + + +def _parse_timestamp( + value: str, + naive_timezone: tzinfo | None, + row_number: int, +) -> datetime: + try: + normalized = value.strip().replace("/", "-").replace(" ", "T", 1) + if "T" not in normalized: + raise ValueError + parsed = datetime.fromisoformat(normalized) + if parsed.tzinfo is None or parsed.utcoffset() is None: + if naive_timezone is None: + raise DispatchScadaParseError( + f"row {row_number}: offset-free timestamp requires naive_timezone" + ) + if ( + type(naive_timezone) is not timezone + or naive_timezone.utcoffset(None) != timedelta(hours=10) + ): + raise DispatchScadaParseError( + f"row {row_number}: invalid naive_timezone" + ) + parsed = parsed.replace(tzinfo=naive_timezone) + if parsed.utcoffset() is None: + raise ValueError + return parsed.astimezone(timezone.utc) + except DispatchScadaParseError: + raise + except (AttributeError, TypeError, ValueError, OverflowError): + raise DispatchScadaParseError(f"row {row_number}: invalid timestamp") from None + + +def _parse_power(value: str, row_number: int) -> float: + try: + return float(value.strip()) + except (AttributeError, TypeError, ValueError, OverflowError): + raise DispatchScadaParseError(f"row {row_number}: invalid power value") from None + + +def _record_from_row( + row: list[str], + row_number: int, + *, + source_artifact_id: str, + ingestion_version: int, + correction_version: int, + naive_timezone: tzinfo | None, +) -> GeneratorPower5m: + try: + return GeneratorPower5m( + generator_id=row[5].strip(), + interval_start=_parse_timestamp(row[4], naive_timezone, row_number), + power_mw=_parse_power(row[6], row_number), + source_id=source_artifact_id, + source_timestamp=_parse_timestamp(row[7], naive_timezone, row_number), + ingestion_version=ingestion_version, + correction_version=correction_version, + ) + except DispatchScadaParseError: + raise + except (AttributeError, TypeError, ValueError, OverflowError): + raise DispatchScadaParseError(f"row {row_number}: invalid data fields") from None + + +def _validate_metadata( + row: list[str], + *, + source_artifact_id: str, +) -> None: + if len(row) != 10: + raise DispatchScadaParseError("row 1: invalid metadata field count") + if tuple(row[:5]) != _METADATA_PREFIX or row[8] != "DISPATCHSCADA": + raise DispatchScadaParseError("row 1: invalid metadata structure") + if row[7] != source_artifact_id: + raise DispatchScadaParseError("row 1: source sequence does not match artifact") + if not row[5].strip() or not row[6].strip() or not row[9].strip(): + raise DispatchScadaParseError("row 1: malformed metadata fields") + try: + datetime.strptime(f"{row[5]} {row[6]}", "%Y/%m/%d %H:%M:%S") + except (TypeError, ValueError, OverflowError): + raise DispatchScadaParseError("row 1: invalid metadata timestamp") from None + + +def _validate_header(row: list[str]) -> None: + if len(row) != len(_HEADER): + raise DispatchScadaParseError("row 2: invalid header field count") + if tuple(row) != _HEADER: + raise DispatchScadaParseError("row 2: invalid header") + + +def _validate_footer(rows: list[list[str]]) -> None: + row_number = len(rows) + footer = rows[-1] + if len(footer) != 3 or tuple(footer[:2]) != _FOOTER_PREFIX: + raise DispatchScadaParseError(f"row {row_number}: invalid footer") + try: + footer_count = int(footer[2]) + except (TypeError, ValueError, OverflowError): + raise DispatchScadaParseError(f"row {row_number}: invalid footer count") from None + if str(footer_count) != footer[2] or footer_count != len(rows): + raise DispatchScadaParseError(f"row {row_number}: footer count mismatch") + + +def parse_dispatch_scada_csv( + payload: str, + *, + source_artifact_id: str, + ingestion_version: int, + correction_version: int = 0, + naive_timezone: tzinfo | None = None, +) -> tuple[GeneratorPower5m, ...]: + """Parse one complete canonical Dispatch SCADA CSV member. + + The returned records are UTC-normalized and sorted by their logical key. + Offset-free source timestamps require an explicit ``naive_timezone``. + """ + + rows = _read_rows(payload) + if not rows: + raise DispatchScadaParseError("row 1: missing metadata row") + _validate_metadata(rows[0], source_artifact_id=source_artifact_id) + if len(rows) < 2: + raise DispatchScadaParseError("row 2: missing header row") + _validate_header(rows[1]) + if len(rows) < 3: + raise DispatchScadaParseError("row 3: missing footer row") + _validate_footer(rows) + + data_rows = rows[2:-1] + if not data_rows: + raise DispatchScadaParseError("row 3: report contains no data rows") + + records: list[GeneratorPower5m] = [] + seen_keys: set[tuple[str, datetime]] = set() + for row_number, row in enumerate(data_rows, start=3): + if len(row) != 8: + raise DispatchScadaParseError(f"row {row_number}: invalid data field count") + if tuple(row[:4]) != _DATA_PREFIX: + raise DispatchScadaParseError(f"row {row_number}: invalid data structure") + record = _record_from_row( + row, + row_number, + source_artifact_id=source_artifact_id, + ingestion_version=ingestion_version, + correction_version=correction_version, + naive_timezone=naive_timezone, + ) + if record.logical_key in seen_keys: + raise DispatchScadaParseError(f"row {row_number}: duplicate logical key") + seen_keys.add(record.logical_key) + records.append(record) + + return tuple(sorted(records, key=lambda record: record.logical_key)) diff --git a/app/backend/batterywatch_api/dispatch_scada_ingestion.py b/app/backend/batterywatch_api/dispatch_scada_ingestion.py new file mode 100644 index 0000000..518a512 --- /dev/null +++ b/app/backend/batterywatch_api/dispatch_scada_ingestion.py @@ -0,0 +1,343 @@ +"""Atomic persistence for verified raw Dispatch SCADA observations.""" + +from __future__ import annotations + +from collections.abc import Mapping +from contextlib import contextmanager +from dataclasses import dataclass +from datetime import datetime, timedelta +from typing import Any, Iterable, Iterator, Protocol + + +@dataclass(frozen=True, slots=True) +class DispatchScadaIngestionResult: + raw_observation_count: int + mapped_power_count: int + replayed: bool + + +class DispatchScadaConflictError(Exception): + """Raised when an artifact identity is already bound to different evidence.""" + + +@dataclass(frozen=True, slots=True) +class DispatchScadaArtifactReceipt: + source_artifact_id: str + source_url: str + zip_filename: str + csv_member_name: str + report_timestamp: datetime + zip_sha256: str + raw_zip: bytes + + +@dataclass(frozen=True, slots=True) +class RawDispatchScadaObservation: + source_artifact_id: str + duid: str + interval_start: datetime + power_mw: float + source_timestamp: datetime + ingestion_version: int = 0 + correction_version: int = 0 + + +class _Cursor(Protocol): + def execute(self, statement: str, parameters: tuple[Any, ...]) -> None: ... + def fetchone(self) -> tuple[Any, ...] | None: ... + def close(self) -> None: ... + + +class _Connection(Protocol): + def cursor(self) -> _Cursor: ... + def commit(self) -> None: ... + def rollback(self) -> None: ... + + +@contextmanager +def _managed_cursor(connection: _Connection) -> Iterator[_Cursor]: + cursor = connection.cursor() + try: + yield cursor + finally: + cursor.close() + + +_ARTIFACT_INSERT_SQL = """ +INSERT INTO dispatch_scada_artifacts ( + source_artifact_id, source_url, zip_filename, csv_member_name, + report_timestamp, zip_sha256, raw_zip +) +VALUES (%s, %s, %s, %s, %s, %s, %s) +ON CONFLICT DO NOTHING RETURNING 1 +""" + +_ARTIFACT_SELECT_SQL = """ +SELECT source_artifact_id, source_url, zip_filename, csv_member_name, + report_timestamp, zip_sha256, raw_zip +FROM dispatch_scada_artifacts +WHERE source_artifact_id = %s +""" + +_OBSERVATION_INSERT_SQL = """ +INSERT INTO raw_dispatch_scada_observations ( + source_artifact_id, duid, interval_start, power_mw, source_timestamp, + ingestion_version, correction_version +) +VALUES (%s, %s, %s, %s, %s, %s, %s) +""" + +_GENERATOR_UPSERT_SQL = """ +INSERT INTO generators ( + generator_id, + site_name, + region, + capacity_mw, + storage_capacity_mwh, + data_start, + data_end, + source_id, + source_timestamp, + ingestion_version, + correction_version +) +VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) +ON CONFLICT (generator_id) DO UPDATE +SET site_name = EXCLUDED.site_name, + region = EXCLUDED.region, + capacity_mw = EXCLUDED.capacity_mw, + storage_capacity_mwh = EXCLUDED.storage_capacity_mwh, + data_start = EXCLUDED.data_start, + data_end = EXCLUDED.data_end, + source_id = EXCLUDED.source_id, + source_timestamp = EXCLUDED.source_timestamp, + ingestion_version = EXCLUDED.ingestion_version, + correction_version = EXCLUDED.correction_version, + updated_at = CURRENT_TIMESTAMP +WHERE (EXCLUDED.correction_version, EXCLUDED.ingestion_version, EXCLUDED.source_timestamp) + > (generators.correction_version, generators.ingestion_version, generators.source_timestamp) +RETURNING 1 +""" + +_POWER_UPSERT_SQL = """ +INSERT INTO generator_power_5m ( + generator_id, + interval_start, + power_mw, + source_id, + source_timestamp, + ingestion_version, + correction_version +) +VALUES (%s, %s, %s, %s, %s, %s, %s) +ON CONFLICT (generator_id, interval_start) DO UPDATE +SET power_mw = EXCLUDED.power_mw, + source_id = EXCLUDED.source_id, + source_timestamp = EXCLUDED.source_timestamp, + ingestion_version = EXCLUDED.ingestion_version, + correction_version = EXCLUDED.correction_version +WHERE (EXCLUDED.correction_version, EXCLUDED.ingestion_version, EXCLUDED.source_timestamp) + > (generator_power_5m.correction_version, + generator_power_5m.ingestion_version, generator_power_5m.source_timestamp) +RETURNING 1 +""" + +_GENERATOR_BOUNDS_UPDATE_SQL = """ +UPDATE generators +SET data_start = LEAST(COALESCE(generators.data_start, bounds.data_start), bounds.data_start), + data_end = GREATEST(COALESCE(generators.data_end, bounds.data_end), bounds.data_end), + updated_at = CURRENT_TIMESTAMP +FROM (VALUES (%s, %s, %s)) AS bounds(data_start, data_end, generator_id) +WHERE generators.generator_id = bounds.generator_id +""" + + +def _record_value(record: Any, *names: str) -> Any: + if isinstance(record, Mapping): + for name in names: + if name in record: + return record[name] + for name in names: + try: + return getattr(record, name) + except AttributeError: + pass + raise AttributeError(f"record has none of: {', '.join(names)}") + + +def _generator_id(record: Any) -> str: + return _record_value(record, "generator_id", "duid") + + +def _power_key(record: Any) -> tuple[Any, Any]: + return ( + _generator_id(record), + _record_value(record, "interval_start", "timestamp"), + ) + + +def _receipt_parameters(receipt: DispatchScadaArtifactReceipt) -> tuple[Any, ...]: + return ( + receipt.source_artifact_id, + receipt.source_url, + receipt.zip_filename, + receipt.csv_member_name, + receipt.report_timestamp, + receipt.zip_sha256, + receipt.raw_zip, + ) + + +def _normalized_bytes(value: Any) -> bytes: + if isinstance(value, memoryview): + return value.tobytes() + return bytes(value) + + +def _receipt_matches( + receipt: DispatchScadaArtifactReceipt, stored: tuple[Any, ...] +) -> bool: + if len(stored) != 7: + return False + expected = _receipt_parameters(receipt) + return expected[:6] == stored[:6] and _normalized_bytes(expected[6]) == _normalized_bytes( + stored[6] + ) + + +class PostgreSQLDispatchScadaIngestor: + def __init__(self, connection: _Connection): + self._connection = connection + + def ingest( + self, + receipt: DispatchScadaArtifactReceipt, + observations: Iterable[RawDispatchScadaObservation], + generators: Iterable[Any] = (), + power_records: Iterable[Any] = (), + ) -> DispatchScadaIngestionResult: + try: + materialized = tuple(observations) + if not materialized: + raise ValueError("observations must not be empty") + for observation in materialized: + if observation.source_artifact_id != receipt.source_artifact_id: + raise ValueError("observation source artifact does not match receipt") + + generator_rows = tuple(generators) + generator_by_id: dict[Any, Any] = {} + for generator in generator_rows: + generator_id = _generator_id(generator) + if generator_id in generator_by_id: + raise ValueError("duplicate generator metadata key") + generator_by_id[generator_id] = generator + + power_rows = tuple(power_records) + power_keys: set[tuple[Any, Any]] = set() + power_by_generator: dict[Any, list[Any]] = {} + for power in power_rows: + key = _power_key(power) + if key in power_keys: + raise ValueError("duplicate generator power key") + power_keys.add(key) + if key[0] not in generator_by_id: + raise ValueError("power record generator is not in metadata") + power_by_generator.setdefault(key[0], []).append(power) + + replayed = False + with _managed_cursor(self._connection) as cursor: + cursor.execute(_ARTIFACT_INSERT_SQL, _receipt_parameters(receipt)) + if cursor.fetchone() is None: + cursor.execute( + _ARTIFACT_SELECT_SQL, (receipt.source_artifact_id,) + ) + stored = cursor.fetchone() + if stored is None or not _receipt_matches(receipt, stored): + raise DispatchScadaConflictError( + "dispatch SCADA artifact conflicts with stored evidence" + ) + replayed = True + else: + for observation in materialized: + cursor.execute( + _OBSERVATION_INSERT_SQL, + ( + observation.source_artifact_id, + observation.duid, + observation.interval_start, + observation.power_mw, + observation.source_timestamp, + observation.ingestion_version, + observation.correction_version, + ), + ) + + for generator in generator_rows: + generator_id = _generator_id(generator) + cursor.execute( + _GENERATOR_UPSERT_SQL, + ( + generator_id, + _record_value(generator, "site_name"), + _record_value(generator, "region"), + _record_value(generator, "capacity_mw"), + _record_value(generator, "storage_capacity_mwh"), + _record_value(generator, "data_start"), + _record_value(generator, "data_end"), + _record_value(generator, "source_id"), + _record_value(generator, "source_timestamp"), + _record_value(generator, "ingestion_version"), + _record_value(generator, "correction_version"), + ), + ) + cursor.fetchone() + + for power in power_rows: + cursor.execute( + _POWER_UPSERT_SQL, + ( + _generator_id(power), + _record_value(power, "interval_start", "timestamp"), + _record_value(power, "power_mw"), + _record_value(power, "source_id"), + _record_value(power, "source_timestamp"), + _record_value(power, "ingestion_version"), + _record_value(power, "correction_version"), + ), + ) + cursor.fetchone() + + for generator_id, generator_power_rows in power_by_generator.items(): + interval_starts = tuple( + _record_value(power, "interval_start", "timestamp") + for power in generator_power_rows + ) + cursor.execute( + _GENERATOR_BOUNDS_UPDATE_SQL, + ( + min(interval_starts), + max(interval_starts) + timedelta(minutes=5), + generator_id, + ), + ) + self._connection.commit() + if replayed: + return DispatchScadaIngestionResult(0, 0, True) + return DispatchScadaIngestionResult( + len(materialized), len(power_rows), False + ) + except Exception: + try: + self._connection.rollback() + except Exception: + pass + raise + + +__all__ = [ + "DispatchScadaArtifactReceipt", + "DispatchScadaConflictError", + "DispatchScadaIngestionResult", + "PostgreSQLDispatchScadaIngestor", + "RawDispatchScadaObservation", +] diff --git a/app/backend/batterywatch_api/fixtures.py b/app/backend/batterywatch_api/fixtures.py new file mode 100644 index 0000000..38aeac6 --- /dev/null +++ b/app/backend/batterywatch_api/fixtures.py @@ -0,0 +1,56 @@ +"""Deterministic five-minute fixture data for the first API slice.""" + +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from typing import Literal, TypedDict + +FIXTURE_DUID = "BWTEST1" +FIXTURE_REGION = "NSW1" +INTERVAL = timedelta(minutes=5) +START = datetime(2026, 1, 1, tzinfo=timezone.utc) + + +@dataclass(frozen=True) +class FixturePoint: + timestamp: datetime + power_mw: float + price_aud_per_mwh: float | None + soc_percent: float | None + + +class GeneratorMetadata(TypedDict): + duid: str + site_name: str + region: str + capacity_mw: float + storage_capacity_mwh: float + data_start: datetime + data_end: datetime + data_status: Literal["fixture"] + + +# The first five intervals intentionally exercise discharge, charging, zero, +# negative-price discharge, and missing-price handling. +_POWER = (1.5, -1.0, 0.0, 2.0, 1.0, 0.0, -0.5, 0.75, 0.0, 1.25, -0.75, 0.0) +_PRICE = (100.0, 80.0, 50.0, -20.0, None, 40.0, 30.0, 120.0, 25.0, 90.0, 70.0, 60.0) +_SOC = (45.0, 42.0, 42.0, 48.0, None, 50.0, 47.0, 49.0, 49.0, 53.0, 51.0, 51.0) + + +def generator_metadata() -> GeneratorMetadata: + return { + "duid": FIXTURE_DUID, + "site_name": "BatteryWatch Fixture", + "region": FIXTURE_REGION, + "capacity_mw": 2.0, + "storage_capacity_mwh": 4.0, + "data_start": START, + "data_end": START + len(_POWER) * INTERVAL, + "data_status": "fixture", + } + + +def fixture_points() -> list[FixturePoint]: + return [ + FixturePoint(START + index * INTERVAL, power, price, soc) + for index, (power, price, soc) in enumerate(zip(_POWER, _PRICE, _SOC)) + ] diff --git a/app/backend/batterywatch_api/main.py b/app/backend/batterywatch_api/main.py new file mode 100644 index 0000000..497c1e6 --- /dev/null +++ b/app/backend/batterywatch_api/main.py @@ -0,0 +1,433 @@ +"""Standalone BatteryWatch HTTP API.""" + +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import datetime, timezone +import os +from pathlib import Path + +from fastapi import APIRouter, FastAPI, HTTPException, Query, Request +from fastapi.middleware.cors import CORSMiddleware +from fastapi.staticfiles import StaticFiles + +from .fixtures import FIXTURE_DUID, INTERVAL, fixture_points, generator_metadata +from .runtime import ( + ConnectionFactory, + DatabaseHealthTracer, + RepositoryProvider, + configured_database_url, + database_repository_provider, +) +from .models import ( + Coverage, + EstimateMetadata, + Generator, + GeneratorListResponse, + HealthResponse, + ProvenanceMetadata, + SeriesPoint, + SeriesResponse, + SeriesSummary, +) + +router = APIRouter() + +MAX_WINDOW_SECONDS = 7 * 24 * 60 * 60 + + +def _utc(value: datetime, field: str) -> datetime: + if value.tzinfo is None: + raise HTTPException(status_code=400, detail=f"{field} must include a UTC offset") + return value.astimezone(timezone.utc) + + +def _point(raw) -> SeriesPoint: + energy = raw.power_mw * 5 / 60 + status = "missing" if raw.price_aud_per_mwh is None else ( + "negative" if raw.price_aud_per_mwh < 0 else "available" + ) + if raw.price_aud_per_mwh is None: + gross = charging = net = None + elif raw.power_mw > 0: + gross = energy * raw.price_aud_per_mwh + charging = 0.0 + net = gross + elif raw.power_mw < 0: + gross = 0.0 + charging = abs(energy) * raw.price_aud_per_mwh + net = -charging + else: + gross = charging = net = 0.0 + return SeriesPoint( + timestamp=raw.timestamp, + power_mw=raw.power_mw, + soc_percent=raw.soc_percent, + price_aud_per_mwh=raw.price_aud_per_mwh, + energy_mwh=energy, + gross_value_aud=gross, + charging_cost_aud=charging, + net_energy_value_aud=net, + price_status=status, + ) + + +def _database_generator(record) -> Generator: + def optional_utc(value: datetime | None) -> datetime | None: + if value is None: + return None + if value.tzinfo is None or value.utcoffset() is None: + raise ValueError("database generator bounds must include a UTC offset") + return value.astimezone(timezone.utc) + + return Generator( + duid=record.generator_id, + site_name=record.site_name, + region=record.region, + capacity_mw=record.capacity_mw, + storage_capacity_mwh=record.storage_capacity_mwh, + data_start=optional_utc(record.data_start), + data_end=optional_utc(record.data_end), + data_status="database", + ) + + +def _tracer_is_healthy(tracer: object) -> bool: + try: + if callable(tracer): + return bool(tracer()) + check = getattr(tracer, "check", None) + return callable(check) and bool(check()) + except Exception: + return False + + +@router.get("/api/health", response_model=HealthResponse) +def health(request: Request) -> HealthResponse: + if request.app.state.data_mode == "database": + if not _tracer_is_healthy(request.app.state.health_tracer): + raise HTTPException(status_code=503, detail="Database health unavailable") + return HealthResponse(service="batterywatch-api", data_mode="database") + return HealthResponse(service="batterywatch-api") + + +def _ensure_fixture_runtime(request: Request) -> None: + if request.app.state.data_mode != "fixture": + raise HTTPException(status_code=503, detail="Database data reads unavailable") + + +def _database_bounds(start: str | None, end: str | None) -> tuple[datetime, datetime]: + if start is None or end is None: + raise HTTPException( + status_code=400, + detail="Database series requires explicit start and end", + ) + try: + parsed_start = datetime.fromisoformat(start.replace("Z", "+00:00")) + parsed_end = datetime.fromisoformat(end.replace("Z", "+00:00")) + except ValueError: + raise HTTPException( + status_code=400, + detail="start and end must be ISO-8601 timestamps", + ) from None + requested_start = _utc(parsed_start, "start") + requested_end = _utc(parsed_end, "end") + if requested_end <= requested_start: + raise HTTPException(status_code=400, detail="end must be after start") + if (requested_end - requested_start).total_seconds() > MAX_WINDOW_SECONDS: + raise HTTPException( + status_code=400, + detail="Requested range exceeds the seven-day limit", + ) + return requested_start, requested_end + + +def _series_response( + generator: Generator, + requested_start: datetime, + requested_end: datetime, + raw_points, + provenance: ProvenanceMetadata, +) -> SeriesResponse: + points = [ + _point(item) + for item in raw_points + if requested_start <= item.timestamp < requested_end + ] + priced = [item for item in points if item.price_aud_per_mwh is not None] + soc_points = [item for item in points if item.soc_percent is not None] + exported = sum(item.energy_mwh for item in points if item.energy_mwh > 0) + imported = sum(-item.energy_mwh for item in points if item.energy_mwh < 0) + gross = sum(item.gross_value_aud or 0 for item in points) + charging = sum(item.charging_cost_aud or 0 for item in points) + return SeriesResponse( + generator=generator, + requested_start=requested_start, + requested_end=requested_end, + points=points, + summary=SeriesSummary( + interval_count=len(points), + interval_hours=5 / 60, + total_energy_mwh=sum(item.energy_mwh for item in points), + exported_energy_mwh=exported, + imported_energy_mwh=imported, + gross_value_aud=gross, + charging_cost_aud=charging, + net_energy_value_aud=gross - charging, + ), + coverage=Coverage( + total_intervals=len(points), + price_intervals=len(priced), + missing_price_intervals=len(points) - len(priced), + price_coverage_percent=(len(priced) / len(points) * 100) if points else 0, + soc_intervals=len(soc_points), + missing_soc_intervals=len(points) - len(soc_points), + soc_coverage_percent=(len(soc_points) / len(points) * 100) if points else 0, + ), + estimate=EstimateMetadata( + label="Estimated gross energy value", + disclaimer="Estimate only; excludes efficiency losses, auxiliary use, degradation, FCAS, network charges, contracts, tax, and fees.", + calculation="energy_mwh = power_mw × 5/60; value_aud = energy_mwh × price_aud_per_mwh", + ), + provenance=provenance, + ) + + +@dataclass(frozen=True, slots=True) +class _DatabasePoint: + timestamp: datetime + power_mw: float + soc_percent: float | None + price_aud_per_mwh: float | None + + +def _database_points( + power_rows, + soc_rows, + price_rows, + requested_start: datetime, + requested_end: datetime, +) -> list[_DatabasePoint]: + soc_by_timestamp = { + row.timestamp: row.soc_percent + for row in soc_rows + if requested_start <= row.timestamp < requested_end + } + price_by_timestamp = { + row.timestamp: row.price_aud_per_mwh + for row in price_rows + if requested_start <= row.timestamp < requested_end + } + return [ + _DatabasePoint( + timestamp=row.timestamp, + power_mw=row.power_mw, + soc_percent=soc_by_timestamp.get(row.timestamp), + price_aud_per_mwh=price_by_timestamp.get(row.timestamp), + ) + for row in power_rows + if requested_start <= row.timestamp < requested_end + ] + + +def _safe_source_label(rows, fallback: str) -> str: + source_ids = { + row.source_id + for row in rows + if isinstance(getattr(row, "source_id", None), str) + and 0 < len(row.source_id) <= 128 + and row.source_id.isascii() + and all(character.isalnum() or character in "._-" for character in row.source_id) + } + return ", ".join(sorted(source_ids)) or fallback + + +def _database_series( + request: Request, + generator: str, + start: str | None, + end: str | None, +) -> SeriesResponse: + if ( + (start is None or end is None) + and not request.app.state.database_url + and not request.app.state.repository_provider_injected + ): + raise HTTPException(status_code=503, detail="Database series unavailable") + requested_start, requested_end = _database_bounds(start, end) + found = False + result: SeriesResponse | None = None + try: + with request.app.state.repository_provider() as repository: + metadata = repository.read_generator(generator) + if metadata is not None: + found = True + power_rows = repository.list_power( + generator, start=requested_start, end=requested_end + ) + soc_rows = repository.list_soc( + generator, start=requested_start, end=requested_end + ) + price_rows = repository.list_prices( + metadata.region, start=requested_start, end=requested_end + ) + result = _series_response( + _database_generator(metadata), + requested_start, + requested_end, + _database_points( + power_rows, + soc_rows, + price_rows, + requested_start, + requested_end, + ), + ProvenanceMetadata( + data_mode="database", + power_source=_safe_source_label( + power_rows, "database generator_power_5m" + ), + price_source=_safe_source_label( + price_rows, "database nem_price_5m" + ), + soc_source=_safe_source_label( + soc_rows, "database generator_soc_5m" + ), + sign_convention="positive discharge/export; negative charge/import", + calculation_version="estimate-v1", + ), + ) + except Exception: + raise HTTPException(status_code=503, detail="Database series unavailable") from None + if not found: + raise HTTPException(status_code=404, detail="Unknown generator") + assert result is not None + return result + + +@router.get("/api/generators", response_model=GeneratorListResponse) +def generators(request: Request) -> GeneratorListResponse: + if request.app.state.data_mode == "fixture": + return GeneratorListResponse(generators=[Generator(**generator_metadata())]) + try: + with request.app.state.repository_provider() as repository: + records = repository.list_generators() + return GeneratorListResponse( + generators=[_database_generator(record) for record in records] + ) + except Exception: + raise HTTPException( + status_code=503, detail="Database generators unavailable" + ) from None + + +@router.get("/api/series", response_model=SeriesResponse) +def series( + request: Request, + generator: str = Query(..., min_length=1), + start: str | None = None, + end: str | None = None, +) -> SeriesResponse: + if request.app.state.data_mode == "database": + return _database_series(request, generator, start, end) + _ensure_fixture_runtime(request) + if generator != FIXTURE_DUID: + raise HTTPException(status_code=404, detail="Unknown generator") + raw_points = fixture_points() + default_start = raw_points[0].timestamp + default_end = raw_points[-1].timestamp + INTERVAL + try: + requested_start = _utc(datetime.fromisoformat(start.replace("Z", "+00:00")), "start") if start else default_start + requested_end = _utc(datetime.fromisoformat(end.replace("Z", "+00:00")), "end") if end else default_end + except ValueError as exc: + raise HTTPException(status_code=400, detail="start and end must be ISO-8601 timestamps") from exc + if requested_end <= requested_start: + raise HTTPException(status_code=400, detail="end must be after start") + if (requested_end - requested_start).total_seconds() > MAX_WINDOW_SECONDS: + raise HTTPException(status_code=400, detail="Requested range exceeds the seven-day limit") + + return _series_response( + Generator(**generator_metadata()), + requested_start, + requested_end, + raw_points, + ProvenanceMetadata( + power_source="deterministic five-minute fixture", + price_source="AEMO RRP-shaped deterministic fixture", + soc_source="deterministic fixture; nullable when unavailable", + sign_convention="positive discharge/export; negative charge/import", + calculation_version="estimate-v1", + ), + ) + + +frontend_dist = Path(__file__).resolve().parents[2] / "frontend" / "dist" + + +def create_app( + mode: str = "fixture", + health_tracer: object | None = None, + database_url: str | None = None, + connection_factory: ConnectionFactory | None = None, + data_mode: str | None = None, + repository_provider: RepositoryProvider | None = None, +) -> FastAPI: + selected_mode = mode if data_mode is None else data_mode + if selected_mode not in {"fixture", "database"}: + raise ValueError("mode must be fixture or database") + application = FastAPI(title="BatteryWatch API", version="0.1.0") + application.add_middleware( + CORSMiddleware, + allow_origins=["http://localhost:5173", "http://127.0.0.1:5173"], + allow_methods=["GET"], + allow_headers=["*"], + ) + configured_url = configured_database_url(database_url) if selected_mode == "database" else None + application.state.data_mode = selected_mode + application.state.database_url = configured_url + application.state.connection_factory = connection_factory + application.state.repository_provider_injected = repository_provider is not None + application.state.repository_provider = ( + repository_provider + if repository_provider is not None + else ( + (lambda: database_repository_provider(configured_url, connection_factory)) + if selected_mode == "database" + else None + ) + ) + application.state.health_tracer = ( + health_tracer + if health_tracer is not None + else DatabaseHealthTracer(configured_url, connection_factory) + if selected_mode == "database" + else None + ) + application.include_router(router) + if frontend_dist.is_dir(): + application.mount("/", StaticFiles(directory=frontend_dist, html=True), name="frontend") + return application + + +def create_runtime_app( + environ: Mapping[str, str] | None = None, + *, + health_tracer: object | None = None, + connection_factory: ConnectionFactory | None = None, + repository_provider: RepositoryProvider | None = None, +) -> FastAPI: + """Create the deployed application from explicit fail-closed environment mode.""" + + environment = os.environ if environ is None else environ + database_url = environment.get("BATTERYWATCH_DATABASE_URL") or environment.get( + "DATABASE_URL" + ) + return create_app( + data_mode=environment.get("BATTERYWATCH_DATA_MODE", "fixture"), + database_url=database_url, + health_tracer=health_tracer, + connection_factory=connection_factory, + repository_provider=repository_provider, + ) + + +app = create_runtime_app() \ No newline at end of file diff --git a/app/backend/batterywatch_api/models.py b/app/backend/batterywatch_api/models.py new file mode 100644 index 0000000..c83756c --- /dev/null +++ b/app/backend/batterywatch_api/models.py @@ -0,0 +1,109 @@ +"""Typed response models for the BatteryWatch API.""" + +from datetime import datetime, timezone +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, field_serializer + + +def utc_iso(value: datetime) -> str: + return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + + +class ApiModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + +class Generator(ApiModel): + duid: str + site_name: str + region: str + capacity_mw: float = Field(gt=0) + storage_capacity_mwh: float = Field(gt=0) + data_start: datetime | None + data_end: datetime | None + data_status: Literal["fixture", "database"] = "fixture" + + @field_serializer("data_start", "data_end") + def serialize_bounds(self, value: datetime | None) -> str | None: + return None if value is None else utc_iso(value) + + +class GeneratorListResponse(ApiModel): + generators: list[Generator] + + +class HealthResponse(ApiModel): + status: Literal["ok"] = "ok" + service: str + data_mode: Literal["fixture", "database"] = "fixture" + + +class SeriesPoint(ApiModel): + timestamp: datetime + power_mw: float + soc_percent: float | None + price_aud_per_mwh: float | None + energy_mwh: float + gross_value_aud: float | None + charging_cost_aud: float | None + net_energy_value_aud: float | None + price_status: Literal["available", "negative", "missing"] + + @field_serializer("timestamp") + def serialize_timestamp(self, value: datetime) -> str: + return utc_iso(value) + + +class SeriesSummary(ApiModel): + interval_count: int = Field(ge=0) + interval_hours: float = Field(gt=0) + total_energy_mwh: float + exported_energy_mwh: float = Field(ge=0) + imported_energy_mwh: float = Field(ge=0) + gross_value_aud: float + charging_cost_aud: float + net_energy_value_aud: float + + +class Coverage(ApiModel): + total_intervals: int = Field(ge=0) + price_intervals: int = Field(ge=0) + missing_price_intervals: int = Field(ge=0) + price_coverage_percent: float = Field(ge=0, le=100) + soc_intervals: int = Field(ge=0) + missing_soc_intervals: int = Field(ge=0) + soc_coverage_percent: float = Field(ge=0, le=100) + + +class EstimateMetadata(ApiModel): + is_estimate: Literal[True] = True + label: str + disclaimer: str + calculation: str + + +class ProvenanceMetadata(ApiModel): + data_mode: Literal["deterministic_fixture", "database"] = "deterministic_fixture" + power_source: str + price_source: str + soc_source: str + timezone: Literal["UTC"] = "UTC" + interval_minutes: Literal[5] = 5 + sign_convention: str + calculation_version: str + + +class SeriesResponse(ApiModel): + generator: Generator + requested_start: datetime + requested_end: datetime + points: list[SeriesPoint] + summary: SeriesSummary + coverage: Coverage + estimate: EstimateMetadata + provenance: ProvenanceMetadata + + @field_serializer("requested_start", "requested_end") + def serialize_bounds(self, value: datetime) -> str: + return utc_iso(value) diff --git a/app/backend/batterywatch_api/nemweb_dispatch_prices.py b/app/backend/batterywatch_api/nemweb_dispatch_prices.py new file mode 100644 index 0000000..ed17a8b --- /dev/null +++ b/app/backend/batterywatch_api/nemweb_dispatch_prices.py @@ -0,0 +1,284 @@ +"""Official NEMWeb DispatchIS regional-price artifact adapter.""" + +from collections.abc import Callable +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from hashlib import sha256 +from html.parser import HTMLParser +from io import BytesIO +import re +from urllib.parse import urljoin, urlsplit +import zlib +from zipfile import ZIP_DEFLATED, ZIP_STORED, BadZipFile, LargeZipFile, ZipFile + +from .aemo import parse_dispatch_price_mms_csv +from .nemweb_http import NemwebHttpResource, fetch_nemweb_resource +from .storage import RegionalPrice5m + + +_INDEX_PATH = "/REPORTS/CURRENT/DispatchIS_Reports/" +_OBSERVED_INDEX_PATH = "/Reports/CURRENT/DispatchIS_Reports/" +DISPATCH_PRICE_INDEX_URL = "https://www.nemweb.com.au" + _INDEX_PATH +DISPATCH_PRICE_INDEX_MAX_BYTES = 2 * 1024 * 1024 +DISPATCH_PRICE_ARTIFACT_MAX_BYTES = 16 * 1024 * 1024 +_MAX_CSV_BYTES = 16 * 1024 * 1024 +_MAX_COMPRESSION_RATIO = 100 +_NEM_TIMEZONE = timezone(timedelta(hours=10)) +_FILENAME_RE = re.compile( + r"PUBLIC_DISPATCHIS_(?P[0-9]{12})_(?P[0-9]{1,32})\.zip" +) + + +class NemwebDispatchPriceError(ValueError): + """Raised when a public DispatchIS source input is not canonical.""" + + +@dataclass(frozen=True, slots=True) +class DispatchPriceArtifactRef: + url: str + zip_filename: str + source_artifact_id: str + report_timestamp: datetime + + +@dataclass(frozen=True, slots=True) +class DispatchPriceArtifact: + reference: DispatchPriceArtifactRef + csv_member_name: str + csv_payload: str + zip_sha256: str + raw_zip: bytes + + +@dataclass(frozen=True, slots=True) +class DispatchPriceCollection: + artifact: DispatchPriceArtifact + records: tuple[RegionalPrice5m, ...] + + +class _HrefCollector(HTMLParser): + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.hrefs: list[str] = [] + + def handle_starttag( + self, + tag: str, + attrs: list[tuple[str, str | None]], + ) -> None: + if tag != "a": + return + for name, value in attrs: + if name == "href" and value is not None: + self.hrefs.append(value) + + +def discover_dispatch_price_artifacts( + index_html: str, + *, + index_url: str, +) -> tuple[DispatchPriceArtifactRef, ...]: + """Return canonical current DispatchIS references in chronological order.""" + + if index_url != DISPATCH_PRICE_INDEX_URL or not isinstance(index_html, str): + raise NemwebDispatchPriceError("invalid DispatchIS index") + try: + encoded_size = len(index_html.encode("utf-8")) + except UnicodeEncodeError as exc: + raise NemwebDispatchPriceError("invalid DispatchIS index") from exc + if encoded_size == 0 or encoded_size > DISPATCH_PRICE_INDEX_MAX_BYTES: + raise NemwebDispatchPriceError("invalid DispatchIS index") + + parser = _HrefCollector() + parser.feed(index_html) + expected_origin = urlsplit(index_url) + references: dict[str, DispatchPriceArtifactRef] = {} + source_urls: dict[str, str] = {} + for href in parser.hrefs: + try: + url = urljoin(index_url, href) + parts = urlsplit(url) + except ValueError: + continue + if (parts.scheme, parts.netloc) != (expected_origin.scheme, expected_origin.netloc): + continue + source_path = next( + ( + path + for path in (_INDEX_PATH, _OBSERVED_INDEX_PATH) + if parts.path.startswith(path) + ), + None, + ) + if source_path is None: + continue + filename = parts.path[len(source_path):] + if not filename or "/" in filename or parts.query or parts.fragment: + continue + match = _FILENAME_RE.fullmatch(filename) + if match is None: + continue + try: + report_timestamp = datetime.strptime( + match.group("timestamp"), + "%Y%m%d%H%M", + ).replace(tzinfo=_NEM_TIMEZONE).astimezone(timezone.utc) + except (OverflowError, ValueError): + continue + source_id = match.group("source_id") + canonical_url = DISPATCH_PRICE_INDEX_URL + filename + existing_url = source_urls.get(source_id) + if existing_url is not None and existing_url != canonical_url: + raise NemwebDispatchPriceError("conflicting DispatchIS artifact references") + source_urls[source_id] = canonical_url + references[canonical_url] = DispatchPriceArtifactRef( + url=canonical_url, + zip_filename=filename, + source_artifact_id=source_id, + report_timestamp=report_timestamp, + ) + + if not references: + raise NemwebDispatchPriceError("invalid DispatchIS index") + return tuple(sorted( + references.values(), + key=lambda item: ( + item.report_timestamp, + int(item.source_artifact_id), + item.zip_filename, + ), + )) + + +def extract_dispatch_price_zip( + reference: DispatchPriceArtifactRef, + zip_payload: bytes, +) -> DispatchPriceArtifact: + """Extract one canonical DispatchIS CSV member with bounded resources.""" + + if ( + not isinstance(reference, DispatchPriceArtifactRef) + or type(zip_payload) is not bytes + or not zip_payload + or len(zip_payload) > DISPATCH_PRICE_ARTIFACT_MAX_BYTES + ): + raise NemwebDispatchPriceError("invalid DispatchIS ZIP") + match = _FILENAME_RE.fullmatch(reference.zip_filename) + if match is None: + raise NemwebDispatchPriceError("invalid DispatchIS ZIP") + try: + expected_timestamp = datetime.strptime( + match.group("timestamp"), + "%Y%m%d%H%M", + ).replace(tzinfo=_NEM_TIMEZONE).astimezone(timezone.utc) + except (OverflowError, ValueError) as exc: + raise NemwebDispatchPriceError("invalid DispatchIS ZIP") from exc + if ( + reference.url != DISPATCH_PRICE_INDEX_URL + reference.zip_filename + or reference.source_artifact_id != match.group("source_id") + or reference.report_timestamp != expected_timestamp + ): + raise NemwebDispatchPriceError("invalid DispatchIS ZIP") + + try: + with ZipFile(BytesIO(zip_payload)) as archive: + members = archive.infolist() + expected_member = reference.zip_filename.removesuffix(".zip") + ".CSV" + if ( + len(members) != 1 + or members[0].is_dir() + or members[0].filename != expected_member + or members[0].orig_filename != expected_member + ): + raise NemwebDispatchPriceError("invalid DispatchIS ZIP") + member = members[0] + if ( + member.flag_bits & 1 + or member.compress_type not in (ZIP_STORED, ZIP_DEFLATED) + or member.file_size <= 0 + or member.file_size > _MAX_CSV_BYTES + or member.compress_size <= 0 + or member.file_size > member.compress_size * _MAX_COMPRESSION_RATIO + ): + raise NemwebDispatchPriceError("invalid DispatchIS ZIP") + with archive.open(member) as member_stream: + raw_csv = member_stream.read(_MAX_CSV_BYTES + 1) + has_more_data = bool(member_stream.read(1)) + if ( + len(raw_csv) > _MAX_CSV_BYTES + or has_more_data + or len(raw_csv) != member.file_size + ): + raise NemwebDispatchPriceError("invalid DispatchIS ZIP") + csv_payload = raw_csv.decode("utf-8-sig") + if not csv_payload or "\x00" in csv_payload: + raise NemwebDispatchPriceError("invalid DispatchIS ZIP") + except NemwebDispatchPriceError: + raise + except ( + BadZipFile, + EOFError, + LargeZipFile, + NotImplementedError, + RuntimeError, + UnicodeDecodeError, + zlib.error, + ) as exc: + raise NemwebDispatchPriceError("invalid DispatchIS ZIP") from exc + + return DispatchPriceArtifact( + reference=reference, + csv_member_name=member.filename, + csv_payload=csv_payload, + zip_sha256=sha256(zip_payload).hexdigest(), + raw_zip=zip_payload, + ) + + +def collect_latest_dispatch_prices( + ingestion_version: int, + correction_version: int = 0, + *, + fetch: Callable[..., NemwebHttpResource] = fetch_nemweb_resource, +) -> DispatchPriceCollection: + """Fetch, verify and parse the latest official five-region price report.""" + + index_resource = fetch( + DISPATCH_PRICE_INDEX_URL, + max_bytes=DISPATCH_PRICE_INDEX_MAX_BYTES, + ) + try: + index_html = index_resource.body.decode("utf-8") + except UnicodeDecodeError as exc: + raise NemwebDispatchPriceError("invalid DispatchIS index") from exc + references = discover_dispatch_price_artifacts( + index_html, + index_url=DISPATCH_PRICE_INDEX_URL, + ) + latest = references[-1] + artifact_resource = fetch( + latest.url, + max_bytes=DISPATCH_PRICE_ARTIFACT_MAX_BYTES, + ) + artifact = extract_dispatch_price_zip(latest, artifact_resource.body) + records = parse_dispatch_price_mms_csv( + artifact.csv_payload, + source_id=artifact.reference.source_artifact_id, + ingestion_version=ingestion_version, + correction_version=correction_version, + ) + return DispatchPriceCollection(artifact=artifact, records=records) + + +__all__ = [ + "DISPATCH_PRICE_ARTIFACT_MAX_BYTES", + "DISPATCH_PRICE_INDEX_MAX_BYTES", + "DISPATCH_PRICE_INDEX_URL", + "DispatchPriceArtifact", + "DispatchPriceArtifactRef", + "DispatchPriceCollection", + "NemwebDispatchPriceError", + "collect_latest_dispatch_prices", + "discover_dispatch_price_artifacts", + "extract_dispatch_price_zip", +] diff --git a/app/backend/batterywatch_api/nemweb_dispatch_scada.py b/app/backend/batterywatch_api/nemweb_dispatch_scada.py new file mode 100644 index 0000000..67b5471 --- /dev/null +++ b/app/backend/batterywatch_api/nemweb_dispatch_scada.py @@ -0,0 +1,227 @@ +"""Pure source metadata handling for NEMWeb Dispatch SCADA artifacts.""" + +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from hashlib import sha256 +from html.parser import HTMLParser +from io import BytesIO +import re +from urllib.parse import urljoin, urlsplit +import zlib +from zipfile import ZIP_DEFLATED, ZIP_STORED, BadZipFile, LargeZipFile, ZipFile + + +_INDEX_PATH = "/REPORTS/CURRENT/Dispatch_SCADA/" +_OBSERVED_INDEX_PATH = "/Reports/CURRENT/Dispatch_SCADA/" +_OFFICIAL_INDEX_URL = "https://www.nemweb.com.au" + _INDEX_PATH +_MAX_INDEX_BYTES = 2 * 1024 * 1024 +_MAX_ZIP_BYTES = 16 * 1024 * 1024 +_MAX_CSV_BYTES = 8 * 1024 * 1024 +_MAX_COMPRESSION_RATIO = 100 +_NEM_TIMEZONE = timezone(timedelta(hours=10)) +_FILENAME_RE = re.compile( + r"PUBLIC_DISPATCHSCADA_(?P[0-9]{12})_(?P[0-9]{1,32})\.zip" +) + + +@dataclass(frozen=True) +class DispatchScadaArtifactRef: + """Canonical identity for one current Dispatch SCADA ZIP.""" + + url: str + zip_filename: str + source_artifact_id: str + report_timestamp: datetime + + +class NemwebDispatchScadaError(ValueError): + """Raised when a public NEMWeb artifact source input is not canonical.""" + + +@dataclass(frozen=True) +class DispatchScadaArtifact: + """One verified ZIP and decoded CSV member with immutable provenance.""" + + reference: DispatchScadaArtifactRef + csv_member_name: str + csv_payload: str + zip_sha256: str + raw_zip: bytes + + +class _HrefCollector(HTMLParser): + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.hrefs: list[str] = [] + + def handle_starttag( + self, tag: str, attrs: list[tuple[str, str | None]] + ) -> None: + if tag != "a": + return + for name, value in attrs: + if name == "href" and value is not None: + self.hrefs.append(value) + + +def discover_dispatch_scada_artifacts( + index_html: str, *, index_url: str +) -> tuple[DispatchScadaArtifactRef, ...]: + """Return canonical current Dispatch SCADA references in source order.""" + + if index_url != _OFFICIAL_INDEX_URL or type(index_html) is not str: + raise NemwebDispatchScadaError("invalid Dispatch SCADA index") + try: + encoded_size = len(index_html.encode("utf-8")) + except UnicodeEncodeError: + raise NemwebDispatchScadaError("invalid Dispatch SCADA index") from None + if encoded_size == 0 or encoded_size > _MAX_INDEX_BYTES: + raise NemwebDispatchScadaError("invalid Dispatch SCADA index") + + parser = _HrefCollector() + parser.feed(index_html) + expected_origin = urlsplit(index_url) + references: dict[str, DispatchScadaArtifactRef] = {} + source_urls: dict[str, str] = {} + + for href in parser.hrefs: + try: + url = urljoin(index_url, href) + parts = urlsplit(url) + except ValueError: + continue + if (parts.scheme, parts.netloc) != (expected_origin.scheme, expected_origin.netloc): + continue + source_path = next( + ( + candidate + for candidate in (_INDEX_PATH, _OBSERVED_INDEX_PATH) + if parts.path.startswith(candidate) + ), + None, + ) + if source_path is None: + continue + filename = parts.path[len(source_path) :] + if not filename or "/" in filename or parts.query or parts.fragment: + continue + match = _FILENAME_RE.fullmatch(filename) + if match is None: + continue + try: + report_timestamp = datetime.strptime( + match.group("timestamp"), "%Y%m%d%H%M" + ).replace(tzinfo=_NEM_TIMEZONE).astimezone(timezone.utc) + except (ValueError, OverflowError): + continue + source_id = match.group("source_id") + canonical_url = _OFFICIAL_INDEX_URL + filename + existing_url = source_urls.get(source_id) + if existing_url is not None and existing_url != canonical_url: + raise NemwebDispatchScadaError( + "conflicting Dispatch SCADA artifact references" + ) + source_urls[source_id] = canonical_url + references[canonical_url] = DispatchScadaArtifactRef( + url=canonical_url, + zip_filename=filename, + source_artifact_id=source_id, + report_timestamp=report_timestamp, + ) + + if not references: + raise NemwebDispatchScadaError("invalid Dispatch SCADA index") + + return tuple( + sorted( + references.values(), + key=lambda item: ( + item.report_timestamp, + int(item.source_artifact_id), + item.zip_filename, + ), + ) + ) + + +def extract_dispatch_scada_zip( + reference: DispatchScadaArtifactRef, + zip_payload: bytes, +) -> DispatchScadaArtifact: + """Extract one canonical Dispatch SCADA CSV member.""" + + if ( + type(reference) is not DispatchScadaArtifactRef + or type(zip_payload) is not bytes + or not zip_payload + or len(zip_payload) > _MAX_ZIP_BYTES + ): + raise NemwebDispatchScadaError("invalid Dispatch SCADA ZIP") + match = _FILENAME_RE.fullmatch(reference.zip_filename) + if match is None: + raise NemwebDispatchScadaError("invalid Dispatch SCADA ZIP") + try: + expected_timestamp = datetime.strptime( + match.group("timestamp"), "%Y%m%d%H%M" + ).replace(tzinfo=_NEM_TIMEZONE).astimezone(timezone.utc) + except (ValueError, OverflowError): + raise NemwebDispatchScadaError("invalid Dispatch SCADA ZIP") from None + if ( + reference.url != _OFFICIAL_INDEX_URL + reference.zip_filename + or reference.source_artifact_id != match.group("source_id") + or reference.report_timestamp != expected_timestamp + ): + raise NemwebDispatchScadaError("invalid Dispatch SCADA ZIP") + try: + with ZipFile(BytesIO(zip_payload)) as archive: + members = archive.infolist() + expected_member = reference.zip_filename.removesuffix(".zip") + ".CSV" + if ( + len(members) != 1 + or members[0].is_dir() + or members[0].filename != expected_member + or members[0].orig_filename != expected_member + ): + raise NemwebDispatchScadaError("invalid Dispatch SCADA ZIP") + member = members[0] + if ( + member.flag_bits & 1 + or member.compress_type not in (ZIP_STORED, ZIP_DEFLATED) + or member.file_size <= 0 + or member.file_size > _MAX_CSV_BYTES + or member.compress_size <= 0 + or member.file_size + > member.compress_size * _MAX_COMPRESSION_RATIO + ): + raise NemwebDispatchScadaError("invalid Dispatch SCADA ZIP") + with archive.open(member) as member_stream: + raw_csv = member_stream.read(_MAX_CSV_BYTES + 1) + has_more_data = bool(member_stream.read(1)) + if ( + len(raw_csv) > _MAX_CSV_BYTES + or has_more_data + or len(raw_csv) != member.file_size + ): + raise NemwebDispatchScadaError("invalid Dispatch SCADA ZIP") + payload = raw_csv.decode("utf-8-sig") + if not payload or "\x00" in payload: + raise NemwebDispatchScadaError("invalid Dispatch SCADA ZIP") + except NemwebDispatchScadaError: + raise + except ( + BadZipFile, + EOFError, + LargeZipFile, + RuntimeError, + UnicodeDecodeError, + NotImplementedError, + zlib.error, + ): + raise NemwebDispatchScadaError("invalid Dispatch SCADA ZIP") from None + return DispatchScadaArtifact( + reference=reference, + csv_member_name=member.filename, + csv_payload=payload, + zip_sha256=sha256(zip_payload).hexdigest(), + raw_zip=zip_payload, + ) diff --git a/app/backend/batterywatch_api/nemweb_http.py b/app/backend/batterywatch_api/nemweb_http.py new file mode 100644 index 0000000..9b1c1d1 --- /dev/null +++ b/app/backend/batterywatch_api/nemweb_http.py @@ -0,0 +1,174 @@ +"""Bounded HTTP transport for official NEMWeb collector resources.""" + +from collections.abc import Callable +from dataclasses import dataclass +from http.client import HTTPException +import math +import re +from typing import Any +from urllib.parse import urlsplit +from urllib.error import URLError +from urllib.request import HTTPRedirectHandler, Request, build_opener + + +_USER_AGENT = "BatteryWatch-Collector/0.1" +_INDEX_URLS = frozenset(( + "https://www.nemweb.com.au/REPORTS/CURRENT/Dispatch_SCADA/", + "https://www.nemweb.com.au/REPORTS/CURRENT/DispatchIS_Reports/", +)) +_MAX_RESOURCE_BYTES = 16 * 1024 * 1024 +_ARTIFACT_PATH_RE = re.compile( + r"(?:" + r"/REPORTS/CURRENT/Dispatch_SCADA/" + r"PUBLIC_DISPATCHSCADA_[0-9]{12}_[0-9]{1,32}\.zip" + r"|" + r"/REPORTS/CURRENT/DispatchIS_Reports/" + r"PUBLIC_DISPATCHIS_[0-9]{12}_[0-9]{1,32}\.zip" + r")" +) + + +class NemwebHttpError(ValueError): + """Safe public failure raised for an unusable NEMWeb response.""" + + +class _NoRedirectHandler(HTTPRedirectHandler): + def redirect_request( + self, + req: Request, + fp: Any, + code: int, + msg: str, + headers: Any, + newurl: str, + ) -> Any: + raise NemwebHttpError("unusable NEMWeb response") + + +_DEFAULT_OPENER = build_opener(_NoRedirectHandler()) + + +@dataclass(frozen=True) +class NemwebHttpResource: + requested_url: str + resolved_url: str + body: bytes + content_type: str | None + etag: str | None + last_modified: str | None + + +def _safe_optional_header(headers: Any, name: str) -> str | None: + value = headers.get(name) + if value is None: + return None + if ( + type(value) is not str + or len(value) > 1024 + or any(ord(character) < 32 or ord(character) == 127 for character in value) + ): + raise NemwebHttpError("unusable NEMWeb response") + return value + + +def fetch_nemweb_resource( + url: str, + *, + max_bytes: int, + timeout_seconds: float = 30.0, + opener: Callable[..., Any] = _DEFAULT_OPENER.open, +) -> NemwebHttpResource: + """Fetch one NEMWeb resource through an injectable HTTPS opener.""" + + valid_url = False + if type(url) is str: + try: + parts = urlsplit(url) + except ValueError as error: + raise NemwebHttpError("invalid NEMWeb request") from error + valid_url = url in _INDEX_URLS or ( + parts.scheme == "https" + and parts.netloc == "www.nemweb.com.au" + and not parts.query + and not parts.fragment + and _ARTIFACT_PATH_RE.fullmatch(parts.path) is not None + ) + valid_limit = ( + type(max_bytes) is int and 0 < max_bytes <= _MAX_RESOURCE_BYTES + ) + valid_timeout = ( + (type(timeout_seconds) is int and 0 < timeout_seconds <= 60) + or ( + type(timeout_seconds) is float + and math.isfinite(timeout_seconds) + and 0 < timeout_seconds <= 60 + ) + ) + if not valid_url or not valid_limit or not valid_timeout: + raise NemwebHttpError("invalid NEMWeb request") + + try: + request = Request( + url, + headers={ + "User-Agent": _USER_AGENT, + "Accept": "text/html,application/zip,application/octet-stream", + }, + ) + with opener(request, timeout=timeout_seconds) as response: + resolved_url = response.geturl() + if response.status != 200 or resolved_url != url: + raise NemwebHttpError("unusable NEMWeb response") + + content_length_header = _safe_optional_header( + response.headers, "Content-Length" + ) + declared_length: int | None = None + if content_length_header is not None: + if ( + not content_length_header.isascii() + or not content_length_header.isdecimal() + ): + raise NemwebHttpError("unusable NEMWeb response") + declared_length = int(content_length_header) + if declared_length > max_bytes: + raise NemwebHttpError("unusable NEMWeb response") + + content_encoding = _safe_optional_header( + response.headers, "Content-Encoding" + ) + if content_encoding is not None and content_encoding.lower() != "identity": + raise NemwebHttpError("unusable NEMWeb response") + + content_type = _safe_optional_header(response.headers, "Content-Type") + etag = _safe_optional_header(response.headers, "ETag") + last_modified = _safe_optional_header( + response.headers, "Last-Modified" + ) + body = response.read(max_bytes + 1) + if ( + type(body) is not bytes + or not body + or len(body) > max_bytes + or (declared_length is not None and len(body) != declared_length) + ): + raise NemwebHttpError("unusable NEMWeb response") + return NemwebHttpResource( + requested_url=url, + resolved_url=resolved_url, + body=body, + content_type=content_type, + etag=etag, + last_modified=last_modified, + ) + except NemwebHttpError: + raise + except ( + EOFError, + HTTPException, + URLError, + TimeoutError, + OSError, + ValueError, + ) as error: + raise NemwebHttpError("unusable NEMWeb response") from error diff --git a/app/backend/batterywatch_api/runtime.py b/app/backend/batterywatch_api/runtime.py new file mode 100644 index 0000000..b84ee72 --- /dev/null +++ b/app/backend/batterywatch_api/runtime.py @@ -0,0 +1,91 @@ +"""Runtime selection seams for fixture and database-backed requests.""" + +from __future__ import annotations + +import importlib +import os +from collections.abc import Callable, Iterator +from contextlib import AbstractContextManager, contextmanager +from typing import Any + +from .storage import PostgreSQLRepository, StorageRepository + +ConnectionFactory = Callable[[str], Any] +RepositoryProvider = Callable[[], AbstractContextManager[StorageRepository]] + + +@contextmanager +def database_repository_provider( + database_url: str | None, + connection_factory: ConnectionFactory | None = None, +) -> Iterator[StorageRepository]: + """Yield one PostgreSQL repository while owning one request connection.""" + + if not database_url: + raise RuntimeError("Database URL is not configured") + factory = connection_factory or _default_connection_factory + connection = factory(database_url) + try: + yield PostgreSQLRepository(connection) + finally: + connection.close() + + +def configured_database_url(explicit: str | None = None) -> str | None: + """Return explicit database configuration or a supported environment value.""" + + if explicit is not None: + return explicit + return os.getenv("BATTERYWATCH_DATABASE_URL") or os.getenv("DATABASE_URL") + + +class DatabaseHealthTracer: + """Probe a configured database without importing its driver eagerly.""" + + def __init__( + self, + database_url: str | None, + connection_factory: ConnectionFactory | None = None, + ) -> None: + self._database_url = database_url + self._connection_factory = connection_factory + + def check(self) -> bool: + return self() + + def __call__(self) -> bool: + if not self._database_url: + return False + + try: + connection_factory = self._connection_factory or _default_connection_factory + connection = connection_factory(self._database_url) + try: + cursor = connection.cursor() + try: + cursor.execute("SELECT 1") + return True + finally: + cursor.close() + finally: + connection.close() + except Exception: + return False + + +def _default_connection_factory(database_url: str) -> Any: + try: + driver = importlib.import_module("psycopg") + connect = driver.connect + except Exception: + return None + return connect(database_url) + + +__all__ = [ + "ConnectionFactory", + "RepositoryProvider", + "DatabaseHealthTracer", + "configured_database_url", + "database_repository_provider", +] diff --git a/app/backend/batterywatch_api/storage.py b/app/backend/batterywatch_api/storage.py new file mode 100644 index 0000000..a16fcf7 --- /dev/null +++ b/app/backend/batterywatch_api/storage.py @@ -0,0 +1,1024 @@ +"""Persistence contracts and repository implementations for BatteryWatch. + +The repositories store one effective record per logical generator/region interval. +The PostgreSQL adapter uses a DB-API-compatible connection supplied by its caller; +it deliberately does not import a driver or create a connection by itself. +""" + +from __future__ import annotations + +from contextlib import contextmanager +from dataclasses import dataclass +from datetime import datetime, timezone +from math import isfinite +from collections.abc import Hashable, Iterable, Iterator, MutableMapping +from typing import Any, Literal, Protocol, TypeVar + +UTC = timezone.utc +PriceStatus = Literal["available", "negative", "missing"] +_VALID_PRICE_STATUSES = {"available", "negative", "missing"} + + +def utc_timestamp(value: datetime, field: str = "timestamp") -> datetime: + """Return an aware timestamp in UTC; reject ambiguous naive timestamps.""" + + if value.tzinfo is None or value.utcoffset() is None: + raise ValueError(f"{field} must include a UTC offset") + return value.astimezone(UTC) + + +def aligned_5m(value: datetime, field: str = "interval_start") -> datetime: + """Normalize an interval start to UTC and require an exact five-minute boundary.""" + + normalized = utc_timestamp(value, field) + if normalized.minute % 5 or normalized.second or normalized.microsecond: + raise ValueError(f"{field} must be aligned to a five-minute boundary") + return normalized + + +def _text(value: str, field: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"{field} must be a non-empty string") + return value + + +def _number(value: float, field: str) -> float: + if isinstance(value, bool): + raise ValueError(f"{field} must be numeric") + try: + normalized = float(value) + except (TypeError, ValueError) as exc: + raise ValueError(f"{field} must be numeric") from exc + if not isfinite(normalized): + raise ValueError(f"{field} must be finite") + return normalized + + +def _optional_number(value: float | None, field: str) -> float | None: + return None if value is None else _number(value, field) + + +def _version(value: int, field: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise ValueError(f"{field} must be a non-negative integer") + return value + + +def _flags(value: Iterable[str]) -> tuple[str, ...]: + if isinstance(value, str): + raise ValueError("quality_flags must be an iterable of strings, not a string") + normalized = tuple(_text(flag, "quality flag") for flag in value) + return normalized + + +def _provenance_values( + source_id: str, + source_timestamp: datetime, + ingestion_version: int, + correction_version: int, +) -> tuple[str, datetime, int, int]: + return ( + _text(source_id, "source_id"), + utc_timestamp(source_timestamp, "source_timestamp"), + _version(ingestion_version, "ingestion_version"), + _version(correction_version, "correction_version"), + ) + + +@dataclass(frozen=True, slots=True) +class RecordProvenance: + """Source and revision information retained with every persisted record.""" + + source_id: str + source_timestamp: datetime + ingestion_version: int + correction_version: int = 0 + + def __post_init__(self) -> None: + source_id, source_timestamp, ingestion, correction = _provenance_values( + self.source_id, + self.source_timestamp, + self.ingestion_version, + self.correction_version, + ) + object.__setattr__(self, "source_id", source_id) + object.__setattr__(self, "source_timestamp", source_timestamp) + object.__setattr__(self, "ingestion_version", ingestion) + object.__setattr__(self, "correction_version", correction) + + +class _VersionedRecord: + __slots__ = () + + source_id: str + source_timestamp: datetime + ingestion_version: int + correction_version: int + + @property + def provenance(self) -> RecordProvenance: + return RecordProvenance( + source_id=self.source_id, + source_timestamp=self.source_timestamp, + ingestion_version=self.ingestion_version, + correction_version=self.correction_version, + ) + + +@dataclass(frozen=True, slots=True) +class GeneratorMetadata(_VersionedRecord): + """Stable generator metadata and its registry provenance.""" + + generator_id: str + site_name: str + region: str + capacity_mw: float + storage_capacity_mwh: float + source_id: str + source_timestamp: datetime + ingestion_version: int + correction_version: int = 0 + data_start: datetime | None = None + data_end: datetime | None = None + + def __post_init__(self) -> None: + object.__setattr__(self, "generator_id", _text(self.generator_id, "generator_id")) + object.__setattr__(self, "site_name", _text(self.site_name, "site_name")) + object.__setattr__(self, "region", _text(self.region, "region")) + object.__setattr__(self, "capacity_mw", _number(self.capacity_mw, "capacity_mw")) + object.__setattr__( + self, + "storage_capacity_mwh", + _number(self.storage_capacity_mwh, "storage_capacity_mwh"), + ) + if self.capacity_mw <= 0 or self.storage_capacity_mwh <= 0: + raise ValueError("generator capacities must be positive") + source_id, source_timestamp, ingestion, correction = _provenance_values( + self.source_id, + self.source_timestamp, + self.ingestion_version, + self.correction_version, + ) + object.__setattr__(self, "source_id", source_id) + object.__setattr__(self, "source_timestamp", source_timestamp) + object.__setattr__(self, "ingestion_version", ingestion) + object.__setattr__(self, "correction_version", correction) + start = None if self.data_start is None else utc_timestamp(self.data_start, "data_start") + end = None if self.data_end is None else utc_timestamp(self.data_end, "data_end") + if start is not None and end is not None and end <= start: + raise ValueError("data_end must be after data_start") + object.__setattr__(self, "data_start", start) + object.__setattr__(self, "data_end", end) + + @property + def logical_key(self) -> tuple[str]: + return (self.generator_id,) + + @property + def duid(self) -> str: + """NEM terminology alias for the stable generator identity.""" + + return self.generator_id + + +@dataclass(frozen=True, slots=True) +class GeneratorPower5m(_VersionedRecord): + """Five-minute generator power; positive exports and negative imports.""" + + generator_id: str + interval_start: datetime + power_mw: float + source_id: str + source_timestamp: datetime + ingestion_version: int + correction_version: int = 0 + + def __post_init__(self) -> None: + object.__setattr__(self, "generator_id", _text(self.generator_id, "generator_id")) + object.__setattr__(self, "interval_start", aligned_5m(self.interval_start)) + object.__setattr__(self, "power_mw", _number(self.power_mw, "power_mw")) + source_id, source_timestamp, ingestion, correction = _provenance_values( + self.source_id, + self.source_timestamp, + self.ingestion_version, + self.correction_version, + ) + object.__setattr__(self, "source_id", source_id) + object.__setattr__(self, "source_timestamp", source_timestamp) + object.__setattr__(self, "ingestion_version", ingestion) + object.__setattr__(self, "correction_version", correction) + + @property + def logical_key(self) -> tuple[str, datetime]: + return (self.generator_id, self.interval_start) + + @property + def timestamp(self) -> datetime: + return self.interval_start + + +@dataclass(frozen=True, slots=True) +class GeneratorSoc5m(_VersionedRecord): + """Five-minute SOC observation; ``None`` means unavailable, never inferred.""" + + generator_id: str + interval_start: datetime + soc_percent: float | None + source_id: str + source_timestamp: datetime + ingestion_version: int + correction_version: int = 0 + quality_flags: tuple[str, ...] = () + + def __post_init__(self) -> None: + object.__setattr__(self, "generator_id", _text(self.generator_id, "generator_id")) + object.__setattr__(self, "interval_start", aligned_5m(self.interval_start)) + soc = _optional_number(self.soc_percent, "soc_percent") + if soc is not None and not 0 <= soc <= 100: + raise ValueError("soc_percent must be between 0 and 100") + object.__setattr__(self, "soc_percent", soc) + source_id, source_timestamp, ingestion, correction = _provenance_values( + self.source_id, + self.source_timestamp, + self.ingestion_version, + self.correction_version, + ) + object.__setattr__(self, "source_id", source_id) + object.__setattr__(self, "source_timestamp", source_timestamp) + object.__setattr__(self, "ingestion_version", ingestion) + object.__setattr__(self, "correction_version", correction) + object.__setattr__(self, "quality_flags", _flags(self.quality_flags)) + + @property + def logical_key(self) -> tuple[str, datetime]: + return (self.generator_id, self.interval_start) + + @property + def timestamp(self) -> datetime: + return self.interval_start + + @property + def is_available(self) -> bool: + return self.soc_percent is not None + + @property + def soc_status(self) -> Literal["available", "missing"]: + return "available" if self.is_available else "missing" + + +@dataclass(frozen=True, slots=True) +class RegionalPrice5m(_VersionedRecord): + """Five-minute regional price with explicit missing/negative status and flags.""" + + region: str + interval_start: datetime + price_aud_per_mwh: float | None + price_status: PriceStatus + source_id: str + source_timestamp: datetime + ingestion_version: int + correction_version: int = 0 + quality_flags: tuple[str, ...] = () + intervention: int = 0 + apc_flag: int = 0 + market_suspended: bool = False + + def __post_init__(self) -> None: + object.__setattr__(self, "region", _text(self.region, "region")) + object.__setattr__(self, "interval_start", aligned_5m(self.interval_start)) + if self.price_status not in _VALID_PRICE_STATUSES: + raise ValueError("price_status must be available, negative, or missing") + price = _optional_number(self.price_aud_per_mwh, "price_aud_per_mwh") + expected_status = "missing" if price is None else ("negative" if price < 0 else "available") + if self.price_status != expected_status: + raise ValueError("price_status does not match price_aud_per_mwh") + object.__setattr__(self, "price_aud_per_mwh", price) + source_id, source_timestamp, ingestion, correction = _provenance_values( + self.source_id, + self.source_timestamp, + self.ingestion_version, + self.correction_version, + ) + object.__setattr__(self, "source_id", source_id) + object.__setattr__(self, "source_timestamp", source_timestamp) + object.__setattr__(self, "ingestion_version", ingestion) + object.__setattr__(self, "correction_version", correction) + object.__setattr__(self, "quality_flags", _flags(self.quality_flags)) + object.__setattr__(self, "intervention", _version(self.intervention, "intervention")) + object.__setattr__(self, "apc_flag", _version(self.apc_flag, "apc_flag")) + if not isinstance(self.market_suspended, bool): + raise ValueError("market_suspended must be a boolean") + + @property + def logical_key(self) -> tuple[str, datetime]: + return (self.region, self.interval_start) + + @property + def timestamp(self) -> datetime: + return self.interval_start + + @property + def status(self) -> PriceStatus: + return self.price_status + + @property + def is_available(self) -> bool: + return self.price_aud_per_mwh is not None + + @property + def is_missing(self) -> bool: + return not self.is_available + + @property + def is_negative(self) -> bool: + return self.price_aud_per_mwh is not None and self.price_aud_per_mwh < 0 + + +# Short aliases make the contract convenient without losing the table-shaped names. +GeneratorRecord = GeneratorMetadata +PowerRecord = GeneratorPower5m +SocRecord = GeneratorSoc5m +PriceRecord = RegionalPrice5m + + +class StorageRepository(Protocol): + """Replaceable persistence boundary used by later database-backed slices.""" + + def upsert_generator(self, record: GeneratorMetadata) -> bool: ... + + def upsert_power(self, record: GeneratorPower5m) -> bool: ... + + def upsert_soc(self, record: GeneratorSoc5m) -> bool: ... + + def upsert_price(self, record: RegionalPrice5m) -> bool: ... + + def read_generator(self, generator_id: str) -> GeneratorMetadata | None: ... + + def list_generators(self) -> tuple[GeneratorMetadata, ...]: ... + + def list_power( + self, + generator_id: str, + start: datetime | None = None, + end: datetime | None = None, + ) -> tuple[GeneratorPower5m, ...]: ... + + def list_soc( + self, + generator_id: str, + start: datetime | None = None, + end: datetime | None = None, + ) -> tuple[GeneratorSoc5m, ...]: ... + + def list_prices( + self, + region: str, + start: datetime | None = None, + end: datetime | None = None, + ) -> tuple[RegionalPrice5m, ...]: ... + + def read_power(self, generator_id: str, interval_start: datetime) -> GeneratorPower5m | None: ... + + def read_soc(self, generator_id: str, interval_start: datetime) -> GeneratorSoc5m | None: ... + + def read_price(self, region: str, interval_start: datetime) -> RegionalPrice5m | None: ... + + +Repository = StorageRepository +_Key = TypeVar("_Key", bound=Hashable) +_Record = TypeVar("_Record", bound=_VersionedRecord) + + +def _revision(record: _VersionedRecord) -> tuple[int, int, datetime]: + return (record.correction_version, record.ingestion_version, record.source_timestamp) + + +def _upsert(store: MutableMapping[_Key, _Record], key: _Key, record: _Record) -> bool: + current = store.get(key) + if current is not None and _revision(record) <= _revision(current): + return False + store[key] = record + return True + + +def _window(start: datetime | None, end: datetime | None) -> tuple[datetime | None, datetime | None]: + normalized_start = None if start is None else utc_timestamp(start, "start") + normalized_end = None if end is None else utc_timestamp(end, "end") + if normalized_start is not None and normalized_end is not None and normalized_end < normalized_start: + raise ValueError("end must not precede start") + return normalized_start, normalized_end + + +class InMemoryRepository: + """Deterministic one-effective-record repository for tests and local seams. + + Upsert returns ``True`` for a new or newer effective record and ``False`` for + an exact replay or a stale revision. Logical keys contain only the stable + dimension and aligned interval; provenance remains on the stored value. + """ + + def __init__(self) -> None: + self._generators: dict[tuple[str], GeneratorMetadata] = {} + self._power: dict[tuple[str, datetime], GeneratorPower5m] = {} + self._soc: dict[tuple[str, datetime], GeneratorSoc5m] = {} + self._prices: dict[tuple[str, datetime], RegionalPrice5m] = {} + + def upsert_generator(self, record: GeneratorMetadata) -> bool: + return _upsert(self._generators, record.logical_key, record) + + insert_generator = upsert_generator + + def upsert_power(self, record: GeneratorPower5m) -> bool: + return _upsert(self._power, record.logical_key, record) + + insert_power = upsert_power + + def upsert_soc(self, record: GeneratorSoc5m) -> bool: + return _upsert(self._soc, record.logical_key, record) + + insert_soc = upsert_soc + + def upsert_price(self, record: RegionalPrice5m) -> bool: + return _upsert(self._prices, record.logical_key, record) + + insert_price = upsert_price + + def read_generator(self, generator_id: str) -> GeneratorMetadata | None: + return self._generators.get((_text(generator_id, "generator_id"),)) + + def read_power(self, generator_id: str, interval_start: datetime) -> GeneratorPower5m | None: + return self._power.get((_text(generator_id, "generator_id"), aligned_5m(interval_start))) + + def read_soc(self, generator_id: str, interval_start: datetime) -> GeneratorSoc5m | None: + return self._soc.get((_text(generator_id, "generator_id"), aligned_5m(interval_start))) + + def read_price(self, region: str, interval_start: datetime) -> RegionalPrice5m | None: + return self._prices.get((_text(region, "region"), aligned_5m(interval_start))) + + def list_generators(self) -> tuple[GeneratorMetadata, ...]: + return tuple(self._generators[key] for key in sorted(self._generators)) + + def list_power( + self, + generator_id: str, + start: datetime | None = None, + end: datetime | None = None, + ) -> tuple[GeneratorPower5m, ...]: + dimension = _text(generator_id, "generator_id") + normalized_start, normalized_end = _window(start, end) + values = [ + record + for (record_dimension, interval_start), record in self._power.items() + if record_dimension == dimension + and (normalized_start is None or interval_start >= normalized_start) + and (normalized_end is None or interval_start < normalized_end) + ] + return tuple(sorted(values, key=lambda record: record.interval_start)) + + def list_soc( + self, + generator_id: str, + start: datetime | None = None, + end: datetime | None = None, + ) -> tuple[GeneratorSoc5m, ...]: + dimension = _text(generator_id, "generator_id") + normalized_start, normalized_end = _window(start, end) + values = [ + record + for (record_dimension, interval_start), record in self._soc.items() + if record_dimension == dimension + and (normalized_start is None or interval_start >= normalized_start) + and (normalized_end is None or interval_start < normalized_end) + ] + return tuple(sorted(values, key=lambda record: record.interval_start)) + + def list_prices( + self, + region: str, + start: datetime | None = None, + end: datetime | None = None, + ) -> tuple[RegionalPrice5m, ...]: + dimension = _text(region, "region") + normalized_start, normalized_end = _window(start, end) + values = [ + record + for (record_dimension, interval_start), record in self._prices.items() + if record_dimension == dimension + and (normalized_start is None or interval_start >= normalized_start) + and (normalized_end is None or interval_start < normalized_end) + ] + return tuple(sorted(values, key=lambda record: record.interval_start)) + + def count_power(self, generator_id: str) -> int: + return len(self.list_power(generator_id)) + + def count_soc(self, generator_id: str) -> int: + return len(self.list_soc(generator_id)) + + def count_prices(self, region: str) -> int: + return len(self.list_prices(region)) + + +class PostgreSQLCursor(Protocol): + """Small cursor surface required by :class:`PostgreSQLRepository`.""" + + def execute(self, statement: str, parameters: tuple[Any, ...]) -> None: ... + + def fetchone(self) -> tuple[Any, ...] | None: ... + + def fetchall(self) -> Iterable[tuple[Any, ...]]: ... + + def close(self) -> None: ... + + +class PostgreSQLConnection(Protocol): + """Small connection surface accepted by the PostgreSQL adapter.""" + + def cursor(self) -> PostgreSQLCursor: ... + + def commit(self) -> None: ... + + def rollback(self) -> None: ... + + +@contextmanager +def _managed_cursor(connection: PostgreSQLConnection) -> Iterator[PostgreSQLCursor]: + """Yield a cursor and close it without owning the caller's connection.""" + + cursor = connection.cursor() + try: + yield cursor + finally: + cursor.close() + + +_GENERATOR_UPSERT_SQL = """ +INSERT INTO generators ( + generator_id, + site_name, + region, + capacity_mw, + storage_capacity_mwh, + data_start, + data_end, + source_id, + source_timestamp, + ingestion_version, + correction_version +) +VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) +ON CONFLICT (generator_id) DO UPDATE +SET site_name = EXCLUDED.site_name, + region = EXCLUDED.region, + capacity_mw = EXCLUDED.capacity_mw, + storage_capacity_mwh = EXCLUDED.storage_capacity_mwh, + data_start = EXCLUDED.data_start, + data_end = EXCLUDED.data_end, + source_id = EXCLUDED.source_id, + source_timestamp = EXCLUDED.source_timestamp, + ingestion_version = EXCLUDED.ingestion_version, + correction_version = EXCLUDED.correction_version, + updated_at = CURRENT_TIMESTAMP +WHERE (EXCLUDED.correction_version, EXCLUDED.ingestion_version, EXCLUDED.source_timestamp) + > (generators.correction_version, generators.ingestion_version, generators.source_timestamp) +RETURNING 1 +""" + +_POWER_UPSERT_SQL = """ +INSERT INTO generator_power_5m ( + generator_id, + interval_start, + power_mw, + source_id, + source_timestamp, + ingestion_version, + correction_version +) +VALUES (%s, %s, %s, %s, %s, %s, %s) +ON CONFLICT (generator_id, interval_start) DO UPDATE +SET power_mw = EXCLUDED.power_mw, + source_id = EXCLUDED.source_id, + source_timestamp = EXCLUDED.source_timestamp, + ingestion_version = EXCLUDED.ingestion_version, + correction_version = EXCLUDED.correction_version +WHERE (EXCLUDED.correction_version, EXCLUDED.ingestion_version, EXCLUDED.source_timestamp) + > (generator_power_5m.correction_version, + generator_power_5m.ingestion_version, + generator_power_5m.source_timestamp) +RETURNING 1 +""" + +_SOC_UPSERT_SQL = """ +INSERT INTO generator_soc_5m ( + generator_id, + interval_start, + soc_percent, + source_id, + source_timestamp, + ingestion_version, + correction_version, + quality_flags +) +VALUES (%s, %s, %s, %s, %s, %s, %s, %s) +ON CONFLICT (generator_id, interval_start) DO UPDATE +SET soc_percent = EXCLUDED.soc_percent, + source_id = EXCLUDED.source_id, + source_timestamp = EXCLUDED.source_timestamp, + ingestion_version = EXCLUDED.ingestion_version, + correction_version = EXCLUDED.correction_version, + quality_flags = EXCLUDED.quality_flags +WHERE (EXCLUDED.correction_version, EXCLUDED.ingestion_version, EXCLUDED.source_timestamp) + > (generator_soc_5m.correction_version, + generator_soc_5m.ingestion_version, + generator_soc_5m.source_timestamp) +RETURNING 1 +""" + +_PRICE_UPSERT_SQL = """ +INSERT INTO nem_price_5m ( + region, + interval_start, + price_aud_per_mwh, + price_status, + intervention, + apc_flag, + market_suspended, + source_id, + source_timestamp, + ingestion_version, + correction_version, + quality_flags +) +VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) +ON CONFLICT (region, interval_start) DO UPDATE +SET price_aud_per_mwh = EXCLUDED.price_aud_per_mwh, + price_status = EXCLUDED.price_status, + intervention = EXCLUDED.intervention, + apc_flag = EXCLUDED.apc_flag, + market_suspended = EXCLUDED.market_suspended, + source_id = EXCLUDED.source_id, + source_timestamp = EXCLUDED.source_timestamp, + ingestion_version = EXCLUDED.ingestion_version, + correction_version = EXCLUDED.correction_version, + quality_flags = EXCLUDED.quality_flags +WHERE (EXCLUDED.correction_version, EXCLUDED.ingestion_version, EXCLUDED.source_timestamp) + > (nem_price_5m.correction_version, + nem_price_5m.ingestion_version, + nem_price_5m.source_timestamp) +RETURNING 1 +""" + +_GENERATOR_READ_SQL = """ +SELECT generator_id, site_name, region, capacity_mw, storage_capacity_mwh, + source_id, source_timestamp, ingestion_version, correction_version, + data_start, data_end +FROM generators +WHERE generator_id = %s +""" + +_GENERATORS_LIST_SQL = """ +SELECT generator_id, site_name, region, capacity_mw, storage_capacity_mwh, + source_id, source_timestamp, ingestion_version, correction_version, + data_start, data_end +FROM generators +ORDER BY generator_id ASC +""" + +_POWER_READ_SQL = """ +SELECT generator_id, interval_start, power_mw, source_id, source_timestamp, + ingestion_version, correction_version +FROM generator_power_5m +WHERE generator_id = %s AND interval_start = %s +""" + +_SOC_READ_SQL = """ +SELECT generator_id, interval_start, soc_percent, source_id, source_timestamp, + ingestion_version, correction_version, quality_flags +FROM generator_soc_5m +WHERE generator_id = %s AND interval_start = %s +""" + +_PRICE_READ_SQL = """ +SELECT region, interval_start, price_aud_per_mwh, price_status, + intervention, apc_flag, market_suspended, source_id, source_timestamp, + ingestion_version, correction_version, quality_flags +FROM nem_price_5m +WHERE region = %s AND interval_start = %s +""" + +_POWER_LIST_SQL = """ +SELECT generator_id, interval_start, power_mw, source_id, source_timestamp, + ingestion_version, correction_version +FROM generator_power_5m +WHERE generator_id = %s + AND (%s::timestamptz IS NULL OR interval_start >= %s::timestamptz) + AND (%s::timestamptz IS NULL OR interval_start < %s::timestamptz) +ORDER BY interval_start ASC +""" + +_SOC_LIST_SQL = """ +SELECT generator_id, interval_start, soc_percent, source_id, source_timestamp, + ingestion_version, correction_version, quality_flags +FROM generator_soc_5m +WHERE generator_id = %s + AND (%s::timestamptz IS NULL OR interval_start >= %s::timestamptz) + AND (%s::timestamptz IS NULL OR interval_start < %s::timestamptz) +ORDER BY interval_start ASC +""" + +_PRICE_LIST_SQL = """ +SELECT region, interval_start, price_aud_per_mwh, price_status, + intervention, apc_flag, market_suspended, source_id, source_timestamp, + ingestion_version, correction_version, quality_flags +FROM nem_price_5m +WHERE region = %s + AND (%s::timestamptz IS NULL OR interval_start >= %s::timestamptz) + AND (%s::timestamptz IS NULL OR interval_start < %s::timestamptz) +ORDER BY interval_start ASC +""" + + +def _generator_from_row(row: tuple[Any, ...]) -> GeneratorMetadata: + return GeneratorMetadata( + generator_id=row[0], + site_name=row[1], + region=row[2], + capacity_mw=row[3], + storage_capacity_mwh=row[4], + source_id=row[5], + source_timestamp=row[6], + ingestion_version=row[7], + correction_version=row[8], + data_start=row[9], + data_end=row[10], + ) + + +def _power_from_row(row: tuple[Any, ...]) -> GeneratorPower5m: + return GeneratorPower5m( + generator_id=row[0], + interval_start=row[1], + power_mw=row[2], + source_id=row[3], + source_timestamp=row[4], + ingestion_version=row[5], + correction_version=row[6], + ) + + +def _soc_from_row(row: tuple[Any, ...]) -> GeneratorSoc5m: + return GeneratorSoc5m( + generator_id=row[0], + interval_start=row[1], + soc_percent=row[2], + source_id=row[3], + source_timestamp=row[4], + ingestion_version=row[5], + correction_version=row[6], + quality_flags=tuple(row[7] or ()), + ) + + +def _price_from_row(row: tuple[Any, ...]) -> RegionalPrice5m: + return RegionalPrice5m( + region=row[0], + interval_start=row[1], + price_aud_per_mwh=row[2], + price_status=row[3], + intervention=row[4], + apc_flag=row[5], + market_suspended=row[6], + source_id=row[7], + source_timestamp=row[8], + ingestion_version=row[9], + correction_version=row[10], + quality_flags=tuple(row[11] or ()), + ) + + +class PostgreSQLRepository: + """PostgreSQL implementation of the S2a storage boundary. + + The supplied connection is owned by the caller. Writes commit one + transaction each and return ``True`` only when PostgreSQL inserted or + replaced the effective row; an exact replay or stale revision returns + ``False`` because the guarded ``ON CONFLICT`` update returns no row. + """ + + def __init__(self, connection: PostgreSQLConnection): + self._connection = connection + + def _write(self, statement: str, parameters: tuple[Any, ...]) -> bool: + try: + with _managed_cursor(self._connection) as cursor: + cursor.execute(statement, parameters) + applied = cursor.fetchone() is not None + self._connection.commit() + return applied + except Exception: + try: + self._connection.rollback() + except Exception: + pass + raise + + def _one(self, statement: str, parameters: tuple[Any, ...]) -> tuple[Any, ...] | None: + with _managed_cursor(self._connection) as cursor: + cursor.execute(statement, parameters) + return cursor.fetchone() + + def _many(self, statement: str, parameters: tuple[Any, ...]) -> list[tuple[Any, ...]]: + with _managed_cursor(self._connection) as cursor: + cursor.execute(statement, parameters) + return list(cursor.fetchall()) + + def upsert_generator(self, record: GeneratorMetadata) -> bool: + return self._write( + _GENERATOR_UPSERT_SQL, + ( + record.generator_id, + record.site_name, + record.region, + record.capacity_mw, + record.storage_capacity_mwh, + record.data_start, + record.data_end, + record.source_id, + record.source_timestamp, + record.ingestion_version, + record.correction_version, + ), + ) + + insert_generator = upsert_generator + + def upsert_power(self, record: GeneratorPower5m) -> bool: + return self._write( + _POWER_UPSERT_SQL, + ( + record.generator_id, + record.interval_start, + record.power_mw, + record.source_id, + record.source_timestamp, + record.ingestion_version, + record.correction_version, + ), + ) + + insert_power = upsert_power + + def upsert_soc(self, record: GeneratorSoc5m) -> bool: + return self._write( + _SOC_UPSERT_SQL, + ( + record.generator_id, + record.interval_start, + record.soc_percent, + record.source_id, + record.source_timestamp, + record.ingestion_version, + record.correction_version, + list(record.quality_flags), + ), + ) + + insert_soc = upsert_soc + + def upsert_price(self, record: RegionalPrice5m) -> bool: + return self._write( + _PRICE_UPSERT_SQL, + ( + record.region, + record.interval_start, + record.price_aud_per_mwh, + record.price_status, + record.intervention, + record.apc_flag, + record.market_suspended, + record.source_id, + record.source_timestamp, + record.ingestion_version, + record.correction_version, + list(record.quality_flags), + ), + ) + + insert_price = upsert_price + + def read_generator(self, generator_id: str) -> GeneratorMetadata | None: + row = self._one(_GENERATOR_READ_SQL, (_text(generator_id, "generator_id"),)) + return None if row is None else _generator_from_row(row) + + def read_power(self, generator_id: str, interval_start: datetime) -> GeneratorPower5m | None: + row = self._one( + _POWER_READ_SQL, + (_text(generator_id, "generator_id"), aligned_5m(interval_start)), + ) + return None if row is None else _power_from_row(row) + + def read_soc(self, generator_id: str, interval_start: datetime) -> GeneratorSoc5m | None: + row = self._one( + _SOC_READ_SQL, + (_text(generator_id, "generator_id"), aligned_5m(interval_start)), + ) + return None if row is None else _soc_from_row(row) + + def read_price(self, region: str, interval_start: datetime) -> RegionalPrice5m | None: + row = self._one( + _PRICE_READ_SQL, + (_text(region, "region"), aligned_5m(interval_start)), + ) + return None if row is None else _price_from_row(row) + + def list_generators(self) -> tuple[GeneratorMetadata, ...]: + return tuple(_generator_from_row(row) for row in self._many(_GENERATORS_LIST_SQL, ())) + + @staticmethod + def _window_parameters( + dimension: str, + start: datetime | None, + end: datetime | None, + ) -> tuple[Any, ...]: + return (dimension, start, start, end, end) + + def list_power( + self, + generator_id: str, + start: datetime | None = None, + end: datetime | None = None, + ) -> tuple[GeneratorPower5m, ...]: + dimension = _text(generator_id, "generator_id") + normalized_start, normalized_end = _window(start, end) + values = [ + _power_from_row(row) + for row in self._many( + _POWER_LIST_SQL, + self._window_parameters(dimension, normalized_start, normalized_end), + ) + ] + return tuple(sorted(values, key=lambda record: record.interval_start)) + + def list_soc( + self, + generator_id: str, + start: datetime | None = None, + end: datetime | None = None, + ) -> tuple[GeneratorSoc5m, ...]: + dimension = _text(generator_id, "generator_id") + normalized_start, normalized_end = _window(start, end) + values = [ + _soc_from_row(row) + for row in self._many( + _SOC_LIST_SQL, + self._window_parameters(dimension, normalized_start, normalized_end), + ) + ] + return tuple(sorted(values, key=lambda record: record.interval_start)) + + def list_prices( + self, + region: str, + start: datetime | None = None, + end: datetime | None = None, + ) -> tuple[RegionalPrice5m, ...]: + dimension = _text(region, "region") + normalized_start, normalized_end = _window(start, end) + values = [ + _price_from_row(row) + for row in self._many( + _PRICE_LIST_SQL, + self._window_parameters(dimension, normalized_start, normalized_end), + ) + ] + return tuple(sorted(values, key=lambda record: record.interval_start)) + + def count_power(self, generator_id: str) -> int: + return len(self.list_power(generator_id)) + + def count_soc(self, generator_id: str) -> int: + return len(self.list_soc(generator_id)) + + def count_prices(self, region: str) -> int: + return len(self.list_prices(region)) + +__all__ = [ + "GeneratorMetadata", + "GeneratorPower5m", + "GeneratorSoc5m", + "RegionalPrice5m", + "RecordProvenance", + "StorageRepository", + "Repository", + "InMemoryRepository", + "PostgreSQLRepository", + "PostgreSQLConnection", + "PostgreSQLCursor", + "GeneratorRecord", + "PowerRecord", + "SocRecord", + "PriceRecord", + "PriceStatus", + "utc_timestamp", + "aligned_5m", +] diff --git a/app/backend/requirements.txt b/app/backend/requirements.txt new file mode 100644 index 0000000..67b386f --- /dev/null +++ b/app/backend/requirements.txt @@ -0,0 +1,5 @@ +fastapi>=0.115,<1 +pydantic>=2.7,<3 +httpx>=0.27,<1 +uvicorn[standard]>=0.30,<1 +psycopg[binary]>=3.2,<4 diff --git a/app/backend/tests/__init__.py b/app/backend/tests/__init__.py new file mode 100644 index 0000000..c36a1c5 --- /dev/null +++ b/app/backend/tests/__init__.py @@ -0,0 +1 @@ +"""BatteryWatch API tests.""" diff --git a/app/backend/tests/test_aemo.py b/app/backend/tests/test_aemo.py new file mode 100644 index 0000000..7c43dd1 --- /dev/null +++ b/app/backend/tests/test_aemo.py @@ -0,0 +1,275 @@ +"""Tests for the fixture-backed AEMO dispatch-price parser.""" + +from datetime import datetime, timedelta, timezone +import unittest + +from batterywatch_api.aemo import ( + AemoParseError, + parse_dispatch_price_csv, + parse_dispatch_price_mms_csv, +) + + +UTC = timezone.utc +SOURCE_TIME = datetime(2026, 1, 1, 0, 6, tzinfo=UTC) +MMS_SOURCE_ID = "0000000000000001" +MMS_HEADER = ( + "I", "DISPATCH", "PRICE", "5", "SETTLEMENTDATE", "RUNNO", "REGIONID", + "DISPATCHINTERVAL", "INTERVENTION", "RRP", "EEP", "ROP", "APCFLAG", + "MARKETSUSPENDEDFLAG", "LASTCHANGED", "RAISE6SECRRP", "RAISE6SECROP", + "RAISE6SECAPCFLAG", "RAISE60SECRRP", "RAISE60SECROP", "RAISE60SECAPCFLAG", + "RAISE5MINRRP", "RAISE5MINROP", "RAISE5MINAPCFLAG", "RAISEREGRRP", + "RAISEREGROP", "RAISEREGAPCFLAG", "LOWER6SECRRP", "LOWER6SECROP", + "LOWER6SECAPCFLAG", "LOWER60SECRRP", "LOWER60SECROP", "LOWER60SECAPCFLAG", + "LOWER5MINRRP", "LOWER5MINROP", "LOWER5MINAPCFLAG", "LOWERREGRRP", + "LOWERREGROP", "LOWERREGAPCFLAG", "PRICE_STATUS", "PRE_AP_ENERGY_PRICE", + "PRE_AP_RAISE6_PRICE", "PRE_AP_RAISE60_PRICE", "PRE_AP_RAISE5MIN_PRICE", + "PRE_AP_RAISEREG_PRICE", "PRE_AP_LOWER6_PRICE", "PRE_AP_LOWER60_PRICE", + "PRE_AP_LOWER5MIN_PRICE", "PRE_AP_LOWERREG_PRICE", "RAISE1SECRRP", + "RAISE1SECROP", "RAISE1SECAPCFLAG", "LOWER1SECRRP", "LOWER1SECROP", + "LOWER1SECAPCFLAG", "PRE_AP_RAISE1_PRICE", "PRE_AP_LOWER1_PRICE", + "CUMUL_PRE_AP_ENERGY_PRICE", "CUMUL_PRE_AP_RAISE6_PRICE", + "CUMUL_PRE_AP_RAISE60_PRICE", "CUMUL_PRE_AP_RAISE5MIN_PRICE", + "CUMUL_PRE_AP_RAISEREG_PRICE", "CUMUL_PRE_AP_LOWER6_PRICE", + "CUMUL_PRE_AP_LOWER60_PRICE", "CUMUL_PRE_AP_LOWER5MIN_PRICE", + "CUMUL_PRE_AP_LOWERREG_PRICE", "CUMUL_PRE_AP_RAISE1_PRICE", + "CUMUL_PRE_AP_LOWER1_PRICE", "OCD_STATUS", "MII_STATUS", +) + + +def _mms_row( + region: str, + rrp: str, + *, + interval: str = "2026/08/30 12:05:00", + intervention: str = "0", + apc_flag: str = "0", + suspended: str = "0", + price_status: str = "FIRM", +) -> str: + values = [""] * len(MMS_HEADER) + values[:4] = ["D", "DISPATCH", "PRICE", "5"] + values[MMS_HEADER.index("SETTLEMENTDATE")] = interval + values[MMS_HEADER.index("RUNNO")] = "1" + values[MMS_HEADER.index("REGIONID")] = region + values[MMS_HEADER.index("DISPATCHINTERVAL")] = "20260830097" + values[MMS_HEADER.index("INTERVENTION")] = intervention + values[MMS_HEADER.index("RRP")] = rrp + values[MMS_HEADER.index("APCFLAG")] = apc_flag + values[MMS_HEADER.index("MARKETSUSPENDEDFLAG")] = suspended + values[MMS_HEADER.index("LASTCHANGED")] = "2026/08/30 12:00:11" + values[MMS_HEADER.index("PRICE_STATUS")] = price_status + return ",".join(values) + + +def _mms_payload(*rows: str, source_id: str = MMS_SOURCE_ID) -> str: + report_rows = [ + f"C,NEMP.WORLD,DISPATCHIS,AEMO,PUBLIC,2026/08/30,12:05:15,{source_id},DISPATCHIS,0000000000000000", + ",".join(MMS_HEADER), + *rows, + ] + report_rows.append(f"C,END OF REPORT,{len(report_rows) + 1}") + return "\n".join(report_rows) + "\n" + + +class AemoDispatchPriceParserTests(unittest.TestCase): + def test_parses_complete_mms_price_batch_with_provenance_and_source_status(self): + payload = _mms_payload( + _mms_row("NSW1", "-4.93554", intervention="1", apc_flag="1"), + _mms_row("QLD1", "-4.22661"), + _mms_row("SA1", "-4.32012"), + _mms_row("TAS1", "-4.94962"), + _mms_row("VIC1", "-4.8574"), + ) + + records = parse_dispatch_price_mms_csv( + payload, + source_id=MMS_SOURCE_ID, + ingestion_version=7, + correction_version=2, + ) + + self.assertEqual(tuple(record.region for record in records), ( + "NSW1", "QLD1", "SA1", "TAS1", "VIC1", + )) + self.assertEqual( + tuple(record.interval_start for record in records), + (datetime(2026, 8, 30, 2, 5, tzinfo=UTC),) * 5, + ) + self.assertEqual(records[0].price_aud_per_mwh, -4.93554) + self.assertEqual(records[0].price_status, "negative") + self.assertEqual(records[0].source_timestamp, datetime(2026, 8, 30, 2, 0, 11, tzinfo=UTC)) + self.assertEqual(records[0].quality_flags, ( + "runno=1", "intervention=1", "apcflag=1", "aemo_price_status=FIRM", + )) + self.assertEqual((records[0].intervention, records[0].apc_flag), (1, 1)) + self.assertFalse(records[0].market_suspended) + + def test_mms_price_parser_rejects_incomplete_or_unsafe_reports(self): + rows = [ + _mms_row("NSW1", "10"), + _mms_row("QLD1", "11"), + _mms_row("SA1", "12"), + _mms_row("TAS1", "13"), + _mms_row("VIC1", "14"), + ] + valid = _mms_payload(*rows) + invalid_payloads = ( + valid.replace("C,END OF REPORT,8", "C,END OF REPORT,7"), + valid.replace("I,DISPATCH,PRICE,5", "I,DISPATCH,PRICE,4", 1), + valid.replace(",VIC1,20260830097", ",NSW1,20260830097", 1), + valid.replace("2026/08/30 12:05:00", "2026/08/30 12:10:00", 1), + valid.replace(",NSW1,20260830097", ",NSW2,20260830097", 1), + valid.replace(",0,10,", ",-1,10,", 1), + valid.replace(",10,", ",nan,", 1), + valid.replace(",FIRM,", ",bad status,", 1), + valid.replace("DISPATCHIS,AEMO", "DISPATCHIS\x00,AEMO", 1), + _mms_payload(*rows[:-1]), + _mms_payload(*rows, source_id="0000000000000002"), + ) + + for payload in invalid_payloads: + with self.subTest(payload=payload[:50]): + with self.assertRaises(AemoParseError): + parse_dispatch_price_mms_csv( + payload, + source_id=MMS_SOURCE_ID, + ingestion_version=7, + ) + + def test_mms_parser_ignores_empty_fields_in_unrelated_tables(self) -> None: + rows = _mms_payload( + _mms_row("NSW1", "10"), + _mms_row("QLD1", "11"), + _mms_row("SA1", "12"), + _mms_row("TAS1", "13"), + _mms_row("VIC1", "14"), + ).splitlines() + rows.insert(-1, "D,DISPATCH,UNIT_SOLUTION,5,2026/08/30 12:05:00,,") + rows[-1] = f"C,END OF REPORT,{len(rows)}" + + records = parse_dispatch_price_mms_csv( + "\n".join(rows) + "\n", + source_id=MMS_SOURCE_ID, + ingestion_version=7, + ) + + self.assertEqual(len(records), 5) + + def test_parses_rows_into_typed_prices_with_provenance_flags(self): + csv_text = """SETTLEMENTDATE,REGIONID,RRP,INTERVENTION,APCFLAG,RUNNO +2026/01/01 10:00:00,NSW1,125.50,0,0,1 +2026/01/01 10:05:00,NSW1,-10.25,1,4,2 +""" + + records = parse_dispatch_price_csv( + csv_text, + source_id="aemo-dispatch-fixture", + source_timestamp=SOURCE_TIME, + ingestion_version=7, + correction_version=2, + naive_timezone=timezone(timedelta(hours=10)), + ) + + self.assertEqual(len(records), 2) + self.assertEqual(records[0].interval_start, datetime(2026, 1, 1, 0, 0, tzinfo=UTC)) + self.assertEqual(records[0].price_aud_per_mwh, 125.5) + self.assertEqual(records[0].price_status, "available") + self.assertEqual(records[0].provenance.source_id, "aemo-dispatch-fixture") + self.assertEqual(records[0].provenance.ingestion_version, 7) + self.assertEqual(records[0].provenance.correction_version, 2) + self.assertIn("runno=1", records[0].quality_flags) + self.assertIn("intervention=1", records[1].quality_flags) + self.assertIn("apcflag=4", records[1].quality_flags) + self.assertEqual(records[1].price_status, "negative") + + def test_maps_aemo_price_flags(self): + csv_text = """SETTLEMENTDATE,REGIONID,RRP,INTERVENTION,APCFLAG,MARKETSUSPENDEDFLAG +2026/01/01 10:05:00,NSW1,125.50,1,4,1 +""" + + records = parse_dispatch_price_csv( + csv_text, + source_id="aemo-dispatch-fixture", + source_timestamp=SOURCE_TIME, + ingestion_version=7, + naive_timezone=timezone(timedelta(hours=10)), + ) + + actual = tuple( + getattr(records[0], field, None) + for field in ("intervention", "apc_flag", "market_suspended") + ) + self.assertEqual(actual, (1, 4, True)) + + def test_rejects_invalid_control_flags(self): + cases = ( + ("INTERVENTION", "-1"), + ("APCFLAG", "-1"), + ("MARKETSUSPENDEDFLAG", "-1"), + ("MARKETSUSPENDEDFLAG", "2"), + ) + + for field, value in cases: + with self.subTest(field=field, value=value): + values = { + "INTERVENTION": "0", + "APCFLAG": "0", + "MARKETSUSPENDEDFLAG": "0", + } + values[field] = value + csv_text = f"""SETTLEMENTDATE,REGIONID,RRP,INTERVENTION,APCFLAG,MARKETSUSPENDEDFLAG +2026/01/01 10:05:00,NSW1,125.50,{values['INTERVENTION']},{values['APCFLAG']},{values['MARKETSUSPENDEDFLAG']} +""" + + with self.assertRaises(AemoParseError): + parse_dispatch_price_csv( + csv_text, + source_id="aemo-dispatch-fixture", + source_timestamp=SOURCE_TIME, + ingestion_version=7, + naive_timezone=timezone(timedelta(hours=10)), + ) + + def test_blank_rrp_is_missing_sorted_rows(self): + csv_text = """SETTLEMENTDATE,REGIONID,RRP,INTERVENTION,APCFLAG,RUNNO +2026-01-01T10:10:00+10:00,NSW1,,0,0,3 +2026-01-01T10:00:00+10:00,NSW1,0,0,0,1 +""" + + records = parse_dispatch_price_csv( + csv_text, + source_id="aemo-dispatch-fixture", + source_timestamp=SOURCE_TIME, + ingestion_version=1, + ) + + self.assertEqual([record.interval_start for record in records], [ + datetime(2026, 1, 1, 0, 0, tzinfo=UTC), + datetime(2026, 1, 1, 0, 10, tzinfo=UTC), + ]) + self.assertEqual(records[0].price_aud_per_mwh, 0.0) + self.assertIsNone(records[1].price_aud_per_mwh) + self.assertEqual(records[1].price_status, "missing") + + def test_malformed_or_inconsistent_rows_fail_closed(self): + cases = ( + "REGIONID,RRP\nNSW1,10\n", + "SETTLEMENTDATE,REGIONID,RRP\nnot-a-time,NSW1,10\n", + "SETTLEMENTDATE,REGIONID,RRP\n2026/01/01 10:01:00,NSW1,10\n", + "SETTLEMENTDATE,REGIONID,RRP\n2026/01/01 10:00:00,,10\n", + ) + + for csv_text in cases: + with self.subTest(csv_text=csv_text): + with self.assertRaises(AemoParseError): + parse_dispatch_price_csv( + csv_text, + source_id="aemo-dispatch-fixture", + source_timestamp=SOURCE_TIME, + ingestion_version=1, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/backend/tests/test_api.py b/app/backend/tests/test_api.py new file mode 100644 index 0000000..ad21e71 --- /dev/null +++ b/app/backend/tests/test_api.py @@ -0,0 +1,47 @@ +"""Public HTTP contract tests for the fixture API.""" + +import unittest + +from fastapi.testclient import TestClient + +from batterywatch_api.main import app + + +class BatteryWatchApiTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.client = TestClient(app) + + def test_health_and_generator_contract(self): + self.assertEqual(self.client.get("/api/health").status_code, 200) + response = self.client.get("/api/generators") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json()["generators"][0]["duid"], "BWTEST1") + + def test_series_calculates_signs_and_labels_estimates(self): + response = self.client.get("/api/series", params={"generator": "BWTEST1"}) + self.assertEqual(response.status_code, 200) + body = response.json() + self.assertEqual(len(body["points"]), 12) + self.assertAlmostEqual(body["points"][0]["energy_mwh"], 0.125) + self.assertAlmostEqual(body["points"][1]["energy_mwh"], -5 / 60) + self.assertEqual(body["points"][3]["price_status"], "negative") + self.assertIsNone(body["points"][4]["price_aud_per_mwh"]) + self.assertEqual(body["points"][4]["price_status"], "missing") + self.assertTrue(body["estimate"]["is_estimate"]) + self.assertIn("Estimate only", body["estimate"]["disclaimer"]) + + def test_unknown_generator_and_invalid_ranges_fail_closed(self): + self.assertEqual(self.client.get("/api/series?generator=NOPE").status_code, 404) + self.assertEqual( + self.client.get("/api/series", params={"generator": "BWTEST1", "start": "2026-01-01T01:00:00Z", "end": "2026-01-01T00:00:00Z"}).status_code, + 400, + ) + self.assertEqual( + self.client.get("/api/series", params={"generator": "BWTEST1", "start": "2026-01-01T00:00:00Z", "end": "2026-01-09T00:00:00Z"}).status_code, + 400, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/backend/tests/test_api_database_generators.py b/app/backend/tests/test_api_database_generators.py new file mode 100644 index 0000000..041c8cb --- /dev/null +++ b/app/backend/tests/test_api_database_generators.py @@ -0,0 +1,427 @@ +"""Focused database generator-list tracer tests.""" + +from contextlib import contextmanager +from datetime import datetime, timedelta, timezone +import inspect +import unittest + +from fastapi.testclient import TestClient + +import batterywatch_api.main as main +from batterywatch_api.storage import ( + GeneratorMetadata, + InMemoryRepository, + StorageRepository, +) + + +class DatabaseGeneratorApiTests(unittest.TestCase): + def test_factory_accepts_a_repository_provider(self): + self.assertIn("repository_provider", inspect.signature(main.create_app).parameters) + + def test_storage_repository_declares_list_generators(self): + self.assertTrue(callable(getattr(StorageRepository, "list_generators", None))) + + def test_database_generators_maps_repository_metadata_as_database_utc_values(self): + record = GeneratorMetadata( + generator_id="DB-1", + site_name="Database Battery", + region="QLD1", + capacity_mw=3.5, + storage_capacity_mwh=7.0, + source_id="registry", + source_timestamp=datetime(2026, 1, 1, 10, 0, tzinfo=timezone(timedelta(hours=10))), + ingestion_version=1, + data_start=datetime(2026, 1, 1, 10, 5, tzinfo=timezone(timedelta(hours=10))), + data_end=datetime(2026, 1, 1, 11, 5, tzinfo=timezone(timedelta(hours=10))), + ) + + @contextmanager + def provider(): + class Repository(InMemoryRepository): + def list_generators(self): + return (record,) + + yield Repository() + + response = TestClient( + main.create_app( + mode="database", + health_tracer=lambda: True, + repository_provider=provider, + ) + ).get("/api/generators") + + self.assertEqual(response.status_code, 200) + self.assertEqual( + response.json(), + { + "generators": [ + { + "duid": "DB-1", + "site_name": "Database Battery", + "region": "QLD1", + "capacity_mw": 3.5, + "storage_capacity_mwh": 7.0, + "data_start": "2026-01-01T00:05:00Z", + "data_end": "2026-01-01T01:05:00Z", + "data_status": "database", + } + ] + }, + ) + + def test_default_database_provider_is_lazy(self): + calls = [] + + def connect(database_url): + calls.append(database_url) + raise AssertionError("connection must not be opened during app creation") + + main.create_app( + mode="database", + database_url="server-side-configured-url", + connection_factory=connect, + health_tracer=lambda: True, + ) + + self.assertEqual(calls, []) + + def test_default_database_provider_closes_connection_on_success(self): + row = ( + "DB-DEFAULT", + "Default Battery", + "VIC1", + 2.0, + 4.0, + "registry", + datetime(2026, 1, 1, tzinfo=timezone.utc), + 1, + 0, + datetime(2026, 1, 1, tzinfo=timezone.utc), + datetime(2026, 1, 1, 1, tzinfo=timezone.utc), + ) + calls = [] + + class Cursor: + def __init__(self): + self.closed = False + + def execute(self, statement, parameters): + self.parameters = parameters + + def fetchall(self): + return [row] + + def close(self): + self.closed = True + + class Connection: + def __init__(self): + self.cursor_instance = Cursor() + self.closed = False + + def cursor(self): + return self.cursor_instance + + def close(self): + self.closed = True + + connection = Connection() + + def connect(database_url): + calls.append(database_url) + return connection + + application = main.create_app( + mode="database", + database_url="server-side-configured-url", + connection_factory=connect, + health_tracer=lambda: True, + ) + response = TestClient(application).get("/api/generators") + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json()["generators"][0]["duid"], "DB-DEFAULT") + self.assertEqual(calls, ["server-side-configured-url"]) + self.assertTrue(connection.cursor_instance.closed) + self.assertTrue(connection.closed) + + def test_default_database_provider_closes_connection_on_query_failure(self): + failure_marker = "default-query-marker" + url_marker = "postgresql://default-secret.invalid/batterywatch" + + class Cursor: + def __init__(self): + self.closed = False + + def execute(self, statement, parameters): + raise RuntimeError(f"{failure_marker}: {url_marker}") + + def close(self): + self.closed = True + + class Connection: + def __init__(self): + self.cursor_instance = Cursor() + self.closed = False + + def cursor(self): + return self.cursor_instance + + def close(self): + self.closed = True + + connection = Connection() + response = TestClient( + main.create_app( + mode="database", + database_url=url_marker, + connection_factory=lambda _: connection, + health_tracer=lambda: True, + ) + ).get("/api/generators") + + self.assertEqual( + response.json(), {"detail": "Database generators unavailable"} + ) + self.assertEqual(response.status_code, 503) + self.assertNotIn(failure_marker, response.text) + self.assertNotIn(url_marker, response.text) + self.assertNotIn("BWTEST1", response.text) + self.assertTrue(connection.cursor_instance.closed) + self.assertTrue(connection.closed) + + def test_database_generators_returns_generic_503_for_provider_failure(self): + failure_marker = "provider-failure-marker" + url_marker = "postgresql://secret-marker.invalid/batterywatch" + + def provider(): + raise RuntimeError(f"{failure_marker}: {url_marker}") + + response = TestClient( + main.create_app( + mode="database", + database_url=url_marker, + health_tracer=lambda: True, + repository_provider=provider, + ) + ).get("/api/generators") + + self.assertEqual( + response.json(), {"detail": "Database generators unavailable"} + ) + self.assertEqual(response.status_code, 503) + self.assertNotIn(failure_marker, response.text) + self.assertNotIn(url_marker, response.text) + self.assertNotIn("BWTEST1", response.text) + + def test_database_generators_returns_generic_503_for_context_entry_failure(self): + failure_marker = "context-entry-marker" + url_marker = "postgresql://entry-secret.invalid/batterywatch" + + class FailingContext: + def __enter__(self): + raise RuntimeError(f"{failure_marker}: {url_marker}") + + def __exit__(self, exc_type, exc_value, traceback): + return False + + response = TestClient( + main.create_app( + mode="database", + database_url=url_marker, + health_tracer=lambda: True, + repository_provider=lambda: FailingContext(), + ) + ).get("/api/generators") + + self.assertEqual( + response.json(), {"detail": "Database generators unavailable"} + ) + self.assertEqual(response.status_code, 503) + self.assertNotIn(failure_marker, response.text) + self.assertNotIn(url_marker, response.text) + self.assertNotIn("BWTEST1", response.text) + + def test_database_generators_returns_generic_503_for_list_failure(self): + failure_marker = "list-query-marker" + url_marker = "postgresql://query-secret.invalid/batterywatch" + + @contextmanager + def provider(): + class Repository(InMemoryRepository): + def list_generators(self): + raise RuntimeError(f"{failure_marker}: {url_marker}") + + yield Repository() + + response = TestClient( + main.create_app( + mode="database", + database_url=url_marker, + health_tracer=lambda: True, + repository_provider=provider, + ) + ).get("/api/generators") + + self.assertEqual( + response.json(), {"detail": "Database generators unavailable"} + ) + self.assertEqual(response.status_code, 503) + self.assertNotIn(failure_marker, response.text) + self.assertNotIn(url_marker, response.text) + self.assertNotIn("BWTEST1", response.text) + + def test_database_generators_returns_generic_503_for_mapping_failure(self): + failure_marker = "mapping-failure-marker" + url_marker = "postgresql://mapping-secret.invalid/batterywatch" + + class InvalidRecord: + generator_id = "DB-BAD" + site_name = "Bad Battery" + region = "NSW1" + capacity_mw = 1.0 + storage_capacity_mwh = 2.0 + @property + def data_start(self): + raise RuntimeError(f"{failure_marker}: {url_marker}") + + data_end = None + + @contextmanager + def provider(): + class Repository(InMemoryRepository): + def list_generators(self) -> tuple[GeneratorMetadata, ...]: + return (InvalidRecord(),) # type: ignore[reportReturnType] + + yield Repository() + + response = TestClient( + main.create_app( + mode="database", + database_url=url_marker, + health_tracer=lambda: True, + repository_provider=provider, + ) + ).get("/api/generators") + + self.assertEqual( + response.json(), {"detail": "Database generators unavailable"} + ) + self.assertEqual(response.status_code, 503) + self.assertNotIn(failure_marker, response.text) + self.assertNotIn(url_marker, response.text) + self.assertNotIn("BWTEST1", response.text) + + def test_database_generators_returns_generic_503_for_context_exit_failure(self): + failure_marker = "context-exit-marker" + url_marker = "postgresql://exit-secret.invalid/batterywatch" + + @contextmanager + def provider(): + class Repository(InMemoryRepository): + def list_generators(self): + return () + + try: + yield Repository() + finally: + raise RuntimeError(f"{failure_marker}: {url_marker}") + + response = TestClient( + main.create_app( + mode="database", + database_url=url_marker, + health_tracer=lambda: True, + repository_provider=provider, + ) + ).get("/api/generators") + + self.assertEqual( + response.json(), {"detail": "Database generators unavailable"} + ) + self.assertEqual(response.status_code, 503) + self.assertNotIn(failure_marker, response.text) + self.assertNotIn(url_marker, response.text) + self.assertNotIn("BWTEST1", response.text) + + def test_database_generators_returns_null_for_unavailable_bounds(self): + record = GeneratorMetadata( + generator_id="DB-2", + site_name="Partial Battery", + region="NSW1", + capacity_mw=1.0, + storage_capacity_mwh=2.0, + source_id="registry", + source_timestamp=datetime(2026, 1, 1, tzinfo=timezone.utc), + ingestion_version=1, + data_start=None, + data_end=None, + ) + + @contextmanager + def provider(): + class Repository(InMemoryRepository): + def list_generators(self): + return (record,) + + yield Repository() + + response = TestClient( + main.create_app( + mode="database", + health_tracer=lambda: True, + repository_provider=provider, + ) + ).get("/api/generators") + + self.assertEqual(response.status_code, 200) + generator = response.json()["generators"][0] + self.assertIsNone(generator["data_start"]) + self.assertIsNone(generator["data_end"]) + + def test_database_generators_uses_injected_provider(self): + events = [] + + class Repository(InMemoryRepository): + def list_generators(self): + events.append("list") + return () + + @contextmanager + def provider(): + events.append("enter") + yield Repository() + events.append("exit") + + response = TestClient( + main.create_app( + mode="database", + health_tracer=lambda: True, + repository_provider=provider, + ) + ).get("/api/generators") + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json(), {"generators": []}) + self.assertEqual(events, ["enter", "list", "exit"]) + + def test_fixture_generators_do_not_invoke_a_database_provider(self): + calls = [] + + def provider(): + calls.append("called") + raise AssertionError("fixture mode must not use the database provider") + + response = TestClient( + main.create_app(repository_provider=provider) + ).get("/api/generators") + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json()["generators"][0]["duid"], "BWTEST1") + self.assertEqual(calls, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/backend/tests/test_api_database_series.py b/app/backend/tests/test_api_database_series.py new file mode 100644 index 0000000..4db2438 --- /dev/null +++ b/app/backend/tests/test_api_database_series.py @@ -0,0 +1,761 @@ +"""Focused database-series tracer tests.""" + +from contextlib import AbstractContextManager, contextmanager +from datetime import datetime, timedelta, timezone +import inspect +import unittest +from typing import cast + +from fastapi.testclient import TestClient + +import batterywatch_api.main as main +from batterywatch_api.storage import ( + GeneratorMetadata, + GeneratorPower5m, + GeneratorSoc5m, + InMemoryRepository, + RegionalPrice5m, + StorageRepository, +) + + +class DatabaseSeriesApiTests(unittest.TestCase): + def test_storage_repository_declares_bounded_series_list_methods(self): + for name in ("list_power", "list_soc", "list_prices"): + method = getattr(StorageRepository, name, None) + self.assertTrue(callable(method), name) + assert callable(method) + parameters = list(inspect.signature(method).parameters) + self.assertEqual(parameters[2:], ["start", "end"]) + self.assertIsNone(inspect.signature(method).parameters["start"].default) + self.assertIsNone(inspect.signature(method).parameters["end"].default) + + def test_database_series_reads_generator_metadata_through_provider(self): + record = GeneratorMetadata( + generator_id="DB-1", + site_name="Database Battery", + region="QLD1", + capacity_mw=3.5, + storage_capacity_mwh=7.0, + source_id="registry", + source_timestamp=datetime(2026, 1, 1, tzinfo=timezone.utc), + ingestion_version=1, + ) + + @contextmanager + def provider(): + class Repository(InMemoryRepository): + def read_generator(self, generator_id): + self.read_generator_id = generator_id + return record + + yield Repository() + + response = TestClient( + main.create_app( + mode="database", + health_tracer=lambda: True, + repository_provider=provider, + ) + ).get( + "/api/series", + params={ + "generator": "DB-1", + "start": "2026-01-01T00:00:00Z", + "end": "2026-01-01T00:05:00Z", + }, + ) + + self.assertEqual(response.status_code, 200) + + def test_database_series_unknown_generator_is_not_replaced_by_fixture(self): + events = [] + + @contextmanager + def provider(): + class Repository(InMemoryRepository): + def read_generator(self, generator_id): + events.append(("read_generator", generator_id)) + return None + + def list_power(self, generator_id, start=None, end=None): + raise AssertionError("unknown generators must not read series") + + yield Repository() + + response = TestClient( + main.create_app( + mode="database", + health_tracer=lambda: True, + repository_provider=provider, + ) + ).get( + "/api/series", + params={ + "generator": "NOPE", + "start": "2026-01-01T00:00:00Z", + "end": "2026-01-01T00:05:00Z", + }, + ) + + self.assertEqual(response.status_code, 404) + + def test_database_series_list_failure_is_generic(self): + failure_marker = "series-list-marker" + secret_marker = "postgresql://list-secret.invalid/batterywatch" + metadata = GeneratorMetadata( + generator_id="DB-FAIL", + site_name="Failure Battery", + region="NSW1", + capacity_mw=1.0, + storage_capacity_mwh=2.0, + source_id="registry", + source_timestamp=datetime(2026, 1, 1, tzinfo=timezone.utc), + ingestion_version=1, + ) + + @contextmanager + def provider(): + class Repository(InMemoryRepository): + def read_generator(self, generator_id): + return metadata + + def list_power(self, generator_id, start=None, end=None): + raise RuntimeError(f"{failure_marker}: {secret_marker}") + + yield Repository() + + response = TestClient( + main.create_app( + mode="database", + health_tracer=lambda: True, + repository_provider=provider, + ) + ).get( + "/api/series", + params={ + "generator": "DB-FAIL", + "start": "2026-01-01T00:00:00Z", + "end": "2026-01-01T00:05:00Z", + }, + ) + + self.assertEqual( + response.json(), {"detail": "Database series unavailable"} + ) + self.assertEqual(response.status_code, 503) + self.assertNotIn(failure_marker, response.text) + self.assertNotIn(secret_marker, response.text) + + def test_database_series_mapping_failure_is_generic(self): + failure_marker = "series-mapping-marker" + secret_marker = "postgresql://mapping-secret.invalid/batterywatch" + + class InvalidMetadata: + generator_id = "DB-BAD" + site_name = "Bad Battery" + region = "NSW1" + capacity_mw = 1.0 + storage_capacity_mwh = 2.0 + + @property + def data_start(self): + raise RuntimeError(f"{failure_marker}: {secret_marker}") + + data_end = None + + @contextmanager + def provider(): + class Repository(InMemoryRepository): + def read_generator(self, generator_id): + return cast(GeneratorMetadata, InvalidMetadata()) + + yield Repository() + + response = TestClient( + main.create_app( + mode="database", + health_tracer=lambda: True, + repository_provider=provider, + ) + ).get( + "/api/series", + params={ + "generator": "DB-BAD", + "start": "2026-01-01T00:00:00Z", + "end": "2026-01-01T00:05:00Z", + }, + ) + + self.assertEqual( + response.json(), {"detail": "Database series unavailable"} + ) + self.assertEqual(response.status_code, 503) + self.assertNotIn(failure_marker, response.text) + self.assertNotIn(secret_marker, response.text) + + def test_database_series_context_exit_failure_is_generic(self): + failure_marker = "series-exit-marker" + secret_marker = "postgresql://exit-secret.invalid/batterywatch" + metadata = GeneratorMetadata( + generator_id="DB-EXIT", + site_name="Exit Battery", + region="NSW1", + capacity_mw=1.0, + storage_capacity_mwh=2.0, + source_id="registry", + source_timestamp=datetime(2026, 1, 1, tzinfo=timezone.utc), + ingestion_version=1, + ) + + @contextmanager + def provider(): + class Repository(InMemoryRepository): + def read_generator(self, generator_id): + return metadata + + def list_power(self, generator_id, start=None, end=None): + return () + + def list_soc(self, generator_id, start=None, end=None): + return () + + def list_prices(self, region, start=None, end=None): + return () + + try: + yield Repository() + finally: + raise RuntimeError(f"{failure_marker}: {secret_marker}") + + response = TestClient( + main.create_app( + mode="database", + health_tracer=lambda: True, + repository_provider=provider, + ) + ).get( + "/api/series", + params={ + "generator": "DB-EXIT", + "start": "2026-01-01T00:00:00Z", + "end": "2026-01-01T00:05:00Z", + }, + ) + + self.assertEqual( + response.json(), {"detail": "Database series unavailable"} + ) + self.assertEqual(response.status_code, 503) + self.assertNotIn(failure_marker, response.text) + self.assertNotIn(secret_marker, response.text) + + def test_database_series_context_entry_failure_is_generic(self): + failure_marker = "series-entry-marker" + secret_marker = "postgresql://entry-secret.invalid/batterywatch" + + class FailingContext: + def __enter__(self): + raise RuntimeError(f"{failure_marker}: {secret_marker}") + + def __exit__(self, exc_type, exc_value, traceback): + return False + + response = TestClient( + main.create_app( + mode="database", + health_tracer=lambda: True, + repository_provider=lambda: FailingContext(), + ) + ).get( + "/api/series", + params={ + "generator": "DB-1", + "start": "2026-01-01T00:00:00Z", + "end": "2026-01-01T00:05:00Z", + }, + ) + + self.assertEqual( + response.json(), {"detail": "Database series unavailable"} + ) + + def test_database_series_provider_failure_is_generic_and_no_fixture_fallback(self): + failure_marker = "series-provider-marker" + secret_marker = "postgresql://series-secret.invalid/batterywatch" + + def provider(): + raise RuntimeError(f"{failure_marker}: {secret_marker}") + + response = TestClient( + main.create_app( + mode="database", + health_tracer=lambda: True, + repository_provider=provider, + ) + ).get( + "/api/series", + params={ + "generator": "BWTEST1", + "start": "2026-01-01T00:00:00Z", + "end": "2026-01-01T00:05:00Z", + }, + ) + + self.assertEqual( + response.json(), {"detail": "Database series unavailable"} + ) + self.assertEqual(response.status_code, 503) + self.assertNotIn(failure_marker, response.text) + self.assertNotIn(secret_marker, response.text) + self.assertNotIn("BWTEST1", response.text) + + def test_database_series_requires_explicit_start_and_end(self): + provider_calls = [] + + @contextmanager + def provider(): + provider_calls.append("called") + raise AssertionError("bounds must be validated before the provider") + yield # pragma: no cover + + response = TestClient( + main.create_app( + mode="database", + health_tracer=lambda: True, + repository_provider=provider, + ) + ).get("/api/series", params={"generator": "DB-1"}) + + self.assertEqual(response.status_code, 400) + + def test_database_series_rejects_non_increasing_bounds(self): + def provider() -> AbstractContextManager[StorageRepository]: + raise AssertionError("bounds must be validated before the provider") + + response = TestClient( + main.create_app( + mode="database", + health_tracer=lambda: True, + repository_provider=provider, + ) + ).get( + "/api/series", + params={ + "generator": "DB-UTC", + "start": "2026-01-01T00:05:00Z", + "end": "2026-01-01T00:05:00Z", + }, + ) + + self.assertEqual(response.status_code, 400) + + def test_database_series_rejects_ranges_over_seven_days(self): + def provider() -> AbstractContextManager[StorageRepository]: + raise AssertionError("bounds must be validated before the provider") + + response = TestClient( + main.create_app( + mode="database", + health_tracer=lambda: True, + repository_provider=provider, + ) + ).get( + "/api/series", + params={ + "generator": "DB-UTC", + "start": "2026-01-01T00:00:00Z", + "end": "2026-01-08T00:00:01Z", + }, + ) + + self.assertEqual(response.status_code, 400) + + def test_database_series_accepts_non_aligned_utc_query_bounds(self): + metadata = GeneratorMetadata( + generator_id="DB-NONALIGNED", + site_name="Non-aligned Battery", + region="NSW1", + capacity_mw=1.0, + storage_capacity_mwh=2.0, + source_id="registry", + source_timestamp=datetime(2026, 1, 1, tzinfo=timezone.utc), + ingestion_version=1, + ) + power = GeneratorPower5m( + generator_id="DB-NONALIGNED", + interval_start=datetime(2026, 1, 1, 0, 5, tzinfo=timezone.utc), + power_mw=0.0, + source_id="dispatch", + source_timestamp=datetime(2026, 1, 1, 0, 5, tzinfo=timezone.utc), + ingestion_version=1, + ) + + @contextmanager + def provider(): + repository = InMemoryRepository() + repository.upsert_generator(metadata) + repository.upsert_power(power) + yield repository + + response = TestClient( + main.create_app( + mode="database", + health_tracer=lambda: True, + repository_provider=provider, + ) + ).get( + "/api/series", + params={ + "generator": "DB-NONALIGNED", + "start": "2026-01-01T00:01:00Z", + "end": "2026-01-01T00:06:00Z", + }, + ) + + self.assertEqual(response.status_code, 200) + self.assertEqual( + response.json()["points"], + [ + { + "timestamp": "2026-01-01T00:05:00Z", + "power_mw": 0.0, + "soc_percent": None, + "price_aud_per_mwh": None, + "energy_mwh": 0.0, + "gross_value_aud": None, + "charging_cost_aud": None, + "net_energy_value_aud": None, + "price_status": "missing", + } + ], + ) + + def test_database_series_reads_all_three_bounded_repository_series(self): + start = datetime(2026, 1, 1, tzinfo=timezone.utc) + end = start + timedelta(minutes=20) + metadata = GeneratorMetadata( + generator_id="DB-BOUND", + site_name="Bounded Battery", + region="NSW1", + capacity_mw=1.0, + storage_capacity_mwh=2.0, + source_id="registry", + source_timestamp=start, + ingestion_version=1, + ) + power = GeneratorPower5m( + generator_id="DB-BOUND", + interval_start=start, + power_mw=0.0, + source_id="dispatch", + source_timestamp=start, + ingestion_version=1, + ) + soc = GeneratorSoc5m( + generator_id="DB-BOUND", + interval_start=start, + soc_percent=None, + source_id="telemetry", + source_timestamp=start, + ingestion_version=1, + ) + price = RegionalPrice5m( + region="NSW1", + interval_start=start, + price_aud_per_mwh=None, + price_status="missing", + source_id="rrp", + source_timestamp=start, + ingestion_version=1, + ) + calls = [] + + @contextmanager + def provider(): + class Repository(InMemoryRepository): + def read_generator(self, generator_id): + return metadata + + def list_power(self, generator_id, start=None, end=None): + calls.append(("power", generator_id, start, end)) + return (power,) + + def list_soc(self, generator_id, start=None, end=None): + calls.append(("soc", generator_id, start, end)) + return (soc,) + + def list_prices(self, region, start=None, end=None): + calls.append(("prices", region, start, end)) + return (price,) + + yield Repository() + + response = TestClient( + main.create_app( + mode="database", + health_tracer=lambda: True, + repository_provider=provider, + ) + ).get( + "/api/series", + params={ + "generator": "DB-BOUND", + "start": start.isoformat().replace("+00:00", "Z"), + "end": end.isoformat().replace("+00:00", "Z"), + }, + ) + + self.assertEqual(calls, [ + ("power", "DB-BOUND", start, end), + ("soc", "DB-BOUND", start, end), + ("prices", "NSW1", start, end), + ]) + + def test_database_series_uses_power_rows_with_exact_nullable_joins(self): + start = datetime(2026, 1, 1, tzinfo=timezone.utc) + end = start + timedelta(minutes=15) + metadata = GeneratorMetadata( + generator_id="DB-JOIN", + site_name="Join Battery", + region="NSW1", + capacity_mw=2.0, + storage_capacity_mwh=4.0, + source_id="registry", + source_timestamp=start, + ingestion_version=1, + ) + power = tuple( + GeneratorPower5m( + generator_id="DB-JOIN", + interval_start=start + timedelta(minutes=5 * index), + power_mw=value, + source_id="dispatch", + source_timestamp=start, + ingestion_version=1, + ) + for index, value in enumerate((0.0, 2.0, -1.0)) + ) + soc = ( + GeneratorSoc5m( + generator_id="DB-JOIN", + interval_start=start, + soc_percent=None, + source_id="telemetry", + source_timestamp=start, + ingestion_version=1, + ), + GeneratorSoc5m( + generator_id="DB-JOIN", + interval_start=start + timedelta(minutes=5), + soc_percent=55.0, + source_id="telemetry", + source_timestamp=start, + ingestion_version=1, + ), + ) + prices = tuple( + RegionalPrice5m( + region="NSW1", + interval_start=start + timedelta(minutes=5 * index), + price_aud_per_mwh=value, + price_status=( + "missing" + if value is None + else "negative" + if value < 0 + else "available" + ), + source_id="rrp", + source_timestamp=start, + ingestion_version=1, + ) + for index, value in enumerate((10.0, -20.0, None)) + ) + + @contextmanager + def provider(): + class Repository(InMemoryRepository): + def read_generator(self, generator_id): + return metadata + + def list_power(self, generator_id, start=None, end=None): + return power + + def list_soc(self, generator_id, start=None, end=None): + return soc + + def list_prices(self, region, start=None, end=None): + return prices + + yield Repository() + + response = TestClient( + main.create_app( + mode="database", + health_tracer=lambda: True, + repository_provider=provider, + ) + ).get( + "/api/series", + params={ + "generator": "DB-JOIN", + "start": start.isoformat().replace("+00:00", "Z"), + "end": end.isoformat().replace("+00:00", "Z"), + }, + ) + + body = response.json() + self.assertEqual( + [point["timestamp"] for point in body["points"]], + [ + "2026-01-01T00:00:00Z", + "2026-01-01T00:05:00Z", + "2026-01-01T00:10:00Z", + ], + ) + self.assertEqual([point["power_mw"] for point in body["points"]], [0.0, 2.0, -1.0]) + self.assertIsNone(body["points"][0]["soc_percent"]) + self.assertEqual(body["points"][1]["soc_percent"], 55.0) + self.assertIsNone(body["points"][2]["soc_percent"]) + self.assertEqual(body["points"][1]["price_status"], "negative") + self.assertIsNone(body["points"][2]["price_aud_per_mwh"]) + self.assertEqual(body["points"][2]["price_status"], "missing") + self.assertEqual(body["provenance"]["data_mode"], "database") + self.assertEqual(body["provenance"]["power_source"], "dispatch") + self.assertEqual(body["provenance"]["price_source"], "rrp") + self.assertEqual(body["provenance"]["soc_source"], "telemetry") + + def test_database_series_reuses_estimate_math_and_coverage(self): + start = datetime(2026, 1, 1, tzinfo=timezone.utc) + end = start + timedelta(minutes=15) + metadata = GeneratorMetadata( + generator_id="DB-ESTIMATE", + site_name="Estimate Battery", + region="NSW1", + capacity_mw=2.0, + storage_capacity_mwh=4.0, + source_id="registry", + source_timestamp=start, + ingestion_version=1, + ) + power = tuple( + GeneratorPower5m( + generator_id="DB-ESTIMATE", + interval_start=start + timedelta(minutes=5 * index), + power_mw=value, + source_id="dispatch", + source_timestamp=start, + ingestion_version=1, + ) + for index, value in enumerate((1.5, -1.0, 0.0)) + ) + soc = tuple( + GeneratorSoc5m( + generator_id="DB-ESTIMATE", + interval_start=start + timedelta(minutes=5 * index), + soc_percent=value, + source_id="telemetry", + source_timestamp=start, + ingestion_version=1, + ) + for index, value in enumerate((45.0, None, 50.0)) + ) + prices = tuple( + RegionalPrice5m( + region="NSW1", + interval_start=start + timedelta(minutes=5 * index), + price_aud_per_mwh=value, + price_status="missing" if value is None else "available", + source_id="rrp", + source_timestamp=start, + ingestion_version=1, + ) + for index, value in enumerate((100.0, 80.0, None)) + ) + + @contextmanager + def provider(): + class Repository(InMemoryRepository): + def read_generator(self, generator_id): + return metadata + + def list_power(self, generator_id, start=None, end=None): + return power + + def list_soc(self, generator_id, start=None, end=None): + return soc + + def list_prices(self, region, start=None, end=None): + return prices + + yield Repository() + + response = TestClient( + main.create_app( + mode="database", + health_tracer=lambda: True, + repository_provider=provider, + ) + ).get( + "/api/series", + params={ + "generator": "DB-ESTIMATE", + "start": start.isoformat().replace("+00:00", "Z"), + "end": end.isoformat().replace("+00:00", "Z"), + }, + ) + + body = response.json() + self.assertAlmostEqual(body["points"][0]["energy_mwh"], 0.125) + self.assertAlmostEqual(body["points"][0]["gross_value_aud"], 12.5) + self.assertAlmostEqual(body["points"][1]["charging_cost_aud"], 80 / 12) + self.assertIsNone(body["points"][2]["gross_value_aud"]) + self.assertEqual(body["coverage"]["missing_price_intervals"], 1) + self.assertEqual(body["coverage"]["missing_soc_intervals"], 1) + self.assertAlmostEqual(body["summary"]["exported_energy_mwh"], 0.125) + self.assertAlmostEqual(body["summary"]["imported_energy_mwh"], 1 / 12) + self.assertAlmostEqual(body["summary"]["net_energy_value_aud"], 12.5 - 80 / 12) + + def test_database_series_normalizes_offset_aware_bounds_to_utc(self): + record = GeneratorMetadata( + generator_id="DB-UTC", + site_name="UTC Battery", + region="NSW1", + capacity_mw=1.0, + storage_capacity_mwh=2.0, + source_id="registry", + source_timestamp=datetime(2026, 1, 1, tzinfo=timezone.utc), + ingestion_version=1, + ) + + @contextmanager + def provider(): + class Repository(InMemoryRepository): + def read_generator(self, generator_id): + return record + + yield Repository() + + response = TestClient( + main.create_app( + mode="database", + health_tracer=lambda: True, + repository_provider=provider, + ) + ).get( + "/api/series", + params={ + "generator": "DB-UTC", + "start": "2026-01-01T10:00:00+10:00", + "end": "2026-01-01T10:05:00+10:00", + }, + ) + + self.assertEqual( + response.json()["requested_start"], "2026-01-01T00:00:00Z" + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/backend/tests/test_battery_assets.py b/app/backend/tests/test_battery_assets.py new file mode 100644 index 0000000..7318e5d --- /dev/null +++ b/app/backend/tests/test_battery_assets.py @@ -0,0 +1,78 @@ +"""Tests for strict reviewed battery asset configuration.""" + +from __future__ import annotations + +import json +from pathlib import Path +from tempfile import TemporaryDirectory +import unittest + +from batterywatch_api.battery_assets import load_battery_assets + + +CONFIG = Path(__file__).resolve().parents[2] / "config" / "battery_assets.json" + + +class BatteryAssetTests(unittest.TestCase): + def test_reviewed_config_loads_64_unique_current_batteries(self) -> None: + assets = load_battery_assets(CONFIG) + duids = {asset.duid for asset in assets} + + self.assertEqual( + (len(assets), len(duids), "HPR1" in duids), + (64, 64, True), + ) + self.assertTrue( + { + "ERB01", + "ERB02", + "LDBESS1", + "MREHA1", + "MREHA2", + "MREHA3", + "SNB01", + "SNB02", + "STABESS1", + "WTAHB1", + }.issubset(duids) + ) + + def test_reviewed_config_excludes_only_unobservable_or_incomplete_registry_rows(self) -> None: + payload = json.loads(CONFIG.read_text(encoding="utf-8")) + + self.assertEqual( + {item["duid"] for item in payload["excluded"]}, + {"KEPBL1", "MOORABS1", "NESBESS1", "NESBESS2", "WILLBES1"}, + ) + + def test_invalid_or_duplicate_assets_fail_closed(self) -> None: + valid = json.loads(CONFIG.read_text(encoding="utf-8")) + cases: list[dict[str, object]] = [] + + duplicate = json.loads(json.dumps(valid)) + duplicate["assets"].append(dict(duplicate["assets"][0])) + cases.append(duplicate) + + unknown_key = json.loads(json.dumps(valid)) + unknown_key["assets"][0]["unexpected"] = True + cases.append(unknown_key) + + bad_timestamp = json.loads(json.dumps(valid)) + bad_timestamp["assets"][0]["source_timestamp"] = "2025-03-14" + cases.append(bad_timestamp) + + bad_capacity = json.loads(json.dumps(valid)) + bad_capacity["assets"][0]["capacity_mw"] = 0 + cases.append(bad_capacity) + + with TemporaryDirectory() as directory: + path = Path(directory) / "assets.json" + for index, payload in enumerate(cases): + with self.subTest(index=index): + path.write_text(json.dumps(payload), encoding="utf-8") + with self.assertRaises(ValueError): + load_battery_assets(path) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/backend/tests/test_collector.py b/app/backend/tests/test_collector.py new file mode 100644 index 0000000..1b34983 --- /dev/null +++ b/app/backend/tests/test_collector.py @@ -0,0 +1,151 @@ +"""Tests for one latest Dispatch SCADA collection cycle.""" + +from dataclasses import dataclass +from datetime import datetime, timezone +from hashlib import sha256 +from io import BytesIO +import unittest +from zipfile import ZIP_DEFLATED, ZipFile + +from batterywatch_api.collector import ( + DISPATCH_SCADA_ARTIFACT_MAX_BYTES, + DISPATCH_SCADA_INDEX_MAX_BYTES, + DISPATCH_SCADA_INDEX_URL, + DispatchScadaCollection, + collect_latest_dispatch_scada, +) +from batterywatch_api.nemweb_dispatch_scada import ( + DispatchScadaArtifact, + DispatchScadaArtifactRef, +) +from batterywatch_api.nemweb_http import NemwebHttpResource +from batterywatch_api.storage import GeneratorPower5m + + +NEWER_FILENAME = "PUBLIC_DISPATCHSCADA_202608291205_0000000000000002.zip" +NEWER_URL = DISPATCH_SCADA_INDEX_URL + NEWER_FILENAME +NEWER_SOURCE_ID = "0000000000000002" + + +@dataclass +class FakeFetch: + resources: dict[str, NemwebHttpResource] + + def __post_init__(self) -> None: + self.calls: list[tuple[str, int]] = [] + + def __call__(self, url: str, *, max_bytes: int) -> NemwebHttpResource: + self.calls.append((url, max_bytes)) + return self.resources[url] + + +def _make_resource(url: str, body: bytes, content_type: str) -> NemwebHttpResource: + return NemwebHttpResource( + requested_url=url, + resolved_url=url, + body=body, + content_type=content_type, + etag=None, + last_modified=None, + ) + + +def _make_zip(filename: str, csv_payload: str) -> tuple[bytes, str]: + member_name = filename.removesuffix(".zip") + ".CSV" + buffer = BytesIO() + with ZipFile(buffer, "w", compression=ZIP_DEFLATED) as archive: + archive.writestr(member_name, csv_payload.encode("utf-8")) + return buffer.getvalue(), member_name + + +class CollectLatestDispatchScadaTests(unittest.TestCase): + def test_collects_latest_canonical_artifact_and_records(self) -> None: + index_html = f""" + + newer + older + + """ + csv_payload = f"""C,NEMP.WORLD,DISPATCHSCADA,AEMO,PUBLIC,2026/08/29,12:05:15,{NEWER_SOURCE_ID},DISPATCHSCADA,0000000000000001 +I,DISPATCH,UNIT_SCADA,1,SETTLEMENTDATE,DUID,SCADAVALUE,LASTCHANGED +D,DISPATCH,UNIT_SCADA,1,"2026/08/29 12:05:00",BWTR2,0,"2026/08/29 12:05:11" +D,DISPATCH,UNIT_SCADA,1,"2026/08/29 12:00:00",BWTR1,-4.5,"2026/08/29 12:00:11" +C,END OF REPORT,5 +""" + zip_payload, member_name = _make_zip(NEWER_FILENAME, csv_payload) + fetch = FakeFetch( + resources={ + DISPATCH_SCADA_INDEX_URL: _make_resource( + DISPATCH_SCADA_INDEX_URL, + index_html.encode("utf-8"), + "text/html; charset=utf-8", + ), + NEWER_URL: _make_resource( + NEWER_URL, + zip_payload, + "application/zip", + ), + } + ) + + reference = DispatchScadaArtifactRef( + url=NEWER_URL, + zip_filename=NEWER_FILENAME, + source_artifact_id=NEWER_SOURCE_ID, + report_timestamp=datetime(2026, 8, 29, 2, 5, tzinfo=timezone.utc), + ) + expected = ( + DispatchScadaCollection( + artifact=DispatchScadaArtifact( + reference=reference, + csv_member_name=member_name, + csv_payload=csv_payload, + zip_sha256=sha256(zip_payload).hexdigest(), + raw_zip=zip_payload, + ), + records=( + GeneratorPower5m( + generator_id="BWTR1", + interval_start=datetime( + 2026, 8, 29, 2, 0, tzinfo=timezone.utc + ), + power_mw=-4.5, + source_id=NEWER_SOURCE_ID, + source_timestamp=datetime( + 2026, 8, 29, 2, 0, 11, tzinfo=timezone.utc + ), + ingestion_version=7, + correction_version=2, + ), + GeneratorPower5m( + generator_id="BWTR2", + interval_start=datetime( + 2026, 8, 29, 2, 5, tzinfo=timezone.utc + ), + power_mw=0.0, + source_id=NEWER_SOURCE_ID, + source_timestamp=datetime( + 2026, 8, 29, 2, 5, 11, tzinfo=timezone.utc + ), + ingestion_version=7, + correction_version=2, + ), + ), + ), + ( + (DISPATCH_SCADA_INDEX_URL, DISPATCH_SCADA_INDEX_MAX_BYTES), + (NEWER_URL, DISPATCH_SCADA_ARTIFACT_MAX_BYTES), + ), + ) + + result = collect_latest_dispatch_scada( + ingestion_version=7, + correction_version=2, + fetch=fetch, + ) + + self.assertEqual((result, tuple(fetch.calls)), expected) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/backend/tests/test_collector_service.py b/app/backend/tests/test_collector_service.py new file mode 100644 index 0000000..b8da199 --- /dev/null +++ b/app/backend/tests/test_collector_service.py @@ -0,0 +1,328 @@ +"""Tests for the separate Dispatch SCADA collector runtime.""" + +from __future__ import annotations + +from contextlib import redirect_stdout +from datetime import datetime, timezone +from io import StringIO +from pathlib import Path +from typing import Any +import unittest + +from batterywatch_api.battery_assets import BatteryAsset +from batterywatch_api.collector import DispatchScadaCollection +from batterywatch_api.collector_service import ( + CollectorCycleResult, + main, + run_collection_cycle, + run_database_cycle, + run_price_collection_cycle, + run_polling_loop, +) +from batterywatch_api.dispatch_price_ingestion import DispatchPriceIngestionResult +from batterywatch_api.dispatch_scada_ingestion import DispatchScadaIngestionResult +from batterywatch_api.nemweb_dispatch_prices import ( + DispatchPriceArtifact, + DispatchPriceArtifactRef, + DispatchPriceCollection, +) +from batterywatch_api.nemweb_dispatch_scada import ( + DispatchScadaArtifact, + DispatchScadaArtifactRef, +) +from batterywatch_api.storage import GeneratorPower5m, RegionalPrice5m + + +UTC = timezone.utc +REPORT_TIME = datetime(2026, 8, 29, 2, 5, tzinfo=UTC) +SOURCE_TIME = datetime(2026, 8, 29, 2, 5, 11, tzinfo=UTC) +ARTIFACT_ID = "0000000000000042" + + +def collection() -> DispatchScadaCollection: + filename = f"PUBLIC_DISPATCHSCADA_202608291205_{ARTIFACT_ID}.zip" + reference = DispatchScadaArtifactRef( + url="https://www.nemweb.com.au/REPORTS/CURRENT/Dispatch_SCADA/" + filename, + zip_filename=filename, + source_artifact_id=ARTIFACT_ID, + report_timestamp=REPORT_TIME, + ) + return DispatchScadaCollection( + artifact=DispatchScadaArtifact( + reference=reference, + csv_member_name=filename.removesuffix(".zip") + ".CSV", + csv_payload="validated-csv", + zip_sha256="a" * 64, + raw_zip=b"validated-zip", + ), + records=( + GeneratorPower5m("BAT1", REPORT_TIME, -4.5, ARTIFACT_ID, SOURCE_TIME, 0), + GeneratorPower5m("UNMAPPED", REPORT_TIME, 8.0, ARTIFACT_ID, SOURCE_TIME, 0), + ), + ) + + +def price_collection() -> DispatchPriceCollection: + filename = f"PUBLIC_DISPATCHIS_202608291205_{ARTIFACT_ID}.zip" + reference = DispatchPriceArtifactRef( + url="https://www.nemweb.com.au/REPORTS/CURRENT/DispatchIS_Reports/" + filename, + zip_filename=filename, + source_artifact_id=ARTIFACT_ID, + report_timestamp=REPORT_TIME, + ) + records = tuple( + RegionalPrice5m( + region=region, + interval_start=REPORT_TIME, + price_aud_per_mwh=float(index + 40), + price_status="available", + source_id=ARTIFACT_ID, + source_timestamp=SOURCE_TIME, + ingestion_version=42, + quality_flags=("aemo_price_status=FIRM",), + ) + for index, region in enumerate(("NSW1", "QLD1", "SA1", "TAS1", "VIC1")) + ) + return DispatchPriceCollection( + artifact=DispatchPriceArtifact( + reference=reference, + csv_member_name=filename.removesuffix(".zip") + ".CSV", + csv_payload="validated-dispatchis-csv", + zip_sha256="b" * 64, + raw_zip=b"validated-dispatchis-zip", + ), + records=records, + ) + + +class CapturingIngestor: + def __init__(self, connection) -> None: + self.connection = connection + self.call: tuple[Any, ...] | None = None + + def ingest(self, receipt, observations, generators=(), power_records=()): + self.call = (receipt, tuple(observations), tuple(generators), tuple(power_records)) + return DispatchScadaIngestionResult(len(self.call[1]), len(self.call[3]), False) + + +class CapturingPriceIngestor: + def __init__(self, connection) -> None: + self.connection = connection + self.call: tuple[Any, ...] | None = None + + def ingest(self, receipt, records): + self.call = (receipt, tuple(records)) + return DispatchPriceIngestionResult(len(self.call[1]), False) + + +class ClosingConnection: + def __init__(self) -> None: + self.close_calls = 0 + + def close(self) -> None: + self.close_calls += 1 + + +class CollectorServiceTests(unittest.TestCase): + def test_cycle_persists_all_raw_rows_and_only_reviewed_battery_power(self) -> None: + collected = collection() + asset = BatteryAsset( + duid="BAT1", + site_name="Battery One", + region="NSW1", + capacity_mw=10.0, + storage_capacity_mwh=20.0, + source_id="reviewed-registry", + source_timestamp=datetime(2025, 3, 31, tzinfo=UTC), + ) + captured: list[CapturingIngestor] = [] + + def collect(*, ingestion_version, correction_version=0): + self.assertEqual((ingestion_version, correction_version), (0, 0)) + return collected + + def ingestor_factory(connection): + ingestor = CapturingIngestor(connection) + captured.append(ingestor) + return ingestor + + result = run_collection_cycle( + object(), + (asset,), + collect=collect, + ingestor_factory=ingestor_factory, + ) + + self.assertEqual(result, DispatchScadaIngestionResult(2, 1, False)) + self.assertEqual(len(captured), 1) + assert captured[0].call is not None + receipt, raw_rows, generators, power_rows = captured[0].call + self.assertEqual(receipt.source_artifact_id, ARTIFACT_ID) + self.assertEqual(receipt.raw_zip, b"validated-zip") + self.assertEqual(tuple(row.duid for row in raw_rows), ("BAT1", "UNMAPPED")) + self.assertEqual(tuple(row.ingestion_version for row in raw_rows), (42, 42)) + self.assertEqual(tuple(row.generator_id for row in power_rows), ("BAT1",)) + self.assertEqual(power_rows[0].ingestion_version, 42) + self.assertEqual(tuple(row.generator_id for row in generators), ("BAT1",)) + self.assertEqual(generators[0].source_id, "reviewed-registry") + self.assertEqual(generators[0].ingestion_version, 1) + + def test_price_cycle_persists_raw_artifact_and_five_regions(self) -> None: + captured: list[CapturingPriceIngestor] = [] + + def ingestor_factory(connection): + ingestor = CapturingPriceIngestor(connection) + captured.append(ingestor) + return ingestor + + result = run_price_collection_cycle( + object(), + collect=lambda **kwargs: price_collection(), + ingestor_factory=ingestor_factory, + ) + + self.assertEqual(result, DispatchPriceIngestionResult(5, False)) + assert captured[0].call is not None + receipt, records = captured[0].call + self.assertEqual(receipt.raw_zip, b"validated-dispatchis-zip") + self.assertEqual(receipt.source_artifact_id, ARTIFACT_ID) + self.assertEqual( + tuple(record.region for record in records), + ("NSW1", "QLD1", "SA1", "TAS1", "VIC1"), + ) + + def test_database_cycle_closes_connection_when_ingestion_fails(self) -> None: + connection = ClosingConnection() + failure = RuntimeError("database write failed") + asset = BatteryAsset( + "BAT1", "Battery One", "NSW1", 10, 20, + "reviewed-registry", datetime(2025, 3, 31, tzinfo=UTC), + ) + + def connect(database_url, *, connect_timeout): + self.assertEqual((database_url, connect_timeout), ("postgresql://private", 10)) + return connection + + class FailingIngestor: + def __init__(self, unused_connection) -> None: + pass + + def ingest(self, *args, **kwargs): + raise failure + + with self.assertRaises(RuntimeError) as raised: + run_database_cycle( + "postgresql://private", + (asset,), + connect=connect, + collect=lambda **kwargs: collection(), + collect_prices=lambda **kwargs: self.fail("price collection must not run"), + ingestor_factory=FailingIngestor, + ) + + self.assertIs(raised.exception, failure) + self.assertEqual(connection.close_calls, 1) + + def test_polling_loop_runs_immediately_then_stops_during_wait(self) -> None: + cycle_calls = [] + waits = [] + + def cycle(): + cycle_calls.append("cycle") + return CollectorCycleResult( + DispatchScadaIngestionResult(2, 1, False), + DispatchPriceIngestionResult(5, False), + ) + + def wait(seconds): + waits.append(seconds) + return True + + result = run_polling_loop(cycle, interval_seconds=300, wait=wait) + + self.assertEqual( + result, + CollectorCycleResult( + DispatchScadaIngestionResult(2, 1, False), + DispatchPriceIngestionResult(5, False), + ), + ) + self.assertEqual((cycle_calls, waits), (["cycle"], [300])) + + def test_once_entrypoint_uses_environment_and_closes_connection(self) -> None: + connection = ClosingConnection() + captured: list[CapturingIngestor] = [] + + def connect(database_url, *, connect_timeout): + self.assertEqual((database_url, connect_timeout), ("postgresql://private", 10)) + return connection + + def ingestor_factory(active_connection): + ingestor = CapturingIngestor(active_connection) + captured.append(ingestor) + return ingestor + + output = StringIO() + with redirect_stdout(output): + exit_code = main( + ["--once"], + environ={ + "BATTERYWATCH_DATABASE_URL": "postgresql://private", + "BATTERYWATCH_ASSETS_PATH": str( + Path(__file__).resolve().parents[2] + / "config" + / "battery_assets.json" + ), + }, + connect=connect, + collect=lambda **kwargs: collection(), + collect_prices=lambda **kwargs: price_collection(), + ingestor_factory=ingestor_factory, + price_ingestor_factory=CapturingPriceIngestor, + ) + + self.assertEqual(exit_code, 0) + self.assertEqual(connection.close_calls, 1) + self.assertEqual(len(captured), 1) + self.assertIn('"status": "ok"', output.getvalue()) + self.assertIn('"price_count": 5', output.getvalue()) + + def test_cli_safety_arguments_override_environment_file_values(self) -> None: + connection = ClosingConnection() + + def connect(database_url, *, connect_timeout): + self.assertEqual((database_url, connect_timeout), ("postgresql://private", 10)) + return connection + + output = StringIO() + with redirect_stdout(output): + exit_code = main( + [ + "--once", + "--assets-path", + str( + Path(__file__).resolve().parents[2] + / "config" + / "battery_assets.json" + ), + "--interval-seconds", + "300", + ], + environ={ + "BATTERYWATCH_DATABASE_URL": "postgresql://private", + "BATTERYWATCH_ASSETS_PATH": "/unreviewed/assets.json", + "BATTERYWATCH_COLLECT_INTERVAL_SECONDS": "invalid", + }, + connect=connect, + collect=lambda **kwargs: collection(), + collect_prices=lambda **kwargs: price_collection(), + ingestor_factory=CapturingIngestor, + price_ingestor_factory=CapturingPriceIngestor, + ) + + self.assertEqual(exit_code, 0) + self.assertEqual(connection.close_calls, 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/backend/tests/test_dispatch_price_ingestion.py b/app/backend/tests/test_dispatch_price_ingestion.py new file mode 100644 index 0000000..426fbde --- /dev/null +++ b/app/backend/tests/test_dispatch_price_ingestion.py @@ -0,0 +1,234 @@ +"""Tests for atomic official DispatchIS price ingestion.""" + +from datetime import datetime, timedelta, timezone +from pathlib import Path +import unittest + +from batterywatch_api.dispatch_price_ingestion import ( + DispatchPriceArtifactConflictError, + DispatchPriceArtifactReceipt, + DispatchPriceIngestionResult, + PostgreSQLDispatchPriceIngestor, +) +from batterywatch_api.storage import RegionalPrice5m + + +UTC = timezone.utc +REPORT_TIMESTAMP = datetime(2026, 8, 30, 2, 5, tzinfo=UTC) +SOURCE_TIMESTAMP = REPORT_TIMESTAMP + timedelta(seconds=4) +INTERVAL = REPORT_TIMESTAMP +ARTIFACT_ID = "0000000000000042" +REGIONS = ("NSW1", "QLD1", "SA1", "TAS1", "VIC1") + + +class FakeCursor: + def __init__(self, connection) -> None: + self.connection = connection + + def execute(self, statement, parameters) -> None: + self.connection.executions.append((statement, tuple(parameters))) + if len(self.connection.executions) == self.connection.fail_on_execute: + raise self.connection.failure + + def fetchone(self): + if self.connection.fetchone_results: + return self.connection.fetchone_results.pop(0) + return (1,) + + def close(self) -> None: + self.connection.closed_cursors += 1 + + +class FakeConnection: + def __init__(self, *, fail_on_execute=None, failure=None, fetchone_results=()) -> None: + self.executions = [] + self.fail_on_execute = fail_on_execute + self.failure = failure + self.fetchone_results = list(fetchone_results) + self.cursor_calls = self.closed_cursors = 0 + self.commits = self.rollbacks = 0 + self._cursor = FakeCursor(self) + + def cursor(self): + self.cursor_calls += 1 + return self._cursor + + def commit(self) -> None: + self.commits += 1 + + def rollback(self) -> None: + self.rollbacks += 1 + + +def receipt() -> DispatchPriceArtifactReceipt: + filename = f"PUBLIC_DISPATCHIS_202608301205_{ARTIFACT_ID}.zip" + return DispatchPriceArtifactReceipt( + source_artifact_id=ARTIFACT_ID, + source_url=( + "https://www.nemweb.com.au/REPORTS/CURRENT/DispatchIS_Reports/" + + filename + ), + zip_filename=filename, + csv_member_name=filename.removesuffix(".zip") + ".CSV", + report_timestamp=REPORT_TIMESTAMP, + zip_sha256="a" * 64, + raw_zip=b"official-dispatchis-zip", + ) + + +def prices() -> tuple[RegionalPrice5m, ...]: + return tuple( + RegionalPrice5m( + region=region, + interval_start=INTERVAL, + price_aud_per_mwh=-10.0 if region == "SA1" else float(index + 40), + price_status="negative" if region == "SA1" else "available", + intervention=0, + apc_flag=0, + market_suspended=False, + source_id=ARTIFACT_ID, + source_timestamp=SOURCE_TIMESTAMP, + ingestion_version=int(ARTIFACT_ID), + correction_version=0, + quality_flags=(), + ) + for index, region in enumerate(REGIONS) + ) + + +class PostgreSQLDispatchPriceIngestorTests(unittest.TestCase): + def test_inserts_receipt_then_five_prices_in_one_commit(self) -> None: + connection = FakeConnection() + + result = PostgreSQLDispatchPriceIngestor(connection).ingest(receipt(), prices()) + + self.assertEqual(result, DispatchPriceIngestionResult(5, False)) + self.assertEqual( + (connection.cursor_calls, connection.closed_cursors, + connection.commits, connection.rollbacks), + (1, 1, 1, 0), + ) + self.assertEqual(len(connection.executions), 6) + self.assertIn("dispatch_price_artifacts", connection.executions[0][0]) + self.assertTrue(all( + "INSERT INTO nem_price_5m" in statement + for statement, _ in connection.executions[1:] + )) + self.assertEqual( + tuple(parameters[0] for _, parameters in connection.executions[1:]), + REGIONS, + ) + self.assertTrue(all("RETURNING 1" in statement for statement, _ in connection.executions)) + + def test_exact_artifact_replay_is_a_clean_noop(self) -> None: + artifact = receipt() + stored = ( + artifact.source_artifact_id, + artifact.source_url, + artifact.zip_filename, + artifact.csv_member_name, + artifact.report_timestamp, + artifact.zip_sha256, + memoryview(artifact.raw_zip), + ) + connection = FakeConnection(fetchone_results=(None, stored)) + + result = PostgreSQLDispatchPriceIngestor(connection).ingest(artifact, prices()) + + self.assertEqual(result, DispatchPriceIngestionResult(0, True)) + self.assertEqual(len(connection.executions), 2) + self.assertEqual((connection.commits, connection.rollbacks), (1, 0)) + + def test_conflicting_artifact_identity_rolls_back(self) -> None: + artifact = receipt() + stored = ( + artifact.source_artifact_id, + artifact.source_url, + artifact.zip_filename, + artifact.csv_member_name, + artifact.report_timestamp, + "b" * 64, + artifact.raw_zip, + ) + connection = FakeConnection(fetchone_results=(None, stored)) + + with self.assertRaises(DispatchPriceArtifactConflictError): + PostgreSQLDispatchPriceIngestor(connection).ingest(artifact, prices()) + + self.assertEqual((connection.commits, connection.rollbacks), (0, 1)) + + def test_rejects_incomplete_region_set_before_database_write(self) -> None: + connection = FakeConnection() + + with self.assertRaisesRegex(ValueError, "five canonical regions"): + PostgreSQLDispatchPriceIngestor(connection).ingest(receipt(), prices()[:-1]) + + self.assertEqual((connection.executions, connection.commits, connection.rollbacks), ([], 0, 1)) + + def test_rejects_records_not_bound_to_artifact_interval_and_version(self) -> None: + from dataclasses import replace + + mismatched_interval = tuple( + replace(record, interval_start=INTERVAL + timedelta(minutes=5)) + for record in prices() + ) + mismatched_version = tuple( + replace(record, ingestion_version=1) + for record in prices() + ) + + for records, message in ( + (mismatched_interval, "report timestamp"), + (mismatched_version, "artifact version"), + ): + with self.subTest(message=message): + connection = FakeConnection() + with self.assertRaisesRegex(ValueError, message): + PostgreSQLDispatchPriceIngestor(connection).ingest(receipt(), records) + self.assertEqual(connection.executions, []) + + def test_mid_batch_failure_rolls_back_and_reraises_same_error(self) -> None: + failure = RuntimeError("price write failed") + connection = FakeConnection(fail_on_execute=4, failure=failure) + same_error = False + + try: + PostgreSQLDispatchPriceIngestor(connection).ingest(receipt(), prices()) + except RuntimeError as raised: + same_error = raised is failure + else: + self.fail("expected price write failure") + + self.assertTrue(same_error) + self.assertEqual((connection.commits, connection.rollbacks), (0, 1)) + + +class DispatchPriceMigrationTests(unittest.TestCase): + def test_backup_script_passes_connection_uri_explicitly_to_pg_dump(self) -> None: + root = Path(__file__).resolve().parents[2] + script = (root / "deploy" / "backup-verify.sh").read_text(encoding="utf-8") + + self.assertIn('database_url=$BATTERYWATCH_DATABASE_URL', script) + self.assertIn('pg_dump --dbname="$database_url" --format=custom', script) + self.assertNotIn('export PGDATABASE=', script) + + def test_deploy_migration_applies_and_verifies_price_artifact_table(self) -> None: + app_root = Path(__file__).resolve().parents[2] + migration = (app_root / "migrations" / "003_dispatch_price_artifacts.sql").read_text( + encoding="utf-8" + ) + migrate_script = (app_root / "deploy" / "migrate.sh").read_text(encoding="utf-8") + + self.assertIn("CREATE TABLE IF NOT EXISTS dispatch_price_artifacts", migration) + self.assertIn("raw_zip BYTEA NOT NULL", migration) + self.assertIn("PUBLIC_DISPATCHIS_", migration) + self.assertIn("003_dispatch_price_artifacts.sql", migrate_script) + self.assertIn("SELECT count(*) = 7", migrate_script) + self.assertIn("'dispatch_price_artifacts'", migrate_script) + self.assertIn("database_url=$BATTERYWATCH_DATABASE_URL", migrate_script) + self.assertEqual(migrate_script.count('--dbname="$database_url"'), 4) + self.assertNotIn("export PGDATABASE=", migrate_script) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/backend/tests/test_dispatch_scada.py b/app/backend/tests/test_dispatch_scada.py new file mode 100644 index 0000000..e700a3f --- /dev/null +++ b/app/backend/tests/test_dispatch_scada.py @@ -0,0 +1,435 @@ +"""Tests for the strict canonical Dispatch SCADA parser.""" + +from datetime import datetime, timedelta, timezone +from typing import Any +import unittest +from unittest.mock import Mock +from zoneinfo import ZoneInfo + +from batterywatch_api.dispatch_scada import ( + DispatchScadaParseError, + parse_dispatch_scada_csv, +) +from batterywatch_api.storage import GeneratorPower5m + + +_SOURCE_ARTIFACT_ID = "0000000535047618" +_NAIVE_TIMEZONE = timezone(timedelta(hours=10)) +_CANONICAL_PAYLOAD = """C,NEMP.WORLD,DISPATCHSCADA,AEMO,PUBLIC,2026/08/29,14:10:15,0000000535047618,DISPATCHSCADA,0000000535047612 +I,DISPATCH,UNIT_SCADA,1,SETTLEMENTDATE,DUID,SCADAVALUE,LASTCHANGED +D,DISPATCH,UNIT_SCADA,1,"2026/08/29 14:10:00",BWTR2,0,"2026/08/29 14:10:11" +D,DISPATCH,UNIT_SCADA,1,"2026/08/29 14:05:00",BWTR1,-12.460880,"2026/08/29 14:05:11" +C,END OF REPORT,5 +""" + + +class DispatchScadaParserTests(unittest.TestCase): + def _parse( + self, + payload: str = _CANONICAL_PAYLOAD, + *, + naive_timezone: Any = _NAIVE_TIMEZONE, + ): + return parse_dispatch_scada_csv( + payload, + source_artifact_id=_SOURCE_ARTIFACT_ID, + ingestion_version=7, + correction_version=2, + naive_timezone=naive_timezone, + ) + + def test_rejects_offset_free_timestamp_without_timezone(self): + with self.assertRaises(DispatchScadaParseError): + self._parse(naive_timezone=None) + + def test_rejects_wrong_metadata_discriminator(self): + payload = _CANONICAL_PAYLOAD.replace( + "C,NEMP.WORLD,DISPATCHSCADA,AEMO,PUBLIC", + "X,NEMP.WORLD,DISPATCHSCADA,AEMO,PUBLIC", + 1, + ) + + with self.assertRaises(DispatchScadaParseError): + self._parse(payload) + + def test_rejects_wrong_metadata_source(self): + payload = _CANONICAL_PAYLOAD.replace("NEMP.WORLD", "OTHER.SOURCE", 1) + + with self.assertRaises(DispatchScadaParseError): + self._parse(payload) + + def test_rejects_wrong_metadata_table(self): + payload = _CANONICAL_PAYLOAD.replace( + "NEMP.WORLD,DISPATCHSCADA,AEMO", + "NEMP.WORLD,OTHER_TABLE,AEMO", + 1, + ) + + with self.assertRaises(DispatchScadaParseError): + self._parse(payload) + + def test_rejects_metadata_source_sequence_mismatch(self): + payload = _CANONICAL_PAYLOAD.replace( + _SOURCE_ARTIFACT_ID, + "0000000535047619", + 1, + ) + + with self.assertRaises(DispatchScadaParseError): + self._parse(payload) + + def test_rejects_wrong_metadata_report_table(self): + payload = _CANONICAL_PAYLOAD.replace( + "DISPATCHSCADA,0000000535047612", + "OTHER_TABLE,0000000535047612", + 1, + ) + + with self.assertRaises(DispatchScadaParseError): + self._parse(payload) + + def test_rejects_wrong_header_discriminator(self): + payload = _CANONICAL_PAYLOAD.replace( + "I,DISPATCH,UNIT_SCADA,1,SETTLEMENTDATE", + "X,DISPATCH,UNIT_SCADA,1,SETTLEMENTDATE", + 1, + ) + + with self.assertRaises(DispatchScadaParseError): + self._parse(payload) + + def test_rejects_wrong_header_source(self): + payload = _CANONICAL_PAYLOAD.replace( + "I,DISPATCH,UNIT_SCADA", + "I,OTHER,UNIT_SCADA", + 1, + ) + + with self.assertRaises(DispatchScadaParseError): + self._parse(payload) + + def test_rejects_wrong_header_table(self): + payload = _CANONICAL_PAYLOAD.replace( + "I,DISPATCH,UNIT_SCADA", + "I,DISPATCH,OTHER_TABLE", + 1, + ) + + with self.assertRaises(DispatchScadaParseError): + self._parse(payload) + + def test_rejects_wrong_header_version(self): + payload = _CANONICAL_PAYLOAD.replace( + "I,DISPATCH,UNIT_SCADA,1,SETTLEMENTDATE", + "I,DISPATCH,UNIT_SCADA,2,SETTLEMENTDATE", + 1, + ) + + with self.assertRaises(DispatchScadaParseError): + self._parse(payload) + + def test_rejects_wrong_header_columns(self): + payload = _CANONICAL_PAYLOAD.replace( + "SETTLEMENTDATE,DUID,SCADAVALUE,LASTCHANGED", + "SETTLEMENTDATE,BAD_DUID,SCADAVALUE,LASTCHANGED", + 1, + ) + + with self.assertRaises(DispatchScadaParseError): + self._parse(payload) + + def test_rejects_wrong_metadata_row_length(self): + payload = _CANONICAL_PAYLOAD.replace( + "DISPATCHSCADA,0000000535047612\nI,", + "DISPATCHSCADA\nI,", + 1, + ) + + with self.assertRaises(DispatchScadaParseError): + self._parse(payload) + + def test_rejects_wrong_header_row_length(self): + payload = _CANONICAL_PAYLOAD.replace( + ",SCADAVALUE,LASTCHANGED\nD,", + ",SCADAVALUE\nD,", + 1, + ) + + with self.assertRaises(DispatchScadaParseError): + self._parse(payload) + + def test_rejects_wrong_metadata_publisher(self): + payload = _CANONICAL_PAYLOAD.replace( + "DISPATCHSCADA,AEMO,PUBLIC", + "DISPATCHSCADA,OTHER,PUBLIC", + 1, + ) + + with self.assertRaises(DispatchScadaParseError): + self._parse(payload) + + def test_rejects_wrong_metadata_visibility(self): + payload = _CANONICAL_PAYLOAD.replace( + "DISPATCHSCADA,AEMO,PUBLIC", + "DISPATCHSCADA,AEMO,PRIVATE", + 1, + ) + + with self.assertRaises(DispatchScadaParseError): + self._parse(payload) + + def test_rejects_wrong_data_discriminator(self): + payload = _CANONICAL_PAYLOAD.replace( + "D,DISPATCH,UNIT_SCADA,1", + "X,DISPATCH,UNIT_SCADA,1", + 1, + ) + + with self.assertRaises(DispatchScadaParseError): + self._parse(payload) + + def test_rejects_wrong_data_source(self): + payload = _CANONICAL_PAYLOAD.replace( + "D,DISPATCH,UNIT_SCADA,1", + "D,OTHER,UNIT_SCADA,1", + 1, + ) + + with self.assertRaises(DispatchScadaParseError): + self._parse(payload) + + def test_rejects_wrong_data_table(self): + payload = _CANONICAL_PAYLOAD.replace( + "D,DISPATCH,UNIT_SCADA,1", + "D,DISPATCH,OTHER_TABLE,1", + 1, + ) + + with self.assertRaises(DispatchScadaParseError): + self._parse(payload) + + def test_rejects_wrong_data_version(self): + payload = _CANONICAL_PAYLOAD.replace( + "D,DISPATCH,UNIT_SCADA,1", + "D,DISPATCH,UNIT_SCADA,2", + 1, + ) + + with self.assertRaises(DispatchScadaParseError): + self._parse(payload) + + def test_rejects_wrong_data_row_length(self): + payload = _CANONICAL_PAYLOAD.replace( + '"2026/08/29 14:10:11"\nD,', + '"2026/08/29 14:10:11",EXTRA\nD,', + 1, + ) + + with self.assertRaises(DispatchScadaParseError): + self._parse(payload) + + def test_rejects_non_numeric_power(self): + payload = _CANONICAL_PAYLOAD.replace("-12.460880", "not-a-number", 1) + + with self.assertRaises(DispatchScadaParseError): + self._parse(payload) + + def test_rejects_non_finite_power(self): + payload = _CANONICAL_PAYLOAD.replace("-12.460880", "nan", 1) + + with self.assertRaises(DispatchScadaParseError): + self._parse(payload) + + def test_rejects_invalid_naive_timezone(self): + with self.assertRaises(DispatchScadaParseError): + self._parse(naive_timezone="not-a-timezone") + + def test_rejects_dst_aware_naive_timezone(self): + with self.assertRaises(DispatchScadaParseError): + self._parse(naive_timezone=ZoneInfo("Australia/Sydney")) + + def test_rejects_wrong_fixed_naive_timezone(self): + with self.assertRaises(DispatchScadaParseError): + self._parse(naive_timezone=timezone(timedelta(hours=11))) + + def test_rejects_bad_timestamp_timezone_offset(self): + payload = _CANONICAL_PAYLOAD.replace( + "2026/08/29 14:10:00", + "2026/08/29 14:10:00+99:00", + 1, + ) + + with self.assertRaises(DispatchScadaParseError): + self._parse(payload) + + def test_rejects_invalid_interval_timestamp(self): + payload = _CANONICAL_PAYLOAD.replace( + "2026/08/29 14:10:00", + "not-a-timestamp", + 1, + ) + + with self.assertRaises(DispatchScadaParseError): + self._parse(payload) + + def test_rejects_misaligned_interval_timestamp(self): + payload = _CANONICAL_PAYLOAD.replace( + "2026/08/29 14:10:00", + "2026/08/29 14:06:00", + 1, + ) + + with self.assertRaises(DispatchScadaParseError): + self._parse(payload) + + def test_rejects_invalid_source_timestamp(self): + payload = _CANONICAL_PAYLOAD.replace( + "2026/08/29 14:10:11", + "not-a-timestamp", + 1, + ) + + with self.assertRaises(DispatchScadaParseError): + self._parse(payload) + + def test_rejects_blank_duid(self): + payload = _CANONICAL_PAYLOAD.replace( + ",BWTR1,-12.460880", + ",,-12.460880", + 1, + ) + + with self.assertRaises(DispatchScadaParseError): + self._parse(payload) + + def test_rejects_infinite_power(self): + payload = _CANONICAL_PAYLOAD.replace("-12.460880", "inf", 1) + + with self.assertRaises(DispatchScadaParseError): + self._parse(payload) + + def test_rejects_missing_footer(self): + payload = _CANONICAL_PAYLOAD.replace("C,END OF REPORT,5\n", "", 1) + + with self.assertRaises(DispatchScadaParseError): + self._parse(payload) + + def test_rejects_footer_with_extra_field(self): + payload = _CANONICAL_PAYLOAD.replace( + "C,END OF REPORT,5", + "C,END OF REPORT,5,EXTRA", + 1, + ) + + with self.assertRaises(DispatchScadaParseError): + self._parse(payload) + + def test_rejects_footer_with_missing_field(self): + payload = _CANONICAL_PAYLOAD.replace( + "C,END OF REPORT,5", + "C,END OF REPORT", + 1, + ) + + with self.assertRaises(DispatchScadaParseError): + self._parse(payload) + + def test_rejects_footer_count_mismatch(self): + payload = _CANONICAL_PAYLOAD.replace("C,END OF REPORT,5", "C,END OF REPORT,4", 1) + + with self.assertRaises(DispatchScadaParseError): + self._parse(payload) + + def test_rejects_non_numeric_footer_count(self): + payload = _CANONICAL_PAYLOAD.replace("C,END OF REPORT,5", "C,END OF REPORT,total", 1) + + with self.assertRaises(DispatchScadaParseError): + self._parse(payload) + + def test_rejects_invalid_footer_marker(self): + payload = _CANONICAL_PAYLOAD.replace( + "C,END OF REPORT,5", + "C,END OF DATA,5", + 1, + ) + + with self.assertRaises(DispatchScadaParseError): + self._parse(payload) + + def test_rejects_member_with_no_data_rows(self): + payload = "\n".join( + _CANONICAL_PAYLOAD.splitlines()[:2] + ["C,END OF REPORT,3"] + ) + "\n" + + with self.assertRaises(DispatchScadaParseError): + self._parse(payload) + + def test_rejects_metadata_only_envelope(self): + payload = _CANONICAL_PAYLOAD.splitlines()[0] + "\n" + + with self.assertRaises(DispatchScadaParseError): + self._parse(payload) + + def test_rejects_metadata_and_header_only_envelope(self): + payload = "\n".join(_CANONICAL_PAYLOAD.splitlines()[:2]) + "\n" + + with self.assertRaises(DispatchScadaParseError): + self._parse(payload) + + def test_rejects_duplicate_duid_and_interval(self): + payload = _CANONICAL_PAYLOAD.replace("BWTR2", "BWTR1", 1) + payload = payload.replace( + "2026/08/29 14:05:00", + "2026/08/29 14:10:00", + 1, + ) + + with self.assertRaises(DispatchScadaParseError): + self._parse(payload) + + def test_rejects_empty_payload_with_public_error(self): + with self.assertRaises(DispatchScadaParseError): + self._parse("") + + def test_rejects_non_string_payload_with_public_error(self): + with self.assertRaises(DispatchScadaParseError): + self._parse(Mock(spec=str)) + + def test_rejects_malformed_csv(self): + payload = _CANONICAL_PAYLOAD + 'D,"unterminated\n' + + with self.assertRaises(DispatchScadaParseError): + self._parse(payload) + + def test_rejects_invalid_metadata_timestamp(self): + payload = _CANONICAL_PAYLOAD.replace("2026/08/29", "not-a-date", 1) + + with self.assertRaises(DispatchScadaParseError): + self._parse(payload) + + def test_parses_complete_member_deterministically(self): + actual = self._parse() + + self.assertEqual( + actual, + ( + GeneratorPower5m( + generator_id="BWTR1", + interval_start=datetime(2026, 8, 29, 4, 5, tzinfo=timezone.utc), + power_mw=-12.460880, + source_id="0000000535047618", + source_timestamp=datetime(2026, 8, 29, 4, 5, 11, tzinfo=timezone.utc), + ingestion_version=7, + correction_version=2, + ), + GeneratorPower5m( + generator_id="BWTR2", + interval_start=datetime(2026, 8, 29, 4, 10, tzinfo=timezone.utc), + power_mw=0.0, + source_id="0000000535047618", + source_timestamp=datetime(2026, 8, 29, 4, 10, 11, tzinfo=timezone.utc), + ingestion_version=7, + correction_version=2, + ), + ), + ) diff --git a/app/backend/tests/test_dispatch_scada_ingestion.py b/app/backend/tests/test_dispatch_scada_ingestion.py new file mode 100644 index 0000000..1f9d6a5 --- /dev/null +++ b/app/backend/tests/test_dispatch_scada_ingestion.py @@ -0,0 +1,104 @@ +"""Static contract tests for the raw Dispatch SCADA migration.""" + +from pathlib import Path +import unittest + + +class DispatchScadaIngestionSchemaTests(unittest.TestCase): + def test_migration_defines_raw_dispatch_scada_schema(self) -> None: + migration = ( + Path(__file__).resolve().parents[2] + / "migrations" + / "002_dispatch_scada_raw_ingestion.sql" + ).read_text(encoding="utf-8") + required_fragments = { + "artifact_table": "CREATE TABLE IF NOT EXISTS dispatch_scada_artifacts", + "artifact_id": "source_artifact_id TEXT PRIMARY KEY", + "artifact_id_non_empty_numeric": ( + "source_artifact_id <> ''" + " AND source_artifact_id ~ '^[0-9]+$'" + ), + "canonical_https_url": ( + "source_url TEXT NOT NULL" + " CHECK (source_url <> '' AND source_url ~ '^https://')" + ), + "canonical_nemweb_url": ( + "source_url = 'https://www.nemweb.com.au/REPORTS/CURRENT/Dispatch_SCADA/'" + " || zip_filename" + ), + "zip_filename": "zip_filename TEXT NOT NULL CHECK (btrim(zip_filename) <> '')", + "canonical_zip_filename": ( + "zip_filename ~ ('^PUBLIC_DISPATCHSCADA_[0-9]{12}_'" + " || source_artifact_id || '[.]zip$')" + ), + "csv_member_name": ( + "csv_member_name TEXT NOT NULL" + " CHECK (btrim(csv_member_name) <> '')" + ), + "canonical_csv_member_name": ( + "csv_member_name = left(zip_filename, -4) || '.CSV'" + ), + "report_timestamp": "report_timestamp TIMESTAMPTZ NOT NULL", + "sha256": ( + "zip_sha256 TEXT NOT NULL" + " CHECK (zip_sha256 ~ '^[0-9a-f]{64}$')" + ), + "raw_zip": "raw_zip BYTEA NOT NULL CHECK (octet_length(raw_zip) > 0)", + "artifact_stored_at": ( + "stored_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP" + ), + "observation_table": ( + "CREATE TABLE IF NOT EXISTS raw_dispatch_scada_observations" + ), + "observation_artifact_id": "source_artifact_id TEXT NOT NULL", + "artifact_foreign_key": ( + "REFERENCES dispatch_scada_artifacts (source_artifact_id)" + " ON DELETE RESTRICT" + ), + "duid": "duid TEXT NOT NULL CHECK (btrim(duid) <> '')", + "interval": "interval_start TIMESTAMPTZ NOT NULL", + "interval_minute_alignment": ( + "date_trunc('minute', interval_start) = interval_start" + ), + "interval_five_minute_alignment": ( + "EXTRACT(MINUTE FROM interval_start)::INTEGER % 5 = 0" + ), + "power": "power_mw DOUBLE PRECISION NOT NULL", + "finite_power": ( + "power_mw::TEXT NOT IN ('NaN', 'Infinity', '-Infinity')" + ), + "source_timestamp": "source_timestamp TIMESTAMPTZ NOT NULL", + "ingestion_version": ( + "ingestion_version BIGINT NOT NULL DEFAULT 0" + " CHECK (ingestion_version >= 0)" + ), + "correction_version": ( + "correction_version BIGINT NOT NULL DEFAULT 0" + " CHECK (correction_version >= 0)" + ), + "observation_stored_at": ( + "stored_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP" + ), + "artifact_specific_key": ( + "PRIMARY KEY (source_artifact_id, duid, interval_start)" + ), + "artifact_time_index": ( + "CREATE INDEX IF NOT EXISTS dispatch_scada_artifacts_report_timestamp_idx" + " ON dispatch_scada_artifacts (report_timestamp DESC)" + ), + "observation_time_index": ( + "CREATE INDEX IF NOT EXISTS raw_dispatch_scada_observations_interval_idx" + " ON raw_dispatch_scada_observations (interval_start DESC)" + ), + } + self.assertEqual( + {name: fragment in migration for name, fragment in required_fragments.items()}, + {name: True for name in required_fragments}, + ) + self.assertNotIn("generators", migration.lower()) + self.assertNotIn("generator_power_5m", migration.lower()) + self.assertNotIn("hypertable", migration.lower()) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/backend/tests/test_dispatch_scada_ingestion_repository.py b/app/backend/tests/test_dispatch_scada_ingestion_repository.py new file mode 100644 index 0000000..b13290b --- /dev/null +++ b/app/backend/tests/test_dispatch_scada_ingestion_repository.py @@ -0,0 +1,285 @@ +from datetime import datetime, timedelta, timezone +import unittest + +from batterywatch_api.dispatch_scada_ingestion import ( + DispatchScadaArtifactReceipt, + DispatchScadaConflictError, + DispatchScadaIngestionResult, + PostgreSQLDispatchScadaIngestor, + RawDispatchScadaObservation, +) +from batterywatch_api.storage import GeneratorMetadata, GeneratorPower5m + + +UTC = timezone.utc +REPORT_TIMESTAMP = datetime(2026, 1, 1, 0, 0, tzinfo=UTC) +INTERVAL_ONE = datetime(2026, 1, 1, 0, 0, tzinfo=UTC) +INTERVAL_TWO = datetime(2026, 1, 1, 0, 5, tzinfo=UTC) +SOURCE_TIMESTAMP = datetime(2026, 1, 1, 0, 0, 3, tzinfo=UTC) + + +class FakeCursor: + def __init__(self, connection): + self.connection = connection + + def execute(self, statement, parameters): + self.connection.executions.append((statement, tuple(parameters))) + if len(self.connection.executions) == self.connection.fail_on_execute: + raise self.connection.failure + + def fetchone(self): + if self.connection.fetchone_results: + return self.connection.fetchone_results.pop(0) + return (1,) + + def close(self): + self.connection.closed_cursors += 1 + + +class FakeConnection: + def __init__(self, *, fail_on_execute=None, failure=None, fetchone_results=None): + self.executions = [] + self.fail_on_execute = fail_on_execute + self.failure = failure + self.fetchone_results = list(fetchone_results or ()) + self.cursor_calls = self.closed_cursors = 0 + self.commits = self.rollbacks = 0 + self._cursor = FakeCursor(self) + + def cursor(self): + self.cursor_calls += 1 + return self._cursor + + def commit(self): + self.commits += 1 + + def rollback(self): + self.rollbacks += 1 + + +def receipt(): + filename = "PUBLIC_DISPATCHSCADA_202601010000_123.zip" + return DispatchScadaArtifactReceipt( + "123", + "https://www.nemweb.com.au/REPORTS/CURRENT/Dispatch_SCADA/" + filename, + filename, + filename.removesuffix(".zip") + ".CSV", + REPORT_TIMESTAMP, + "a" * 64, + b"zip-payload", + ) + + +def observations(): + return ( + RawDispatchScadaObservation( + "123", "BAT-1", INTERVAL_ONE, 1.25, SOURCE_TIMESTAMP + ), + RawDispatchScadaObservation( + "123", "BAT-2", INTERVAL_TWO, 0.0, SOURCE_TIMESTAMP, 2, 1 + ), + ) + + +class PostgreSQLDispatchScadaIngestorReplayTests(unittest.TestCase): + def test_exact_replay_is_a_clean_noop(self): + artifact = receipt() + stored = ( + artifact.source_artifact_id, + artifact.source_url, + artifact.zip_filename, + artifact.csv_member_name, + artifact.report_timestamp, + artifact.zip_sha256, + memoryview(artifact.raw_zip), + ) + connection = FakeConnection(fetchone_results=[None, stored]) + + result = PostgreSQLDispatchScadaIngestor(connection).ingest( + artifact, observations() + ) + + self.assertEqual( + (result, len(connection.executions), connection.cursor_calls, + connection.closed_cursors, connection.commits, connection.rollbacks), + (DispatchScadaIngestionResult(0, 0, True), 2, 1, 1, 1, 0), + ) + self.assertIn("ON CONFLICT DO NOTHING RETURNING 1", connection.executions[0][0]) + self.assertIn("SELECT", connection.executions[1][0]) + + +class PostgreSQLDispatchScadaIngestorConflictTests(unittest.TestCase): + def test_immutable_conflict_rolls_back_once(self): + artifact = receipt() + stored = ( + artifact.source_artifact_id, + artifact.source_url, + artifact.zip_filename, + artifact.csv_member_name, + artifact.report_timestamp, + "b" * 64, + artifact.raw_zip, + ) + connection = FakeConnection(fetchone_results=[None, stored]) + + with self.assertRaises(DispatchScadaConflictError): + PostgreSQLDispatchScadaIngestor(connection).ingest(artifact, observations()) + + self.assertEqual( + (len(connection.executions), connection.executions[1][1], + connection.cursor_calls, connection.closed_cursors, + connection.commits, connection.rollbacks), + (2, (artifact.source_artifact_id,), 1, 1, 0, 1), + ) + + +class PostgreSQLDispatchScadaIngestorMappingTests(unittest.TestCase): + def test_persists_raw_rows_then_guarded_mapping_in_one_transaction(self): + artifact = receipt() + generator = GeneratorMetadata( + "BAT-1", "Site One", "NSW", 10, 20, "registry", SOURCE_TIMESTAMP, 3, 2 + ) + power = GeneratorPower5m( + "BAT-1", INTERVAL_ONE, 1.25, "dispatch", SOURCE_TIMESTAMP, 4, 1 + ) + connection = FakeConnection() + + result = PostgreSQLDispatchScadaIngestor(connection).ingest( + artifact, + observations(), + generators=(generator,), + power_records=(power,), + ) + + self.assertEqual( + tuple( + ( + "artifact" if "dispatch_scada_artifacts" in statement + else "observation" if "raw_dispatch_scada_observations" in statement + else "generator" if "INSERT INTO generators" in statement + else "bounds" if "UPDATE generators" in statement + else "power", + parameters, + ) + for statement, parameters in connection.executions + ), + ( + ("artifact", ("123", artifact.source_url, artifact.zip_filename, + artifact.csv_member_name, REPORT_TIMESTAMP, "a" * 64, + b"zip-payload")), + ("observation", ("123", "BAT-1", INTERVAL_ONE, 1.25, + SOURCE_TIMESTAMP, 0, 0)), + ("observation", ("123", "BAT-2", INTERVAL_TWO, 0.0, + SOURCE_TIMESTAMP, 2, 1)), + ("generator", ("BAT-1", "Site One", "NSW", 10.0, 20.0, + None, None, + "registry", SOURCE_TIMESTAMP, 3, 2)), + ("power", ("BAT-1", INTERVAL_ONE, 1.25, "dispatch", + SOURCE_TIMESTAMP, 4, 1)), + ("bounds", (INTERVAL_ONE, INTERVAL_ONE + timedelta(minutes=5), + "BAT-1")), + ), + ) + self.assertEqual( + (result, connection.cursor_calls, connection.closed_cursors, + connection.commits, connection.rollbacks), + (DispatchScadaIngestionResult(2, 1, False), 1, 1, 1, 0), + ) + self.assertTrue(all("%s" in statement for statement, _ in connection.executions)) + self.assertTrue(all( + "ON CONFLICT" in statement + for statement, _ in (connection.executions[0], connection.executions[3], + connection.executions[4]) + )) + self.assertIn("LEAST", connection.executions[5][0]) + self.assertIn("GREATEST", connection.executions[5][0]) + + +class PostgreSQLDispatchScadaIngestorMappedFailureTests(unittest.TestCase): + def test_mapped_write_failure_rolls_back_once_and_reraises_same_error(self): + failure = RuntimeError("mapped power write failed") + generator = GeneratorMetadata( + "BAT-1", "Site One", "NSW", 10, 20, "registry", SOURCE_TIMESTAMP, 3, 2 + ) + power = GeneratorPower5m( + "BAT-1", INTERVAL_ONE, 1.25, "dispatch", SOURCE_TIMESTAMP, 4, 1 + ) + connection = FakeConnection(fail_on_execute=5, failure=failure) + same_error = False + + try: + PostgreSQLDispatchScadaIngestor(connection).ingest( + receipt(), + observations(), + generators=(generator,), + power_records=(power,), + ) + except RuntimeError as raised: + same_error = raised is failure + else: + self.fail("expected the mapped power write failure") + + self.assertEqual( + (same_error, connection.commits, connection.rollbacks, + connection.cursor_calls, connection.closed_cursors), + (True, 0, 1, 1, 1), + ) + + +class PostgreSQLDispatchScadaIngestorSuccessTests(unittest.TestCase): + def test_inserts_artifact_then_two_observations_in_one_commit(self): + artifact = receipt() + batch = observations() + connection = FakeConnection() + + accepted = PostgreSQLDispatchScadaIngestor(connection).ingest(artifact, batch) + executions = tuple( + ( + "artifact" if "dispatch_scada_artifacts" in statement else "observation", + parameters, + "%s" in statement, + "ON CONFLICT" not in statement, + ) + for statement, parameters in connection.executions + ) + self.assertEqual( + (accepted, executions, connection.cursor_calls, connection.closed_cursors, + connection.commits, connection.rollbacks), + ( + DispatchScadaIngestionResult(2, 0, False), + ( + ("artifact", ("123", artifact.source_url, artifact.zip_filename, + artifact.csv_member_name, REPORT_TIMESTAMP, "a" * 64, + b"zip-payload"), True, False), + ("observation", ("123", "BAT-1", INTERVAL_ONE, 1.25, + SOURCE_TIMESTAMP, 0, 0), True, True), + ("observation", ("123", "BAT-2", INTERVAL_TWO, 0.0, + SOURCE_TIMESTAMP, 2, 1), True, True), + ), + 1, 1, 1, 0, + ), + ) + + +class PostgreSQLDispatchScadaIngestorFailureTests(unittest.TestCase): + def test_mid_batch_execute_failure_rolls_back_once_and_reraises_same_error(self): + failure = RuntimeError("mid-batch execute failed") + connection = FakeConnection(fail_on_execute=3, failure=failure) + same_error = False + + try: + PostgreSQLDispatchScadaIngestor(connection).ingest(receipt(), observations()) + except RuntimeError as raised: + same_error = raised is failure + else: + self.fail("expected the mid-batch execute failure") + + self.assertEqual( + (same_error, connection.commits, connection.rollbacks, + connection.cursor_calls, connection.closed_cursors), + (True, 0, 1, 1, 1), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/backend/tests/test_nemweb_dispatch_prices.py b/app/backend/tests/test_nemweb_dispatch_prices.py new file mode 100644 index 0000000..849e486 --- /dev/null +++ b/app/backend/tests/test_nemweb_dispatch_prices.py @@ -0,0 +1,159 @@ +"""Tests for the official NEMWeb DispatchIS regional-price adapter.""" + +from dataclasses import FrozenInstanceError +from datetime import datetime, timezone +from hashlib import sha256 +from io import BytesIO +import unittest +from unittest.mock import patch +from zipfile import ZIP_DEFLATED, ZipFile + +import batterywatch_api.nemweb_dispatch_prices as source +from batterywatch_api.nemweb_http import NemwebHttpResource +from batterywatch_api.storage import RegionalPrice5m + + +INDEX_URL = "https://www.nemweb.com.au/REPORTS/CURRENT/DispatchIS_Reports/" +FILENAME = "PUBLIC_DISPATCHIS_202608301205_0000000535164870.zip" +ARTIFACT_URL = INDEX_URL + FILENAME +SOURCE_ID = "0000000535164870" + + +def _resource(url: str, body: bytes) -> NemwebHttpResource: + return NemwebHttpResource( + requested_url=url, + resolved_url=url, + body=body, + content_type=None, + etag=None, + last_modified=None, + ) + + +def _zip(payload: str, *, member_name: str | None = None) -> tuple[bytes, str]: + expected = FILENAME.removesuffix(".zip") + ".CSV" + member = member_name or expected + buffer = BytesIO() + with ZipFile(buffer, "w", compression=ZIP_DEFLATED) as archive: + archive.writestr(member, payload.encode("utf-8")) + return buffer.getvalue(), expected + + +class NemwebDispatchPriceAdapterTests(unittest.TestCase): + def test_discovers_canonical_artifacts_and_extracts_verified_member(self) -> None: + index_html = f""" + older + newer + duplicate + external + """ + references = source.discover_dispatch_price_artifacts( + index_html, + index_url=INDEX_URL, + ) + self.assertEqual(tuple(item.source_artifact_id for item in references), ( + "0000000535164869", SOURCE_ID, + )) + reference = references[-1] + self.assertEqual(reference.report_timestamp, datetime(2026, 8, 30, 2, 5, tzinfo=timezone.utc)) + with self.assertRaises(FrozenInstanceError): + reference.url = "changed" # type: ignore[misc] + + csv_payload = "C,NEMP.WORLD,DISPATCHIS,AEMO,PUBLIC\nC,END OF REPORT,2\n" + zip_payload, member_name = _zip(csv_payload) + artifact = source.extract_dispatch_price_zip(reference, zip_payload) + self.assertEqual( + ( + artifact.csv_member_name, + artifact.csv_payload, + artifact.zip_sha256, + artifact.raw_zip, + ), + ( + member_name, + csv_payload, + sha256(zip_payload).hexdigest(), + zip_payload, + ), + ) + + def test_collects_latest_artifact_and_delegates_strict_parser(self) -> None: + index_html = f'latest' + csv_payload = "C,NEMP.WORLD,DISPATCHIS,AEMO,PUBLIC\nC,END OF REPORT,2\n" + zip_payload, _member_name = _zip(csv_payload) + calls: list[tuple[str, int]] = [] + + def fetch(url: str, *, max_bytes: int) -> NemwebHttpResource: + calls.append((url, max_bytes)) + body = index_html.encode("utf-8") if url == INDEX_URL else zip_payload + return _resource(url, body) + + expected_records = ( + RegionalPrice5m( + region="NSW1", + interval_start=datetime(2026, 8, 30, 2, 5, tzinfo=timezone.utc), + price_aud_per_mwh=-4.9, + price_status="negative", + source_id=SOURCE_ID, + source_timestamp=datetime(2026, 8, 30, 2, 0, 11, tzinfo=timezone.utc), + ingestion_version=7, + correction_version=2, + ), + ) + with patch.object( + source, + "parse_dispatch_price_mms_csv", + return_value=expected_records, + ) as parser: + result = source.collect_latest_dispatch_prices( + ingestion_version=7, + correction_version=2, + fetch=fetch, + ) + + self.assertEqual(result.records, expected_records) + self.assertEqual(result.artifact.reference.source_artifact_id, SOURCE_ID) + self.assertEqual(calls, [ + (INDEX_URL, source.DISPATCH_PRICE_INDEX_MAX_BYTES), + (ARTIFACT_URL, source.DISPATCH_PRICE_ARTIFACT_MAX_BYTES), + ]) + parser.assert_called_once_with( + csv_payload, + source_id=SOURCE_ID, + ingestion_version=7, + correction_version=2, + ) + + def test_rejects_noncanonical_or_unsafe_source_inputs(self) -> None: + invalid_indexes = ( + "", + 'nested', + 'bad date', + ) + for index_html in invalid_indexes: + with self.subTest(index_html=index_html): + with self.assertRaises(source.NemwebDispatchPriceError): + source.discover_dispatch_price_artifacts( + index_html, + index_url=INDEX_URL, + ) + + reference = source.DispatchPriceArtifactRef( + url=ARTIFACT_URL, + zip_filename=FILENAME, + source_artifact_id=SOURCE_ID, + report_timestamp=datetime(2026, 8, 30, 2, 5, tzinfo=timezone.utc), + ) + bad_archives = ( + b"not a zip", + _zip("valid", member_name="../" + FILENAME.removesuffix(".zip") + ".CSV")[0], + _zip("valid\x00bad")[0], + ) + for payload in bad_archives: + with self.subTest(size=len(payload)): + with self.assertRaises(source.NemwebDispatchPriceError): + source.extract_dispatch_price_zip(reference, payload) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/backend/tests/test_nemweb_dispatch_scada.py b/app/backend/tests/test_nemweb_dispatch_scada.py new file mode 100644 index 0000000..910d84f --- /dev/null +++ b/app/backend/tests/test_nemweb_dispatch_scada.py @@ -0,0 +1,547 @@ +"""Tests for strict NEMWeb Dispatch SCADA source discovery.""" + +from dataclasses import FrozenInstanceError, replace +from datetime import datetime, timezone +from hashlib import sha256 +from io import BytesIO +import struct +from typing import Any +import unittest +from unittest.mock import patch +from zipfile import ZIP_BZIP2, ZIP_DEFLATED, ZIP_STORED, ZipFile + +import batterywatch_api.nemweb_dispatch_scada as source + + +INDEX_URL = "https://www.nemweb.com.au/REPORTS/CURRENT/Dispatch_SCADA/" + + +def _make_zip(member_name: str, payload: bytes) -> bytes: + return _make_zip_members((member_name, payload)) + + +def _make_zip_members( + *members: tuple[str, bytes], compression: int = ZIP_DEFLATED +) -> bytes: + buffer = BytesIO() + with ZipFile(buffer, "w", compression=compression) as archive: + for member_name, payload in members: + archive.writestr(member_name, payload) + return buffer.getvalue() + + +def _suffix_raw_zip_member_name(zip_payload: bytes, suffix: bytes) -> bytes: + mutated = bytearray(zip_payload) + local_name_length = struct.unpack_from(" bytes: + mutated = bytearray(zip_payload) + name_length = struct.unpack_from(" bytes: + mutated = bytearray(zip_payload) + local_flags = struct.unpack_from(" None: + index_html = """ + + unrelated + other origin + later + duplicate + earlier + + """ + discover = getattr(source, "discover_dispatch_scada_artifacts", lambda *_args, **_kwargs: ()) + + actual = discover(index_html, index_url=INDEX_URL) + + self.assertEqual( + tuple( + ( + item.url, + item.zip_filename, + item.source_artifact_id, + item.report_timestamp, + ) + for item in actual + ), + ( + ( + INDEX_URL + "PUBLIC_DISPATCHSCADA_202501011200_3.zip", + "PUBLIC_DISPATCHSCADA_202501011200_3.zip", + "3", + datetime(2025, 1, 1, 2, 0, tzinfo=timezone.utc), + ), + ( + INDEX_URL + "PUBLIC_DISPATCHSCADA_202501020304_20.zip", + "PUBLIC_DISPATCHSCADA_202501020304_20.zip", + "20", + datetime(2025, 1, 1, 17, 4, tzinfo=timezone.utc), + ), + ), + ) + with self.assertRaises(FrozenInstanceError): + actual[0].url = "changed" # type: ignore[misc] + + def test_rejects_empty_index_with_public_error(self) -> None: + public_error = getattr(source, "NemwebDispatchScadaError", AssertionError) + + with self.assertRaises(public_error): + source.discover_dispatch_scada_artifacts("", index_url=INDEX_URL) + + def test_rejects_noncanonical_index_url(self) -> None: + index_html = 'file' + + for index_url in ( + "http://www.nemweb.com.au/REPORTS/CURRENT/Dispatch_SCADA/", + "https://nemweb.com.au/REPORTS/CURRENT/Dispatch_SCADA/", + "https://www.nemweb.com.au/REPORTS/CURRENT/Dispatch_SCADA/?page=1", + "https://www.nemweb.com.au/REPORTS/ARCHIVE/Dispatch_SCADA/", + ): + with self.subTest(index_url=index_url): + with self.assertRaises(source.NemwebDispatchScadaError): + source.discover_dispatch_scada_artifacts( + index_html, index_url=index_url + ) + + def test_normalizes_invalid_index_payloads_to_public_error(self) -> None: + invalid_payloads: tuple[Any, ...] = ( + None, + b"not text", + "x" * (2 * 1024 * 1024 + 1), + "\ud800", + ) + + for index_html in invalid_payloads: + with self.subTest(payload_type=type(index_html).__name__): + try: + source.discover_dispatch_scada_artifacts( + index_html, # type: ignore[arg-type] + index_url=INDEX_URL, + ) + except Exception as error: + error_type: type[BaseException] | None = type(error) + else: + error_type = None + self.assertIs(error_type, source.NemwebDispatchScadaError) + + def test_rejects_index_without_canonical_artifacts(self) -> None: + index_html = """ + unrelated + nested + bad date + """ + + with self.assertRaises(source.NemwebDispatchScadaError): + source.discover_dispatch_scada_artifacts( + index_html, + index_url=INDEX_URL, + ) + + def test_rejects_conflicting_source_sequence(self) -> None: + index_html = """ + first + conflict + """ + + with self.assertRaises(source.NemwebDispatchScadaError): + source.discover_dispatch_scada_artifacts( + index_html, + index_url=INDEX_URL, + ) + + def test_accepts_observed_official_root_relative_href(self) -> None: + filename = "PUBLIC_DISPATCHSCADA_202608271535_0000000534726419.zip" + index_html = ( + 'artifact' + ) + + try: + references = source.discover_dispatch_scada_artifacts( + index_html, + index_url=INDEX_URL, + ) + except source.NemwebDispatchScadaError: + actual: tuple[tuple[str, str], ...] = () + else: + actual = tuple((item.url, item.zip_filename) for item in references) + + self.assertEqual(actual, ((INDEX_URL + filename, filename),)) + + def test_ignores_malformed_href_without_losing_valid_artifact(self) -> None: + filename = "PUBLIC_DISPATCHSCADA_202501011200_3.zip" + index_html = ( + 'malformed' + f'valid' + ) + + try: + references = source.discover_dispatch_scada_artifacts( + index_html, + index_url=INDEX_URL, + ) + except Exception as error: + actual: tuple[tuple[str, str], ...] = ((type(error).__name__, ""),) + else: + actual = tuple((item.url, item.zip_filename) for item in references) + + self.assertEqual(actual, ((INDEX_URL + filename, filename),)) + + def test_normalizes_oversized_numeric_source_id_to_public_error(self) -> None: + index_html = ( + 'oversized sequence' + ) + + try: + source.discover_dispatch_scada_artifacts(index_html, index_url=INDEX_URL) + except Exception as error: + error_type: type[BaseException] | None = type(error) + else: + error_type = None + + self.assertIs(error_type, source.NemwebDispatchScadaError) + + def test_normalizes_utc_underflow_timestamp_to_public_error(self) -> None: + index_html = ( + 'underflow' + ) + + try: + source.discover_dispatch_scada_artifacts(index_html, index_url=INDEX_URL) + except Exception as error: + error_type: type[BaseException] | None = type(error) + else: + error_type = None + + self.assertIs(error_type, source.NemwebDispatchScadaError) + + def test_filters_external_origin_using_protected_path(self) -> None: + index_html = ( + 'external' + ) + + with self.assertRaises(source.NemwebDispatchScadaError): + source.discover_dispatch_scada_artifacts( + index_html, + index_url=INDEX_URL, + ) + + def test_orders_equal_timestamps_by_numeric_source_id(self) -> None: + index_html = """ + twelve + three + """ + + references = source.discover_dispatch_scada_artifacts( + index_html, + index_url=INDEX_URL, + ) + + self.assertEqual( + tuple(item.source_artifact_id for item in references), + ("3", "12"), + ) + + +class ExtractDispatchScadaZipTests(unittest.TestCase): + def test_extracts_one_canonical_member_with_immutable_provenance(self) -> None: + reference = source.DispatchScadaArtifactRef( + url=( + INDEX_URL + + "PUBLIC_DISPATCHSCADA_202501011200_0000000000000003.zip" + ), + zip_filename="PUBLIC_DISPATCHSCADA_202501011200_0000000000000003.zip", + source_artifact_id="0000000000000003", + report_timestamp=datetime(2025, 1, 1, 2, 0, tzinfo=timezone.utc), + ) + member_name = reference.zip_filename.removesuffix(".zip") + ".CSV" + csv_payload = b"C,NEMP.WORLD,DISPATCHSCADA\r\n" + zip_payload = _make_zip(member_name, csv_payload) + artifact = source.extract_dispatch_scada_zip(reference, zip_payload) + actual = ( + artifact.reference, + artifact.csv_member_name, + artifact.csv_payload, + artifact.zip_sha256, + artifact.raw_zip, + ) + + self.assertEqual( + actual, + ( + reference, + member_name, + csv_payload.decode("utf-8"), + sha256(zip_payload).hexdigest(), + zip_payload, + ), + ) + with self.assertRaises(FrozenInstanceError): + artifact.csv_payload = "changed" # type: ignore[misc] + + def test_normalizes_invalid_zip_public_inputs(self) -> None: + reference = source.DispatchScadaArtifactRef( + url=INDEX_URL + "PUBLIC_DISPATCHSCADA_202501011200_3.zip", + zip_filename="PUBLIC_DISPATCHSCADA_202501011200_3.zip", + source_artifact_id="3", + report_timestamp=datetime(2025, 1, 1, 2, 0, tzinfo=timezone.utc), + ) + member_name = reference.zip_filename.removesuffix(".zip") + ".CSV" + valid_zip = _make_zip(member_name, b"C,NEMP.WORLD,DISPATCHSCADA\r\n") + cases: tuple[tuple[Any, Any], ...] = ( + (object(), valid_zip), + (reference, "not bytes"), + (reference, b""), + (reference, b"x" * ((16 * 1024 * 1024) + 1)), + ) + + for bad_reference, bad_payload in cases: + with self.subTest( + reference_type=type(bad_reference).__name__, + payload_type=type(bad_payload).__name__, + ): + try: + source.extract_dispatch_scada_zip( + bad_reference, # type: ignore[arg-type] + bad_payload, # type: ignore[arg-type] + ) + except Exception as error: + error_type: type[BaseException] | None = type(error) + else: + error_type = None + self.assertIs(error_type, source.NemwebDispatchScadaError) + + + def test_rejects_invalid_archive_structure_with_public_error(self) -> None: + reference = source.DispatchScadaArtifactRef( + url=INDEX_URL + "PUBLIC_DISPATCHSCADA_202501011200_3.zip", + zip_filename="PUBLIC_DISPATCHSCADA_202501011200_3.zip", + source_artifact_id="3", + report_timestamp=datetime(2025, 1, 1, 2, 0, tzinfo=timezone.utc), + ) + expected_member = reference.zip_filename.removesuffix(".zip") + ".CSV" + cases = ( + b"not a zip", + _make_zip_members(), + _make_zip_members( + (expected_member, b"valid"), + ("extra.txt", b"extra"), + ), + _make_zip("WRONG.CSV", b"wrong"), + _make_zip(expected_member.lower(), b"wrong case"), + _make_zip("../" + expected_member, b"traversal"), + _make_zip("nested/" + expected_member, b"nested"), + ) + + for zip_payload in cases: + with self.subTest(size=len(zip_payload)): + try: + source.extract_dispatch_scada_zip(reference, zip_payload) + except Exception as error: + error_type: type[BaseException] | None = type(error) + else: + error_type = None + self.assertIs(error_type, source.NemwebDispatchScadaError) + + + def test_rejects_unsafe_member_content_with_public_error(self) -> None: + reference = source.DispatchScadaArtifactRef( + url=INDEX_URL + "PUBLIC_DISPATCHSCADA_202501011200_3.zip", + zip_filename="PUBLIC_DISPATCHSCADA_202501011200_3.zip", + source_artifact_id="3", + report_timestamp=datetime(2025, 1, 1, 2, 0, tzinfo=timezone.utc), + ) + expected_member = reference.zip_filename.removesuffix(".zip") + ".CSV" + cases = ( + _make_zip(expected_member, b""), + _make_zip(expected_member, b"\xff"), + _make_zip(expected_member, b"valid\x00invalid"), + _make_zip_members( + (expected_member, b"unsupported compression"), + compression=ZIP_BZIP2, + ), + _make_zip(expected_member, b"A" * (1024 * 1024)), + _make_zip(expected_member, b"B" * ((8 * 1024 * 1024) + 1)), + ) + + for zip_payload in cases: + with self.subTest(size=len(zip_payload)): + try: + source.extract_dispatch_scada_zip(reference, zip_payload) + except Exception as error: + error_type: type[BaseException] | None = type(error) + else: + error_type = None + self.assertIs(error_type, source.NemwebDispatchScadaError) + + + def test_rejects_forged_artifact_reference_provenance(self) -> None: + reference = source.DispatchScadaArtifactRef( + url=INDEX_URL + "PUBLIC_DISPATCHSCADA_202501011200_3.zip", + zip_filename="PUBLIC_DISPATCHSCADA_202501011200_3.zip", + source_artifact_id="3", + report_timestamp=datetime(2025, 1, 1, 2, 0, tzinfo=timezone.utc), + ) + member_name = reference.zip_filename.removesuffix(".zip") + ".CSV" + zip_payload = _make_zip(member_name, b"C,NEMP.WORLD,DISPATCHSCADA\r\n") + cases = ( + replace(reference, url="https://evil.example/" + reference.zip_filename), + replace(reference, source_artifact_id="4"), + replace( + reference, + report_timestamp=datetime(2025, 1, 1, 2, 5, tzinfo=timezone.utc), + ), + ) + + for forged_reference in cases: + with self.subTest(reference=forged_reference): + try: + source.extract_dispatch_scada_zip(forged_reference, zip_payload) + except Exception as error: + error_type: type[BaseException] | None = type(error) + else: + error_type = None + self.assertIs(error_type, source.NemwebDispatchScadaError) + + + def test_rejects_raw_nul_suffixed_member_name(self) -> None: + reference = source.DispatchScadaArtifactRef( + url=( + INDEX_URL + + "PUBLIC_DISPATCHSCADA_202501011200_0000000000000003.zip" + ), + zip_filename="PUBLIC_DISPATCHSCADA_202501011200_0000000000000003.zip", + source_artifact_id="0000000000000003", + report_timestamp=datetime(2025, 1, 1, 2, 0, tzinfo=timezone.utc), + ) + member_name = reference.zip_filename.removesuffix(".zip") + ".CSV" + zip_payload = _suffix_raw_zip_member_name( + _make_zip(member_name, b"payload"), b"\x00evil" + ) + + with self.assertRaises(source.NemwebDispatchScadaError): + source.extract_dispatch_scada_zip(reference, zip_payload) + + + def test_normalizes_malformed_deflate_to_public_error(self) -> None: + reference = source.DispatchScadaArtifactRef( + url=( + INDEX_URL + + "PUBLIC_DISPATCHSCADA_202501011200_0000000000000003.zip" + ), + zip_filename="PUBLIC_DISPATCHSCADA_202501011200_0000000000000003.zip", + source_artifact_id="0000000000000003", + report_timestamp=datetime(2025, 1, 1, 2, 0, tzinfo=timezone.utc), + ) + member_name = reference.zip_filename.removesuffix(".zip") + ".CSV" + zip_payload = _corrupt_first_compressed_byte( + _make_zip(member_name, b"payload") + ) + + error_type: type[BaseException] | None = None + try: + source.extract_dispatch_scada_zip(reference, zip_payload) + except BaseException as error: + error_type = type(error) + + self.assertIs(error_type, source.NemwebDispatchScadaError) + + + def test_normalizes_crc_and_encryption_failures_to_public_error(self) -> None: + reference = source.DispatchScadaArtifactRef( + url=( + INDEX_URL + + "PUBLIC_DISPATCHSCADA_202501011200_0000000000000003.zip" + ), + zip_filename="PUBLIC_DISPATCHSCADA_202501011200_0000000000000003.zip", + source_artifact_id="0000000000000003", + report_timestamp=datetime(2025, 1, 1, 2, 0, tzinfo=timezone.utc), + ) + member_name = reference.zip_filename.removesuffix(".zip") + ".CSV" + stored_zip = _make_zip_members( + (member_name, b"payload"), compression=ZIP_STORED + ) + cases = ( + _corrupt_first_compressed_byte(stored_zip), + _mark_zip_member_encrypted(_make_zip(member_name, b"payload")), + ) + + for zip_payload in cases: + with self.subTest(): + with self.assertRaises(source.NemwebDispatchScadaError): + source.extract_dispatch_scada_zip(reference, zip_payload) + + + def test_normalizes_zip_stream_eof_to_public_error(self) -> None: + class EofZipFile: + def __init__(self, *_args: object, **_kwargs: object) -> None: + pass + + def __enter__(self) -> "EofZipFile": + raise EOFError + + def __exit__(self, *_args: object) -> None: + return None + + reference = source.DispatchScadaArtifactRef( + url=( + INDEX_URL + + "PUBLIC_DISPATCHSCADA_202501011200_0000000000000003.zip" + ), + zip_filename="PUBLIC_DISPATCHSCADA_202501011200_0000000000000003.zip", + source_artifact_id="0000000000000003", + report_timestamp=datetime(2025, 1, 1, 2, 0, tzinfo=timezone.utc), + ) + member_name = reference.zip_filename.removesuffix(".zip") + ".CSV" + + error_type: type[BaseException] | None = None + with patch.object(source, "ZipFile", EofZipFile): + try: + source.extract_dispatch_scada_zip( + reference, _make_zip(member_name, b"payload") + ) + except BaseException as error: + error_type = type(error) + + self.assertIs(error_type, source.NemwebDispatchScadaError) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/backend/tests/test_nemweb_http.py b/app/backend/tests/test_nemweb_http.py new file mode 100644 index 0000000..92e0a20 --- /dev/null +++ b/app/backend/tests/test_nemweb_http.py @@ -0,0 +1,319 @@ +"""Tests for the bounded official NEMWeb HTTPS adapter.""" + +from dataclasses import FrozenInstanceError +from http.client import HTTPMessage, IncompleteRead +from io import BytesIO +from typing import Any +import unittest +from unittest.mock import patch +from urllib.error import URLError +from urllib.request import Request +from urllib.response import addinfourl + +import batterywatch_api.nemweb_http as http + + +INDEX_URL = "https://www.nemweb.com.au/REPORTS/CURRENT/Dispatch_SCADA/" +PRICE_INDEX_URL = "https://www.nemweb.com.au/REPORTS/CURRENT/DispatchIS_Reports/" +PRICE_ARTIFACT_URL = ( + PRICE_INDEX_URL + "PUBLIC_DISPATCHIS_202608301205_0000000535164870.zip" +) + + +class FakeResponse: + def __init__( + self, + body: bytes, + *, + url: str = INDEX_URL, + status: int = 200, + headers: dict[str, str] | None = None, + ) -> None: + self._body = body + self._url = url + self.status = status + self.headers = headers or {} + self.read_sizes: list[int] = [] + + def __enter__(self) -> "FakeResponse": + return self + + def __exit__(self, *_args: object) -> None: + return None + + def geturl(self) -> str: + return self._url + + def read(self, size: int = -1) -> bytes: + self.read_sizes.append(size) + return self._body if size < 0 else self._body[:size] + + +class FakeOpener: + def __init__(self, response: FakeResponse) -> None: + self.response = response + self.calls: list[tuple[Request, float]] = [] + + def __call__(self, request: Request, *, timeout: float) -> FakeResponse: + self.calls.append((request, timeout)) + return self.response + + +class RaisingOpener: + def __init__(self, error: BaseException) -> None: + self.error = error + + def __call__(self, _request: Request, *, timeout: float) -> FakeResponse: + del timeout + raise self.error + + +class ReadErrorResponse(FakeResponse): + def __init__(self, error: BaseException) -> None: + super().__init__(b"payload") + self.error = error + + def read(self, size: int = -1) -> bytes: + del size + raise self.error + + +class FetchNemwebResourceTests(unittest.TestCase): + def test_fetches_official_dispatch_price_index_and_artifact(self) -> None: + for url, body in ( + (PRICE_INDEX_URL, b"price index"), + (PRICE_ARTIFACT_URL, b"PK price artifact"), + ): + with self.subTest(url=url): + opener = FakeOpener(FakeResponse(body, url=url)) + result = http.fetch_nemweb_resource( + url, + max_bytes=1024, + opener=opener, + ) + self.assertEqual((result.requested_url, result.body), (url, body)) + + def test_fetches_official_resource_with_immutable_metadata(self) -> None: + response = FakeResponse( + b"index", + headers={ + "Content-Type": "text/html; charset=utf-8", + "ETag": '"index-v1"', + "Last-Modified": "Sat, 29 Aug 2026 06:55:00 GMT", + }, + ) + opener = FakeOpener(response) + fetch: Any = getattr(http, "fetch_nemweb_resource", None) + result: Any = ( + fetch( + INDEX_URL, + max_bytes=1024, + timeout_seconds=7.5, + opener=opener, + ) + if callable(fetch) + else None + ) + + actual = ( + ( + result.requested_url, + result.resolved_url, + result.body, + result.content_type, + result.etag, + result.last_modified, + response.read_sizes, + opener.calls[0][0].full_url, + opener.calls[0][0].get_header("User-agent"), + opener.calls[0][1], + ) + if result is not None + else None + ) + self.assertEqual( + actual, + ( + INDEX_URL, + INDEX_URL, + b"index", + "text/html; charset=utf-8", + '"index-v1"', + "Sat, 29 Aug 2026 06:55:00 GMT", + [1025], + INDEX_URL, + "BatteryWatch-Collector/0.1", + 7.5, + ), + ) + with self.assertRaises(FrozenInstanceError): + result.body = b"changed" # type: ignore[misc] + + def test_normalizes_malformed_url_parse_failures_to_public_error(self) -> None: + error_type: type[BaseException] | None = None + try: + http.fetch_nemweb_resource( + "https://[invalid", + max_bytes=1024, + opener=FakeOpener(FakeResponse(b"payload")), + ) + except BaseException as error: + error_type = type(error) + self.assertIs(error_type, http.NemwebHttpError) + + def test_normalizes_huge_integer_timeout_to_public_error(self) -> None: + error_type: type[BaseException] | None = None + try: + http.fetch_nemweb_resource( + INDEX_URL, + max_bytes=1024, + timeout_seconds=10**10000, + opener=FakeOpener(FakeResponse(b"payload")), + ) + except BaseException as error: + error_type = type(error) + self.assertIs(error_type, http.NemwebHttpError) + + def test_default_transport_refuses_redirects_before_target_request(self) -> None: + default_transport = getattr( + http.fetch_nemweb_resource, "__kwdefaults__", {} + ).get("opener") + default_opener = getattr(default_transport, "__self__", None) + self.assertIsNotNone(default_opener) + redirected_url = ( + INDEX_URL + + "PUBLIC_DISPATCHSCADA_202608291700_0000000535067000.zip" + ) + requested_urls: list[str] = [] + + def fake_resource_open( + request: Request, _data: Any = None + ) -> addinfourl: + requested_urls.append(request.full_url) + headers = HTTPMessage() + headers["Location"] = redirected_url + response = addinfourl( + BytesIO(b"redirect"), + headers, + request.full_url, + code=302, + ) + setattr(response, "msg", "Found") + return response + + with patch.object( + default_opener, "_open", side_effect=fake_resource_open + ): + with self.assertRaises(http.NemwebHttpError): + http.fetch_nemweb_resource(INDEX_URL, max_bytes=1024) + + self.assertEqual(requested_urls, [INDEX_URL]) + + def test_normalizes_invalid_request_arguments_to_public_error(self) -> None: + response = FakeResponse(b"payload") + invalid_cases: tuple[tuple[object, object, object], ...] = ( + (None, 1024, 10.0), + ("http://www.nemweb.com.au/REPORTS/CURRENT/Dispatch_SCADA/", 1024, 10.0), + ("https://nemweb.com.au/REPORTS/CURRENT/Dispatch_SCADA/", 1024, 10.0), + ("https://www.nemweb.com.au:443/REPORTS/CURRENT/Dispatch_SCADA/", 1024, 10.0), + (INDEX_URL + "?latest=1", 1024, 10.0), + (INDEX_URL + "#fragment", 1024, 10.0), + ("https://www.nemweb.com.au/REPORTS/CURRENT/Dispatch_SCADA/not-canonical.zip", 1024, 10.0), + (INDEX_URL, True, 10.0), + (INDEX_URL, 0, 10.0), + (INDEX_URL, 16 * 1024 * 1024 + 1, 10.0), + (INDEX_URL, 1024, True), + (INDEX_URL, 1024, 0.0), + (INDEX_URL, 1024, 61.0), + (INDEX_URL, 1024, float("inf")), + ) + + for url, max_bytes, timeout_seconds in invalid_cases: + with self.subTest(url=url, max_bytes=max_bytes, timeout=timeout_seconds): + error_type: type[BaseException] | None = None + try: + http.fetch_nemweb_resource( + url, # type: ignore[arg-type] + max_bytes=max_bytes, # type: ignore[arg-type] + timeout_seconds=timeout_seconds, # type: ignore[arg-type] + opener=FakeOpener(response), + ) + except BaseException as error: + error_type = type(error) + self.assertIs(error_type, http.NemwebHttpError) + + def test_normalizes_transport_status_and_redirect_failures(self) -> None: + redirected_url = ( + INDEX_URL + + "PUBLIC_DISPATCHSCADA_202608291700_0000000535067000.zip" + ) + openers: tuple[object, ...] = ( + FakeOpener(FakeResponse(b"payload", status=404)), + FakeOpener(FakeResponse(b"payload", url=redirected_url)), + RaisingOpener(URLError("source unavailable")), + RaisingOpener(TimeoutError("timed out")), + RaisingOpener(OSError("socket failure")), + ) + + for opener in openers: + with self.subTest(opener=type(opener).__name__): + error_type: type[BaseException] | None = None + try: + http.fetch_nemweb_resource( + INDEX_URL, + max_bytes=1024, + opener=opener, # type: ignore[arg-type] + ) + except BaseException as error: + error_type = type(error) + self.assertIs(error_type, http.NemwebHttpError) + + + def test_rejects_unbounded_or_unsafe_response_content(self) -> None: + invalid_responses = ( + FakeResponse(b""), + FakeResponse(b"x" * 1025), + FakeResponse(b"payload", headers={"Content-Length": "1025"}), + FakeResponse(b"payload", headers={"Content-Length": "invalid"}), + FakeResponse(b"payload", headers={"Content-Length": "3"}), + FakeResponse(b"payload", headers={"Content-Encoding": "gzip"}), + FakeResponse(b"payload", headers={"ETag": "unsafe\nvalue"}), + FakeResponse(b"payload", headers={"Last-Modified": "x" * 1025}), + ) + + for response in invalid_responses: + with self.subTest(headers=response.headers, body_size=len(response._body)): + error_type: type[BaseException] | None = None + try: + http.fetch_nemweb_resource( + INDEX_URL, + max_bytes=1024, + opener=FakeOpener(response), + ) + except BaseException as error: + error_type = type(error) + self.assertIs(error_type, http.NemwebHttpError) + + + def test_normalizes_protocol_read_failures(self) -> None: + errors: tuple[BaseException, ...] = ( + IncompleteRead(b"partial", 10), + EOFError(), + ) + + for read_error in errors: + with self.subTest(error=type(read_error).__name__): + error_type: type[BaseException] | None = None + try: + http.fetch_nemweb_resource( + INDEX_URL, + max_bytes=1024, + opener=FakeOpener(ReadErrorResponse(read_error)), + ) + except BaseException as error: + error_type = type(error) + self.assertIs(error_type, http.NemwebHttpError) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/backend/tests/test_postgresql_storage.py b/app/backend/tests/test_postgresql_storage.py new file mode 100644 index 0000000..f76933a --- /dev/null +++ b/app/backend/tests/test_postgresql_storage.py @@ -0,0 +1,383 @@ +"""Focused fake-connection tests for the S2b PostgreSQL storage adapter.""" + +from datetime import datetime, timedelta, timezone +from decimal import Decimal +import unittest + +from batterywatch_api.storage import ( + GeneratorMetadata, + GeneratorPower5m, + GeneratorSoc5m, + PostgreSQLRepository, + RegionalPrice5m, +) + + +UTC = timezone.utc +INTERVAL = datetime(2026, 1, 1, 0, 5, tzinfo=UTC) +SOURCE_TIME = datetime(2026, 1, 1, 0, 5, 3, tzinfo=UTC) +END = datetime(2026, 1, 1, 1, 5, tzinfo=UTC) + + +class FakeCursor: + def __init__(self, connection): + self.connection = connection + self.rows = [] + + def execute(self, statement, parameters): + self.connection.executions.append((statement, tuple(parameters))) + if self.connection.error is not None: + raise self.connection.error + self.rows = list(self.connection.responses.pop(0)) if self.connection.responses else [] + + def fetchone(self): + return self.rows.pop(0) if self.rows else None + + def fetchall(self): + rows, self.rows = self.rows, [] + return rows + + def close(self): + self.connection.closed_cursors += 1 + + +class FakeConnection: + def __init__(self, responses=(), error=None): + self.responses = list(responses) + self.error = error + self.executions = [] + self.commits = 0 + self.rollbacks = 0 + self.closed_cursors = 0 + + def cursor(self): + return FakeCursor(self) + + def commit(self): + self.commits += 1 + + def rollback(self): + self.rollbacks += 1 + + +def generator_record(**overrides): + values = { + "generator_id": "BAT-1", + "site_name": "Test Battery", + "region": "NSW1", + "capacity_mw": 2.0, + "storage_capacity_mwh": 4.0, + "source_id": "registry-v1", + "source_timestamp": SOURCE_TIME, + "ingestion_version": 3, + "correction_version": 1, + "data_start": INTERVAL, + "data_end": END, + } + values.update(overrides) + return GeneratorMetadata(**values) + + +def power_record(**overrides): + values = { + "generator_id": "BAT-1", + "interval_start": INTERVAL, + "power_mw": 0.0, + "source_id": "nem-dispatch", + "source_timestamp": SOURCE_TIME, + "ingestion_version": 1, + "correction_version": 0, + } + values.update(overrides) + return GeneratorPower5m(**values) + + +def soc_record(**overrides): + values = { + "generator_id": "BAT-1", + "interval_start": INTERVAL, + "soc_percent": None, + "source_id": "battery-telemetry", + "source_timestamp": SOURCE_TIME, + "ingestion_version": 1, + "correction_version": 0, + "quality_flags": ("unavailable",), + } + values.update(overrides) + return GeneratorSoc5m(**values) + + +def price_record(**overrides): + values = { + "region": "NSW1", + "interval_start": INTERVAL, + "price_aud_per_mwh": None, + "price_status": "missing", + "source_id": "nem-rrp", + "source_timestamp": SOURCE_TIME, + "ingestion_version": 1, + "correction_version": 0, + "quality_flags": ("unavailable",), + } + values.update(overrides) + return RegionalPrice5m(**values) + + +class PostgreSQLRepositoryWriteTests(unittest.TestCase): + def test_generator_upsert_uses_parameterized_bindings(self): + connection = FakeConnection(responses=([("applied",)],)) + repository = PostgreSQLRepository(connection) + record = generator_record() + + self.assertTrue(repository.upsert_generator(record)) + self.assertEqual(connection.commits, 1) + statement, parameters = connection.executions[0] + self.assertIn("INSERT INTO generators", statement) + self.assertIn("ON CONFLICT (generator_id)", statement) + self.assertIn("%s", statement) + self.assertNotIn(record.generator_id, statement) + self.assertNotIn(record.site_name, statement) + self.assertEqual( + parameters, + ( + "BAT-1", + "Test Battery", + "NSW1", + 2.0, + 4.0, + INTERVAL, + END, + "registry-v1", + SOURCE_TIME, + 3, + 1, + ), + ) + + def test_measurement_upserts_bind_zero_unavailable_values(self): + connection = FakeConnection( + responses=([("applied",)], [ + ("applied",), + ], [ + ("applied",), + ]) + ) + repository = PostgreSQLRepository(connection) + + power = power_record() + soc = soc_record() + price = price_record() + self.assertTrue(repository.upsert_power(power)) + self.assertTrue(repository.upsert_soc(soc)) + self.assertTrue(repository.upsert_price(price)) + + power_sql, power_params = connection.executions[0] + soc_sql, soc_params = connection.executions[1] + price_sql, price_params = connection.executions[2] + self.assertIn("INSERT INTO generator_power_5m", power_sql) + self.assertIn("INSERT INTO generator_soc_5m", soc_sql) + self.assertIn("INSERT INTO nem_price_5m", price_sql) + self.assertEqual(power_params[2], 0.0) + self.assertIsNone(soc_params[2]) + self.assertIsNone(price_params[2]) + self.assertEqual(price_params[3], "missing") + for statement in (power_sql, soc_sql, price_sql): + self.assertIn("%s", statement) + self.assertNotIn("BAT-1", statement) + self.assertNotIn("NSW1", statement) + + def test_price_upsert_binds_aemo_flags(self): + connection = FakeConnection(responses=([("applied",)],)) + repository = PostgreSQLRepository(connection) + record = price_record( + price_aud_per_mwh=12.5, + price_status="available", + intervention=1, + apc_flag=4, + market_suspended=True, + ) + + self.assertTrue(repository.upsert_price(record)) + statement, parameters = connection.executions[0] + self.assertIn("intervention", statement) + self.assertIn("apc_flag", statement) + self.assertIn("market_suspended", statement) + self.assertIn("intervention = EXCLUDED.intervention", statement) + self.assertIn("apc_flag = EXCLUDED.apc_flag", statement) + self.assertIn("market_suspended = EXCLUDED.market_suspended", statement) + self.assertEqual( + parameters, + ( + "NSW1", + INTERVAL, + 12.5, + "available", + 1, + 4, + True, + "nem-rrp", + SOURCE_TIME, + 1, + 0, + ["unavailable"], + ), + ) + + def test_returning_row_distinguishes_replay_from_newer_correction(self): + connection = FakeConnection( + responses=([("applied",)], [], [("applied",)]) + ) + repository = PostgreSQLRepository(connection) + original = power_record(power_mw=1.25, ingestion_version=1) + replay = power_record(power_mw=1.25, ingestion_version=1) + corrected = power_record( + power_mw=1.5, + ingestion_version=2, + correction_version=1, + source_timestamp=SOURCE_TIME + timedelta(minutes=1), + ) + + self.assertTrue(repository.upsert_power(original)) + self.assertFalse(repository.upsert_power(replay)) + self.assertTrue(repository.upsert_power(corrected)) + self.assertEqual(connection.commits, 3) + statement, _ = connection.executions[0] + self.assertIn("WHERE (EXCLUDED.correction_version", statement) + self.assertIn("EXCLUDED.source_timestamp", statement) + + def test_database_failure_rolls_back_propagates(self): + failure = RuntimeError("database unavailable") + connection = FakeConnection(error=failure) + repository = PostgreSQLRepository(connection) + + with self.assertRaises(RuntimeError) as raised: + repository.upsert_power(power_record()) + self.assertIs(raised.exception, failure) + self.assertEqual(connection.commits, 0) + self.assertEqual(connection.rollbacks, 1) + self.assertEqual(connection.closed_cursors, 1) + + +class PostgreSQLRepositoryReadTests(unittest.TestCase): + def test_reads_map_rows_preserve_zero_none_flags(self): + connection = FakeConnection( + responses=( + [ + ( + "BAT-1", + "Test Battery", + "NSW1", + Decimal("2.0"), + Decimal("4.0"), + "registry-v1", + SOURCE_TIME, + 3, + 1, + INTERVAL, + END, + ) + ], + [("BAT-1", INTERVAL, Decimal("0.0"), "nem-dispatch", SOURCE_TIME, 1, 0)], + [("BAT-1", INTERVAL, None, "battery-telemetry", SOURCE_TIME, 1, 0, ["unavailable"])], + [("NSW1", INTERVAL, Decimal("0.0"), "available", 0, 0, False, "nem-rrp", SOURCE_TIME, 1, 0, [])], + ) + ) + repository = PostgreSQLRepository(connection) + + generator = repository.read_generator("BAT-1") + power = repository.read_power("BAT-1", INTERVAL) + soc = repository.read_soc("BAT-1", INTERVAL) + price = repository.read_price("NSW1", INTERVAL) + + self.assertEqual(generator, generator_record()) + self.assertEqual(power, power_record()) + self.assertEqual(soc, soc_record()) + self.assertEqual(price, price_record(price_aud_per_mwh=0.0, price_status="available", quality_flags=())) + self.assertIsNotNone(power) + self.assertIsNotNone(soc) + self.assertIsNotNone(price) + assert power is not None + assert soc is not None + assert price is not None + self.assertEqual(power.power_mw, 0.0) + self.assertIsNone(soc.soc_percent) + self.assertEqual(price.price_aud_per_mwh, 0.0) + self.assertEqual(connection.closed_cursors, 4) + + def test_price_reads_map_aemo_flags(self): + row = ( + "NSW1", + INTERVAL, + Decimal("12.5"), + "available", + 1, + 4, + True, + "nem-rrp", + SOURCE_TIME, + 1, + 0, + [], + ) + connection = FakeConnection(responses=([row], [row])) + repository = PostgreSQLRepository(connection) + + try: + point = repository.read_price("NSW1", INTERVAL) + values = repository.list_prices( + "NSW1", start=INTERVAL, end=INTERVAL + timedelta(minutes=5) + ) + except Exception as exc: + self.fail(f"price flag row mapping failed: {exc}") + + self.assertIsNotNone(point) + assert point is not None + expected = (1, 4, True) + self.assertEqual( + (point.intervention, point.apc_flag, point.market_suspended), + expected, + ) + self.assertEqual( + [ + (item.intervention, item.apc_flag, item.market_suspended) + for item in values + ], + [expected], + ) + + def test_missing_read_list_window_sorted(self): + earlier = datetime(2026, 1, 1, 0, 0, tzinfo=UTC) + connection = FakeConnection( + responses=( + [], + [ + ("BAT-1", INTERVAL, 1.0, "nem-dispatch", SOURCE_TIME, 1, 0), + ("BAT-1", earlier, 0.5, "nem-dispatch", SOURCE_TIME, 1, 0), + ], + ) + ) + repository = PostgreSQLRepository(connection) + start = datetime(2026, 1, 1, 10, 0, tzinfo=timezone(timedelta(hours=10))) + end = datetime(2026, 1, 1, 10, 10, tzinfo=timezone(timedelta(hours=10))) + + self.assertIsNone(repository.read_power("BAT-1", INTERVAL)) + values = repository.list_power("BAT-1", start=start, end=end) + + self.assertEqual([item.interval_start for item in values], [earlier, INTERVAL]) + statement, parameters = connection.executions[1] + self.assertIn("ORDER BY interval_start ASC", statement) + self.assertEqual(parameters, ("BAT-1", earlier, earlier, INTERVAL + timedelta(minutes=5), INTERVAL + timedelta(minutes=5))) + + def test_read_database_failure_is_not_silenced(self): + failure = RuntimeError("query failed") + connection = FakeConnection(error=failure) + repository = PostgreSQLRepository(connection) + + with self.assertRaises(RuntimeError) as raised: + repository.read_price("NSW1", INTERVAL) + self.assertIs(raised.exception, failure) + self.assertEqual(connection.closed_cursors, 1) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/app/backend/tests/test_runtime_health.py b/app/backend/tests/test_runtime_health.py new file mode 100644 index 0000000..619f948 --- /dev/null +++ b/app/backend/tests/test_runtime_health.py @@ -0,0 +1,314 @@ +"""Focused runtime-selection and health tracer tests.""" + +import inspect +import os +import unittest +from unittest.mock import patch + +from fastapi.testclient import TestClient + +from batterywatch_api.database_main import create_database_application +import batterywatch_api.main as main + + +class _ProbeCursor: + def __init__(self, error=None): + self.statements = [] + self.error = error + self.closed = False + + def execute(self, statement): + self.statements.append(statement) + if self.error is not None: + raise self.error + + def close(self): + self.closed = True + + +class _ProbeConnection: + def __init__(self, cursor): + self.cursor_instance = cursor + self.closed = False + + def cursor(self): + return self.cursor_instance + + def close(self): + self.closed = True + + +class RuntimeHealthTests(unittest.TestCase): + def test_main_exposes_application_factory(self): + self.assertTrue(hasattr(main, "create_app")) + + def test_factory_returns_a_fresh_fixture_application(self): + self.assertIsNot(main.create_app(), main.app) + + def test_factory_defaults_to_fixture_runtime_without_database_configuration(self): + application = main.create_app() + self.assertEqual(getattr(application.state, "data_mode", None), "fixture") + + def test_factory_accepts_an_explicit_runtime_mode(self): + self.assertIn("mode", inspect.signature(main.create_app).parameters) + + def test_factory_accepts_data_mode_configuration(self): + self.assertIn("data_mode", inspect.signature(main.create_app).parameters) + + def test_runtime_application_reads_explicit_database_mode_from_environment(self): + application = main.create_runtime_app( + { + "BATTERYWATCH_DATA_MODE": "database", + "BATTERYWATCH_DATABASE_URL": "postgresql://private", + }, + health_tracer=lambda: True, + ) + + response = TestClient(application).get("/api/health") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json()["data_mode"], "database") + + def test_database_entrypoint_cannot_be_switched_to_fixture_by_environment(self): + application = create_database_application( + { + "BATTERYWATCH_DATA_MODE": "fixture", + "BATTERYWATCH_DATABASE_URL": "postgresql://private", + }, + health_tracer=lambda: True, + ) + + response = TestClient(application).get("/api/health") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json()["data_mode"], "database") + + def test_database_mode_health_fails_closed_without_configuration(self): + with patch.dict( + os.environ, + {"BATTERYWATCH_DATABASE_URL": "", "DATABASE_URL": ""}, + ): + response = TestClient(main.create_app(mode="database")).get("/api/health") + self.assertEqual(response.status_code, 503) + + def test_database_health_tracer_exposes_a_safe_check(self): + tracer = main.DatabaseHealthTracer(None) + self.assertFalse(getattr(tracer, "check", lambda: True)()) + + def test_factory_accepts_an_injected_health_tracer(self): + self.assertIn("health_tracer", inspect.signature(main.create_app).parameters) + + def test_database_health_uses_an_injected_healthy_tracer(self): + def healthy_tracer(): + return True + + response = TestClient( + main.create_app(mode="database", health_tracer=healthy_tracer) + ).get("/api/health") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json()["data_mode"], "database") + + def test_database_health_accepts_a_tracer_object(self): + class HealthyTracer: + def check(self): + return True + + response = TestClient( + main.create_app(mode="database", health_tracer=HealthyTracer()) + ).get("/api/health") + self.assertEqual(response.status_code, 200) + + def test_factory_accepts_database_configuration(self): + self.assertIn("database_url", inspect.signature(main.create_app).parameters) + + def test_factory_accepts_an_injected_connection_factory(self): + self.assertIn("connection_factory", inspect.signature(main.create_app).parameters) + + def test_database_health_traces_an_injected_connection(self): + calls = [] + + class Cursor: + def execute(self, statement): + self.statement = statement + + def fetchone(self): + return (1,) + + def close(self): + pass + + class Connection: + def __init__(self): + self.closed = False + + def cursor(self): + return Cursor() + + def close(self): + self.closed = True + + def connect(database_url): + calls.append(database_url) + return Connection() + + response = TestClient( + main.create_app( + mode="database", + database_url="configured", + connection_factory=connect, + ) + ).get("/api/health") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json()["data_mode"], "database") + self.assertEqual(calls, ["configured"]) + + def test_database_health_returns_generic_503_when_psycopg_import_fails(self): + import_marker = "driver-import-marker" + url_marker = "postgresql://database-url-marker.invalid/batterywatch" + application = main.create_app( + mode="database", database_url=url_marker + ) + + with patch( + "batterywatch_api.runtime.importlib.import_module", + side_effect=ImportError(import_marker), + ): + response = TestClient(application).get("/api/health") + + self.assertEqual(response.status_code, 503) + self.assertEqual( + response.json(), {"detail": "Database health unavailable"} + ) + self.assertNotIn("BWTEST1", response.text) + self.assertNotIn(import_marker, response.text) + self.assertNotIn(url_marker, response.text) + + def test_database_health_returns_generic_503_when_connection_factory_fails(self): + failure_marker = "connection-factory-marker" + url_marker = "postgresql://database-url-marker.invalid/batterywatch" + + def connect(database_url): + raise RuntimeError(f"{failure_marker}: {database_url}") + + response = TestClient( + main.create_app( + mode="database", + database_url=url_marker, + connection_factory=connect, + ) + ).get("/api/health") + + self.assertEqual(response.status_code, 503) + self.assertEqual( + response.json(), {"detail": "Database health unavailable"} + ) + self.assertNotIn("BWTEST1", response.text) + self.assertNotIn(failure_marker, response.text) + self.assertNotIn(url_marker, response.text) + + def test_database_health_returns_generic_503_when_query_fails(self): + failure_marker = "query-failure-marker" + url_marker = "postgresql://database-url-marker.invalid/batterywatch" + cursor = _ProbeCursor( + error=RuntimeError(f"{failure_marker}: {url_marker}") + ) + connection = _ProbeConnection(cursor) + + response = TestClient( + main.create_app( + mode="database", + database_url=url_marker, + connection_factory=lambda _: connection, + ) + ).get("/api/health") + + self.assertEqual(response.status_code, 503) + self.assertEqual( + response.json(), {"detail": "Database health unavailable"} + ) + self.assertNotIn("BWTEST1", response.text) + self.assertNotIn(failure_marker, response.text) + self.assertNotIn(url_marker, response.text) + + def test_database_health_probe_executes_exactly_select_one(self): + cursor = _ProbeCursor() + connection = _ProbeConnection(cursor) + tracer = main.DatabaseHealthTracer( + "database-url", + connection_factory=lambda _: connection, + ) + + self.assertTrue(tracer.check()) + self.assertEqual(cursor.statements, ["SELECT 1"]) + + def test_database_health_probe_closes_cursor_and_connection_on_success(self): + cursor = _ProbeCursor() + connection = _ProbeConnection(cursor) + tracer = main.DatabaseHealthTracer( + "database-url", + connection_factory=lambda _: connection, + ) + + self.assertTrue(tracer.check()) + self.assertTrue(cursor.closed) + self.assertTrue(connection.closed) + + def test_database_health_probe_closes_cursor_and_connection_on_query_failure(self): + cursor = _ProbeCursor(error=RuntimeError("query failure")) + connection = _ProbeConnection(cursor) + tracer = main.DatabaseHealthTracer( + "database-url", + connection_factory=lambda _: connection, + ) + + self.assertFalse(tracer.check()) + self.assertTrue(cursor.closed) + self.assertTrue(connection.closed) + + def test_database_mode_can_read_configuration_from_environment(self): + calls = [] + + class Cursor: + def execute(self, statement): + pass + + def close(self): + pass + + class Connection: + def cursor(self): + return Cursor() + + def close(self): + pass + + def connect(database_url): + calls.append(database_url) + return Connection() + + with patch.dict( + os.environ, {"BATTERYWATCH_DATABASE_URL": "configured"} + ): + response = TestClient( + main.create_app(mode="database", connection_factory=connect) + ).get("/api/health") + self.assertEqual(response.status_code, 200) + self.assertEqual(calls, ["configured"]) + + def test_factory_rejects_an_unknown_runtime_mode(self): + with self.assertRaises(ValueError): + main.create_app(mode="unsupported") + + def test_database_mode_does_not_fall_back_to_fixture_generators(self): + response = TestClient( + main.create_app(mode="database", health_tracer=lambda: True) + ).get("/api/generators") + self.assertEqual(response.status_code, 503) + + def test_database_mode_does_not_fall_back_to_fixture_series(self): + response = TestClient( + main.create_app(mode="database", health_tracer=lambda: True) + ).get("/api/series", params={"generator": "BWTEST1"}) + self.assertEqual(response.status_code, 503) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/backend/tests/test_storage.py b/app/backend/tests/test_storage.py new file mode 100644 index 0000000..37c6c29 --- /dev/null +++ b/app/backend/tests/test_storage.py @@ -0,0 +1,179 @@ +"""Persistence contract tests for the S2a storage seam.""" + +from datetime import datetime, timedelta, timezone +from pathlib import Path +import unittest + +from batterywatch_api.storage import ( + GeneratorMetadata, + GeneratorPower5m, + GeneratorSoc5m, + InMemoryRepository, + RegionalPrice5m, +) + + +UTC = timezone.utc +INTERVAL = datetime(2026, 1, 1, 0, 5, tzinfo=UTC) +SOURCE_TIME = datetime(2026, 1, 1, 0, 5, 3, tzinfo=UTC) + + +class StorageContractTests(unittest.TestCase): + def setUp(self): + self.repository = InMemoryRepository() + + def test_records_normalize_utc_and_require_aligned_five_minute_intervals(self): + offset = timezone(timedelta(hours=10)) + record = GeneratorPower5m( + generator_id="BAT-1", + interval_start=datetime(2026, 1, 1, 10, 5, tzinfo=offset), + power_mw=0.0, + source_id="nem-dispatch", + source_timestamp=datetime(2026, 1, 1, 10, 5, 3, tzinfo=offset), + ingestion_version=1, + ) + + self.assertEqual(record.interval_start, INTERVAL) + self.assertEqual(record.interval_start.tzinfo, UTC) + self.assertEqual(record.source_timestamp, SOURCE_TIME) + with self.assertRaises(ValueError): + GeneratorPower5m( + generator_id="BAT-1", + interval_start=datetime(2026, 1, 1, 0, 6, tzinfo=UTC), + power_mw=1.0, + source_id="nem-dispatch", + source_timestamp=SOURCE_TIME, + ingestion_version=1, + ) + with self.assertRaises(ValueError): + GeneratorPower5m( + generator_id="BAT-1", + interval_start=datetime(2026, 1, 1, 0, 5), + power_mw=1.0, + source_id="nem-dispatch", + source_timestamp=SOURCE_TIME, + ingestion_version=1, + ) + + def test_insert_and_read_back_preserves_zero_and_unavailable_values(self): + generator = GeneratorMetadata( + generator_id="BAT-1", + site_name="Test Battery", + region="NSW1", + capacity_mw=2.0, + storage_capacity_mwh=4.0, + source_id="registry-v1", + source_timestamp=SOURCE_TIME, + ingestion_version=1, + ) + power = GeneratorPower5m( + generator_id="BAT-1", + interval_start=INTERVAL, + power_mw=0.0, + source_id="nem-dispatch", + source_timestamp=SOURCE_TIME, + ingestion_version=1, + ) + soc = GeneratorSoc5m( + generator_id="BAT-1", + interval_start=INTERVAL, + soc_percent=None, + source_id="battery-telemetry", + source_timestamp=SOURCE_TIME, + ingestion_version=1, + ) + price = RegionalPrice5m( + region="NSW1", + interval_start=INTERVAL, + price_aud_per_mwh=None, + price_status="missing", + source_id="nem-rrp", + source_timestamp=SOURCE_TIME, + ingestion_version=1, + quality_flags=("unavailable",), + ) + + self.assertTrue(self.repository.insert_generator(generator)) + self.assertTrue(self.repository.insert_power(power)) + self.assertTrue(self.repository.insert_soc(soc)) + self.assertTrue(self.repository.insert_price(price)) + self.assertEqual(self.repository.read_generator("BAT-1"), generator) + self.assertEqual(self.repository.read_power("BAT-1", INTERVAL), power) + self.assertEqual(self.repository.read_soc("BAT-1", INTERVAL), soc) + self.assertEqual(self.repository.read_price("NSW1", INTERVAL), price) + read_power = self.repository.read_power("BAT-1", INTERVAL) + read_soc = self.repository.read_soc("BAT-1", INTERVAL) + read_price = self.repository.read_price("NSW1", INTERVAL) + self.assertIsNotNone(read_power) + self.assertIsNotNone(read_soc) + self.assertIsNotNone(read_price) + assert read_power is not None + assert read_soc is not None + assert read_price is not None + self.assertEqual(read_power.power_mw, 0.0) + self.assertIsNone(read_soc.soc_percent) + self.assertIsNone(read_price.price_aud_per_mwh) + self.assertTrue(read_price.is_missing) + + def test_replaying_source_record_is_idempotent(self): + record = self._power(power_mw=1.25) + + self.assertTrue(self.repository.insert_power(record)) + self.assertFalse(self.repository.insert_power(record)) + self.assertEqual(self.repository.count_power("BAT-1"), 1) + self.assertEqual(self.repository.list_power("BAT-1"), (record,)) + + def test_later_correction_replaces_one_effective_interval_and_stale_data_does_not_regress(self): + original = self._power(power_mw=1.25, ingestion_version=1, correction_version=0) + corrected = self._power( + power_mw=1.5, + ingestion_version=2, + correction_version=1, + source_timestamp=SOURCE_TIME + timedelta(minutes=1), + ) + stale = self._power(power_mw=0.5, ingestion_version=3, correction_version=0) + + self.assertTrue(self.repository.insert_power(original)) + self.assertTrue(self.repository.insert_power(corrected)) + self.assertFalse(self.repository.insert_power(stale)) + self.assertEqual(self.repository.count_power("BAT-1"), 1) + self.assertEqual(self.repository.read_power("BAT-1", INTERVAL), corrected) + + def _power(self, *, power_mw, ingestion_version=1, correction_version=0, source_timestamp=SOURCE_TIME): + return GeneratorPower5m( + generator_id="BAT-1", + interval_start=INTERVAL, + power_mw=power_mw, + source_id="nem-dispatch", + source_timestamp=source_timestamp, + ingestion_version=ingestion_version, + correction_version=correction_version, + ) + + +class MigrationContractTests(unittest.TestCase): + def test_initial_migration_declares_v1_five_minute_tables_and_guards(self): + migration = Path(__file__).resolve().parents[2] / "migrations" / "001_initial_schema.sql" + sql = migration.read_text(encoding="utf-8").lower() + + for table in ( + "generators", + "generator_power_5m", + "generator_soc_5m", + "nem_price_5m", + ): + self.assertIn(f"create table if not exists {table}", sql) + self.assertIn("timestamptz", sql) + self.assertIn("primary key (generator_id, interval_start)", sql) + self.assertIn("unique (generator_id, interval_start, source_id", sql) + self.assertIn("correction_version", sql) + self.assertIn("check", sql) + self.assertIn("price_status", sql) + self.assertIn("soc_percent", sql) + self.assertIn("::text not in ('nan', 'infinity', '-infinity')", sql) + self.assertIn("one effective record per logical key", sql) + self.assertIn("revision history is not retained", sql) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/config/battery_assets.json b/app/config/battery_assets.json new file mode 100644 index 0000000..84a0747 --- /dev/null +++ b/app/config/battery_assets.json @@ -0,0 +1,603 @@ +{ + "schema_version": 1, + "assets": [ + { + "duid": "ADPBA1", + "site_name": "Adelaide Desalination Plant", + "region": "SA1", + "capacity_mw": 7.76, + "storage_capacity_mwh": 12.6, + "source_id": "aemo-generation-information-2025", + "source_timestamp": "2025-02-28T00:00:00Z" + }, + { + "duid": "BALB1", + "site_name": "Ballarat Energy Storage System", + "region": "VIC1", + "capacity_mw": 30.0, + "storage_capacity_mwh": 30.0, + "source_id": "aemo-generation-information-2025", + "source_timestamp": "2025-03-11T00:00:00Z" + }, + { + "duid": "BBATTERY1", + "site_name": "Bouldercombe Battery project", + "region": "QLD1", + "capacity_mw": 50.0, + "storage_capacity_mwh": 100.0, + "source_id": "aemo-generation-information-2025", + "source_timestamp": "2025-03-18T00:00:00Z" + }, + { + "duid": "BHB1", + "site_name": "Broken Hill", + "region": "NSW1", + "capacity_mw": 50.0, + "storage_capacity_mwh": 50.0, + "source_id": "aemo-generation-information-2025", + "source_timestamp": "2025-03-11T00:00:00Z" + }, + { + "duid": "BLYTHB1", + "site_name": "Blyth BESS", + "region": "SA1", + "capacity_mw": 200.0, + "storage_capacity_mwh": 400.0, + "source_id": "aemo-generation-information-2025", + "source_timestamp": "2025-03-24T00:00:00Z" + }, + { + "duid": "BOWWBA1", + "site_name": "Bolivar Waste Water Treatment", + "region": "SA1", + "capacity_mw": 3.08, + "storage_capacity_mwh": 5.04, + "source_id": "aemo-generation-information-2025", + "source_timestamp": "2025-03-05T00:00:00Z" + }, + { + "duid": "BRDDBES1", + "site_name": "Broadsound Energy Park", + "region": "QLD1", + "capacity_mw": 180.0, + "storage_capacity_mwh": 360.0, + "source_id": "aemo-nem-registration-list-2026-08-25", + "source_timestamp": "2026-08-25T00:00:00Z" + }, + { + "duid": "BRNDBES1", + "site_name": "Brendale BESS", + "region": "QLD1", + "capacity_mw": 205.0, + "storage_capacity_mwh": 410.0, + "source_id": "aemo-nem-registration-list-2026-08-25", + "source_timestamp": "2026-08-25T00:00:00Z" + }, + { + "duid": "BULBES1", + "site_name": "Bulgana Green Power Hub - BESS", + "region": "VIC1", + "capacity_mw": 20.0, + "storage_capacity_mwh": 34.0, + "source_id": "aemo-generation-information-2025", + "source_timestamp": "2024-12-03T00:00:00Z" + }, + { + "duid": "BUNGAMB1", + "site_name": "Bungama Battery Energy Storage System", + "region": "SA1", + "capacity_mw": 150.0, + "storage_capacity_mwh": 300.0, + "source_id": "aemo-nem-registration-list-2026-08-25", + "source_timestamp": "2026-08-25T00:00:00Z" + }, + { + "duid": "CAPBES1", + "site_name": "Capital Battery", + "region": "NSW1", + "capacity_mw": 100.32, + "storage_capacity_mwh": 200.0, + "source_id": "aemo-generation-information-2025", + "source_timestamp": "2025-03-17T00:00:00Z" + }, + { + "duid": "CBWWBA1", + "site_name": "Christies Beach Wastewater Treatment Plant", + "region": "SA1", + "capacity_mw": 2.16, + "storage_capacity_mwh": 4.32, + "source_id": "aemo-generation-information-2025", + "source_timestamp": "2025-03-07T00:00:00Z" + }, + { + "duid": "CGBESS01", + "site_name": "Clements Gap BESS", + "region": "SA1", + "capacity_mw": 60.0, + "storage_capacity_mwh": 120.0, + "source_id": "aemo-nem-registration-list-2026-08-25", + "source_timestamp": "2026-08-25T00:00:00Z" + }, + { + "duid": "CHBESS1", + "site_name": "Chinchilla BESS", + "region": "QLD1", + "capacity_mw": 100.0, + "storage_capacity_mwh": 200.0, + "source_id": "aemo-generation-information-2025", + "source_timestamp": "2025-03-14T00:00:00Z" + }, + { + "duid": "DALNTH1", + "site_name": "Dalrymple BESS", + "region": "SA1", + "capacity_mw": 30.0, + "storage_capacity_mwh": 9.0, + "source_id": "aemo-generation-information-2025", + "source_timestamp": "2025-03-11T00:00:00Z" + }, + { + "duid": "DPNTB1", + "site_name": "Darlington Point Energy Storage System", + "region": "NSW1", + "capacity_mw": 25.0, + "storage_capacity_mwh": 50.0, + "source_id": "aemo-generation-information-2025", + "source_timestamp": "2025-03-11T00:00:00Z" + }, + { + "duid": "ERB01", + "site_name": "Eraring Battery Energy Storage System", + "region": "NSW1", + "capacity_mw": 460.0, + "storage_capacity_mwh": 1997.0, + "source_id": "aemo-nem-registration-list-2026-08-25", + "source_timestamp": "2026-08-25T00:00:00Z" + }, + { + "duid": "ERB02", + "site_name": "Eraring Battery Energy Storage System 2", + "region": "NSW1", + "capacity_mw": 240.0, + "storage_capacity_mwh": 1390.0, + "source_id": "aemo-nem-registration-list-2026-08-25", + "source_timestamp": "2026-08-25T00:00:00Z" + }, + { + "duid": "GANNB1", + "site_name": "Gannawarra Energy Storage System", + "region": "VIC1", + "capacity_mw": 25.33, + "storage_capacity_mwh": 50.0, + "source_id": "aemo-generation-information-2025", + "source_timestamp": "2025-03-11T00:00:00Z" + }, + { + "duid": "GREENB1", + "site_name": "Greenbank BESS", + "region": "QLD1", + "capacity_mw": 200.0, + "storage_capacity_mwh": 400.0, + "source_id": "aemo-generation-information-2025", + "source_timestamp": "2025-03-14T00:00:00Z" + }, + { + "duid": "HBESS1", + "site_name": "Hazelwood Battery Energy Storage System (HBESS)", + "region": "VIC1", + "capacity_mw": 200.07, + "storage_capacity_mwh": 162.0, + "source_id": "aemo-generation-information-2025", + "source_timestamp": "2025-03-24T00:00:00Z" + }, + { + "duid": "HPR1", + "site_name": "Hornsdale Power Reserve", + "region": "SA1", + "capacity_mw": 150.0, + "storage_capacity_mwh": 194.0, + "source_id": "aemo-nem-registration-list-2026-08-25", + "source_timestamp": "2026-08-25T00:00:00Z" + }, + { + "duid": "HVWWBA1", + "site_name": "Happy Valley Reservoir", + "region": "SA1", + "capacity_mw": 5.52, + "storage_capacity_mwh": 8.82, + "source_id": "aemo-generation-information-2025", + "source_timestamp": "2025-03-05T00:00:00Z" + }, + { + "duid": "KEPBG1", + "site_name": "Kennedy Energy Park - Phase 1 - Storage", + "region": "QLD1", + "capacity_mw": 2.0, + "storage_capacity_mwh": 4.0, + "source_id": "aemo-generation-information-2025", + "source_timestamp": "2025-03-12T00:00:00Z" + }, + { + "duid": "KESSB1", + "site_name": "Koorangie Energy Storage System", + "region": "VIC1", + "capacity_mw": 185.0, + "storage_capacity_mwh": 370.0, + "source_id": "aemo-nem-registration-list-2026-08-25", + "source_timestamp": "2026-08-25T00:00:00Z" + }, + { + "duid": "LBB1", + "site_name": "Lake Bonney Battery Energy Storage", + "region": "SA1", + "capacity_mw": 25.0, + "storage_capacity_mwh": 52.0, + "source_id": "aemo-generation-information-2025", + "source_timestamp": "2025-03-26T00:00:00Z" + }, + { + "duid": "LDBESS1", + "site_name": "Liddell Battery Energy Storage System", + "region": "NSW1", + "capacity_mw": 500.0, + "storage_capacity_mwh": 1086.2, + "source_id": "aemo-nem-registration-list-2026-08-25", + "source_timestamp": "2026-08-25T00:00:00Z" + }, + { + "duid": "LGAPBS1", + "site_name": "Lincoln Gap Wind Farm Battery", + "region": "SA1", + "capacity_mw": 9.0, + "storage_capacity_mwh": 10.0, + "source_id": "aemo-nem-registration-list-2026-08-25", + "source_timestamp": "2026-08-25T00:00:00Z" + }, + { + "duid": "LIMBESS1", + "site_name": "Limondale Battery", + "region": "NSW1", + "capacity_mw": 50.0, + "storage_capacity_mwh": 536.0, + "source_id": "aemo-nem-registration-list-2026-08-25", + "source_timestamp": "2026-08-25T00:00:00Z" + }, + { + "duid": "LVES1", + "site_name": "Latrobe Valley BESS", + "region": "VIC1", + "capacity_mw": 100.0, + "storage_capacity_mwh": 200.0, + "source_id": "aemo-nem-registration-list-2026-08-25", + "source_timestamp": "2026-08-25T00:00:00Z" + }, + { + "duid": "MANNUMB1", + "site_name": "Mannum Battery Energy Storage System", + "region": "SA1", + "capacity_mw": 100.0, + "storage_capacity_mwh": 200.0, + "source_id": "aemo-nem-registration-list-2026-08-25", + "source_timestamp": "2026-08-25T00:00:00Z" + }, + { + "duid": "MLB01", + "site_name": "Mortlake Battery Energy Storage System", + "region": "NSW1", + "capacity_mw": 300.0, + "storage_capacity_mwh": 650.0, + "source_id": "aemo-nem-registration-list-2026-08-25", + "source_timestamp": "2026-08-25T00:00:00Z" + }, + { + "duid": "MREHA1", + "site_name": "Melbourne Renewable Energy Hub Connection A1", + "region": "VIC1", + "capacity_mw": 200.0, + "storage_capacity_mwh": 400.0, + "source_id": "aemo-nem-registration-list-2026-08-25", + "source_timestamp": "2026-08-25T00:00:00Z" + }, + { + "duid": "MREHA2", + "site_name": "Melbourne Renewable Energy Hub Connection A2", + "region": "VIC1", + "capacity_mw": 200.0, + "storage_capacity_mwh": 400.0, + "source_id": "aemo-nem-registration-list-2026-08-25", + "source_timestamp": "2026-08-25T00:00:00Z" + }, + { + "duid": "MREHA3", + "site_name": "Melbourne Renewable Energy Hub Connection A3", + "region": "VIC1", + "capacity_mw": 200.0, + "storage_capacity_mwh": 800.0, + "source_id": "aemo-nem-registration-list-2026-08-25", + "source_timestamp": "2026-08-25T00:00:00Z" + }, + { + "duid": "MRNBESS1", + "site_name": "Mornington BESS", + "region": "VIC1", + "capacity_mw": 240.0, + "storage_capacity_mwh": 590.0, + "source_id": "aemo-nem-registration-list-2026-08-25", + "source_timestamp": "2026-08-25T00:00:00Z" + }, + { + "duid": "ORABESS1", + "site_name": "Orana BESS", + "region": "NSW1", + "capacity_mw": 415.0, + "storage_capacity_mwh": 1660.0, + "source_id": "aemo-nem-registration-list-2026-08-25", + "source_timestamp": "2026-08-25T00:00:00Z" + }, + { + "duid": "PIBESS1", + "site_name": "Philip Island BESS", + "region": "VIC1", + "capacity_mw": 5.0, + "storage_capacity_mwh": 10.0, + "source_id": "aemo-generation-information-2025", + "source_timestamp": "2025-03-07T00:00:00Z" + }, + { + "duid": "PLBESS1", + "site_name": "Pine Lodge BESS", + "region": "VIC1", + "capacity_mw": 250.0, + "storage_capacity_mwh": 600.0, + "source_id": "aemo-nem-registration-list-2026-08-25", + "source_timestamp": "2026-08-25T00:00:00Z" + }, + { + "duid": "QBYNB1", + "site_name": "Queanbeyan BESS", + "region": "NSW1", + "capacity_mw": 10.0, + "storage_capacity_mwh": 20.0, + "source_id": "aemo-generation-information-2025", + "source_timestamp": "2025-03-13T00:00:00Z" + }, + { + "duid": "QPSFB2", + "site_name": "Quorn Park Solar Hybrid", + "region": "NSW1", + "capacity_mw": 19.0, + "storage_capacity_mwh": 40.0, + "source_id": "aemo-nem-registration-list-2026-08-25", + "source_timestamp": "2026-08-25T00:00:00Z" + }, + { + "duid": "RANGEB1", + "site_name": "Rangebank BESS", + "region": "VIC1", + "capacity_mw": 200.0, + "storage_capacity_mwh": 400.0, + "source_id": "aemo-generation-information-2025", + "source_timestamp": "2025-03-13T00:00:00Z" + }, + { + "duid": "RESS1", + "site_name": "Riverina Energy Storage System 1", + "region": "NSW1", + "capacity_mw": 60.0, + "storage_capacity_mwh": 120.0, + "source_id": "aemo-generation-information-2025", + "source_timestamp": "2025-03-12T00:00:00Z" + }, + { + "duid": "RIVNB2", + "site_name": "Riverina Energy Storage System 2", + "region": "NSW1", + "capacity_mw": 65.0, + "storage_capacity_mwh": 130.0, + "source_id": "aemo-generation-information-2025", + "source_timestamp": "2025-03-11T00:00:00Z" + }, + { + "duid": "SMFBESS1", + "site_name": "Summerfield BESS 1", + "region": "SA1", + "capacity_mw": 120.0, + "storage_capacity_mwh": 480.0, + "source_id": "aemo-nem-registration-list-2026-08-25", + "source_timestamp": "2026-08-25T00:00:00Z" + }, + { + "duid": "SMFBESS2", + "site_name": "Summerfield BESS 2", + "region": "SA1", + "capacity_mw": 120.0, + "storage_capacity_mwh": 480.0, + "source_id": "aemo-nem-registration-list-2026-08-25", + "source_timestamp": "2026-08-25T00:00:00Z" + }, + { + "duid": "SMTHBES1", + "site_name": "Smithfield Battery Energy Storage System, Units 1 - 36", + "region": "NSW1", + "capacity_mw": 65.0, + "storage_capacity_mwh": 130.0, + "source_id": "aemo-nem-registration-list-2026-08-25", + "source_timestamp": "2026-08-25T00:00:00Z" + }, + { + "duid": "SNB01", + "site_name": "Supernode BESS", + "region": "QLD1", + "capacity_mw": 260.0, + "storage_capacity_mwh": 545.0, + "source_id": "aemo-nem-registration-list-2026-08-25", + "source_timestamp": "2026-08-25T00:00:00Z" + }, + { + "duid": "SNB02", + "site_name": "Supernode BESS", + "region": "QLD1", + "capacity_mw": 260.0, + "storage_capacity_mwh": 1090.0, + "source_id": "aemo-nem-registration-list-2026-08-25", + "source_timestamp": "2026-08-25T00:00:00Z" + }, + { + "duid": "STABESS1", + "site_name": "Stanwell Battery Energy Storage System", + "region": "QLD1", + "capacity_mw": 300.0, + "storage_capacity_mwh": 1200.0, + "source_id": "aemo-nem-registration-list-2026-08-25", + "source_timestamp": "2026-08-25T00:00:00Z" + }, + { + "duid": "SWANBBF1", + "site_name": "Swanbank BESS", + "region": "QLD1", + "capacity_mw": 250.0, + "storage_capacity_mwh": 500.0, + "source_id": "aemo-nem-registration-list-2026-08-25", + "source_timestamp": "2026-08-25T00:00:00Z" + }, + { + "duid": "TARBESS1", + "site_name": "Tarong BESS", + "region": "QLD1", + "capacity_mw": 300.0, + "storage_capacity_mwh": 600.0, + "source_id": "aemo-nem-registration-list-2026-08-25", + "source_timestamp": "2026-08-25T00:00:00Z" + }, + { + "duid": "TB2B1", + "site_name": "Tailem Bend Battery Project", + "region": "SA1", + "capacity_mw": 41.5, + "storage_capacity_mwh": 84.0, + "source_id": "aemo-generation-information-2025", + "source_timestamp": "2024-12-11T00:00:00Z" + }, + { + "duid": "TEMPB1", + "site_name": "Templers Battery Energy Storage System", + "region": "SA1", + "capacity_mw": 111.0, + "storage_capacity_mwh": 285.0, + "source_id": "aemo-nem-registration-list-2026-08-25", + "source_timestamp": "2026-08-25T00:00:00Z" + }, + { + "duid": "TIB1", + "site_name": "Torrens Island BESS", + "region": "SA1", + "capacity_mw": 250.7, + "storage_capacity_mwh": 250.0, + "source_id": "aemo-generation-information-2025", + "source_timestamp": "2025-03-11T00:00:00Z" + }, + { + "duid": "TRGBESS1", + "site_name": "Terang BESS", + "region": "VIC1", + "capacity_mw": 100.0, + "storage_capacity_mwh": 200.0, + "source_id": "aemo-nem-registration-list-2026-08-25", + "source_timestamp": "2026-08-25T00:00:00Z" + }, + { + "duid": "ULPBESS1", + "site_name": "Ulinda Park BESS", + "region": "QLD1", + "capacity_mw": 155.0, + "storage_capacity_mwh": 298.0, + "source_id": "aemo-nem-registration-list-2026-08-25", + "source_timestamp": "2026-08-25T00:00:00Z" + }, + { + "duid": "VBB1", + "site_name": "Victorian Big Battery", + "region": "VIC1", + "capacity_mw": 300.0, + "storage_capacity_mwh": 470.0, + "source_id": "aemo-generation-information-2025", + "source_timestamp": "2025-03-26T00:00:00Z" + }, + { + "duid": "WALGRV1", + "site_name": "Wallgrove Grid Battery project", + "region": "NSW1", + "capacity_mw": 50.0, + "storage_capacity_mwh": 75.0, + "source_id": "aemo-generation-information-2025", + "source_timestamp": "2025-03-19T00:00:00Z" + }, + { + "duid": "WANDB1", + "site_name": "Wandoan South BESS", + "region": "QLD1", + "capacity_mw": 100.0, + "storage_capacity_mwh": 150.0, + "source_id": "aemo-generation-information-2025", + "source_timestamp": "2025-03-18T00:00:00Z" + }, + { + "duid": "WDBESS1", + "site_name": "Western Downs Battery", + "region": "QLD1", + "capacity_mw": 254.94, + "storage_capacity_mwh": 510.0, + "source_id": "aemo-generation-information-2025", + "source_timestamp": "2025-03-24T00:00:00Z" + }, + { + "duid": "WDBESS2", + "site_name": "Western Downs Battery Energy Storage System (BESS)", + "region": "QLD1", + "capacity_mw": 255.0, + "storage_capacity_mwh": 510.0, + "source_id": "aemo-nem-registration-list-2026-08-25", + "source_timestamp": "2026-08-25T00:00:00Z" + }, + { + "duid": "WOOLES1", + "site_name": "Woolooga BESS", + "region": "QLD1", + "capacity_mw": 222.0, + "storage_capacity_mwh": 593.0, + "source_id": "aemo-nem-registration-list-2026-08-25", + "source_timestamp": "2026-08-25T00:00:00Z" + }, + { + "duid": "WTAHB1", + "site_name": "Waratah Super Battery", + "region": "NSW1", + "capacity_mw": 850.0, + "storage_capacity_mwh": 1679.0, + "source_id": "aemo-nem-registration-list-2026-08-25", + "source_timestamp": "2026-08-25T00:00:00Z" + } + ], + "excluded": [ + { + "duid": "KEPBL1", + "reason": "not present in latest Dispatch SCADA artifact" + }, + { + "duid": "MOORABS1", + "reason": "not present in latest Dispatch SCADA artifact" + }, + { + "duid": "NESBESS1", + "reason": "AEMO registration workbook reports non-positive maximum storage capacity" + }, + { + "duid": "NESBESS2", + "reason": "AEMO registration workbook reports non-positive maximum storage capacity" + }, + { + "duid": "WILLBES1", + "reason": "AEMO registration workbook reports non-positive maximum storage capacity" + } + ] +} diff --git a/app/deploy/README.md b/app/deploy/README.md new file mode 100644 index 0000000..ca7ad03 --- /dev/null +++ b/app/deploy/README.md @@ -0,0 +1,36 @@ +# BatteryWatch activation runbook + +## Fixed safety boundary + +- PostgreSQL/TimescaleDB remains loopback-only; do not change listeners, firewall, or proxy routes. +- Keep the current fixture API available until backup, restore, migration, collector, API, dashboard, and rollback gates pass. +- The reviewed battery map contains 64 unique current DUIDs. The 35 additions, including `HPR1`, come from the AEMO NEM Registration and Exemption List published 2026-08-25. Five rows remain explicitly excluded because they are not observable in current Dispatch SCADA or have zero/unknown registered storage capacity; do not guess missing capacity. +- SOC remains nullable. Dispatch SCADA MW is public power telemetry, not SOC. +- Do not commit connection strings or copy them into logs/chat. + +## Prerequisites + +1. An approved least-privilege local PostgreSQL role/DSN capable of schema creation and DML on the BatteryWatch database. +2. Non-interactive authority to stop/replace the existing root-managed `batterywatch.service` on port 8080, or an administrator performs that exact cutover locally. +3. `pg_dump`, `pg_restore`, and `psql` available on the target. +4. A release installed at `%h/batterywatch/releases/` with `%h/batterywatch/current` as the reviewed symlink, a venv containing `app/backend/requirements.txt`, and a built `app/frontend/dist`. + +## Gates in order + +1. Record current API health and unit state; confirm response says `data_mode=fixture`. +2. Create a fresh custom-format backup with `backup-verify.sh`; verify its checksum and `pg_restore --list`. +3. Restore that backup into an isolated temporary database, run the engine integrity checks, and prove row-count parity for existing domain tables. Drop the temporary database only after recording redacted parity evidence. +4. Export `BATTERYWATCH_BACKUP_MANIFEST` as the fresh `.sha256` path from step 2, then run `migrate.sh`; it rechecks checksum and age before verifying seven application tables, including immutable DispatchIS price artifacts. +5. Run one collector cycle from the candidate release with `python -m batterywatch_api.collector_service --once`. +6. Query `verify-live.sql` plus `dispatch_price_artifacts`/`nem_price_5m`; require fresh SCADA and DispatchIS artifacts, raw power observations, mapped `generator_power_5m` rows, exactly five current regional price rows, and at least one reviewed battery DUID. +7. Install the two user units under `~/.config/systemd/user/`, reload the user manager, enable/start the collector, and verify two successive five-minute artifacts or an exact replay followed by a new artifact. +8. Stop only the approved root-managed fixture API, start the user `batterywatch-api.service` on port 8080, and require `/api/health` to report `data_mode=database`. +9. Verify `/api/generators`, a bounded `/api/series` request, and the browser chart for one battery. Confirm regional price coverage/value estimates are visible, SOC null coverage remains explicit, and no fixture DUID appears. + +## Host-side rollback + +1. Stop the user `batterywatch-api.service` and `batterywatch-collector.service`. +2. Restart the preserved root-managed fixture `batterywatch.service`. +3. Verify port 8080 and `/api/health` return `data_mode=fixture`. +4. Leave additive raw/effective rows and migration tables intact for evidence; do not drop data during rollback. +5. Point `%h/batterywatch/current` back to the previous immutable release only after service rollback is healthy. diff --git a/app/deploy/backup-verify.sh b/app/deploy/backup-verify.sh new file mode 100755 index 0000000..760e08e --- /dev/null +++ b/app/deploy/backup-verify.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 + +: "${BATTERYWATCH_DATABASE_URL:?BATTERYWATCH_DATABASE_URL is required}" +if [[ $# -ne 1 ]]; then + printf 'usage: %s BACKUP_DIRECTORY\n' "$0" >&2 + exit 64 +fi +backup_dir=$1 +mkdir -p -- "$backup_dir" +stamp=$(date -u +%Y%m%dT%H%M%SZ) +backup="$backup_dir/batterywatch-$stamp.dump" +manifest="$backup.sha256" + +database_url=$BATTERYWATCH_DATABASE_URL +unset BATTERYWATCH_DATABASE_URL + +pg_dump --dbname="$database_url" --format=custom --file="$backup" +unset database_url +pg_restore --list "$backup" >/dev/null +sha256sum "$backup" >"$manifest" +printf 'backup=%s\nmanifest=%s\n' "$backup" "$manifest" diff --git a/app/deploy/batterywatch-api.service b/app/deploy/batterywatch-api.service new file mode 100644 index 0000000..29810ea --- /dev/null +++ b/app/deploy/batterywatch-api.service @@ -0,0 +1,25 @@ +[Unit] +Description=BatteryWatch database-backed API and dashboard +Documentation=file:%h/batterywatch/current/app/deploy/README.md +Wants=network-online.target +After=network-online.target + +[Service] +Type=simple +WorkingDirectory=%h/batterywatch/current/app/backend +EnvironmentFile=%h/.config/batterywatch/runtime.env +ExecStart=%h/batterywatch/current/venv/bin/uvicorn batterywatch_api.database_main:app --host 0.0.0.0 --port 8080 +Restart=on-failure +RestartSec=10s +TimeoutStopSec=20s +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=read-only +# These capability-dependent protections fail before ExecStart under the +# unprivileged systemd user manager on the deployment host. +RestrictSUIDSGID=true +LockPersonality=true + +[Install] +WantedBy=default.target diff --git a/app/deploy/batterywatch-collector.service b/app/deploy/batterywatch-collector.service new file mode 100644 index 0000000..ff7db15 --- /dev/null +++ b/app/deploy/batterywatch-collector.service @@ -0,0 +1,26 @@ +[Unit] +Description=BatteryWatch NEMWeb Dispatch SCADA collector +Documentation=file:%h/batterywatch/current/app/deploy/README.md +Wants=network-online.target +After=network-online.target + +[Service] +Type=simple +WorkingDirectory=%h/batterywatch/current/app/backend +EnvironmentFile=%h/.config/batterywatch/runtime.env +ExecStart=%h/batterywatch/current/venv/bin/python -m batterywatch_api.collector_service --assets-path %h/batterywatch/current/app/config/battery_assets.json --interval-seconds 300 +Restart=on-failure +RestartSec=30s +TimeoutStopSec=20s +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=read-only +# These capability-dependent protections fail before ExecStart under the +# unprivileged systemd user manager on the deployment host. +RestrictSUIDSGID=true +LockPersonality=true +MemoryDenyWriteExecute=true + +[Install] +WantedBy=default.target diff --git a/app/deploy/migrate.sh b/app/deploy/migrate.sh new file mode 100755 index 0000000..5802d48 --- /dev/null +++ b/app/deploy/migrate.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +set -euo pipefail + +: "${BATTERYWATCH_DATABASE_URL:?BATTERYWATCH_DATABASE_URL is required}" +: "${BATTERYWATCH_BACKUP_MANIFEST:?BATTERYWATCH_BACKUP_MANIFEST is required}" +app_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd -P) + +if [[ ! -f "$BATTERYWATCH_BACKUP_MANIFEST" ]]; then + printf 'backup manifest does not exist\n' >&2 + exit 1 +fi +now=$(date +%s) +manifest_mtime=$(stat -c %Y -- "$BATTERYWATCH_BACKUP_MANIFEST") +manifest_age=$((now - manifest_mtime)) +if (( manifest_age < 0 || manifest_age > 3600 )); then + printf 'backup manifest is not fresh (maximum age: 3600 seconds)\n' >&2 + exit 1 +fi +sha256sum --check --status "$BATTERYWATCH_BACKUP_MANIFEST" + +database_url=$BATTERYWATCH_DATABASE_URL +unset BATTERYWATCH_DATABASE_URL + +psql --dbname="$database_url" --no-psqlrc --set=ON_ERROR_STOP=1 \ + --file="$app_dir/migrations/001_initial_schema.sql" +psql --dbname="$database_url" --no-psqlrc --set=ON_ERROR_STOP=1 \ + --file="$app_dir/migrations/002_dispatch_scada_raw_ingestion.sql" +psql --dbname="$database_url" --no-psqlrc --set=ON_ERROR_STOP=1 \ + --file="$app_dir/migrations/003_dispatch_price_artifacts.sql" +psql --dbname="$database_url" --no-psqlrc --set=ON_ERROR_STOP=1 --tuples-only --no-align \ + <<'SQL' +SELECT count(*) = 7 +FROM information_schema.tables +WHERE table_schema = 'public' + AND table_name IN ( + 'generators', 'generator_power_5m', 'generator_soc_5m', 'nem_price_5m', + 'dispatch_scada_artifacts', 'raw_dispatch_scada_observations', + 'dispatch_price_artifacts' + ); +SQL +unset database_url diff --git a/app/deploy/runtime.env.example b/app/deploy/runtime.env.example new file mode 100644 index 0000000..94b7f7d --- /dev/null +++ b/app/deploy/runtime.env.example @@ -0,0 +1,3 @@ +# Copy to ~/.config/batterywatch/runtime.env with mode 0600. +# Set the value locally; never commit or paste it into chat/logs. +BATTERYWATCH_DATABASE_URL=[REDACTED] diff --git a/app/deploy/verify-live.sql b/app/deploy/verify-live.sql new file mode 100644 index 0000000..dece970 --- /dev/null +++ b/app/deploy/verify-live.sql @@ -0,0 +1,10 @@ +\pset tuples_only on +\pset format unaligned +SELECT extversion FROM pg_extension WHERE extname = 'timescaledb'; +SELECT count(*) FROM generators; +SELECT count(*) FROM generator_power_5m; +SELECT count(*) FROM dispatch_scada_artifacts; +SELECT count(*) FROM raw_dispatch_scada_observations; +SELECT max(report_timestamp) FROM dispatch_scada_artifacts; +SELECT max(interval_start) FROM generator_power_5m; +SELECT count(DISTINCT generator_id) FROM generator_power_5m; diff --git a/app/frontend/index.html b/app/frontend/index.html new file mode 100644 index 0000000..8753086 --- /dev/null +++ b/app/frontend/index.html @@ -0,0 +1,13 @@ + + + + + + + BatteryWatch + + +
+ + + diff --git a/app/frontend/package-lock.json b/app/frontend/package-lock.json new file mode 100644 index 0000000..57370e0 --- /dev/null +++ b/app/frontend/package-lock.json @@ -0,0 +1,975 @@ +{ + "name": "batterywatch-dashboard", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "batterywatch-dashboard", + "version": "0.1.0", + "dependencies": { + "echarts": "^6.1.0", + "react": "^18.3.1", + "react-dom": "^18.3.1" + }, + "devDependencies": { + "@types/react": "^18.3.12", + "@types/react-dom": "^18.3.1", + "@vitejs/plugin-react": "^4.3.4", + "typescript": "^5.7.2", + "vite": "^6.0.5" + } + }, + "node_modules/@babel/code-frame": { + "version": "7.29.7", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-validator-identifier": "^7.29.7", + "js-tokens": "^4.0.0", + "picocolors": "^1.1.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/compat-data": { + "version": "7.29.7", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/core": { + "version": "7.29.7", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.29.7", + "@babel/generator": "^7.29.7", + "@babel/helper-compilation-targets": "^7.29.7", + "@babel/helper-module-transforms": "^7.29.7", + "@babel/helpers": "^7.29.7", + "@babel/parser": "^7.29.7", + "@babel/template": "^7.29.7", + "@babel/traverse": "^7.29.7", + "@babel/types": "^7.29.7", + "@jridgewell/remapping": "^2.3.5", + "convert-source-map": "^2.0.0", + "debug": "^4.1.0", + "gensync": "^1.0.0-beta.2", + "json5": "^2.2.3", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/babel" + } + }, + "node_modules/@babel/generator": { + "version": "7.29.8", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.29.8", + "@babel/types": "^7.29.8", + "@jridgewell/gen-mapping": "^0.3.12", + "@jridgewell/trace-mapping": "^0.3.28", + "jsesc": "^3.0.2" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-compilation-targets": { + "version": "7.29.7", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/compat-data": "^7.29.7", + "@babel/helper-validator-option": "^7.29.7", + "browserslist": "^4.24.0", + "lru-cache": "^5.1.1", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-globals": { + "version": "7.29.7", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-imports": { + "version": "7.29.7", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/traverse": "^7.29.7", + "@babel/types": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-transforms": { + "version": "7.29.7", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-module-imports": "^7.29.7", + "@babel/helper-validator-identifier": "^7.29.7", + "@babel/traverse": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-plugin-utils": { + "version": "7.29.7", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-string-parser": { + "version": "7.29.7", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.29.7", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-option": { + "version": "7.29.7", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helpers": { + "version": "7.29.7", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/template": "^7.29.7", + "@babel/types": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.29.8", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.29.8" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/plugin-transform-react-jsx-self": { + "version": "7.29.7", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-react-jsx-source": { + "version": "7.29.7", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/template": { + "version": "7.29.7", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.29.7", + "@babel/parser": "^7.29.7", + "@babel/types": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/traverse": { + "version": "7.29.8", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.29.7", + "@babel/generator": "^7.29.8", + "@babel/helper-globals": "^7.29.7", + "@babel/parser": "^7.29.8", + "@babel/template": "^7.29.7", + "@babel/types": "^7.29.8", + "debug": "^4.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/types": { + "version": "7.29.8", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^7.29.7", + "@babel/helper-validator-identifier": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.25.12", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.13", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.0", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/remapping": { + "version": "2.3.5", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.5", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@napi-rs/lzma-linux-x64-gnu": { + "version": "1.5.1", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^22.20 || ^24.12 || >=25" + } + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.0-beta.27", + "dev": true, + "license": "MIT" + }, + "node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.63.0", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@types/babel__core": { + "version": "7.20.5", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.20.7", + "@babel/types": "^7.20.7", + "@types/babel__generator": "*", + "@types/babel__template": "*", + "@types/babel__traverse": "*" + } + }, + "node_modules/@types/babel__generator": { + "version": "7.27.0", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__template": { + "version": "7.4.4", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.1.0", + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__traverse": { + "version": "7.28.0", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.28.2" + } + }, + "node_modules/@types/estree": { + "version": "1.0.9", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/prop-types": { + "version": "15.7.15", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/react": { + "version": "18.3.31", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/prop-types": "*", + "csstype": "^3.2.2" + } + }, + "node_modules/@types/react-dom": { + "version": "18.3.7", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "^18.0.0" + } + }, + "node_modules/@vitejs/plugin-react": { + "version": "4.7.0", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/core": "^7.28.0", + "@babel/plugin-transform-react-jsx-self": "^7.27.1", + "@babel/plugin-transform-react-jsx-source": "^7.27.1", + "@rolldown/pluginutils": "1.0.0-beta.27", + "@types/babel__core": "^7.20.5", + "react-refresh": "^0.17.0" + }, + "engines": { + "node": "^14.18.0 || >=16.0.0" + }, + "peerDependencies": { + "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0" + } + }, + "node_modules/baseline-browser-mapping": { + "version": "2.11.19", + "dev": true, + "license": "Apache-2.0", + "bin": { + "baseline-browser-mapping": "dist/cli.cjs" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/browserslist": { + "version": "4.28.8", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "baseline-browser-mapping": "^2.11.12", + "caniuse-lite": "^1.0.30001809", + "electron-to-chromium": "^1.5.402", + "node-releases": "^2.0.53", + "update-browserslist-db": "^1.3.0" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + } + }, + "node_modules/caniuse-lite": { + "version": "1.0.30001810", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "CC-BY-4.0" + }, + "node_modules/convert-source-map": { + "version": "2.0.0", + "dev": true, + "license": "MIT" + }, + "node_modules/csstype": { + "version": "3.2.3", + "dev": true, + "license": "MIT" + }, + "node_modules/debug": { + "version": "4.4.3", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/echarts": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/echarts/-/echarts-6.1.0.tgz", + "integrity": "sha512-q0yaFPggC9FUdsWH4blavRWFmxdrIodbkoKNAjJudAI6CA9gNPxHtV2RcZNEepZVlk4yvBYkOkbk6HIVpIyHZA==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "2.3.0", + "zrender": "6.1.0" + } + }, + "node_modules/electron-to-chromium": { + "version": "1.5.415", + "dev": true, + "license": "ISC" + }, + "node_modules/esbuild": { + "version": "0.25.12", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.25.12", + "@esbuild/android-arm": "0.25.12", + "@esbuild/android-arm64": "0.25.12", + "@esbuild/android-x64": "0.25.12", + "@esbuild/darwin-arm64": "0.25.12", + "@esbuild/darwin-x64": "0.25.12", + "@esbuild/freebsd-arm64": "0.25.12", + "@esbuild/freebsd-x64": "0.25.12", + "@esbuild/linux-arm": "0.25.12", + "@esbuild/linux-arm64": "0.25.12", + "@esbuild/linux-ia32": "0.25.12", + "@esbuild/linux-loong64": "0.25.12", + "@esbuild/linux-mips64el": "0.25.12", + "@esbuild/linux-ppc64": "0.25.12", + "@esbuild/linux-riscv64": "0.25.12", + "@esbuild/linux-s390x": "0.25.12", + "@esbuild/linux-x64": "0.25.12", + "@esbuild/netbsd-arm64": "0.25.12", + "@esbuild/netbsd-x64": "0.25.12", + "@esbuild/openbsd-arm64": "0.25.12", + "@esbuild/openbsd-x64": "0.25.12", + "@esbuild/openharmony-arm64": "0.25.12", + "@esbuild/sunos-x64": "0.25.12", + "@esbuild/win32-arm64": "0.25.12", + "@esbuild/win32-ia32": "0.25.12", + "@esbuild/win32-x64": "0.25.12" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/fdir": { + "version": "6.5.0", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/gensync": { + "version": "1.0.0-beta.2", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "license": "MIT" + }, + "node_modules/jsesc": { + "version": "3.1.0", + "dev": true, + "license": "MIT", + "bin": { + "jsesc": "bin/jsesc" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/json5": { + "version": "2.2.3", + "dev": true, + "license": "MIT", + "bin": { + "json5": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/loose-envify": { + "version": "1.4.0", + "license": "MIT", + "dependencies": { + "js-tokens": "^3.0.0 || ^4.0.0" + }, + "bin": { + "loose-envify": "cli.js" + } + }, + "node_modules/lru-cache": { + "version": "5.1.1", + "dev": true, + "license": "ISC", + "dependencies": { + "yallist": "^3.0.2" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "dev": true, + "license": "MIT" + }, + "node_modules/nanoid": { + "version": "3.3.18", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/node-releases": { + "version": "2.0.53", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "dev": true, + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "4.0.7", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/postcss": { + "version": "8.5.26", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.17", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/react": { + "version": "18.3.1", + "license": "MIT", + "dependencies": { + "loose-envify": "^1.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-dom": { + "version": "18.3.1", + "license": "MIT", + "dependencies": { + "loose-envify": "^1.1.0", + "scheduler": "^0.23.2" + }, + "peerDependencies": { + "react": "^18.3.1" + } + }, + "node_modules/react-refresh": { + "version": "0.17.0", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/rollup": { + "version": "4.63.0", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "1.0.9" + }, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "@napi-rs/lzma-linux-x64-gnu": "1.5.1", + "@rollup/rollup-android-arm-eabi": "4.63.0", + "@rollup/rollup-android-arm64": "4.63.0", + "@rollup/rollup-darwin-arm64": "4.63.0", + "@rollup/rollup-darwin-x64": "4.63.0", + "@rollup/rollup-freebsd-arm64": "4.63.0", + "@rollup/rollup-freebsd-x64": "4.63.0", + "@rollup/rollup-linux-arm-gnueabihf": "4.63.0", + "@rollup/rollup-linux-arm-musleabihf": "4.63.0", + "@rollup/rollup-linux-arm64-gnu": "4.63.0", + "@rollup/rollup-linux-arm64-musl": "4.63.0", + "@rollup/rollup-linux-loong64-gnu": "4.63.0", + "@rollup/rollup-linux-loong64-musl": "4.63.0", + "@rollup/rollup-linux-ppc64-gnu": "4.63.0", + "@rollup/rollup-linux-ppc64-musl": "4.63.0", + "@rollup/rollup-linux-riscv64-gnu": "4.63.0", + "@rollup/rollup-linux-riscv64-musl": "4.63.0", + "@rollup/rollup-linux-s390x-gnu": "4.63.0", + "@rollup/rollup-linux-x64-gnu": "4.63.0", + "@rollup/rollup-linux-x64-musl": "4.63.0", + "@rollup/rollup-openbsd-x64": "4.63.0", + "@rollup/rollup-openharmony-arm64": "4.63.0", + "@rollup/rollup-win32-arm64-msvc": "4.63.0", + "@rollup/rollup-win32-ia32-msvc": "4.63.0", + "@rollup/rollup-win32-x64-gnu": "4.63.0", + "@rollup/rollup-win32-x64-msvc": "4.63.0", + "fsevents": "~2.3.2" + } + }, + "node_modules/scheduler": { + "version": "0.23.2", + "license": "MIT", + "dependencies": { + "loose-envify": "^1.1.0" + } + }, + "node_modules/semver": { + "version": "6.3.1", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/tinyglobby": { + "version": "0.2.17", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/tslib": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.3.0.tgz", + "integrity": "sha512-N82ooyxVNm6h1riLCoyS9e3fuJ3AMG2zIZs2Gd1ATcSFjSA23Q0fzjjZeh0jbJvWVDZ0cJT8yaNNaaXHzueNjg==", + "license": "0BSD" + }, + "node_modules/typescript": { + "version": "5.9.3", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/update-browserslist-db": { + "version": "1.3.1", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "escalade": "^3.2.0", + "picocolors": "^1.1.1" + }, + "bin": { + "update-browserslist-db": "cli.js" + }, + "peerDependencies": { + "browserslist": ">= 4.21.0" + } + }, + "node_modules/vite": { + "version": "6.4.3", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "^0.25.0", + "fdir": "^6.4.4", + "picomatch": "^4.0.2", + "postcss": "^8.5.3", + "rollup": "^4.34.9", + "tinyglobby": "^0.2.13" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^18.0.0 || ^20.0.0 || >=22.0.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", + "jiti": ">=1.21.0", + "less": "*", + "lightningcss": "^1.21.0", + "sass": "*", + "sass-embedded": "*", + "stylus": "*", + "sugarss": "*", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/yallist": { + "version": "3.1.1", + "dev": true, + "license": "ISC" + }, + "node_modules/zrender": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/zrender/-/zrender-6.1.0.tgz", + "integrity": "sha512-oEGMDB6pOP2S6OwRR4PdVv610zrjnA3Bh+JnSG12fYJlBKjtNAoEb5fSUoCOOINlH96I2fU38/A2UpRKs67xYQ==", + "license": "BSD-3-Clause", + "dependencies": { + "tslib": "2.3.0" + } + } + } +} diff --git a/app/frontend/package.json b/app/frontend/package.json new file mode 100644 index 0000000..62a4e6f --- /dev/null +++ b/app/frontend/package.json @@ -0,0 +1,22 @@ +{ + "name": "batterywatch-dashboard", + "private": true, + "version": "0.1.0", + "type": "module", + "scripts": { + "build": "vite build", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "echarts": "^6.1.0", + "react": "^18.3.1", + "react-dom": "^18.3.1" + }, + "devDependencies": { + "@types/react": "^18.3.12", + "@types/react-dom": "^18.3.1", + "@vitejs/plugin-react": "^4.3.4", + "typescript": "^5.7.2", + "vite": "^6.0.5" + } +} diff --git a/app/frontend/src/main.tsx b/app/frontend/src/main.tsx new file mode 100644 index 0000000..5a4ac32 --- /dev/null +++ b/app/frontend/src/main.tsx @@ -0,0 +1,68 @@ +import { useEffect, useRef, useState } from "react"; +import { createRoot } from "react-dom/client"; +import * as echarts from "echarts"; +import "./styles.css"; + +type Generator = { duid: string; site_name: string; region: string }; +type Point = { timestamp: string; power_mw: number; price_aud_per_mwh: number | null; net_energy_value_aud: number | null }; +type Series = { generator: Generator; points: Point[]; summary: { exported_energy_mwh: number; imported_energy_mwh: number; net_energy_value_aud: number }; coverage: { price_coverage_percent: number; soc_coverage_percent: number }; estimate: { label: string; disclaimer: string } }; + +function money(value: number) { return new Intl.NumberFormat("en-AU", { style: "currency", currency: "AUD", maximumFractionDigits: 2 }).format(value); } +function number(value: number) { return `${value.toFixed(3)} MWh`; } +function recentWindow() { + const end = new Date(); + const start = new Date(end.getTime() - 24 * 60 * 60 * 1000); + return { start: start.toISOString(), end: end.toISOString() }; +} + +function App() { + const [generators, setGenerators] = useState([]); + const [selected, setSelected] = useState(""); + const [series, setSeries] = useState(null); + const [error, setError] = useState(""); + const chartRef = useRef(null); + + useEffect(() => { fetch("/api/generators").then(async (r) => { if (!r.ok) throw new Error(); return r.json(); }).then((body) => { if (!Array.isArray(body.generators) || body.generators.length === 0) { setError("No battery data is available yet."); return; } setGenerators(body.generators); setSelected(body.generators[0].duid); }).catch(() => setError("Unable to load generators.")); }, []); + useEffect(() => { + if (!selected) return; + setError(""); + setSeries(null); + const window = recentWindow(); + const query = new URLSearchParams({ generator: selected, start: window.start, end: window.end }); + fetch(`/api/series?${query}`).then(async (r) => { if (!r.ok) throw new Error(); return r.json(); }).then(setSeries).catch(() => setError("Unable to load the selected time range.")); + }, [selected]); + useEffect(() => { + if (!chartRef.current || !series) return; + const chart = echarts.init(chartRef.current); + chart.setOption({ + animation: false, + tooltip: { trigger: "axis" }, + legend: { top: 8, data: ["Battery power", "AEMO price", "Net value"] }, + grid: { left: 58, right: 70, top: 48, bottom: 80 }, + xAxis: { type: "time" }, + yAxis: [{ type: "value", name: "Power MW", position: "left" }, { type: "value", name: "AUD/MWh", position: "right" }, { type: "value", name: "AUD", position: "right", offset: 52 }], + dataZoom: [{ type: "inside", throttle: 50 }, { type: "slider", height: 24, bottom: 18 }], + series: [ + { name: "Battery power", type: "line", showSymbol: false, yAxisIndex: 0, lineStyle: { width: 2, color: "#2dd4bf" }, areaStyle: { opacity: 0.12, color: "#2dd4bf" }, data: series.points.map((p) => [p.timestamp, p.power_mw]) }, + { name: "AEMO price", type: "line", showSymbol: false, yAxisIndex: 1, connectNulls: false, lineStyle: { width: 1.5, color: "#fbbf24" }, data: series.points.map((p) => [p.timestamp, p.price_aud_per_mwh]) }, + { name: "Net value", type: "bar", yAxisIndex: 2, itemStyle: { color: "#a78bfa" }, data: series.points.map((p) => [p.timestamp, p.net_energy_value_aud]) }, + ], + }); + const resize = () => chart.resize(); window.addEventListener("resize", resize); + return () => { window.removeEventListener("resize", resize); chart.dispose(); }; + }, [series]); + + return
+

STANDALONE BATTERY ANALYTICS

BatteryWatch

Five-minute battery power, regional price overlays, and transparent energy estimates.

+ {error &&
{error}
} + {!series && !error &&
Loading five-minute data…
} + {series && series.points.length === 0 &&
No observations in the selected 24-hour window.
} + {series && series.points.length > 0 && <> +
Exported energy{number(series.summary.exported_energy_mwh)}
Imported energy{number(series.summary.imported_energy_mwh)}
Estimated net value{money(series.summary.net_energy_value_aud)}
Data coverage{series.coverage.price_coverage_percent.toFixed(0)}% price{series.coverage.soc_coverage_percent.toFixed(0)}% SOC
+

{series.generator.site_name}

{series.points.length} five-minute intervals · zoom and pan enabled

{series.estimate.label}
+ + } +
; +} + +createRoot(document.getElementById("root")!).render(); diff --git a/app/frontend/src/styles.css b/app/frontend/src/styles.css new file mode 100644 index 0000000..82219c5 --- /dev/null +++ b/app/frontend/src/styles.css @@ -0,0 +1,27 @@ +:root { font-family: Inter, ui-sans-serif, system-ui, sans-serif; color: #e2e8f0; background: #07111f; font-synthesis: none; } +* { box-sizing: border-box; } +body { margin: 0; min-width: 320px; } +main { max-width: 1360px; margin: 0 auto; padding: 38px 34px 54px; } +header { display: flex; justify-content: space-between; gap: 28px; align-items: end; border-bottom: 1px solid #1e334d; padding-bottom: 28px; } +.eyebrow { color: #2dd4bf; font-size: 11px; letter-spacing: .16em; font-weight: 700; margin: 0 0 9px; } +h1 { font-size: clamp(32px, 5vw, 56px); line-height: 1; margin: 0; letter-spacing: -.05em; } +.lede { color: #94a3b8; margin: 14px 0 0; max-width: 600px; } +label { color: #94a3b8; font-size: 12px; font-weight: 700; min-width: 270px; } +select { display: block; width: 100%; margin-top: 8px; background: #0d1d31; color: #f8fafc; border: 1px solid #31506e; border-radius: 8px; padding: 11px 12px; } +.cards { display: grid; grid-template-columns: repeat(4, 1fr); gap: 12px; margin: 24px 0; } +.cards article, .panel, .notice { background: #0d1d31; border: 1px solid #1e3854; border-radius: 12px; } +.cards article { padding: 17px 18px; min-height: 98px; } +.cards span, .cards small { color: #94a3b8; font-size: 12px; display: block; } +.cards strong { color: #f8fafc; display: block; font-size: 22px; margin-top: 9px; } +.cards small { margin-top: 3px; } +.panel { padding: 20px 12px 8px; } +.panel-heading { display: flex; justify-content: space-between; align-items: start; padding: 0 10px; gap: 16px; } +h2 { margin: 0; font-size: 18px; } +.panel-heading p { color: #94a3b8; font-size: 13px; margin: 6px 0 0; } +.badge { color: #2dd4bf; background: #113d3e; border: 1px solid #1b6a65; border-radius: 999px; padding: 7px 10px; font-size: 11px; white-space: nowrap; } +.chart { width: 100%; height: 480px; } +.notice { margin-top: 14px; padding: 14px 17px; color: #cbd5e1; font-size: 13px; line-height: 1.5; border-color: #755c20; background: #251f0d; } +.notice strong { color: #fbbf24; } +.error { margin: 24px 0; background: #35161b; border: 1px solid #a33a4b; color: #fecdd3; border-radius: 8px; padding: 14px; } +.loading { color: #94a3b8; padding: 60px 0; text-align: center; } +@media (max-width: 780px) { main { padding: 24px 16px 40px; } header { display: block; } label { display: block; margin-top: 24px; } .cards { grid-template-columns: repeat(2, 1fr); } .panel-heading { display: block; } .badge { display: inline-block; margin-top: 14px; } .chart { height: 390px; } } diff --git a/app/frontend/tsconfig.json b/app/frontend/tsconfig.json new file mode 100644 index 0000000..f094a2e --- /dev/null +++ b/app/frontend/tsconfig.json @@ -0,0 +1,20 @@ +{ + "compilerOptions": { + "target": "ES2020", + "useDefineForClassFields": true, + "lib": ["ES2020", "DOM", "DOM.Iterable"], + "allowJs": false, + "skipLibCheck": true, + "esModuleInterop": true, + "allowSyntheticDefaultImports": true, + "strict": true, + "forceConsistentCasingInFileNames": true, + "module": "ESNext", + "moduleResolution": "Node", + "resolveJsonModule": true, + "isolatedModules": true, + "noEmit": true, + "jsx": "react-jsx" + }, + "include": ["src"] +} diff --git a/app/frontend/vite.config.ts b/app/frontend/vite.config.ts new file mode 100644 index 0000000..cec93e9 --- /dev/null +++ b/app/frontend/vite.config.ts @@ -0,0 +1,10 @@ +import { defineConfig } from "vite"; +import react from "@vitejs/plugin-react"; + +export default defineConfig({ + plugins: [react()], + server: { + port: 5173, + proxy: { "/api": "http://127.0.0.1:8080" }, + }, +}); diff --git a/app/migrations/001_initial_schema.sql b/app/migrations/001_initial_schema.sql new file mode 100644 index 0000000..2c80c97 --- /dev/null +++ b/app/migrations/001_initial_schema.sql @@ -0,0 +1,107 @@ +-- BatteryWatch v1 PostgreSQL/Timescale-ready schema. +-- This migration intentionally does not create the database, role, credentials, +-- or Timescale extension. Provisioning and optional hypertable conversion belong +-- to the separately verified S2b deployment step. +-- +-- Each table stores one effective record per logical key. The guarded upserts +-- replace that effective record only when the revision tuple advances; +-- revision history is not retained by this migration and requires a separate, +-- explicitly designed audit table if it is needed later. + +CREATE TABLE IF NOT EXISTS generators ( + generator_id TEXT PRIMARY KEY, + site_name TEXT NOT NULL, + region TEXT NOT NULL, + capacity_mw DOUBLE PRECISION NOT NULL CHECK ( + capacity_mw > 0 + AND capacity_mw::TEXT NOT IN ('NaN', 'Infinity', '-Infinity') + ), + storage_capacity_mwh DOUBLE PRECISION NOT NULL CHECK ( + storage_capacity_mwh > 0 + AND storage_capacity_mwh::TEXT NOT IN ('NaN', 'Infinity', '-Infinity') + ), + source_id TEXT NOT NULL, + source_timestamp TIMESTAMPTZ NOT NULL, + ingestion_version BIGINT NOT NULL DEFAULT 0 CHECK (ingestion_version >= 0), + correction_version BIGINT NOT NULL DEFAULT 0 CHECK (correction_version >= 0), + data_start TIMESTAMPTZ, + data_end TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + CHECK (data_end IS NULL OR data_start IS NULL OR data_end > data_start) +); + +CREATE TABLE IF NOT EXISTS generator_power_5m ( + generator_id TEXT NOT NULL REFERENCES generators (generator_id), + interval_start TIMESTAMPTZ NOT NULL, + power_mw DOUBLE PRECISION NOT NULL, + source_id TEXT NOT NULL, + source_timestamp TIMESTAMPTZ NOT NULL, + ingestion_version BIGINT NOT NULL CHECK (ingestion_version >= 0), + correction_version BIGINT NOT NULL DEFAULT 0 CHECK (correction_version >= 0), + quality_flags TEXT[] NOT NULL DEFAULT '{}', + PRIMARY KEY (generator_id, interval_start), + UNIQUE (generator_id, interval_start, source_id, source_timestamp, ingestion_version, correction_version), + CHECK (date_trunc('minute', interval_start) = interval_start), + CHECK (EXTRACT(MINUTE FROM interval_start)::INTEGER % 5 = 0), + CHECK (power_mw::TEXT NOT IN ('NaN', 'Infinity', '-Infinity')) +); + +CREATE TABLE IF NOT EXISTS generator_soc_5m ( + generator_id TEXT NOT NULL REFERENCES generators (generator_id), + interval_start TIMESTAMPTZ NOT NULL, + soc_percent DOUBLE PRECISION CHECK ( + soc_percent IS NULL + OR ( + soc_percent BETWEEN 0 AND 100 + AND soc_percent::TEXT NOT IN ('NaN', 'Infinity', '-Infinity') + ) + ), + source_id TEXT NOT NULL, + source_timestamp TIMESTAMPTZ NOT NULL, + ingestion_version BIGINT NOT NULL CHECK (ingestion_version >= 0), + correction_version BIGINT NOT NULL DEFAULT 0 CHECK (correction_version >= 0), + quality_flags TEXT[] NOT NULL DEFAULT '{}', + PRIMARY KEY (generator_id, interval_start), + UNIQUE (generator_id, interval_start, source_id, source_timestamp, ingestion_version, correction_version), + CHECK (date_trunc('minute', interval_start) = interval_start), + CHECK (EXTRACT(MINUTE FROM interval_start)::INTEGER % 5 = 0) +); + +CREATE TABLE IF NOT EXISTS nem_price_5m ( + region TEXT NOT NULL, + interval_start TIMESTAMPTZ NOT NULL, + price_aud_per_mwh NUMERIC(15, 5), + price_status TEXT NOT NULL CHECK (price_status IN ('available', 'negative', 'missing')), + intervention INTEGER NOT NULL DEFAULT 0 CHECK (intervention >= 0), + apc_flag INTEGER NOT NULL DEFAULT 0 CHECK (apc_flag >= 0), + market_suspended BOOLEAN NOT NULL DEFAULT FALSE, + source_id TEXT NOT NULL, + source_timestamp TIMESTAMPTZ NOT NULL, + ingestion_version BIGINT NOT NULL CHECK (ingestion_version >= 0), + correction_version BIGINT NOT NULL DEFAULT 0 CHECK (correction_version >= 0), + quality_flags TEXT[] NOT NULL DEFAULT '{}', + PRIMARY KEY (region, interval_start), + UNIQUE (region, interval_start, source_id, source_timestamp, ingestion_version, correction_version), + CHECK (date_trunc('minute', interval_start) = interval_start), + CHECK (EXTRACT(MINUTE FROM interval_start)::INTEGER % 5 = 0), + CHECK ( + price_aud_per_mwh IS NULL + OR price_aud_per_mwh::TEXT NOT IN ('NaN', 'Infinity', '-Infinity') + ), + CHECK ( + (price_aud_per_mwh IS NULL AND price_status = 'missing') + OR (price_aud_per_mwh IS NOT NULL AND price_status = 'available' AND price_aud_per_mwh >= 0) + OR (price_aud_per_mwh IS NOT NULL AND price_status = 'negative' AND price_aud_per_mwh < 0) + ) +); + +CREATE INDEX IF NOT EXISTS generator_power_5m_time_idx + ON generator_power_5m (interval_start DESC); +CREATE INDEX IF NOT EXISTS generator_soc_5m_time_idx + ON generator_soc_5m (interval_start DESC); +CREATE INDEX IF NOT EXISTS nem_price_5m_time_idx + ON nem_price_5m (interval_start DESC); + +-- On a host with TimescaleDB, a separately reviewed operational migration may +-- convert the three *_5m tables into hypertables after this schema is applied. diff --git a/app/migrations/002_dispatch_scada_raw_ingestion.sql b/app/migrations/002_dispatch_scada_raw_ingestion.sql new file mode 100644 index 0000000..1d2b3be --- /dev/null +++ b/app/migrations/002_dispatch_scada_raw_ingestion.sql @@ -0,0 +1,33 @@ +-- Raw verified Dispatch SCADA evidence is retained before later identity mapping. + +CREATE TABLE IF NOT EXISTS dispatch_scada_artifacts ( + source_artifact_id TEXT PRIMARY KEY CHECK (source_artifact_id <> '' AND source_artifact_id ~ '^[0-9]+$'), + source_url TEXT NOT NULL CHECK (source_url <> '' AND source_url ~ '^https://'), + zip_filename TEXT NOT NULL CHECK (btrim(zip_filename) <> ''), + csv_member_name TEXT NOT NULL CHECK (btrim(csv_member_name) <> ''), + report_timestamp TIMESTAMPTZ NOT NULL, + zip_sha256 TEXT NOT NULL CHECK (zip_sha256 ~ '^[0-9a-f]{64}$'), + raw_zip BYTEA NOT NULL CHECK (octet_length(raw_zip) > 0), + stored_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + CHECK (source_url = 'https://www.nemweb.com.au/REPORTS/CURRENT/Dispatch_SCADA/' || zip_filename), + CHECK (zip_filename ~ ('^PUBLIC_DISPATCHSCADA_[0-9]{12}_' || source_artifact_id || '[.]zip$')), + CHECK (csv_member_name = left(zip_filename, -4) || '.CSV') +); + +CREATE TABLE IF NOT EXISTS raw_dispatch_scada_observations ( + source_artifact_id TEXT NOT NULL REFERENCES dispatch_scada_artifacts (source_artifact_id) ON DELETE RESTRICT, + duid TEXT NOT NULL CHECK (btrim(duid) <> ''), + interval_start TIMESTAMPTZ NOT NULL, + power_mw DOUBLE PRECISION NOT NULL CHECK (power_mw::TEXT NOT IN ('NaN', 'Infinity', '-Infinity')), + source_timestamp TIMESTAMPTZ NOT NULL, + ingestion_version BIGINT NOT NULL DEFAULT 0 CHECK (ingestion_version >= 0), + correction_version BIGINT NOT NULL DEFAULT 0 CHECK (correction_version >= 0), + stored_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (source_artifact_id, duid, interval_start), + CHECK (date_trunc('minute', interval_start) = interval_start), + CHECK (EXTRACT(MINUTE FROM interval_start)::INTEGER % 5 = 0) +); + +CREATE INDEX IF NOT EXISTS dispatch_scada_artifacts_report_timestamp_idx ON dispatch_scada_artifacts (report_timestamp DESC); +CREATE INDEX IF NOT EXISTS raw_dispatch_scada_observations_artifact_interval_idx ON raw_dispatch_scada_observations (source_artifact_id, interval_start DESC); +CREATE INDEX IF NOT EXISTS raw_dispatch_scada_observations_interval_idx ON raw_dispatch_scada_observations (interval_start DESC); diff --git a/app/migrations/003_dispatch_price_artifacts.sql b/app/migrations/003_dispatch_price_artifacts.sql new file mode 100644 index 0000000..6c842ab --- /dev/null +++ b/app/migrations/003_dispatch_price_artifacts.sql @@ -0,0 +1,26 @@ +-- Immutable official DispatchIS artifacts retained before regional price upserts. + +CREATE TABLE IF NOT EXISTS dispatch_price_artifacts ( + source_artifact_id TEXT PRIMARY KEY CHECK ( + source_artifact_id <> '' AND source_artifact_id ~ '^[0-9]+$' + ), + source_url TEXT NOT NULL CHECK (source_url <> '' AND source_url ~ '^https://'), + zip_filename TEXT NOT NULL CHECK (btrim(zip_filename) <> ''), + csv_member_name TEXT NOT NULL CHECK (btrim(csv_member_name) <> ''), + report_timestamp TIMESTAMPTZ NOT NULL, + zip_sha256 TEXT NOT NULL CHECK (zip_sha256 ~ '^[0-9a-f]{64}$'), + raw_zip BYTEA NOT NULL CHECK (octet_length(raw_zip) > 0), + stored_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + CHECK ( + source_url = + 'https://www.nemweb.com.au/REPORTS/CURRENT/DispatchIS_Reports/' || zip_filename + ), + CHECK ( + zip_filename ~ + ('^PUBLIC_DISPATCHIS_[0-9]{12}_' || source_artifact_id || '[.]zip$') + ), + CHECK (csv_member_name = left(zip_filename, -4) || '.CSV') +); + +CREATE INDEX IF NOT EXISTS dispatch_price_artifacts_report_timestamp_idx + ON dispatch_price_artifacts (report_timestamp DESC); From 9a79de51397cdd94971334fd2bfe9a1679be87d8 Mon Sep 17 00:00:00 2001 From: djurcola-agent Date: Sun, 30 Aug 2026 17:37:27 +1000 Subject: [PATCH 02/30] feat: add bounded NEMWeb archive planning and backfill ledger --- .../batterywatch_api/backfill_ledger.py | 245 +++++++ .../batterywatch_api/nemweb_archives.py | 388 +++++++++++ .../tests/fixtures/historical/README.md | 17 + .../dispatch-price-20260830-1500-reduced.csv | 7 + .../dispatch-scada-20260830-1455-reduced.csv | 5 + app/backend/tests/test_backfill_ledger.py | 280 ++++++++ .../tests/test_dispatch_price_ingestion.py | 5 +- app/backend/tests/test_nemweb_archives.py | 617 ++++++++++++++++++ app/deploy/migrate.sh | 8 +- app/docs/historical-analytics-iteration.md | 255 ++++++++ .../004_historical_backfill_ledger.sql | 66 ++ 11 files changed, 1889 insertions(+), 4 deletions(-) create mode 100644 app/backend/batterywatch_api/backfill_ledger.py create mode 100644 app/backend/batterywatch_api/nemweb_archives.py create mode 100644 app/backend/tests/fixtures/historical/README.md create mode 100644 app/backend/tests/fixtures/historical/dispatch-price-20260830-1500-reduced.csv create mode 100644 app/backend/tests/fixtures/historical/dispatch-scada-20260830-1455-reduced.csv create mode 100644 app/backend/tests/test_backfill_ledger.py create mode 100644 app/backend/tests/test_nemweb_archives.py create mode 100644 app/docs/historical-analytics-iteration.md create mode 100644 app/migrations/004_historical_backfill_ledger.sql diff --git a/app/backend/batterywatch_api/backfill_ledger.py b/app/backend/batterywatch_api/backfill_ledger.py new file mode 100644 index 0000000..b63b883 --- /dev/null +++ b/app/backend/batterywatch_api/backfill_ledger.py @@ -0,0 +1,245 @@ +"""Transactional run planning for resumable NEMWeb historical backfills.""" + +from __future__ import annotations + +from contextlib import contextmanager +from dataclasses import dataclass +from datetime import date, datetime, timedelta +import re +from typing import Any, Iterable, Iterator, Protocol + + +@dataclass(frozen=True, slots=True) +class BackfillRunSpec: + run_id: str + requested_start: datetime + requested_end: datetime + ingestion_version: int + + +@dataclass(frozen=True, slots=True) +class BackfillPlanItem: + feed: str + report_date: date + source_url: str + + +@dataclass(frozen=True, slots=True) +class BackfillEnsureResult: + created: bool + resumed: bool + item_count: int + recovered_count: int + + +class BackfillRunConflictError(Exception): + """Raised when an existing run does not match the requested identity.""" + + +class _Cursor(Protocol): + def execute(self, statement: str, parameters: tuple[Any, ...]) -> None: ... + def fetchone(self) -> tuple[Any, ...] | None: ... + def fetchall(self) -> list[tuple[Any, ...]]: ... + def close(self) -> None: ... + + +class _Connection(Protocol): + def cursor(self) -> _Cursor: ... + def commit(self) -> None: ... + def rollback(self) -> None: ... + + +@contextmanager +def _managed_cursor(connection: _Connection) -> Iterator[_Cursor]: + cursor = connection.cursor() + try: + yield cursor + finally: + cursor.close() + + +_RUN_INSERT_SQL = """ +INSERT INTO historical_backfill_runs ( + run_id, requested_start, requested_end, ingestion_version, status +) +VALUES (%s, %s, %s, %s, 'running') +ON CONFLICT DO NOTHING RETURNING 1 +""" + +_RUN_SELECT_SQL = """ +SELECT run_id, requested_start, requested_end, ingestion_version +FROM historical_backfill_runs +WHERE run_id = %s +FOR UPDATE +""" + +_ITEM_INSERT_SQL = """ +INSERT INTO historical_backfill_items ( + run_id, feed, report_date, source_url, status +) +VALUES (%s, %s, %s, %s, 'pending') +""" + +_EVENT_INSERT_SQL = """ +INSERT INTO historical_backfill_events ( + run_id, feed, report_date, event_type, attempt_number +) +VALUES (%s, %s, %s, %s, %s) +""" + +_ITEMS_SELECT_SQL = """ +SELECT feed, report_date, source_url +FROM historical_backfill_items +WHERE run_id = %s +ORDER BY feed, report_date +""" + +_RECOVER_ITEMS_SQL = """ +UPDATE historical_backfill_items +SET status = 'pending', updated_at = CURRENT_TIMESTAMP, started_at = NULL +WHERE run_id = %s AND status = 'running' +RETURNING feed, report_date, attempt_count +""" + +_RESUME_RUN_SQL = """ +UPDATE historical_backfill_runs +SET status = 'running', updated_at = CURRENT_TIMESTAMP, completed_at = NULL +WHERE run_id = %s +""" + +_RUN_ID_PATTERN = re.compile(r"[A-Za-z0-9._-]{1,64}") +_MAX_BIGINT = (1 << 63) - 1 +_FEED_URLS = { + "dispatch_price": ( + "https://www.nemweb.com.au/REPORTS/ARCHIVE/DispatchIS_Reports/", + "PUBLIC_DISPATCHIS_", + ), + "dispatch_scada": ( + "https://www.nemweb.com.au/REPORTS/ARCHIVE/Dispatch_SCADA/", + "PUBLIC_DISPATCHSCADA_", + ), +} + + +def _validate_spec(spec: BackfillRunSpec) -> None: + if _RUN_ID_PATTERN.fullmatch(spec.run_id) is None: + raise ValueError("invalid backfill run id") + if not isinstance(spec.requested_start, datetime) or not isinstance( + spec.requested_end, datetime + ): + raise ValueError("backfill bounds must be datetimes") + if spec.requested_start.utcoffset() != timedelta(0) or spec.requested_end.utcoffset() != timedelta(0): + raise ValueError("backfill bounds must be UTC-aware") + duration = spec.requested_end - spec.requested_start + if duration <= timedelta(0) or duration > timedelta(days=366): + raise ValueError("invalid backfill range") + if type(spec.ingestion_version) is not int or not 0 <= spec.ingestion_version <= _MAX_BIGINT: + raise ValueError("invalid ingestion version") + + +def _validate_items(items: tuple[BackfillPlanItem, ...]) -> None: + if not items: + raise ValueError("planned items must not be empty") + keys: set[tuple[str, date]] = set() + for item in items: + if item.feed not in _FEED_URLS or type(item.report_date) is not date: + raise ValueError("invalid backfill plan item") + key = (item.feed, item.report_date) + if key in keys: + raise ValueError("duplicate backfill plan item") + keys.add(key) + base, prefix = _FEED_URLS[item.feed] + expected_url = f"{base}{prefix}{item.report_date:%Y%m%d}.zip" + if item.source_url != expected_url: + raise ValueError("invalid backfill archive URL") + + +class PostgreSQLBackfillLedger: + """Own the transaction that creates or exactly resumes a backfill run.""" + + def __init__(self, connection: _Connection): + self._connection = connection + + def ensure_run( + self, + spec: BackfillRunSpec, + planned_items: Iterable[BackfillPlanItem], + ) -> BackfillEnsureResult: + materialized = tuple(planned_items) + _validate_spec(spec) + _validate_items(materialized) + items = tuple(sorted(materialized, key=lambda item: (item.feed, item.report_date))) + try: + with _managed_cursor(self._connection) as cursor: + cursor.execute( + _RUN_INSERT_SQL, + ( + spec.run_id, + spec.requested_start, + spec.requested_end, + spec.ingestion_version, + ), + ) + if cursor.fetchone() is not None: + for item in items: + cursor.execute( + _ITEM_INSERT_SQL, + (spec.run_id, item.feed, item.report_date, item.source_url), + ) + cursor.execute( + _EVENT_INSERT_SQL, + (spec.run_id, item.feed, item.report_date, "planned", 0), + ) + result = BackfillEnsureResult(True, False, len(items), 0) + else: + cursor.execute(_RUN_SELECT_SQL, (spec.run_id,)) + stored_run = cursor.fetchone() + expected_run = ( + spec.run_id, + spec.requested_start, + spec.requested_end, + spec.ingestion_version, + ) + if stored_run != expected_run: + raise BackfillRunConflictError("backfill run identity conflicts") + cursor.execute(_ITEMS_SELECT_SQL, (spec.run_id,)) + stored_items = tuple(cursor.fetchall()) + expected_items = tuple( + (item.feed, item.report_date, item.source_url) for item in items + ) + if stored_items != expected_items: + raise BackfillRunConflictError("backfill plan conflicts") + cursor.execute(_RECOVER_ITEMS_SQL, (spec.run_id,)) + recovered = tuple(cursor.fetchall()) + for feed, report_date, attempt_number in recovered: + cursor.execute( + _EVENT_INSERT_SQL, + ( + spec.run_id, + feed, + report_date, + "recovered", + attempt_number, + ), + ) + cursor.execute(_RESUME_RUN_SQL, (spec.run_id,)) + result = BackfillEnsureResult( + False, True, len(items), len(recovered) + ) + self._connection.commit() + return result + except Exception: + try: + self._connection.rollback() + except Exception: + pass + raise + + +__all__ = [ + "BackfillEnsureResult", + "BackfillPlanItem", + "BackfillRunConflictError", + "BackfillRunSpec", + "PostgreSQLBackfillLedger", +] diff --git a/app/backend/batterywatch_api/nemweb_archives.py b/app/backend/batterywatch_api/nemweb_archives.py new file mode 100644 index 0000000..4ed0dd4 --- /dev/null +++ b/app/backend/batterywatch_api/nemweb_archives.py @@ -0,0 +1,388 @@ +"""Bounded planning and validation for historical NEMWeb daily archives.""" + +from __future__ import annotations + +from collections.abc import Iterable, Mapping +from dataclasses import dataclass +from datetime import date, datetime, time, timedelta, timezone +from hashlib import sha256 +from io import BytesIO +import zlib +from typing import Final +from zipfile import ZIP_DEFLATED, ZIP_STORED, BadZipFile, LargeZipFile, ZipFile, ZipInfo + +from .aemo import parse_dispatch_price_mms_csv +from .dispatch_scada import parse_dispatch_scada_csv +from .nemweb_dispatch_prices import ( + DispatchPriceArtifactRef as _PriceRef, + extract_dispatch_price_zip as _extract_price_zip, +) +from .nemweb_dispatch_scada import ( + DispatchScadaArtifactRef as _ScadaRef, + extract_dispatch_scada_zip as _extract_scada_zip, +) + +NEMWEB_ORIGIN: Final = "https://www.nemweb.com.au" +DISPATCH_SCADA_FEED: Final = "dispatch_scada" +DISPATCHIS_PRICE_FEED: Final = "dispatchis_price" +ARCHIVE_SOURCE: Final = "archive" +CURRENT_INDEX_SOURCE: Final = "current_index" +DISPATCH_SCADA_CURRENT_INDEX_URL: Final = NEMWEB_ORIGIN + "/REPORTS/CURRENT/Dispatch_SCADA/" +DISPATCHIS_PRICE_CURRENT_INDEX_URL: Final = NEMWEB_ORIGIN + "/REPORTS/CURRENT/DispatchIS_Reports/" +DISPATCH_SCADA_ARCHIVE_BASE_URL: Final = NEMWEB_ORIGIN + "/REPORTS/ARCHIVE/Dispatch_SCADA/" +DISPATCHIS_PRICE_ARCHIVE_BASE_URL: Final = NEMWEB_ORIGIN + "/REPORTS/ARCHIVE/DispatchIS_Reports/" + +MAX_ARCHIVE_RANGE_DAYS: Final = 366 +MAX_ARCHIVE_ARTIFACTS: Final = (MAX_ARCHIVE_RANGE_DAYS + 1) * 2 +MAX_OUTER_ARCHIVE_BYTES: Final = 128 * 1024 * 1024 +MAX_OUTER_COMPRESSED_BYTES: Final = 128 * 1024 * 1024 +MAX_OUTER_UNCOMPRESSED_BYTES: Final = 512 * 1024 * 1024 +MAX_NESTED_ARCHIVE_BYTES: Final = 16 * 1024 * 1024 +MAX_NESTED_COMPRESSION_RATIO: Final = 100 +MAX_DAILY_INTERVALS: Final = 288 + +_UTC: Final = timezone.utc +_NEM_TIMEZONE: Final = timezone(timedelta(hours=10)) +_FEEDS: Final = (DISPATCH_SCADA_FEED, DISPATCHIS_PRICE_FEED) +_FORMAT_ERRORS: Final = (BadZipFile, EOFError, KeyError, LargeZipFile, + NotImplementedError, OSError, OverflowError, RuntimeError, + UnicodeError, ValueError, zlib.error) + + +class NemwebArchiveError(ValueError): + """Raised when a historical archive request or artifact is unsafe.""" + + +@dataclass(frozen=True, slots=True) +class ArchivePlanItem: + feed: str + report_date: date + source: str + url: str + + +@dataclass(frozen=True, slots=True) +class ArchiveRangePlan: + start: datetime + end: datetime + items: tuple[ArchivePlanItem, ...] + + +@dataclass(frozen=True, slots=True) +class NemwebOuterArchiveArtifact: + feed: str + report_date: date + url: str + filename: str + sha256: str + raw_bytes: bytes + + +@dataclass(frozen=True, slots=True) +class NemwebNestedArchiveArtifact: + outer: NemwebOuterArchiveArtifact + member_name: str + source_artifact_id: str + interval_timestamp: datetime + sha256: str + raw_bytes: bytes + + +@dataclass(frozen=True, slots=True) +class NemwebArchiveExtraction: + outer: NemwebOuterArchiveArtifact + nested: tuple[NemwebNestedArchiveArtifact, ...] + + +def _utc(value: datetime, name: str) -> datetime: + if type(value) is not datetime or value.tzinfo is None: + raise NemwebArchiveError(f"invalid archive {name}") + try: + if value.utcoffset() is None: + raise NemwebArchiveError(f"invalid archive {name}") + return value.astimezone(_UTC) + except (AttributeError, OverflowError, TypeError, ValueError): + raise NemwebArchiveError(f"invalid archive {name}") from None + + +def _availability(values: Mapping[str, Iterable[date]] | None, name: str) -> dict[str, frozenset[date]]: + if values is None: + return {} + if not isinstance(values, Mapping): + raise NemwebArchiveError(f"invalid {name}") + result: dict[str, frozenset[date]] = {} + for feed, entries in values.items(): + if type(feed) is not str or feed not in _FEEDS: + raise NemwebArchiveError(f"unsupported archive feed in {name}") + if isinstance(entries, (str, bytes)): + raise NemwebArchiveError(f"invalid {name}") + try: + materialized = tuple(entries) + except (TypeError, ValueError): + raise NemwebArchiveError(f"invalid {name}") from None + if any(type(entry) is not date for entry in materialized): + raise NemwebArchiveError(f"unsupported archive date in {name}") + result[feed] = frozenset(materialized) + return result + + +def _report_dates(start: datetime, end: datetime) -> tuple[date, ...]: + local_start_at = start.astimezone(_NEM_TIMEZONE) + local_start = local_start_at.date() + if local_start_at.timetz().replace(tzinfo=None) == time.min: + local_start -= timedelta(days=1) + local_end = end.astimezone(_NEM_TIMEZONE) + last = local_end.date() + if local_end.timetz().replace(tzinfo=None) == time.min: + last -= timedelta(days=1) + if last < local_start: + return () + return tuple(local_start + timedelta(days=i) for i in range((last - local_start).days + 1)) + + +def _archive_name(feed: str, report_date: date) -> str: + prefix = "PUBLIC_DISPATCHSCADA_" if feed == DISPATCH_SCADA_FEED else "PUBLIC_DISPATCHIS_" + stamp = f"{report_date.year:04d}{report_date.month:02d}{report_date.day:02d}" + return f"{prefix}{stamp}.zip" + + +def _archive_source(feed: str, report_date: date, archived: Mapping[str, frozenset[date]], current: Mapping[str, frozenset[date]]) -> tuple[str, str]: + if report_date in archived.get(feed, frozenset()): + base = DISPATCH_SCADA_ARCHIVE_BASE_URL if feed == DISPATCH_SCADA_FEED else DISPATCHIS_PRICE_ARCHIVE_BASE_URL + return ARCHIVE_SOURCE, f"{base}{_archive_name(feed, report_date)}" + if report_date in current.get(feed, frozenset()): + url = DISPATCH_SCADA_CURRENT_INDEX_URL if feed == DISPATCH_SCADA_FEED else DISPATCHIS_PRICE_CURRENT_INDEX_URL + return CURRENT_INDEX_SOURCE, url + raise NemwebArchiveError("unsupported or unavailable archive date") + + +def plan_archive_range( + start: datetime, + end: datetime, + *, + feeds: Iterable[str], + archived_dates: Mapping[str, Iterable[date]] | None = None, + current_index_dates: Mapping[str, Iterable[date]] | None = None, + max_artifacts: int = MAX_ARCHIVE_ARTIFACTS, +) -> ArchiveRangePlan: + """Create a deterministic bounded plan for an aware UTC half-open range.""" + start_utc, end_utc = _utc(start, "start"), _utc(end, "end") + if end_utc <= start_utc: + raise NemwebArchiveError("archive range must be increasing") + if end_utc - start_utc > timedelta(days=MAX_ARCHIVE_RANGE_DAYS): + raise NemwebArchiveError("archive range exceeds maximum") + if type(max_artifacts) is not int or not 0 < max_artifacts <= MAX_ARCHIVE_ARTIFACTS: + raise NemwebArchiveError("invalid archive artifact limit") + try: + requested = tuple(feeds) + except (TypeError, ValueError): + raise NemwebArchiveError("invalid archive feeds") from None + if (not requested or any(type(feed) is not str or feed not in _FEEDS for feed in requested) + or len(set(requested)) != len(requested)): + raise NemwebArchiveError("invalid archive feeds") + selected = tuple(feed for feed in _FEEDS if feed in requested) + archived = _availability(archived_dates, "archived dates") + current = _availability(current_index_dates, "current index dates") + dates = _report_dates(start_utc, end_utc) + if len(dates) > MAX_ARCHIVE_RANGE_DAYS: + raise NemwebArchiveError("archive range exceeds maximum") + if not dates or len(dates) * len(selected) > max_artifacts: + raise NemwebArchiveError("archive plan exceeds artifact limit") + items = tuple( + ArchivePlanItem(feed, report_date, source, url) + for report_date in dates + for feed in selected + for source, url in (_archive_source(feed, report_date, archived, current),) + ) + return ArchiveRangePlan(start_utc, end_utc, items) + + +def _outer_identity(item: ArchivePlanItem, outer_url: str | None, outer_filename: str | None) -> tuple[str, str]: + if (type(item) is not ArchivePlanItem or type(item.feed) is not str or item.feed not in _FEEDS + or item.source != ARCHIVE_SOURCE or type(item.report_date) is not date + or type(item.url) is not str): + raise NemwebArchiveError("invalid daily archive plan item") + filename = _archive_name(item.feed, item.report_date) + base = DISPATCH_SCADA_ARCHIVE_BASE_URL if item.feed == DISPATCH_SCADA_FEED else DISPATCHIS_PRICE_ARCHIVE_BASE_URL + expected_url = f"{base}{filename}" + actual_url = item.url if outer_url is None else outer_url + actual_filename = filename if outer_filename is None else outer_filename + if (item.url != expected_url or type(actual_url) is not str + or type(actual_filename) is not str + or actual_url != expected_url or actual_filename != filename): + raise NemwebArchiveError("invalid daily archive provenance") + return filename, expected_url + + +def _nested_identity(feed: str, member_name: str, report_date: date) -> tuple[str, datetime]: + prefix = "PUBLIC_DISPATCHSCADA_" if feed == DISPATCH_SCADA_FEED else "PUBLIC_DISPATCHIS_" + if (type(member_name) is not str or not member_name.startswith(prefix) + or not member_name.endswith(".zip") or "/" in member_name + or "\\" in member_name or "\x00" in member_name): + raise NemwebArchiveError("invalid nested archive member name") + parts = member_name[len(prefix):-4].split("_") + if len(parts) != 2 or len(parts[0]) != 12 or not parts[0].isascii() or not parts[0].isdecimal(): + raise NemwebArchiveError("invalid nested archive member name") + source_id = parts[1] + if not 1 <= len(source_id) <= 32 or not source_id.isascii() or not source_id.isdecimal(): + raise NemwebArchiveError("invalid nested archive member name") + try: + local = datetime.strptime(parts[0], "%Y%m%d%H%M").replace(tzinfo=_NEM_TIMEZONE) + except (OverflowError, ValueError): + raise NemwebArchiveError("invalid nested archive timestamp") from None + local_time = local.timetz().replace(tzinfo=None) + is_report_day_interval = local.date() == report_date and local_time != time.min + is_following_midnight = ( + local.date() == report_date + timedelta(days=1) + and local_time == time.min + ) + if (not is_report_day_interval and not is_following_midnight) or local.minute % 5: + raise NemwebArchiveError("nested archive timestamp is outside report date") + return source_id, local.astimezone(_UTC) + + +def _member_sizes(member: ZipInfo) -> tuple[int, int]: + if (type(member.filename) is not str or type(member.orig_filename) is not str + or member.filename != member.orig_filename or member.is_dir() + or member.flag_bits & 1 or member.compress_type not in (ZIP_STORED, ZIP_DEFLATED) + or type(member.file_size) is not int or type(member.compress_size) is not int + or not 0 < member.file_size <= MAX_NESTED_ARCHIVE_BYTES + or not 0 < member.compress_size <= MAX_NESTED_ARCHIVE_BYTES + or member.file_size > member.compress_size * MAX_NESTED_COMPRESSION_RATIO): + raise NemwebArchiveError("invalid nested archive member") + return member.compress_size, member.file_size + + +def _read_nested_member(archive: ZipFile, member: ZipInfo) -> bytes: + _member_sizes(member) + try: + with archive.open(member) as stream: + payload = stream.read(MAX_NESTED_ARCHIVE_BYTES + 1) + extra = stream.read(1) + except _FORMAT_ERRORS as error: + raise NemwebArchiveError("invalid nested archive member") from error + if (type(payload) is not bytes or len(payload) > MAX_NESTED_ARCHIVE_BYTES + or extra or len(payload) != member.file_size): + raise NemwebArchiveError("invalid nested archive member") + return payload + + +def _validate_inner(feed: str, member_name: str, source_id: str, timestamp: datetime, payload: bytes) -> None: + try: + if feed == DISPATCH_SCADA_FEED: + reference = _ScadaRef( + url=DISPATCH_SCADA_CURRENT_INDEX_URL + member_name, + zip_filename=member_name, source_artifact_id=source_id, + report_timestamp=timestamp, + ) + artifact = _extract_scada_zip(reference, payload) + records = parse_dispatch_scada_csv( + artifact.csv_payload, source_artifact_id=source_id, + ingestion_version=0, correction_version=0, naive_timezone=_NEM_TIMEZONE, + ) + else: + reference = _PriceRef( + url=DISPATCHIS_PRICE_CURRENT_INDEX_URL + member_name, + zip_filename=member_name, source_artifact_id=source_id, + report_timestamp=timestamp, + ) + artifact = _extract_price_zip(reference, payload) + records = parse_dispatch_price_mms_csv( + artifact.csv_payload, source_id=source_id, + ingestion_version=0, correction_version=0, + ) + except _FORMAT_ERRORS as error: + raise NemwebArchiveError("invalid nested archive artifact") from error + if not records or any(record.interval_start != timestamp for record in records): + raise NemwebArchiveError("nested artifact timestamp does not match member") + + +def extract_nested_archive( + item: ArchivePlanItem, + outer_payload: bytes, + *, + outer_url: str | None = None, + outer_filename: str | None = None, + start: datetime | None = None, + end: datetime | None = None, + allow_reduced: bool = False, +) -> NemwebArchiveExtraction: + """Validate a daily archive, including every nested interval ZIP.""" + if type(allow_reduced) is not bool: + raise NemwebArchiveError("invalid reduced archive mode") + if (start is None) != (end is None): + raise NemwebArchiveError("archive filter requires both bounds") + filter_start = filter_end = None + if start is not None and end is not None: + filter_start, filter_end = _utc(start, "filter start"), _utc(end, "filter end") + if filter_end <= filter_start: + raise NemwebArchiveError("archive filter range must be increasing") + filename, url = _outer_identity(item, outer_url, outer_filename) + if type(outer_payload) is not bytes or not outer_payload or len(outer_payload) > MAX_OUTER_ARCHIVE_BYTES: + raise NemwebArchiveError("invalid outer archive payload") + outer = NemwebOuterArchiveArtifact(item.feed, item.report_date, url, filename, + sha256(outer_payload).hexdigest(), outer_payload) + try: + with ZipFile(BytesIO(outer_payload)) as archive: + members = archive.infolist() + if not members or len(members) > MAX_DAILY_INTERVALS: + raise NemwebArchiveError("invalid daily archive member count") + if len({member.filename for member in members}) != len(members): + raise NemwebArchiveError("duplicate daily archive member") + compressed = uncompressed = 0 + specs: list[tuple[ZipInfo, str, str, datetime]] = [] + source_ids: set[str] = set() + timestamps: set[datetime] = set() + for member in members: + source_id, timestamp = _nested_identity(item.feed, member.filename, item.report_date) + if source_id in source_ids or timestamp in timestamps: + raise NemwebArchiveError("duplicate nested archive identity") + compressed_size, file_size = _member_sizes(member) + source_ids.add(source_id) + timestamps.add(timestamp) + compressed += compressed_size + uncompressed += file_size + specs.append((member, member.filename, source_id, timestamp)) + if (compressed > MAX_OUTER_COMPRESSED_BYTES + or uncompressed > MAX_OUTER_UNCOMPRESSED_BYTES + or uncompressed > compressed * MAX_NESTED_COMPRESSION_RATIO): + raise NemwebArchiveError("daily archive resource limits exceeded") + if not allow_reduced: + expected = tuple( + (datetime.combine(item.report_date, time.min, tzinfo=_NEM_TIMEZONE) + + timedelta(minutes=5 * (i + 1))).astimezone(_UTC) + for i in range(MAX_DAILY_INTERVALS) + ) + if tuple(sorted(timestamps)) != expected: + raise NemwebArchiveError("daily archive is not a complete 288-interval report") + nested: list[NemwebNestedArchiveArtifact] = [] + for member, member_name, source_id, timestamp in specs: + payload = _read_nested_member(archive, member) + _validate_inner(item.feed, member_name, source_id, timestamp, payload) + nested.append(NemwebNestedArchiveArtifact( + outer, member_name, source_id, timestamp, + sha256(payload).hexdigest(), payload, + )) + except NemwebArchiveError: + raise + except _FORMAT_ERRORS as error: + raise NemwebArchiveError("invalid outer archive") from error + result = tuple(sorted(nested, key=lambda artifact: ( + artifact.interval_timestamp, artifact.source_artifact_id, artifact.member_name, + ))) + if filter_start is not None and filter_end is not None: + result = tuple(a for a in result if filter_start <= a.interval_timestamp < filter_end) + return NemwebArchiveExtraction(outer, result) + + +__all__ = [ + "ARCHIVE_SOURCE", "ArchivePlanItem", "ArchiveRangePlan", "CURRENT_INDEX_SOURCE", + "DISPATCHIS_PRICE_ARCHIVE_BASE_URL", "DISPATCHIS_PRICE_CURRENT_INDEX_URL", + "DISPATCHIS_PRICE_FEED", "DISPATCH_SCADA_ARCHIVE_BASE_URL", + "DISPATCH_SCADA_CURRENT_INDEX_URL", "DISPATCH_SCADA_FEED", "MAX_ARCHIVE_ARTIFACTS", + "MAX_ARCHIVE_RANGE_DAYS", "MAX_DAILY_INTERVALS", "MAX_NESTED_ARCHIVE_BYTES", + "MAX_NESTED_COMPRESSION_RATIO", "MAX_OUTER_ARCHIVE_BYTES", + "MAX_OUTER_COMPRESSED_BYTES", "MAX_OUTER_UNCOMPRESSED_BYTES", "NemwebArchiveError", + "NemwebArchiveExtraction", "NemwebNestedArchiveArtifact", "NemwebOuterArchiveArtifact", + "NEMWEB_ORIGIN", "extract_nested_archive", "plan_archive_range", +] diff --git a/app/backend/tests/fixtures/historical/README.md b/app/backend/tests/fixtures/historical/README.md new file mode 100644 index 0000000..7c56e0c --- /dev/null +++ b/app/backend/tests/fixtures/historical/README.md @@ -0,0 +1,17 @@ +# Reduced public fixture provenance + +These are deterministic row-reduced derivatives of public AEMO NEMWeb +artifacts fetched on 2026-08-30. No private or credential-bearing data is +present. + +- `dispatch-scada-20260830-1455-reduced.csv` derives from + `PUBLIC_DISPATCHSCADA_202608301455_0000000535210764.zip`, source SHA-256 + `63192068b79e7eab0a0f5aa87d3d22bca53c62786e8df1672e50d2c48500b126`. + It retains the authentic envelope and rows for ADPBA1, BBATTERY1 and HPR1. +- `dispatch-price-20260830-1500-reduced.csv` derives from + `PUBLIC_DISPATCHIS_202608301500_0000000535211318.zip`, source SHA-256 + `fb22ab052a31020fcfcc574b5c2d29c43a2ee4caba903a634ce0980091ac7381`. + It retains the authentic PRICE v5 envelope and all five regions, including + negative prices. + +The reduced fixture trailer row count is recomputed for the retained records. diff --git a/app/backend/tests/fixtures/historical/dispatch-price-20260830-1500-reduced.csv b/app/backend/tests/fixtures/historical/dispatch-price-20260830-1500-reduced.csv new file mode 100644 index 0000000..e54f1c2 --- /dev/null +++ b/app/backend/tests/fixtures/historical/dispatch-price-20260830-1500-reduced.csv @@ -0,0 +1,7 @@ +I,DISPATCH,PRICE,5,SETTLEMENTDATE,RUNNO,REGIONID,DISPATCHINTERVAL,INTERVENTION,RRP,EEP,ROP,APCFLAG,MARKETSUSPENDEDFLAG,LASTCHANGED,RAISE6SECRRP,RAISE6SECROP,RAISE6SECAPCFLAG,RAISE60SECRRP,RAISE60SECROP,RAISE60SECAPCFLAG,RAISE5MINRRP,RAISE5MINROP,RAISE5MINAPCFLAG,RAISEREGRRP,RAISEREGROP,RAISEREGAPCFLAG,LOWER6SECRRP,LOWER6SECROP,LOWER6SECAPCFLAG,LOWER60SECRRP,LOWER60SECROP,LOWER60SECAPCFLAG,LOWER5MINRRP,LOWER5MINROP,LOWER5MINAPCFLAG,LOWERREGRRP,LOWERREGROP,LOWERREGAPCFLAG,PRICE_STATUS,PRE_AP_ENERGY_PRICE,PRE_AP_RAISE6_PRICE,PRE_AP_RAISE60_PRICE,PRE_AP_RAISE5MIN_PRICE,PRE_AP_RAISEREG_PRICE,PRE_AP_LOWER6_PRICE,PRE_AP_LOWER60_PRICE,PRE_AP_LOWER5MIN_PRICE,PRE_AP_LOWERREG_PRICE,RAISE1SECRRP,RAISE1SECROP,RAISE1SECAPCFLAG,LOWER1SECRRP,LOWER1SECROP,LOWER1SECAPCFLAG,PRE_AP_RAISE1_PRICE,PRE_AP_LOWER1_PRICE,CUMUL_PRE_AP_ENERGY_PRICE,CUMUL_PRE_AP_RAISE6_PRICE,CUMUL_PRE_AP_RAISE60_PRICE,CUMUL_PRE_AP_RAISE5MIN_PRICE,CUMUL_PRE_AP_RAISEREG_PRICE,CUMUL_PRE_AP_LOWER6_PRICE,CUMUL_PRE_AP_LOWER60_PRICE,CUMUL_PRE_AP_LOWER5MIN_PRICE,CUMUL_PRE_AP_LOWERREG_PRICE,CUMUL_PRE_AP_RAISE1_PRICE,CUMUL_PRE_AP_LOWER1_PRICE,OCD_STATUS,MII_STATUS +D,DISPATCH,PRICE,5,2026/08/30 15:00:00,1,NSW1,20260830132,0,-6.93755,0,-6.93755,0,0,2026/08/30 14:55:11,0.01,0.01,0,0.01,0.01,0,0.01,0.01,0,1.47,1.47,0,0.01,0.01,0,0.28,0.28,0,0.09,0.09,0,9.73,9.73,0,FIRM,-6.93755,0.01,0.01,0.01,1.47,0.01,0.28,0.09,9.73,0,0,0,0,0,0,0,0,160018.555870,74.65,66.522420,20.86,6803.414730,53.069950,107.020560,62.69,3366.863420,27.65,0.33,NOT_OCD,NOT_MII +D,DISPATCH,PRICE,5,2026/08/30 15:00:00,1,QLD1,20260830132,0,-6,0,-6,0,0,2026/08/30 14:55:11,0.01,0.01,0,0.01,0.01,0,0.01,0.01,0,1.47,1.47,0,0.01,0.01,0,0.28,0.28,0,0.09,0.09,0,9.73,9.73,0,FIRM,-6,0.01,0.01,0.01,1.47,0.01,0.28,0.09,9.73,0,0,0,0,0,0,0,0,131831.533660,72.47,64.24,20.53,6803.414730,53.069950,107.020560,62.69,3366.863420,27.51,0.33,NOT_OCD,NOT_MII +D,DISPATCH,PRICE,5,2026/08/30 15:00:00,1,SA1,20260830132,0,-7.79132,0,-7.79132,0,0,2026/08/30 14:55:11,0.01,0.01,0,0.01,0.01,0,0.01,0.01,0,1.47,1.47,0,0.01,0.01,0,0.28,0.28,0,0.09,0.09,0,9.73,9.73,0,FIRM,-7.79132,0.01,0.01,0.01,1.47,0.01,0.28,0.09,9.73,0,0,0,0,0,0,0,0,169798.399730,100.69,92.772420,32.03,6814.584730,56.219950,110.130830,62.69,3366.863420,31.7,12.86,NOT_OCD,NOT_MII +D,DISPATCH,PRICE,5,2026/08/30 15:00:00,1,TAS1,20260830132,0,-8.20999,0,-8.20999,0,0,2026/08/30 14:55:11,0.38,0.38,0,0.38,0.38,0,0.01,0.01,0,3.6,3.6,0,0.01,0.01,0,0.28,0.28,0,0.09,0.09,0,23.97999,23.97999,0,FIRM,-8.20999,0.38,0.38,0.01,3.6,0.01,0.28,0.09,23.97999,0,0,0,0,0,0,0,0,138921.852820,345.317520,397.277110,42.71,9417.912860,171.185470,139.419840,60.29,6789.6168,26.61,0.27,NOT_OCD,NOT_MII +D,DISPATCH,PRICE,5,2026/08/30 15:00:00,1,VIC1,20260830132,0,-8,0,-8,0,0,2026/08/30 14:55:11,0.01,0.01,0,0.01,0.01,0,0.01,0.01,0,1.47,1.47,0,0.01,0.01,0,0.28,0.28,0,0.09,0.09,0,9.73,9.73,0,FIRM,-8,0.01,0.01,0.01,1.47,0.01,0.28,0.09,9.73,0,0,0,0,0,0,0,0,152256.747340,74.65,66.522420,20.86,6803.414730,53.069950,107.020560,62.69,3366.863420,27.65,0.33,NOT_OCD,NOT_MII +C,END OF REPORT,7 diff --git a/app/backend/tests/fixtures/historical/dispatch-scada-20260830-1455-reduced.csv b/app/backend/tests/fixtures/historical/dispatch-scada-20260830-1455-reduced.csv new file mode 100644 index 0000000..a8cdaa0 --- /dev/null +++ b/app/backend/tests/fixtures/historical/dispatch-scada-20260830-1455-reduced.csv @@ -0,0 +1,5 @@ +I,DISPATCH,UNIT_SCADA,1,SETTLEMENTDATE,DUID,SCADAVALUE,LASTCHANGED +D,DISPATCH,UNIT_SCADA,1,2026/08/30 14:55:00,ADPBA1,0.0340,2026/08/30 14:50:13 +D,DISPATCH,UNIT_SCADA,1,2026/08/30 14:55:00,BBATTERY1,1.461570,2026/08/30 14:50:13 +D,DISPATCH,UNIT_SCADA,1,2026/08/30 14:55:00,HPR1,-0.80,2026/08/30 14:50:13 +C,END OF REPORT,5 diff --git a/app/backend/tests/test_backfill_ledger.py b/app/backend/tests/test_backfill_ledger.py new file mode 100644 index 0000000..2d661d6 --- /dev/null +++ b/app/backend/tests/test_backfill_ledger.py @@ -0,0 +1,280 @@ +"""Contract tests for the resumable historical backfill ledger.""" + +from dataclasses import replace +from datetime import date, datetime, timedelta, timezone +from pathlib import Path +import unittest + +from batterywatch_api.backfill_ledger import ( + BackfillEnsureResult, + BackfillPlanItem, + BackfillRunConflictError, + BackfillRunSpec, + PostgreSQLBackfillLedger, +) + + +UTC = timezone.utc + + +class FakeCursor: + def __init__(self, connection) -> None: + self.connection = connection + + def execute(self, statement, parameters) -> None: + self.connection.executions.append((statement, tuple(parameters))) + if len(self.connection.executions) == self.connection.fail_on_execute: + raise self.connection.failure + + def fetchone(self): + if self.connection.fetchone_results: + return self.connection.fetchone_results.pop(0) + return None + + def fetchall(self): + if self.connection.fetchall_results: + return self.connection.fetchall_results.pop(0) + return [] + + def close(self) -> None: + self.connection.closed_cursors += 1 + + +class FakeConnection: + def __init__( + self, *, fetchone_results=(), fetchalls=(), fail_on_execute=None, failure=None + ) -> None: + self.executions = [] + self.fetchone_results = list(fetchone_results) + self.fetchall_results = list(fetchalls) + self.fail_on_execute = fail_on_execute + self.failure = failure + self.cursor_calls = self.closed_cursors = 0 + self.commits = self.rollbacks = 0 + + def cursor(self): + self.cursor_calls += 1 + return FakeCursor(self) + + def commit(self) -> None: + self.commits += 1 + + def rollback(self) -> None: + self.rollbacks += 1 + + +class BackfillLedgerMigrationTests(unittest.TestCase): + def test_deploys_additive_run_item_and_event_schema(self) -> None: + app_root = Path(__file__).resolve().parents[2] + migration_path = app_root / "migrations" / "004_historical_backfill_ledger.sql" + self.assertTrue(migration_path.exists(), "004 historical ledger migration is missing") + + migration = migration_path.read_text(encoding="utf-8") + migrate_script = (app_root / "deploy" / "migrate.sh").read_text(encoding="utf-8") + upper = migration.upper() + + for table in ( + "historical_backfill_runs", + "historical_backfill_items", + "historical_backfill_events", + ): + self.assertIn(f"CREATE TABLE IF NOT EXISTS {table}", migration) + self.assertIn(f"'{table}'", migrate_script) + + self.assertIn("004_historical_backfill_ledger.sql", migrate_script) + self.assertIn("SELECT count(*) = 10", migrate_script) + self.assertEqual(migrate_script.count('--dbname="$database_url"'), 5) + self.assertIn("ON DELETE RESTRICT", migration) + self.assertIn("event_seq BIGSERIAL PRIMARY KEY", migration) + self.assertIn("historical_backfill_items_claim_idx", migration) + self.assertIn("historical_backfill_events_order_idx", migration) + for forbidden in ("DROP ", "TRUNCATE ", "ALTER TABLE"): + self.assertNotIn(forbidden, upper) + self.assertNotRegex(upper, r"\bDELETE\s+FROM\b") + + +class PostgreSQLBackfillLedgerTests(unittest.TestCase): + def test_ensure_new_run_plans_items_and_events_in_deterministic_order(self) -> None: + spec = BackfillRunSpec( + "run-20260828", + datetime(2026, 8, 27, 14, tzinfo=UTC), + datetime(2026, 8, 29, 14, tzinfo=UTC), + 1, + ) + items = ( + BackfillPlanItem( + "dispatch_scada", + date(2026, 8, 28), + "https://www.nemweb.com.au/REPORTS/ARCHIVE/Dispatch_SCADA/" + "PUBLIC_DISPATCHSCADA_20260828.zip", + ), + BackfillPlanItem( + "dispatch_price", + date(2026, 8, 28), + "https://www.nemweb.com.au/REPORTS/ARCHIVE/DispatchIS_Reports/" + "PUBLIC_DISPATCHIS_20260828.zip", + ), + ) + connection = FakeConnection(fetchone_results=((1,),)) + + result = PostgreSQLBackfillLedger(connection).ensure_run(spec, items) + + self.assertEqual(result, BackfillEnsureResult(True, False, 2, 0)) + self.assertEqual( + (connection.cursor_calls, connection.closed_cursors, + connection.commits, connection.rollbacks), + (1, 1, 1, 0), + ) + self.assertEqual(len(connection.executions), 5) + self.assertIn("INSERT INTO historical_backfill_runs", connection.executions[0][0]) + self.assertEqual( + tuple(parameters[1] for statement, parameters in connection.executions + if "INSERT INTO historical_backfill_items" in statement), + ("dispatch_price", "dispatch_scada"), + ) + self.assertEqual( + tuple(parameters[3] for statement, parameters in connection.executions + if "INSERT INTO historical_backfill_events" in statement), + ("planned", "planned"), + ) + self.assertTrue(all("%s" in statement for statement, _ in connection.executions)) + + def test_exact_resume_recovers_interrupted_item_and_preserves_attempt(self) -> None: + start = datetime(2026, 8, 27, 14, tzinfo=UTC) + end = datetime(2026, 8, 29, 14, tzinfo=UTC) + spec = BackfillRunSpec("run-20260828", start, end, 1) + price = BackfillPlanItem( + "dispatch_price", + date(2026, 8, 28), + "https://www.nemweb.com.au/REPORTS/ARCHIVE/DispatchIS_Reports/" + "PUBLIC_DISPATCHIS_20260828.zip", + ) + scada = BackfillPlanItem( + "dispatch_scada", + date(2026, 8, 28), + "https://www.nemweb.com.au/REPORTS/ARCHIVE/Dispatch_SCADA/" + "PUBLIC_DISPATCHSCADA_20260828.zip", + ) + connection = FakeConnection( + fetchone_results=(None, (spec.run_id, start, end, 1)), + fetchalls=( + ( + (price.feed, price.report_date, price.source_url), + (scada.feed, scada.report_date, scada.source_url), + ), + ((scada.feed, scada.report_date, 3),), + ), + ) + + result = PostgreSQLBackfillLedger(connection).ensure_run(spec, (scada, price)) + + self.assertEqual(result, BackfillEnsureResult(False, True, 2, 1)) + self.assertEqual((connection.commits, connection.rollbacks), (1, 0)) + self.assertEqual((connection.cursor_calls, connection.closed_cursors), (1, 1)) + statements = tuple(statement for statement, _ in connection.executions) + self.assertIn("FROM historical_backfill_runs", statements[1]) + self.assertIn("FOR UPDATE", statements[1]) + self.assertIn("FROM historical_backfill_items", statements[2]) + self.assertIn("UPDATE historical_backfill_items", statements[3]) + self.assertIn("INSERT INTO historical_backfill_events", statements[4]) + self.assertEqual(connection.executions[4][1][-2:], ("recovered", 3)) + self.assertIn("UPDATE historical_backfill_runs", statements[5]) + + def test_changed_existing_run_identity_fails_closed(self) -> None: + start = datetime(2026, 8, 27, 14, tzinfo=UTC) + end = datetime(2026, 8, 29, 14, tzinfo=UTC) + spec = BackfillRunSpec("run-20260828", start, end, 1) + item = BackfillPlanItem( + "dispatch_price", + date(2026, 8, 28), + "https://www.nemweb.com.au/REPORTS/ARCHIVE/DispatchIS_Reports/" + "PUBLIC_DISPATCHIS_20260828.zip", + ) + cases = ( + ( + "run", + (spec.run_id, start, end, 2), + ((item.feed, item.report_date, item.source_url),), + ), + ( + "plan", + (spec.run_id, start, end, 1), + (), + ), + ) + + for label, stored_run, stored_items in cases: + with self.subTest(label=label): + connection = FakeConnection( + fetchone_results=(None, stored_run), + fetchalls=(stored_items, ()), + ) + with self.assertRaises(BackfillRunConflictError): + PostgreSQLBackfillLedger(connection).ensure_run(spec, (item,)) + self.assertEqual((connection.commits, connection.rollbacks), (0, 1)) + self.assertEqual(connection.closed_cursors, 1) + + def test_invalid_inputs_fail_before_sql(self) -> None: + start = datetime(2026, 8, 27, 14, tzinfo=UTC) + end = datetime(2026, 8, 29, 14, tzinfo=UTC) + spec = BackfillRunSpec("run-20260828", start, end, 1) + item = BackfillPlanItem( + "dispatch_price", + date(2026, 8, 28), + "https://www.nemweb.com.au/REPORTS/ARCHIVE/DispatchIS_Reports/" + "PUBLIC_DISPATCHIS_20260828.zip", + ) + cases = ( + (replace(spec, run_id="bad id"), (item,)), + (replace(spec, requested_start=start.replace(tzinfo=None)), (item,)), + (replace(spec, requested_end=start), (item,)), + (replace(spec, requested_end=start + timedelta(days=367)), (item,)), + (replace(spec, ingestion_version=-1), (item,)), + (replace(spec, ingestion_version=1 << 63), (item,)), + (spec, ()), + (spec, (replace(item, feed="regional_soc"),)), + (spec, (replace(item, report_date=datetime(2026, 8, 28)),)), + (spec, (replace(item, source_url="https://example.invalid/archive.zip"),)), + (spec, (item, item)), + ) + + for invalid_spec, invalid_items in cases: + with self.subTest(spec=invalid_spec, items=invalid_items): + connection = FakeConnection() + with self.assertRaises(ValueError): + PostgreSQLBackfillLedger(connection).ensure_run( + invalid_spec, invalid_items + ) + self.assertEqual( + (connection.executions, connection.commits, connection.rollbacks), + ([], 0, 0), + ) + + def test_database_failure_rolls_back_and_reraises_same_error(self) -> None: + spec = BackfillRunSpec( + "run-20260828", + datetime(2026, 8, 27, 14, tzinfo=UTC), + datetime(2026, 8, 29, 14, tzinfo=UTC), + 1, + ) + item = BackfillPlanItem( + "dispatch_price", + date(2026, 8, 28), + "https://www.nemweb.com.au/REPORTS/ARCHIVE/DispatchIS_Reports/" + "PUBLIC_DISPATCHIS_20260828.zip", + ) + failure = RuntimeError("injected database failure") + connection = FakeConnection( + fetchone_results=((1,),), fail_on_execute=3, failure=failure + ) + + with self.assertRaises(RuntimeError) as raised: + PostgreSQLBackfillLedger(connection).ensure_run(spec, (item,)) + + self.assertIs(raised.exception, failure) + self.assertEqual((connection.commits, connection.rollbacks), (0, 1)) + self.assertEqual(connection.closed_cursors, 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/backend/tests/test_dispatch_price_ingestion.py b/app/backend/tests/test_dispatch_price_ingestion.py index 426fbde..404eef2 100644 --- a/app/backend/tests/test_dispatch_price_ingestion.py +++ b/app/backend/tests/test_dispatch_price_ingestion.py @@ -223,10 +223,11 @@ def test_deploy_migration_applies_and_verifies_price_artifact_table(self) -> Non self.assertIn("raw_zip BYTEA NOT NULL", migration) self.assertIn("PUBLIC_DISPATCHIS_", migration) self.assertIn("003_dispatch_price_artifacts.sql", migrate_script) - self.assertIn("SELECT count(*) = 7", migrate_script) + self.assertIn("SELECT count(*) = 10", migrate_script) self.assertIn("'dispatch_price_artifacts'", migrate_script) + self.assertIn("004_historical_backfill_ledger.sql", migrate_script) self.assertIn("database_url=$BATTERYWATCH_DATABASE_URL", migrate_script) - self.assertEqual(migrate_script.count('--dbname="$database_url"'), 4) + self.assertEqual(migrate_script.count('--dbname="$database_url"'), 5) self.assertNotIn("export PGDATABASE=", migrate_script) diff --git a/app/backend/tests/test_nemweb_archives.py b/app/backend/tests/test_nemweb_archives.py new file mode 100644 index 0000000..c44853c --- /dev/null +++ b/app/backend/tests/test_nemweb_archives.py @@ -0,0 +1,617 @@ +"""Tests for the bounded NEMWeb historical archive seam.""" + +from dataclasses import FrozenInstanceError +from datetime import date, datetime, timedelta, timezone +from hashlib import sha256 +from io import BytesIO +from pathlib import Path +from typing import Any +import unittest +from unittest.mock import patch +from zipfile import ZIP_BZIP2, ZIP_DEFLATED, ZipFile, ZipInfo + +from batterywatch_api.aemo import parse_dispatch_price_mms_csv +from batterywatch_api.nemweb_archives import ( + ARCHIVE_SOURCE, + CURRENT_INDEX_SOURCE, + DISPATCHIS_PRICE_FEED, + DISPATCH_SCADA_FEED, + ArchivePlanItem, + NemwebArchiveError, + extract_nested_archive, + plan_archive_range, +) + + +UTC = timezone.utc + + +class ArchiveRangePlannerTests(unittest.TestCase): + def test_plans_intersecting_nem_dates_and_current_fallback_deterministically(self) -> None: + plan = plan_archive_range( + datetime(2026, 8, 29, 14, 30, tzinfo=UTC), + datetime(2026, 8, 31, 14, 30, tzinfo=UTC), + feeds=(DISPATCH_SCADA_FEED, DISPATCHIS_PRICE_FEED), + archived_dates={ + DISPATCH_SCADA_FEED: (date(2026, 8, 30),), + DISPATCHIS_PRICE_FEED: (date(2026, 8, 30),), + }, + current_index_dates={ + DISPATCH_SCADA_FEED: (date(2026, 8, 31), date(2026, 9, 1)), + DISPATCHIS_PRICE_FEED: (date(2026, 8, 31), date(2026, 9, 1)), + }, + ) + + self.assertEqual( + tuple( + (item.feed, item.report_date, item.source, item.url) + for item in plan.items + ), + ( + ( + DISPATCH_SCADA_FEED, + date(2026, 8, 30), + ARCHIVE_SOURCE, + "https://www.nemweb.com.au/REPORTS/ARCHIVE/Dispatch_SCADA/PUBLIC_DISPATCHSCADA_20260830.zip", + ), + ( + DISPATCHIS_PRICE_FEED, + date(2026, 8, 30), + ARCHIVE_SOURCE, + "https://www.nemweb.com.au/REPORTS/ARCHIVE/DispatchIS_Reports/PUBLIC_DISPATCHIS_20260830.zip", + ), + ( + DISPATCH_SCADA_FEED, + date(2026, 8, 31), + CURRENT_INDEX_SOURCE, + "https://www.nemweb.com.au/REPORTS/CURRENT/Dispatch_SCADA/", + ), + ( + DISPATCHIS_PRICE_FEED, + date(2026, 8, 31), + CURRENT_INDEX_SOURCE, + "https://www.nemweb.com.au/REPORTS/CURRENT/DispatchIS_Reports/", + ), + ( + DISPATCH_SCADA_FEED, + date(2026, 9, 1), + CURRENT_INDEX_SOURCE, + "https://www.nemweb.com.au/REPORTS/CURRENT/Dispatch_SCADA/", + ), + ( + DISPATCHIS_PRICE_FEED, + date(2026, 9, 1), + CURRENT_INDEX_SOURCE, + "https://www.nemweb.com.au/REPORTS/CURRENT/DispatchIS_Reports/", + ), + ), + ) + self.assertEqual(plan.start, datetime(2026, 8, 29, 14, 30, tzinfo=UTC)) + self.assertEqual(plan.end, datetime(2026, 8, 31, 14, 30, tzinfo=UTC)) + + def test_midnight_start_includes_previous_market_day_archive(self) -> None: + plan = plan_archive_range( + datetime(2026, 8, 29, 14, 0, tzinfo=UTC), + datetime(2026, 8, 30, 14, 0, tzinfo=UTC), + feeds=(DISPATCH_SCADA_FEED,), + archived_dates={ + DISPATCH_SCADA_FEED: (date(2026, 8, 29), date(2026, 8, 30)), + }, + ) + + self.assertEqual( + tuple(item.report_date for item in plan.items), + (date(2026, 8, 29), date(2026, 8, 30)), + ) + + def test_rejects_unsupported_unavailable_and_excessive_plans(self) -> None: + start = datetime(2026, 8, 30, tzinfo=UTC) + end = datetime(2026, 8, 31, tzinfo=UTC) + with self.assertRaises(NemwebArchiveError): + plan_archive_range(start, end, feeds=("unknown",), archived_dates={}) + with self.assertRaises(NemwebArchiveError): + plan_archive_range(start, end, feeds=(DISPATCH_SCADA_FEED,), archived_dates={}) + with self.assertRaises(NemwebArchiveError): + plan_archive_range( + start, end + timedelta(days=1), + feeds=(DISPATCH_SCADA_FEED,), + archived_dates={ + DISPATCH_SCADA_FEED: (date(2026, 8, 30), date(2026, 8, 31)) + }, + max_artifacts=1, + ) + + def test_rejects_a_plan_spanning_more_than_366_report_dates(self) -> None: + report_dates = tuple(date(2026, 1, 1) + timedelta(days=i) for i in range(367)) + with self.assertRaises(NemwebArchiveError): + plan_archive_range( + datetime(2026, 1, 1, 12, tzinfo=UTC), + datetime(2027, 1, 2, 11, 59, tzinfo=UTC), + feeds=(DISPATCH_SCADA_FEED,), + archived_dates={DISPATCH_SCADA_FEED: report_dates}, + ) + + def test_rejects_invalid_and_excessive_ranges(self) -> None: + valid_dates = {DISPATCH_SCADA_FEED: (date(2026, 8, 30),)} + cases: tuple[tuple[Any, Any], ...] = ( + (datetime(2026, 8, 30), datetime(2026, 8, 30, 1)), + (datetime(2026, 8, 30, tzinfo=UTC), datetime(2026, 8, 30, tzinfo=UTC)), + (datetime(2026, 8, 30, tzinfo=UTC), datetime(2026, 8, 29, 23, tzinfo=UTC)), + (datetime(2026, 8, 30, tzinfo=UTC), datetime(2027, 9, 1, tzinfo=UTC)), + ) + for start, end in cases: + with self.subTest(start=start, end=end), self.assertRaises(NemwebArchiveError): + plan_archive_range( + start, end, feeds=(DISPATCH_SCADA_FEED,), + archived_dates=valid_dates, + ) + with self.assertRaises(NemwebArchiveError): + plan_archive_range( + datetime(2026, 8, 30, tzinfo=UTC), + datetime(2026, 8, 31, tzinfo=UTC), + feeds=(DISPATCH_SCADA_FEED,), + archived_dates=valid_dates, + max_artifacts=0, + ) + + +_FIXTURE_DIR = Path(__file__).parent / "fixtures" / "historical" +_REPORT_DATE = date(2026, 8, 30) +_SCADA_SOURCE_ID = "0000000535210764" +_SCADA_MEMBER = "PUBLIC_DISPATCHSCADA_202608301455_0000000535210764.zip" +_SCADA_URL = ( + "https://www.nemweb.com.au/REPORTS/ARCHIVE/Dispatch_SCADA/" + "PUBLIC_DISPATCHSCADA_20260830.zip" +) +_PRICE_SOURCE_ID = "0000000535211318" +_PRICE_MEMBER = "PUBLIC_DISPATCHIS_202608301500_0000000535211318.zip" +_PRICE_URL = ( + "https://www.nemweb.com.au/REPORTS/ARCHIVE/DispatchIS_Reports/" + "PUBLIC_DISPATCHIS_20260830.zip" +) + + +def _canonical_fixture_csv(filename: str, source_id: str, table: str) -> bytes: + lines = ( + (_FIXTURE_DIR / filename).read_text(encoding="utf-8").splitlines() + ) + footer = lines[-1].split(",") + footer[2] = str(len(lines) + 1) + metadata = ( + f"C,NEMP.WORLD,{table},AEMO,PUBLIC,2026/08/30,15:00:15," + f"{source_id},{table},0000000000000000" + ) + return ("\n".join((metadata, *lines[:-1], ",".join(footer))) + "\n").encode( + "utf-8" + ) + + +def _inner_zip(member_name: str, csv_payload: bytes) -> bytes: + buffer = BytesIO() + with ZipFile(buffer, "w", compression=ZIP_DEFLATED) as archive: + archive.writestr(member_name.removesuffix(".zip") + ".CSV", csv_payload) + return buffer.getvalue() + + +def _scada_csv_at(local_timestamp: str, source_id: str) -> bytes: + payload = _canonical_fixture_csv( + "dispatch-scada-20260830-1455-reduced.csv", source_id, "DISPATCHSCADA" + ) + return payload.replace(b"2026/08/30 14:55:00", local_timestamp.encode()) + + +def _inner_zip_with_extra(member_name: str, csv_payload: bytes) -> bytes: + buffer = BytesIO() + with ZipFile(buffer, "w", compression=ZIP_DEFLATED) as archive: + archive.writestr(member_name.removesuffix(".zip") + ".CSV", csv_payload) + archive.writestr("unexpected.txt", b"extra") + return buffer.getvalue() + + +def _outer_zip(member_name: str, inner_payload: bytes) -> bytes: + return _outer_zip_members(((member_name, inner_payload),)) + + +def _outer_zip_members( + members: tuple[tuple[str, bytes], ...], *, compression: int = ZIP_DEFLATED +) -> bytes: + buffer = BytesIO() + with ZipFile(buffer, "w", compression=compression) as archive: + for member_name, inner_payload in members: + archive.writestr(member_name, inner_payload) + return buffer.getvalue() + + +def _encrypted_outer_zip(member_name: str, inner_payload: bytes) -> bytes: + buffer = BytesIO() + with ZipFile(buffer, "w") as archive: + info = ZipInfo(member_name) + info.flag_bits |= 1 + archive.writestr(info, inner_payload) + return buffer.getvalue() + + +class NestedArchiveExtractionTests(unittest.TestCase): + def test_accepts_following_midnight_as_market_day_final_interval(self) -> None: + source_id = "0000000535210765" + member = f"PUBLIC_DISPATCHSCADA_202608310000_{source_id}.zip" + csv_payload = _scada_csv_at("2026/08/31 00:00:00", source_id) + item = ArchivePlanItem( + feed=DISPATCH_SCADA_FEED, + report_date=_REPORT_DATE, + source=ARCHIVE_SOURCE, + url=_SCADA_URL, + ) + + extraction = extract_nested_archive( + item, + _outer_zip(member, _inner_zip(member, csv_payload)), + allow_reduced=True, + ) + + self.assertEqual( + extraction.nested[0].interval_timestamp, + datetime(2026, 8, 30, 14, 0, tzinfo=UTC), + ) + + def test_rejects_same_day_midnight_outside_market_day_archive(self) -> None: + source_id = "0000000535210765" + member = f"PUBLIC_DISPATCHSCADA_202608300000_{source_id}.zip" + csv_payload = _scada_csv_at("2026/08/30 00:00:00", source_id) + item = ArchivePlanItem( + feed=DISPATCH_SCADA_FEED, + report_date=_REPORT_DATE, + source=ARCHIVE_SOURCE, + url=_SCADA_URL, + ) + + with self.assertRaises(NemwebArchiveError): + extract_nested_archive( + item, + _outer_zip(member, _inner_zip(member, csv_payload)), + allow_reduced=True, + ) + + def test_complete_market_day_sequence_starts_at_five_minutes(self) -> None: + first_id = "0000000535210001" + second_id = "0000000535210002" + first = f"PUBLIC_DISPATCHSCADA_202608300005_{first_id}.zip" + second = f"PUBLIC_DISPATCHSCADA_202608300010_{second_id}.zip" + outer_payload = _outer_zip_members(( + (first, _inner_zip(first, _scada_csv_at("2026/08/30 00:05:00", first_id))), + (second, _inner_zip(second, _scada_csv_at("2026/08/30 00:10:00", second_id))), + )) + item = ArchivePlanItem( + feed=DISPATCH_SCADA_FEED, + report_date=_REPORT_DATE, + source=ARCHIVE_SOURCE, + url=_SCADA_URL, + ) + + with patch("batterywatch_api.nemweb_archives.MAX_DAILY_INTERVALS", 2): + extraction = extract_nested_archive(item, outer_payload) + + self.assertEqual( + tuple(artifact.interval_timestamp for artifact in extraction.nested), + ( + datetime(2026, 8, 29, 14, 5, tzinfo=UTC), + datetime(2026, 8, 29, 14, 10, tzinfo=UTC), + ), + ) + + def test_production_mode_rejects_reduced_archive(self) -> None: + csv_payload = _canonical_fixture_csv( + "dispatch-scada-20260830-1455-reduced.csv", + _SCADA_SOURCE_ID, + "DISPATCHSCADA", + ) + outer_payload = _outer_zip( + _SCADA_MEMBER, + _inner_zip(_SCADA_MEMBER, csv_payload), + ) + item = ArchivePlanItem( + feed=DISPATCH_SCADA_FEED, + report_date=_REPORT_DATE, + source=ARCHIVE_SOURCE, + url=_SCADA_URL, + ) + with self.assertRaises(NemwebArchiveError): + extract_nested_archive(item, outer_payload) + + def test_rejects_wrong_outer_provenance(self) -> None: + csv_payload = _canonical_fixture_csv( + "dispatch-scada-20260830-1455-reduced.csv", + _SCADA_SOURCE_ID, + "DISPATCHSCADA", + ) + outer_payload = _outer_zip( + _SCADA_MEMBER, + _inner_zip(_SCADA_MEMBER, csv_payload), + ) + item = ArchivePlanItem( + feed=DISPATCH_SCADA_FEED, + report_date=_REPORT_DATE, + source=ARCHIVE_SOURCE, + url=_SCADA_URL, + ) + cases: tuple[tuple[ArchivePlanItem, dict[str, Any]], ...] = ( + (item, {"outer_filename": "PUBLIC_DISPATCHSCADA_20260831.zip"}), + (item, {"outer_url": _SCADA_URL.replace("20260830", "20260831")}), + (item, {"outer_url": "https://example.invalid/archive.zip"}), + (ArchivePlanItem( + DISPATCH_SCADA_FEED, _REPORT_DATE, ARCHIVE_SOURCE, + "https://example.invalid/archive.zip", + ), {"outer_url": _SCADA_URL}), + ) + for bad_item, kwargs in cases: + with self.subTest(item=bad_item, kwargs=kwargs), self.assertRaises(NemwebArchiveError): + extract_nested_archive(bad_item, outer_payload, allow_reduced=True, **kwargs) + + def test_rejects_wrong_inner_feed_date_and_timestamp(self) -> None: + item = ArchivePlanItem( + feed=DISPATCH_SCADA_FEED, + report_date=_REPORT_DATE, + source=ARCHIVE_SOURCE, + url=_SCADA_URL, + ) + cases = ( + "PUBLIC_DISPATCHIS_202608301455_0000000535210764.zip", + "PUBLIC_DISPATCHSCADA_202608311455_0000000535210764.zip", + "PUBLIC_DISPATCHSCADA_202608301456_0000000535210764.zip", + ) + for member_name in cases: + with self.subTest(member_name=member_name), self.assertRaises(NemwebArchiveError): + extract_nested_archive( + item, _outer_zip(member_name, b"not an inner zip"), + allow_reduced=True, + ) + + def test_rejects_wrong_inner_csv_provenance(self) -> None: + item = ArchivePlanItem( + feed=DISPATCH_SCADA_FEED, + report_date=_REPORT_DATE, + source=ARCHIVE_SOURCE, + url=_SCADA_URL, + ) + valid = _canonical_fixture_csv( + "dispatch-scada-20260830-1455-reduced.csv", + _SCADA_SOURCE_ID, + "DISPATCHSCADA", + ) + cases = ( + _canonical_fixture_csv( + "dispatch-scada-20260830-1455-reduced.csv", + _SCADA_SOURCE_ID, + "WRONGFEED", + ), + valid.replace(b"2026/08/30 14:55:00", b"2026/08/30 15:00:00"), + valid.replace(b"2026/08/30 14:55:00", b"2026/08/31 14:55:00"), + _canonical_fixture_csv( + "dispatch-scada-20260830-1455-reduced.csv", + "0000000535210765", + "DISPATCHSCADA", + ), + ) + for csv_payload in cases: + with self.subTest(payload=csv_payload[:30]), self.assertRaises(NemwebArchiveError): + extract_nested_archive( + item, _outer_zip(_SCADA_MEMBER, _inner_zip(_SCADA_MEMBER, csv_payload)), + allow_reduced=True, + ) + + def test_rejects_unsafe_or_malformed_nested_members(self) -> None: + item = ArchivePlanItem( + feed=DISPATCH_SCADA_FEED, + report_date=_REPORT_DATE, + source=ARCHIVE_SOURCE, + url=_SCADA_URL, + ) + valid_name = _SCADA_MEMBER + cases = ( + _outer_zip("../" + valid_name, b"not a zip"), + _outer_zip(valid_name + "/", b"not a zip"), + _encrypted_outer_zip(valid_name, b"not a zip"), + _outer_zip_members(((valid_name, b"not a zip"),), compression=ZIP_BZIP2), + _outer_zip(valid_name, b"not a zip"), + ) + for outer_payload in cases: + with self.subTest(size=len(outer_payload)), self.assertRaises(NemwebArchiveError): + extract_nested_archive(item, outer_payload, allow_reduced=True) + + def test_rejects_duplicate_source_ids_and_timestamps(self) -> None: + item = ArchivePlanItem( + feed=DISPATCH_SCADA_FEED, + report_date=_REPORT_DATE, + source=ARCHIVE_SOURCE, + url=_SCADA_URL, + ) + cases = ( + ( + (_SCADA_MEMBER, b"x"), + ("PUBLIC_DISPATCHSCADA_202608301500_0000000535210764.zip", b"x"), + ), + ( + (_SCADA_MEMBER, b"x"), + ("PUBLIC_DISPATCHSCADA_202608301455_0000000535210765.zip", b"x"), + ), + ) + for members in cases: + with self.subTest(members=members), self.assertRaises(NemwebArchiveError): + extract_nested_archive( + item, _outer_zip_members(members), allow_reduced=True, + ) + + def test_rejects_nested_compressed_uncompressed_and_ratio_bounds(self) -> None: + item = ArchivePlanItem( + feed=DISPATCH_SCADA_FEED, + report_date=_REPORT_DATE, + source=ARCHIVE_SOURCE, + url=_SCADA_URL, + ) + payload = _outer_zip(_SCADA_MEMBER, b"A" * 10000) + cases = ( + ("MAX_NESTED_ARCHIVE_BYTES", 1), + ("MAX_NESTED_COMPRESSION_RATIO", 1), + ("MAX_OUTER_COMPRESSED_BYTES", 0), + ("MAX_OUTER_UNCOMPRESSED_BYTES", 0), + ("MAX_OUTER_ARCHIVE_BYTES", len(payload) - 1), + ) + for constant, value in cases: + with self.subTest(constant=constant), patch( + f"batterywatch_api.nemweb_archives.{constant}", value + ), self.assertRaises(NemwebArchiveError): + extract_nested_archive(item, payload, allow_reduced=True) + + def test_rejects_unexpected_outer_or_inner_members(self) -> None: + csv_payload = _canonical_fixture_csv( + "dispatch-scada-20260830-1455-reduced.csv", + _SCADA_SOURCE_ID, + "DISPATCHSCADA", + ) + item = ArchivePlanItem( + feed=DISPATCH_SCADA_FEED, + report_date=_REPORT_DATE, + source=ARCHIVE_SOURCE, + url=_SCADA_URL, + ) + cases = ( + _outer_zip_members(((_SCADA_MEMBER, _inner_zip(_SCADA_MEMBER, csv_payload)), + ("unexpected.txt", b"extra"))), + _outer_zip(_SCADA_MEMBER, _inner_zip_with_extra(_SCADA_MEMBER, csv_payload)), + ) + for payload in cases: + with self.subTest(size=len(payload)), self.assertRaises(NemwebArchiveError): + extract_nested_archive(item, payload, allow_reduced=True) + + def test_filters_exact_half_open_range_only_after_full_validation(self) -> None: + second = "PUBLIC_DISPATCHSCADA_202608301500_0000000535210765.zip" + first_payload = _inner_zip( + _SCADA_MEMBER, _scada_csv_at("2026/08/30 14:55:00", _SCADA_SOURCE_ID) + ) + second_payload = _inner_zip( + second, _scada_csv_at("2026/08/30 15:00:00", "0000000535210765") + ) + item = ArchivePlanItem( + feed=DISPATCH_SCADA_FEED, + report_date=_REPORT_DATE, + source=ARCHIVE_SOURCE, + url=_SCADA_URL, + ) + extraction = extract_nested_archive( + item, + _outer_zip_members(((_SCADA_MEMBER, first_payload), (second, second_payload))), + allow_reduced=True, + start=datetime(2026, 8, 30, 4, 55, tzinfo=UTC), + end=datetime(2026, 8, 30, 5, 0, tzinfo=UTC), + ) + self.assertEqual( + tuple(artifact.interval_timestamp for artifact in extraction.nested), + (datetime(2026, 8, 30, 4, 55, tzinfo=UTC),), + ) + with self.assertRaises(NemwebArchiveError): + extract_nested_archive( + item, + _outer_zip_members(( + (_SCADA_MEMBER, first_payload), (second, b"bad inner zip") + )), + allow_reduced=True, + start=datetime(2026, 8, 30, 4, 55, tzinfo=UTC), + end=datetime(2026, 8, 30, 5, 0, tzinfo=UTC), + ) + + def test_parses_authentic_five_region_price_fixture_and_keeps_negative_rrp(self) -> None: + csv_payload = _canonical_fixture_csv( + "dispatch-price-20260830-1500-reduced.csv", + _PRICE_SOURCE_ID, + "DISPATCHIS", + ) + records = parse_dispatch_price_mms_csv( + csv_payload.decode("utf-8"), + source_id=_PRICE_SOURCE_ID, + ingestion_version=0, + ) + self.assertEqual({record.region for record in records}, {"NSW1", "QLD1", "SA1", "TAS1", "VIC1"}) + self.assertEqual( + next(record for record in records if record.region == "NSW1").price_aud_per_mwh, + -6.93755, + ) + item = ArchivePlanItem( + feed=DISPATCHIS_PRICE_FEED, + report_date=_REPORT_DATE, + source=ARCHIVE_SOURCE, + url=_PRICE_URL, + ) + extraction = extract_nested_archive( + item, _outer_zip(_PRICE_MEMBER, _inner_zip(_PRICE_MEMBER, csv_payload)), + allow_reduced=True, + ) + self.assertEqual(extraction.nested[0].interval_timestamp, + datetime(2026, 8, 30, 5, 0, tzinfo=UTC)) + + def test_extracts_reduced_real_scada_archive_with_outer_provenance(self) -> None: + csv_payload = _canonical_fixture_csv( + "dispatch-scada-20260830-1455-reduced.csv", + _SCADA_SOURCE_ID, + "DISPATCHSCADA", + ) + outer_payload = _outer_zip( + _SCADA_MEMBER, + _inner_zip(_SCADA_MEMBER, csv_payload), + ) + item = ArchivePlanItem( + feed=DISPATCH_SCADA_FEED, + report_date=_REPORT_DATE, + source=ARCHIVE_SOURCE, + url=_SCADA_URL, + ) + + extraction = extract_nested_archive( + item, + outer_payload, + allow_reduced=True, + ) + + self.assertEqual( + ( + extraction.outer.url, + extraction.outer.filename, + extraction.outer.report_date, + extraction.outer.sha256, + extraction.outer.raw_bytes, + ), + ( + _SCADA_URL, + "PUBLIC_DISPATCHSCADA_20260830.zip", + _REPORT_DATE, + sha256(outer_payload).hexdigest(), + outer_payload, + ), + ) + self.assertEqual(len(extraction.nested), 1) + nested = extraction.nested[0] + self.assertEqual( + ( + nested.member_name, + nested.source_artifact_id, + nested.interval_timestamp, + nested.sha256, + nested.raw_bytes, + ), + ( + _SCADA_MEMBER, + _SCADA_SOURCE_ID, + datetime(2026, 8, 30, 4, 55, tzinfo=timezone.utc), + sha256(_inner_zip(_SCADA_MEMBER, csv_payload)).hexdigest(), + _inner_zip(_SCADA_MEMBER, csv_payload), + ), + ) + with self.assertRaises(FrozenInstanceError): + extraction.outer.url = "changed" # type: ignore[misc] + with self.assertRaises(FrozenInstanceError): + nested.member_name = "changed" # type: ignore[misc] + with self.assertRaises(FrozenInstanceError): + extraction.nested = () # type: ignore[misc] + + +if __name__ == "__main__": + unittest.main() diff --git a/app/deploy/migrate.sh b/app/deploy/migrate.sh index 5802d48..8793b92 100755 --- a/app/deploy/migrate.sh +++ b/app/deploy/migrate.sh @@ -27,15 +27,19 @@ psql --dbname="$database_url" --no-psqlrc --set=ON_ERROR_STOP=1 \ --file="$app_dir/migrations/002_dispatch_scada_raw_ingestion.sql" psql --dbname="$database_url" --no-psqlrc --set=ON_ERROR_STOP=1 \ --file="$app_dir/migrations/003_dispatch_price_artifacts.sql" +psql --dbname="$database_url" --no-psqlrc --set=ON_ERROR_STOP=1 \ + --file="$app_dir/migrations/004_historical_backfill_ledger.sql" psql --dbname="$database_url" --no-psqlrc --set=ON_ERROR_STOP=1 --tuples-only --no-align \ <<'SQL' -SELECT count(*) = 7 +SELECT count(*) = 10 FROM information_schema.tables WHERE table_schema = 'public' AND table_name IN ( 'generators', 'generator_power_5m', 'generator_soc_5m', 'nem_price_5m', 'dispatch_scada_artifacts', 'raw_dispatch_scada_observations', - 'dispatch_price_artifacts' + 'dispatch_price_artifacts', + 'historical_backfill_runs', + 'historical_backfill_items', 'historical_backfill_events' ); SQL unset database_url diff --git a/app/docs/historical-analytics-iteration.md b/app/docs/historical-analytics-iteration.md new file mode 100644 index 0000000..d9ced55 --- /dev/null +++ b/app/docs/historical-analytics-iteration.md @@ -0,0 +1,255 @@ +# Historical analytics iteration + +## Baseline and release boundary + +This iteration starts from the exact accepted application source deployed as +`/home/dan/batterywatch/releases/20260830-cc7912e8` on `192.168.20.42`. +The preserved Git baseline is commit `24c1983e530f624a5734883d9608578d386b36ac` +on branch `agent/batterywatch-historical-analytics` in the `djurcola-agent` +fork. The 58 tracked files below `app/` have aggregate SHA-256 +`418ca128c253ff1d3fbfd663fd4c75820cc8f5cdcec30804648c0969a3d84151`. +Owner `main` remains unchanged. The previous immutable release and disabled +`batterywatch.service` remain the rollback target. + +## Verified source semantics + +The design is based on live public AEMO/NEMWeb artifacts and EMMS Data Model +5.5, not inferred telemetry: + +- `Dispatch_SCADA` daily archives contain 288 nested interval ZIPs. Their + `UNIT_SCADA` rows publish signed `SCADAVALUE` by DUID. Positive means grid + export/discharge and negative means grid import/charge in BatteryWatch. +- `DispatchIS_Reports` daily archives contain 288 nested interval ZIPs. Their + `PRICE` rows publish regional RRP, intervention, APC and market-suspension + status. Their `REGIONSUM` v9 rows publish regional + `BDU_INITIAL_ENERGY_STORAGE` in MWh. This is regional aggregate data and is + never an asset SOC. +- `Next_Day_Dispatch` `UNIT_SOLUTION` v6 publishes previous-trading-day, + five-minute per-DUID `INITIAL_ENERGY_STORAGE` in MWh. It is authoritative + next-day individual SOC with publication latency, not near-real-time SOC. + Only non-intervention actual rows are effective; all relevant source control + fields and raw artifacts are retained. +- Tasmania currently has no published regional BDU SOC value. Missing values + remain null and are reported, not inferred. +- Maximum storage capacity is a separate, dated AEMO registry observation. + Percent SOC is calculated only when a reviewed denominator was applicable at + the interval. The response retains MWh, percent, denominator, denominator + effective date and provenance. + +## Architecture seams + +### 1. Archive transport and deterministic planning + +`nemweb_archives.py` owns allow-listed archive/current URLs, bounded HTTPS +fetching, daily/monthly outer-archive validation, safe nested ZIP extraction, +and deterministic UTC-range planning. It returns immutable artifact objects; +it does not parse domain rows or write a database. + +A request is a half-open UTC range `[start, end)`. The planner maps it to the +fixed NEM market timezone (UTC+10), plans only intersecting local report dates, +and filters inner interval artifacts back to the exact UTC range. The API and +operator CLI reject non-aware timestamps, non-increasing ranges, dates outside +feed availability and ranges above the configured hard cap. + +### 2. Append-only evidence and resumable execution + +Migration 004 adds: + +- `nemweb_archive_artifacts`: immutable downloaded outer artifacts, hashes, + source URLs, report dates, fetch/publication metadata and raw bytes; +- `ingestion_runs`: requested range, selected feeds, ingestion version, + lifecycle status, deterministic counters and completion/error summary; +- `ingestion_run_items`: one planned source artifact per run with pending, + running, completed, replayed or failed status, attempts and counts. + +The existing interval artifact/raw tables remain authoritative evidence for +SCADA and DispatchIS. An item is marked complete only after its artifact +transaction commits. On interruption, resuming the same run retries only +non-terminal items. If the data transaction committed before the item status, +existing immutable evidence produces a verified replay and the item becomes +`replayed`. Conflicting bytes for an existing source identity fail closed. +Starting a new run over the same range records a new replay history while the +effective observations remain idempotent. + +The backfill is a separate bounded CLI process. It does not stop, reload or +replace `batterywatch-collector.service`; existing uniqueness and guarded +revision ordering make concurrent live/backfill overlap safe. One database +transaction is used per inner artifact, so work is bounded and resumable. +Summaries are JSON with stable key and feed ordering and contain run ID, +requested/effective coverage, planned/completed/replayed/failed counts and the +next resumable item. + +### 3. Authoritative SOC products + +Migration 005 adds append-only raw `UNIT_SOLUTION` SOC observations, dated +storage-capacity history, richer effective individual SOC fields, and a +separate regional BDU SOC series. It never joins regional SOC to a DUID. + +Individual effective SOC fields include `soc_mwh`, nullable `soc_percent`, +nullable `capacity_mwh`, capacity effective timestamp/source, source artifact, +source `LASTCHANGED`, intervention/run status, publication timestamp/status, +ingestion/correction versions and quality flags. The parser keeps valid finite +MWh including zero, rejects ambiguous duplicate effective rows and retains raw +control fields. + +Regional BDU SOC is parsed from the same DispatchIS artifact as price and is +persisted in its own regional table. The collector extends its existing +DispatchIS cycle rather than creating a second poller, so each interval ZIP is +fetched once. Missing Tasmania remains an explicit null/unavailable regional +observation. + +### 4. Historical query and aggregation + +A new bounded database-only historical endpoint preserves the existing API. +Hard maximum range is 366 days and hard maximum output is 2,500 buckets. Preset +resolution is deterministic and UTC anchored: + +| Range | Resolution | +| --- | --- | +| up to 24 hours | 5 minutes | +| over 24 hours through 7 days | 15 minutes | +| over 7 days through 30 days | 1 hour | +| over 30 days through 366 days | 6 hours | + +An optional resolution may only select an equal or coarser allowed bucket. +Database queries remain bounded by generator/region, range and bucket count. +Long-range responses do not materialize every raw row in Python. + +Each bucket returns mean power plus power minimum/maximum, price mean plus price +minimum/maximum, latest individual SOC plus SOC extrema where meaningful, +separate regional SOC, signed net energy, positive exported-energy magnitude, +positive imported-energy magnitude, observed charging cost, observed export +value and observed net energy value. Monetary and energy totals are summed from +raw five-minute observations, never reconstructed from bucket averages. +Coverage reports expected/observed/missing intervals per source, partial +buckets, earliest/latest source times, publication latency/status, source IDs, +resolution and calculation version. Negative prices and extrema are retained. + +### 5. Estimate scenarios + +Raw observations and observed grid-side calculations are immutable. Financial +values are labelled estimates, never profit or settlement revenue. + +The default scenario assumptions are explicit request parameters with bounded +ranges: charging efficiency, discharging efficiency and degradation cost per +MWh. Scenario output is separate from observed grid-side import/export value +and includes the exact formula and assumptions used. The methodology states +that FCAS revenue, contracts, marginal loss factors, network charges, +auxiliary consumption, taxes, fees and all other unavailable settlement inputs +are excluded. + +### 6. Dashboard + +The dashboard adds 24h, 7d, 30d and validated custom UTC controls. One request +feeds synchronized, zoomable power, regional price and authoritative individual +SOC charts; regional BDU SOC is a separate chart and label. Tooltips preserve +average and extrema and show SOC MWh/percent/capacity provenance and next-day +publication state. + +Summary cards show imported/exported energy, observed charging cost, observed +export value, observed estimated net energy value, separate scenario-adjusted +estimate, price/SOC coverage and missing intervals. Loading, error, no-data, +partial/stale and unavailable-SOC states are explicit. Layout remains responsive. + +## Vertical-slice DAG + +All slices use model-policy profile `implementation-default`, integration owner +Hermes Coder, strict assertion-level RED→GREEN cycles, no commit/push, and +fresh supervisor verification. They run sequentially because they share the +same worktree and public contracts. + +### BW-HIST-S1 — archive power/price tracer and resumable run + +- prerequisites: exact deployed baseline and source-semantics verification +- blocked-by: none +- blocks: BW-HIST-S2, BW-HIST-S3, BW-HIST-S4 +- exclusive files/resources: archive/backfill modules and tests, migration 004, + migration/runbook wiring, real reduced archive fixtures +- concurrency group: sequential-historical +- integration owner: Hermes Coder +- model-policy profile: implementation-default +- targeted verification: focused archive/backfill/migration tests, complete + backend suite, Pyright, compileall, shell syntax and diff check +- tracer: one reduced real SCADA daily archive and one reduced real DispatchIS + daily archive plan, ingest, interrupt, resume and replay end to end +- sizing: deliberate >8-file exception because the public seam includes schema, + transport, CLI and one end-to-end persistence test; stop/re-slice at the + 175k context notice or material scope drift + +### BW-HIST-S2 — individual and regional SOC + +- prerequisites: BW-HIST-S1 +- blocked-by: BW-HIST-S1 +- blocks: BW-HIST-S3, BW-HIST-S4 +- exclusive files/resources: SOC parser/storage/backfill integration and tests, + migration 005, capacity provenance, current DispatchIS collector extension, + real reduced NextDay/REGIONSUM fixtures +- concurrency group: sequential-historical +- integration owner: Hermes Coder +- model-policy profile: implementation-default +- targeted verification: focused SOC/parser/repository/collector/migration tests, + complete backend suite, Pyright, compileall and diff check +- tracer: one real reduced NextDay DUID row and one DispatchIS regional set + through parser, persistence, replay and read-back with null-Tasmania proof +- sizing: deliberate >8-file exception for one authoritative end-to-end seam; + re-slice parser/storage from collector integration if context or diff drifts + +### BW-HIST-S3 — bounded historical API and scenario aggregation + +- prerequisites: BW-HIST-S1 and BW-HIST-S2 +- blocked-by: BW-HIST-S1, BW-HIST-S2 +- blocks: BW-HIST-S4 +- exclusive files/resources: API models, historical query repository/service, + migration 006 aggregate/index structures and focused tests +- concurrency group: sequential-historical +- integration owner: Hermes Coder +- model-policy profile: implementation-default +- targeted verification: focused raw/downsample/bounds/extrema/coverage/scenario + tests, PostgreSQL migration/query tests, complete backend suite, Pyright, + compileall and diff check +- tracer: 30-day database query returning bounded one-hour buckets while exact + raw totals, negative-price extrema and provenance remain correct +- sizing: expected within eight production/test paths; split query and API model + only if the 175k context notice coincides with incomplete behavior + +### BW-HIST-S4 — historical dashboard + +- prerequisites: BW-HIST-S3 +- blocked-by: BW-HIST-S3 +- blocks: deployment and browser acceptance +- exclusive files/resources: frontend API/types/components/styles/tests and + user-facing methodology documentation +- concurrency group: sequential-historical +- integration owner: Hermes Coder +- model-policy profile: implementation-default +- targeted verification: frontend tests/typecheck/build, static DOM verification, + backend suite, Pyright, npm audit and diff check +- tracer: 30-day control renders synchronized real-contract charts, separate + regional SOC, summaries and explicit partial/unavailable states +- sizing: expected within eight frontend paths; split components from final + visual polish if runtime context or visual scope drifts + +## Deployment and acceptance ladder + +1. Freeze and independently review the exact local source hash. +2. Commit/push only with the isolated `djurcola-agent` identity and update an + unmerged PR against `djurcola/BatteryWatch:main`. +3. Record current release/service/database/listener evidence and create a fresh + custom-format PostgreSQL backup. Verify SHA-256, `pg_restore --list`, isolated + restore, database integrity, indexes and domain-count parity. +4. Stage an immutable candidate release without changing `current`; build its + venv/frontend and apply migrations under the backup gate. +5. Run fixture and reduced-real migration/backfill acceptance in an isolated + database, then run candidate API on port 18080 against the real database. +6. Switch `current`, restart only the two BatteryWatch user units, verify + database mode, then execute the bounded 30-day backfill while the live + collector remains enabled and active. +7. Prove data coverage/counts, idempotent replay, deterministic resume summary, + API bounds/downsampling/estimates, browser controls/charts, loopback-only + PostgreSQL and at least two subsequent live collection intervals. +8. Verify rollback by exercising the documented prior-release service path + without deleting additive data; restore the candidate after the rollback + check only if all acceptance gates remain green. + +Cloudflare, the user-managed reverse proxy, unrelated hosts, credentials and +owner main are outside this iteration. diff --git a/app/migrations/004_historical_backfill_ledger.sql b/app/migrations/004_historical_backfill_ledger.sql new file mode 100644 index 0000000..b2b41fb --- /dev/null +++ b/app/migrations/004_historical_backfill_ledger.sql @@ -0,0 +1,66 @@ +-- Resumable operator-controlled historical backfill run state and event history. + +CREATE TABLE IF NOT EXISTS historical_backfill_runs ( + run_id TEXT PRIMARY KEY CHECK ( + length(run_id) BETWEEN 1 AND 64 + AND run_id ~ '^[A-Za-z0-9._-]+$' + ), + requested_start TIMESTAMPTZ NOT NULL, + requested_end TIMESTAMPTZ NOT NULL, + ingestion_version BIGINT NOT NULL CHECK (ingestion_version >= 0), + status TEXT NOT NULL DEFAULT 'running' CHECK ( + status IN ('running', 'completed', 'partial', 'failed') + ), + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + completed_at TIMESTAMPTZ, + CHECK (requested_end > requested_start), + CHECK (requested_end - requested_start <= INTERVAL '366 days') +); + +CREATE TABLE IF NOT EXISTS historical_backfill_items ( + run_id TEXT NOT NULL, + feed TEXT NOT NULL CHECK (feed IN ('dispatch_scada', 'dispatch_price')), + report_date DATE NOT NULL, + source_url TEXT NOT NULL CHECK ( + source_url ~ '^https://www[.]nemweb[.]com[.]au/REPORTS/ARCHIVE/' + ), + status TEXT NOT NULL DEFAULT 'pending' CHECK ( + status IN ('pending', 'running', 'completed', 'failed') + ), + attempt_count BIGINT NOT NULL DEFAULT 0 CHECK (attempt_count >= 0), + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + started_at TIMESTAMPTZ, + completed_at TIMESTAMPTZ, + PRIMARY KEY (run_id, feed, report_date), + CONSTRAINT historical_backfill_items_run_fk + FOREIGN KEY (run_id) + REFERENCES historical_backfill_runs (run_id) + ON DELETE RESTRICT +); + +CREATE TABLE IF NOT EXISTS historical_backfill_events ( + event_seq BIGSERIAL PRIMARY KEY, + run_id TEXT NOT NULL, + feed TEXT NOT NULL CHECK (feed IN ('dispatch_scada', 'dispatch_price')), + report_date DATE NOT NULL, + event_type TEXT NOT NULL CHECK ( + event_type IN ('planned', 'recovered', 'claimed', 'completed', 'failed') + ), + attempt_number BIGINT NOT NULL DEFAULT 0 CHECK (attempt_number >= 0), + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT historical_backfill_events_item_fk + FOREIGN KEY (run_id, feed, report_date) + REFERENCES historical_backfill_items (run_id, feed, report_date) + ON DELETE RESTRICT +); + +CREATE INDEX IF NOT EXISTS historical_backfill_runs_status_idx + ON historical_backfill_runs (status, updated_at, run_id); + +CREATE INDEX IF NOT EXISTS historical_backfill_items_claim_idx + ON historical_backfill_items (run_id, status, feed, report_date); + +CREATE INDEX IF NOT EXISTS historical_backfill_events_order_idx + ON historical_backfill_events (run_id, event_seq); From 081547c51e6ff0c90998f619bdecb6f62e303e5e Mon Sep 17 00:00:00 2001 From: djurcola-agent Date: Sun, 30 Aug 2026 17:59:14 +1000 Subject: [PATCH 03/30] feat: claim historical backfill items deterministically --- .../batterywatch_api/backfill_ledger.py | 80 +++++++++++++++++- app/backend/tests/test_backfill_ledger.py | 82 +++++++++++++++++++ 2 files changed, 160 insertions(+), 2 deletions(-) diff --git a/app/backend/batterywatch_api/backfill_ledger.py b/app/backend/batterywatch_api/backfill_ledger.py index b63b883..f2ea869 100644 --- a/app/backend/batterywatch_api/backfill_ledger.py +++ b/app/backend/batterywatch_api/backfill_ledger.py @@ -24,6 +24,15 @@ class BackfillPlanItem: source_url: str +@dataclass(frozen=True, slots=True) +class BackfillClaim: + run_id: str + feed: str + report_date: date + source_url: str + attempt_number: int + + @dataclass(frozen=True, slots=True) class BackfillEnsureResult: created: bool @@ -94,6 +103,24 @@ def _managed_cursor(connection: _Connection) -> Iterator[_Cursor]: ORDER BY feed, report_date """ +_CLAIM_SELECT_SQL = """ +SELECT feed, report_date, source_url +FROM historical_backfill_items +WHERE run_id = %s AND status IN ('pending', 'failed') +ORDER BY feed, report_date +LIMIT 1 +FOR UPDATE SKIP LOCKED +""" + +_CLAIM_UPDATE_SQL = """ +UPDATE historical_backfill_items +SET status = 'running', attempt_count = attempt_count + 1, + started_at = CURRENT_TIMESTAMP, updated_at = CURRENT_TIMESTAMP +WHERE run_id = %s AND feed = %s AND report_date = %s + AND status IN ('pending', 'failed') +RETURNING feed, report_date, source_url, attempt_count +""" + _RECOVER_ITEMS_SQL = """ UPDATE historical_backfill_items SET status = 'pending', updated_at = CURRENT_TIMESTAMP, started_at = NULL @@ -121,9 +148,13 @@ def _managed_cursor(connection: _Connection) -> Iterator[_Cursor]: } -def _validate_spec(spec: BackfillRunSpec) -> None: - if _RUN_ID_PATTERN.fullmatch(spec.run_id) is None: +def _validate_run_id(run_id: str) -> None: + if not isinstance(run_id, str) or _RUN_ID_PATTERN.fullmatch(run_id) is None: raise ValueError("invalid backfill run id") + + +def _validate_spec(spec: BackfillRunSpec) -> None: + _validate_run_id(spec.run_id) if not isinstance(spec.requested_start, datetime) or not isinstance( spec.requested_end, datetime ): @@ -160,6 +191,50 @@ class PostgreSQLBackfillLedger: def __init__(self, connection: _Connection): self._connection = connection + def claim_next(self, run_id: str) -> BackfillClaim | None: + _validate_run_id(run_id) + try: + with _managed_cursor(self._connection) as cursor: + cursor.execute(_CLAIM_SELECT_SQL, (run_id,)) + selected = cursor.fetchone() + if selected is None: + result = None + else: + feed, report_date, source_url = selected + cursor.execute( + _CLAIM_UPDATE_SQL, + (run_id, feed, report_date), + ) + claimed = cursor.fetchone() + if claimed is None: + raise RuntimeError("claimed backfill item disappeared") + claimed_feed, claimed_date, claimed_url, attempt_number = claimed + cursor.execute( + _EVENT_INSERT_SQL, + ( + run_id, + claimed_feed, + claimed_date, + "claimed", + attempt_number, + ), + ) + result = BackfillClaim( + run_id, + claimed_feed, + claimed_date, + claimed_url, + attempt_number, + ) + self._connection.commit() + return result + except Exception: + try: + self._connection.rollback() + except Exception: + pass + raise + def ensure_run( self, spec: BackfillRunSpec, @@ -237,6 +312,7 @@ def ensure_run( __all__ = [ + "BackfillClaim", "BackfillEnsureResult", "BackfillPlanItem", "BackfillRunConflictError", diff --git a/app/backend/tests/test_backfill_ledger.py b/app/backend/tests/test_backfill_ledger.py index 2d661d6..c8c22a1 100644 --- a/app/backend/tests/test_backfill_ledger.py +++ b/app/backend/tests/test_backfill_ledger.py @@ -6,6 +6,7 @@ import unittest from batterywatch_api.backfill_ledger import ( + BackfillClaim, BackfillEnsureResult, BackfillPlanItem, BackfillRunConflictError, @@ -94,6 +95,87 @@ def test_deploys_additive_run_item_and_event_schema(self) -> None: class PostgreSQLBackfillLedgerTests(unittest.TestCase): + def test_claim_next_claims_deterministic_item_and_appends_event(self) -> None: + run_id = "run-20260828" + report_date = date(2026, 8, 28) + source_url = ( + "https://www.nemweb.com.au/REPORTS/ARCHIVE/DispatchIS_Reports/" + "PUBLIC_DISPATCHIS_20260828.zip" + ) + connection = FakeConnection( + fetchone_results=( + ("dispatch_price", report_date, source_url), + ("dispatch_price", report_date, source_url, 4), + ) + ) + + claim = PostgreSQLBackfillLedger(connection).claim_next(run_id) + + self.assertEqual( + claim, + BackfillClaim(run_id, "dispatch_price", report_date, source_url, 4), + ) + self.assertEqual((connection.commits, connection.rollbacks), (1, 0)) + self.assertEqual((connection.cursor_calls, connection.closed_cursors), (1, 1)) + self.assertEqual(len(connection.executions), 3) + self.assertIn("status IN ('pending', 'failed')", connection.executions[0][0]) + self.assertIn("ORDER BY feed, report_date", connection.executions[0][0]) + self.assertIn("FOR UPDATE SKIP LOCKED", connection.executions[0][0]) + self.assertIn("UPDATE historical_backfill_items", connection.executions[1][0]) + self.assertEqual( + connection.executions[1][1], (run_id, "dispatch_price", report_date) + ) + self.assertEqual( + connection.executions[2][1], + (run_id, "dispatch_price", report_date, "claimed", 4), + ) + + def test_claim_next_database_failure_rolls_back_and_reraises_same_error(self) -> None: + report_date = date(2026, 8, 28) + source_url = ( + "https://www.nemweb.com.au/REPORTS/ARCHIVE/DispatchIS_Reports/" + "PUBLIC_DISPATCHIS_20260828.zip" + ) + failure = RuntimeError("injected database failure") + connection = FakeConnection( + fetchone_results=(("dispatch_price", report_date, source_url),), + fail_on_execute=2, + failure=failure, + ) + + with self.assertRaises(RuntimeError) as raised: + PostgreSQLBackfillLedger(connection).claim_next("run-20260828") + + self.assertIs(raised.exception, failure) + self.assertEqual((connection.commits, connection.rollbacks), (0, 1)) + self.assertEqual(connection.closed_cursors, 1) + + def test_claim_next_rejects_invalid_run_id_before_sql(self) -> None: + connection = FakeConnection() + + with self.assertRaises(ValueError): + PostgreSQLBackfillLedger(connection).claim_next("bad id") + + self.assertEqual( + (connection.executions, connection.cursor_calls, + connection.commits, connection.rollbacks), + ([], 0, 0, 0), + ) + + def test_claim_next_returns_none_when_no_item_is_eligible(self) -> None: + connection = FakeConnection(fetchone_results=(None,)) + + claim = PostgreSQLBackfillLedger(connection).claim_next("run-20260828") + + self.assertIsNone(claim) + self.assertEqual((connection.commits, connection.rollbacks), (1, 0)) + self.assertEqual((connection.cursor_calls, connection.closed_cursors), (1, 1)) + self.assertEqual(len(connection.executions), 1) + self.assertNotIn( + "INSERT INTO historical_backfill_events", + connection.executions[0][0], + ) + def test_ensure_new_run_plans_items_and_events_in_deterministic_order(self) -> None: spec = BackfillRunSpec( "run-20260828", From c7dc9c162388d5feab5b98ca7faff4bf3c7f5786 Mon Sep 17 00:00:00 2001 From: djurcola-agent Date: Sun, 30 Aug 2026 18:18:48 +1000 Subject: [PATCH 04/30] feat: add immutable historical archive artifact schema --- app/backend/tests/test_backfill_ledger.py | 4 +- .../tests/test_dispatch_price_ingestion.py | 4 +- .../test_historical_artifact_migration.py | 61 +++++++++++++++++++ app/deploy/migrate.sh | 7 ++- .../005_historical_source_artifacts.sql | 60 ++++++++++++++++++ 5 files changed, 130 insertions(+), 6 deletions(-) create mode 100644 app/backend/tests/test_historical_artifact_migration.py create mode 100644 app/migrations/005_historical_source_artifacts.sql diff --git a/app/backend/tests/test_backfill_ledger.py b/app/backend/tests/test_backfill_ledger.py index c8c22a1..467ac54 100644 --- a/app/backend/tests/test_backfill_ledger.py +++ b/app/backend/tests/test_backfill_ledger.py @@ -83,8 +83,8 @@ def test_deploys_additive_run_item_and_event_schema(self) -> None: self.assertIn(f"'{table}'", migrate_script) self.assertIn("004_historical_backfill_ledger.sql", migrate_script) - self.assertIn("SELECT count(*) = 10", migrate_script) - self.assertEqual(migrate_script.count('--dbname="$database_url"'), 5) + self.assertIn("SELECT count(*) = 12", migrate_script) + self.assertEqual(migrate_script.count('--dbname="$database_url"'), 6) self.assertIn("ON DELETE RESTRICT", migration) self.assertIn("event_seq BIGSERIAL PRIMARY KEY", migration) self.assertIn("historical_backfill_items_claim_idx", migration) diff --git a/app/backend/tests/test_dispatch_price_ingestion.py b/app/backend/tests/test_dispatch_price_ingestion.py index 404eef2..c1c5c31 100644 --- a/app/backend/tests/test_dispatch_price_ingestion.py +++ b/app/backend/tests/test_dispatch_price_ingestion.py @@ -223,11 +223,11 @@ def test_deploy_migration_applies_and_verifies_price_artifact_table(self) -> Non self.assertIn("raw_zip BYTEA NOT NULL", migration) self.assertIn("PUBLIC_DISPATCHIS_", migration) self.assertIn("003_dispatch_price_artifacts.sql", migrate_script) - self.assertIn("SELECT count(*) = 10", migrate_script) + self.assertIn("SELECT count(*) = 12", migrate_script) self.assertIn("'dispatch_price_artifacts'", migrate_script) self.assertIn("004_historical_backfill_ledger.sql", migrate_script) self.assertIn("database_url=$BATTERYWATCH_DATABASE_URL", migrate_script) - self.assertEqual(migrate_script.count('--dbname="$database_url"'), 5) + self.assertEqual(migrate_script.count('--dbname="$database_url"'), 6) self.assertNotIn("export PGDATABASE=", migrate_script) diff --git a/app/backend/tests/test_historical_artifact_migration.py b/app/backend/tests/test_historical_artifact_migration.py new file mode 100644 index 0000000..cd30f69 --- /dev/null +++ b/app/backend/tests/test_historical_artifact_migration.py @@ -0,0 +1,61 @@ +"""Contract tests for the immutable historical source-artifact schema.""" + +from pathlib import Path +import unittest + + +class HistoricalArtifactMigrationTests(unittest.TestCase): + def test_migration_defines_immutable_artifacts_and_deploys_last(self) -> None: + app_root = Path(__file__).resolve().parents[2] + migration = (app_root / "migrations" / "005_historical_source_artifacts.sql").read_text( + encoding="utf-8" + ) + migrate_script = (app_root / "deploy" / "migrate.sh").read_text(encoding="utf-8") + + self.assertIn("CREATE TABLE IF NOT EXISTS historical_source_artifacts", migration) + self.assertIn( + "artifact_sha256 TEXT PRIMARY KEY CHECK (artifact_sha256 ~ '^[0-9a-f]{64}$')", + migration, + ) + self.assertIn("feed", migration) + self.assertIn("dispatch_price", migration) + self.assertIn("dispatch_scada", migration) + self.assertIn("report_date", migration) + self.assertIn("source_url", migration) + self.assertIn("filename", migration) + self.assertIn("byte_count", migration) + self.assertIn("raw_bytes BYTEA NOT NULL", migration) + self.assertIn("stored_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP", migration) + self.assertIn("UNIQUE (artifact_sha256, feed, report_date)", migration) + self.assertIn("byte_count = octet_length(raw_bytes)", migration) + self.assertIn("DispatchIS_Reports/", migration) + self.assertIn("Dispatch_SCADA/", migration) + self.assertIn("to_char(report_date, 'YYYYMMDD')", migration) + self.assertIn("CREATE TABLE IF NOT EXISTS historical_backfill_item_artifacts", migration) + self.assertIn("PRIMARY KEY (run_id, feed, report_date, attempt_number)", migration) + self.assertIn("attempt_number > 0", migration) + self.assertIn("REFERENCES historical_backfill_items", migration) + self.assertIn("REFERENCES historical_source_artifacts", migration) + self.assertIn("ON DELETE RESTRICT", migration) + self.assertIn("downloaded_at TIMESTAMPTZ NOT NULL", migration) + self.assertIn("source_last_modified TIMESTAMPTZ", migration) + self.assertIn("recorded_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP", migration) + self.assertIn("source_last_modified IS NULL OR source_last_modified <= downloaded_at", migration) + self.assertIn("historical_source_artifacts_feed_report_date_idx", migration) + self.assertIn("historical_backfill_item_artifacts_artifact_sha256_idx", migration) + + self.assertIn("004_historical_backfill_ledger.sql", migrate_script) + self.assertIn("005_historical_source_artifacts.sql", migrate_script) + self.assertIn("SELECT count(*) = 12", migrate_script) + for table in ("historical_source_artifacts", "historical_backfill_item_artifacts"): + self.assertIn(f"'{table}'", migrate_script) + self.assertEqual(migrate_script.count('--dbname="$database_url"'), 6) + + upper = migration.upper() + for forbidden in ("DROP ", "ALTER ", "TRUNCATE "): + self.assertNotIn(forbidden, upper) + self.assertNotRegex(upper, r"\bDELETE\s+FROM\b") + + +if __name__ == "__main__": + unittest.main() diff --git a/app/deploy/migrate.sh b/app/deploy/migrate.sh index 8793b92..33b291e 100755 --- a/app/deploy/migrate.sh +++ b/app/deploy/migrate.sh @@ -29,9 +29,11 @@ psql --dbname="$database_url" --no-psqlrc --set=ON_ERROR_STOP=1 \ --file="$app_dir/migrations/003_dispatch_price_artifacts.sql" psql --dbname="$database_url" --no-psqlrc --set=ON_ERROR_STOP=1 \ --file="$app_dir/migrations/004_historical_backfill_ledger.sql" +psql --dbname="$database_url" --no-psqlrc --set=ON_ERROR_STOP=1 \ + --file="$app_dir/migrations/005_historical_source_artifacts.sql" psql --dbname="$database_url" --no-psqlrc --set=ON_ERROR_STOP=1 --tuples-only --no-align \ <<'SQL' -SELECT count(*) = 10 +SELECT count(*) = 12 FROM information_schema.tables WHERE table_schema = 'public' AND table_name IN ( @@ -39,7 +41,8 @@ WHERE table_schema = 'public' 'dispatch_scada_artifacts', 'raw_dispatch_scada_observations', 'dispatch_price_artifacts', 'historical_backfill_runs', - 'historical_backfill_items', 'historical_backfill_events' + 'historical_backfill_items', 'historical_backfill_events', + 'historical_source_artifacts', 'historical_backfill_item_artifacts' ); SQL unset database_url diff --git a/app/migrations/005_historical_source_artifacts.sql b/app/migrations/005_historical_source_artifacts.sql new file mode 100644 index 0000000..046e587 --- /dev/null +++ b/app/migrations/005_historical_source_artifacts.sql @@ -0,0 +1,60 @@ +-- Immutable official daily archive bytes and their historical backfill attempts. + +CREATE TABLE IF NOT EXISTS historical_source_artifacts ( + artifact_sha256 TEXT PRIMARY KEY CHECK (artifact_sha256 ~ '^[0-9a-f]{64}$'), + feed TEXT NOT NULL CHECK (feed IN ('dispatch_price', 'dispatch_scada')), + report_date DATE NOT NULL, + source_url TEXT NOT NULL, + filename TEXT NOT NULL, + byte_count BIGINT NOT NULL CHECK (byte_count > 0), + raw_bytes BYTEA NOT NULL CHECK (octet_length(raw_bytes) > 0), + stored_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT historical_source_artifacts_archive_identity_ck CHECK ( + ( + feed = 'dispatch_price' + AND source_url = + 'https://www.nemweb.com.au/REPORTS/ARCHIVE/DispatchIS_Reports/' || filename + AND filename = 'PUBLIC_DISPATCHIS_' || to_char(report_date, 'YYYYMMDD') || '.zip' + ) + OR ( + feed = 'dispatch_scada' + AND source_url = + 'https://www.nemweb.com.au/REPORTS/ARCHIVE/Dispatch_SCADA/' || filename + AND filename = 'PUBLIC_DISPATCHSCADA_' || to_char(report_date, 'YYYYMMDD') || '.zip' + ) + ), + CONSTRAINT historical_source_artifacts_byte_count_ck CHECK ( + byte_count = octet_length(raw_bytes) + ), + CONSTRAINT historical_source_artifacts_link_target_uq + UNIQUE (artifact_sha256, feed, report_date) +); + +CREATE TABLE IF NOT EXISTS historical_backfill_item_artifacts ( + run_id TEXT NOT NULL, + feed TEXT NOT NULL CHECK (feed IN ('dispatch_price', 'dispatch_scada')), + report_date DATE NOT NULL, + attempt_number BIGINT NOT NULL CHECK (attempt_number > 0), + artifact_sha256 TEXT NOT NULL, + downloaded_at TIMESTAMPTZ NOT NULL, + source_last_modified TIMESTAMPTZ, + recorded_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (run_id, feed, report_date, attempt_number), + CONSTRAINT historical_backfill_item_artifacts_item_fk + FOREIGN KEY (run_id, feed, report_date) + REFERENCES historical_backfill_items (run_id, feed, report_date) + ON DELETE RESTRICT, + CONSTRAINT historical_backfill_item_artifacts_source_fk + FOREIGN KEY (artifact_sha256, feed, report_date) + REFERENCES historical_source_artifacts (artifact_sha256, feed, report_date) + ON DELETE RESTRICT, + CONSTRAINT historical_backfill_item_artifacts_last_modified_ck CHECK ( + source_last_modified IS NULL OR source_last_modified <= downloaded_at + ) +); + +CREATE INDEX IF NOT EXISTS historical_source_artifacts_feed_report_date_idx + ON historical_source_artifacts (feed, report_date); + +CREATE INDEX IF NOT EXISTS historical_backfill_item_artifacts_artifact_sha256_idx + ON historical_backfill_item_artifacts (artifact_sha256); From a16d95cce2134d311ea3995dab5b5d8c78576b40 Mon Sep 17 00:00:00 2001 From: djurcola-agent Date: Sun, 30 Aug 2026 18:43:27 +1000 Subject: [PATCH 05/30] feat: record first historical archive artifact --- .../batterywatch_api/backfill_artifacts.py | 150 ++++++++++++++++++ app/backend/tests/test_backfill_artifacts.py | 147 +++++++++++++++++ 2 files changed, 297 insertions(+) create mode 100644 app/backend/batterywatch_api/backfill_artifacts.py create mode 100644 app/backend/tests/test_backfill_artifacts.py diff --git a/app/backend/batterywatch_api/backfill_artifacts.py b/app/backend/batterywatch_api/backfill_artifacts.py new file mode 100644 index 0000000..4a6f65d --- /dev/null +++ b/app/backend/batterywatch_api/backfill_artifacts.py @@ -0,0 +1,150 @@ +"""Immutable historical archive registration through a narrow public seam.""" + +from dataclasses import dataclass +from datetime import datetime +from hashlib import sha256 +from typing import Any, Protocol + +from .backfill_ledger import BackfillClaim + + +class _Cursor(Protocol): + def execute(self, statement: str, parameters: tuple[object, ...]) -> None: ... + + def fetchone(self) -> Any: ... + + def close(self) -> None: ... + + +class _Connection(Protocol): + def cursor(self) -> _Cursor: ... + + def commit(self) -> None: ... + + +@dataclass(frozen=True, slots=True) +class BackfillArtifactReceipt: + """Raw archive evidence downloaded for a claimed backfill item.""" + + claim: BackfillClaim + downloaded_at: datetime + source_last_modified: datetime | None + raw_archive: bytes + + +@dataclass(frozen=True, slots=True) +class BackfillArtifactResult: + """Identity of a persisted archive and whether its attempt link was replayed.""" + + artifact_sha256: str + byte_count: int + replayed: bool + + +_ITEM_LOCK_SQL = """ +SELECT source_url, status, attempt_count +FROM historical_backfill_items +WHERE run_id = %s + AND feed = %s + AND report_date = %s +FOR UPDATE +""" + +_ARTIFACT_INSERT_SQL = """ +INSERT INTO historical_source_artifacts ( + artifact_sha256, + feed, + report_date, + source_url, + filename, + byte_count, + raw_archive +) +VALUES (%s, %s, %s, %s, %s, %s, %s) +ON CONFLICT DO NOTHING RETURNING 1 +""" + +_LINK_INSERT_SQL = """ +INSERT INTO historical_backfill_item_artifacts ( + run_id, + feed, + report_date, + attempt_number, + artifact_sha256, + downloaded_at, + source_last_modified +) +VALUES (%s, %s, %s, %s, %s, %s, %s) +ON CONFLICT DO NOTHING RETURNING 1 +""" + +_EVENT_INSERT_SQL = """ +INSERT INTO historical_backfill_events ( + run_id, + feed, + report_date, + event_type, + attempt_number +) +VALUES (%s, %s, %s, %s, %s) +""" + + +class PostgreSQLBackfillArtifactRegistrar: + """Persist first-seen archive evidence for one live backfill claim.""" + + def __init__(self, connection: _Connection) -> None: + self._connection = connection + + def record(self, receipt: BackfillArtifactReceipt) -> BackfillArtifactResult: + claim = receipt.claim + digest = sha256(receipt.raw_archive).hexdigest() + byte_count = len(receipt.raw_archive) + filename = claim.source_url.rsplit("/", 1)[-1] + cursor = self._connection.cursor() + try: + cursor.execute( + _ITEM_LOCK_SQL, + (claim.run_id, claim.feed, claim.report_date), + ) + cursor.fetchone() + cursor.execute( + _ARTIFACT_INSERT_SQL, + ( + digest, + claim.feed, + claim.report_date, + claim.source_url, + filename, + byte_count, + receipt.raw_archive, + ), + ) + cursor.fetchone() + cursor.execute( + _LINK_INSERT_SQL, + ( + claim.run_id, + claim.feed, + claim.report_date, + claim.attempt_number, + digest, + receipt.downloaded_at, + receipt.source_last_modified, + ), + ) + cursor.fetchone() + cursor.execute( + _EVENT_INSERT_SQL, + ( + claim.run_id, + claim.feed, + claim.report_date, + "artifact_recorded", + claim.attempt_number, + ), + ) + self._connection.commit() + return BackfillArtifactResult(digest, byte_count, False) + finally: + cursor.close() diff --git a/app/backend/tests/test_backfill_artifacts.py b/app/backend/tests/test_backfill_artifacts.py new file mode 100644 index 0000000..90f7413 --- /dev/null +++ b/app/backend/tests/test_backfill_artifacts.py @@ -0,0 +1,147 @@ +"""Behavior tests for immutable historical archive artifact registration.""" + +from datetime import date, datetime, timezone +import hashlib +import unittest + +from batterywatch_api.backfill_artifacts import ( + BackfillArtifactReceipt, + BackfillArtifactResult, + PostgreSQLBackfillArtifactRegistrar, +) +from batterywatch_api.backfill_ledger import BackfillClaim + + +UTC = timezone.utc +REPORT_DATE = date(2026, 8, 28) +SOURCE_URL = ( + "https://www.nemweb.com.au/REPORTS/ARCHIVE/DispatchIS_Reports/" + "PUBLIC_DISPATCHIS_20260828.zip" +) + + +class FakeCursor: + def __init__(self, connection) -> None: + self.connection = connection + + def execute(self, statement, parameters) -> None: + self.connection.executions.append((statement, tuple(parameters))) + if len(self.connection.executions) == self.connection.fail_on_execute: + raise self.connection.failure + + def fetchone(self): + if self.connection.fetchone_results: + return self.connection.fetchone_results.pop(0) + return None + + def close(self) -> None: + self.connection.closed_cursors += 1 + + +class FakeConnection: + def __init__(self, *, fetchone_results=(), fail_on_execute=None, failure=None) -> None: + self.fetchone_results = list(fetchone_results) + self.fail_on_execute = fail_on_execute + self.failure = failure + self.executions = [] + self.cursor_calls = self.closed_cursors = 0 + self.commits = self.rollbacks = 0 + + def cursor(self): + self.cursor_calls += 1 + return FakeCursor(self) + + def commit(self) -> None: + self.commits += 1 + + def rollback(self) -> None: + self.rollbacks += 1 + + +def receipt(raw_archive: bytes = b"official archive bytes") -> BackfillArtifactReceipt: + return BackfillArtifactReceipt( + BackfillClaim( + "run-20260828", + "dispatch_price", + REPORT_DATE, + SOURCE_URL, + 3, + ), + datetime(2026, 8, 29, 2, tzinfo=UTC), + datetime(2026, 8, 29, 1, 30, tzinfo=UTC), + raw_archive, + ) + + +class PostgreSQLBackfillArtifactRegistrarTests(unittest.TestCase): + def test_records_content_link_and_event_in_one_transaction(self) -> None: + evidence = receipt() + digest = hashlib.sha256(evidence.raw_archive).hexdigest() + connection = FakeConnection( + fetchone_results=( + (SOURCE_URL, "running", 3), + (1,), + (1,), + ) + ) + + result = PostgreSQLBackfillArtifactRegistrar(connection).record(evidence) + + self.assertEqual( + result, + BackfillArtifactResult(digest, len(evidence.raw_archive), False), + ) + self.assertEqual((connection.commits, connection.rollbacks), (1, 0)) + self.assertEqual((connection.cursor_calls, connection.closed_cursors), (1, 1)) + self.assertEqual(len(connection.executions), 4) + + lock_sql, lock_parameters = connection.executions[0] + self.assertIn("FROM historical_backfill_items", lock_sql) + self.assertIn("FOR UPDATE", lock_sql) + self.assertEqual( + lock_parameters, + ("run-20260828", "dispatch_price", REPORT_DATE), + ) + + artifact_sql, artifact_parameters = connection.executions[1] + self.assertIn("INSERT INTO historical_source_artifacts", artifact_sql) + self.assertIn("ON CONFLICT DO NOTHING RETURNING 1", artifact_sql) + self.assertEqual( + artifact_parameters, + ( + digest, + "dispatch_price", + REPORT_DATE, + SOURCE_URL, + "PUBLIC_DISPATCHIS_20260828.zip", + len(evidence.raw_archive), + evidence.raw_archive, + ), + ) + + link_sql, link_parameters = connection.executions[2] + self.assertIn("INSERT INTO historical_backfill_item_artifacts", link_sql) + self.assertIn("ON CONFLICT DO NOTHING RETURNING 1", link_sql) + self.assertEqual( + link_parameters, + ( + "run-20260828", + "dispatch_price", + REPORT_DATE, + 3, + digest, + evidence.downloaded_at, + evidence.source_last_modified, + ), + ) + + event_sql, event_parameters = connection.executions[3] + self.assertIn("INSERT INTO historical_backfill_events", event_sql) + self.assertEqual( + event_parameters, + ("run-20260828", "dispatch_price", REPORT_DATE, "artifact_recorded", 3), + ) + + +if __name__ == "__main__": + unittest.main() From 9aad7ca010e9eabcf784377a2d5288a9683f94fb Mon Sep 17 00:00:00 2001 From: djurcola-agent Date: Sun, 30 Aug 2026 18:56:27 +1000 Subject: [PATCH 06/30] feat: make archive artifact registration replay-safe --- .../batterywatch_api/backfill_artifacts.py | 157 +++++++++++++-- app/backend/tests/test_backfill_artifacts.py | 189 +++++++++++++++++- 2 files changed, 329 insertions(+), 17 deletions(-) diff --git a/app/backend/batterywatch_api/backfill_artifacts.py b/app/backend/batterywatch_api/backfill_artifacts.py index 4a6f65d..9005c47 100644 --- a/app/backend/batterywatch_api/backfill_artifacts.py +++ b/app/backend/batterywatch_api/backfill_artifacts.py @@ -1,13 +1,29 @@ """Immutable historical archive registration through a narrow public seam.""" from dataclasses import dataclass -from datetime import datetime +from datetime import date, datetime, timedelta from hashlib import sha256 +import re from typing import Any, Protocol from .backfill_ledger import BackfillClaim +MAX_ARCHIVE_BYTES = 512 * 1024 * 1024 +_MAX_ATTEMPT_NUMBER = 2_147_483_647 +_RUN_ID_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:-]{0,127}") +_FEED_URL_PREFIXES = { + "dispatch_scada": ( + "https://www.nemweb.com.au/REPORTS/ARCHIVE/Dispatch_SCADA/" + "PUBLIC_DISPATCHSCADA_" + ), + "dispatch_price": ( + "https://www.nemweb.com.au/REPORTS/ARCHIVE/DispatchIS_Reports/" + "PUBLIC_DISPATCHIS_" + ), +} + + class _Cursor(Protocol): def execute(self, statement: str, parameters: tuple[object, ...]) -> None: ... @@ -21,6 +37,12 @@ def cursor(self) -> _Cursor: ... def commit(self) -> None: ... + def rollback(self) -> None: ... + + +class BackfillArtifactConflictError(ValueError): + """Stored artifact evidence conflicts with the current receipt.""" + @dataclass(frozen=True, slots=True) class BackfillArtifactReceipt: @@ -41,6 +63,48 @@ class BackfillArtifactResult: replayed: bool +def _validate_receipt(receipt: object) -> BackfillArtifactReceipt: + if type(receipt) is not BackfillArtifactReceipt: + raise TypeError("receipt must be BackfillArtifactReceipt") + claim = receipt.claim + if type(claim) is not BackfillClaim: + raise TypeError("receipt claim must be BackfillClaim") + if not isinstance(claim.run_id, str) or _RUN_ID_RE.fullmatch(claim.run_id) is None: + raise ValueError("invalid run_id") + prefix = _FEED_URL_PREFIXES.get(claim.feed) + if prefix is None: + raise ValueError("invalid feed") + if type(claim.report_date) is not date: + raise TypeError("report_date must be a date") + expected_url = f"{prefix}{claim.report_date:%Y%m%d}.zip" + if claim.source_url != expected_url: + raise ValueError("source_url does not match feed and report_date") + if ( + isinstance(claim.attempt_number, bool) + or not isinstance(claim.attempt_number, int) + or not 1 <= claim.attempt_number <= _MAX_ATTEMPT_NUMBER + ): + raise ValueError("invalid attempt_number") + if ( + type(receipt.downloaded_at) is not datetime + or receipt.downloaded_at.utcoffset() != timedelta(0) + ): + raise ValueError("downloaded_at must be UTC-aware") + if receipt.source_last_modified is not None: + if ( + type(receipt.source_last_modified) is not datetime + or receipt.source_last_modified.utcoffset() != timedelta(0) + ): + raise ValueError("source_last_modified must be UTC-aware") + if receipt.source_last_modified > receipt.downloaded_at: + raise ValueError("source_last_modified cannot be after downloaded_at") + if type(receipt.raw_archive) is not bytes: + raise TypeError("raw_archive must be immutable bytes") + if not 1 <= len(receipt.raw_archive) <= MAX_ARCHIVE_BYTES: + raise ValueError("raw_archive size is outside the accepted bounds") + return receipt + + _ITEM_LOCK_SQL = """ SELECT source_url, status, attempt_count FROM historical_backfill_items @@ -64,6 +128,12 @@ class BackfillArtifactResult: ON CONFLICT DO NOTHING RETURNING 1 """ +_ARTIFACT_SELECT_SQL = """ +SELECT feed, report_date, source_url, filename, byte_count, raw_archive +FROM historical_source_artifacts +WHERE artifact_sha256 = %s +""" + _LINK_INSERT_SQL = """ INSERT INTO historical_backfill_item_artifacts ( run_id, @@ -78,6 +148,15 @@ class BackfillArtifactResult: ON CONFLICT DO NOTHING RETURNING 1 """ +_LINK_SELECT_SQL = """ +SELECT artifact_sha256, downloaded_at, source_last_modified +FROM historical_backfill_item_artifacts +WHERE run_id = %s + AND feed = %s + AND report_date = %s + AND attempt_number = %s +""" + _EVENT_INSERT_SQL = """ INSERT INTO historical_backfill_events ( run_id, @@ -97,6 +176,7 @@ def __init__(self, connection: _Connection) -> None: self._connection = connection def record(self, receipt: BackfillArtifactReceipt) -> BackfillArtifactResult: + receipt = _validate_receipt(receipt) claim = receipt.claim digest = sha256(receipt.raw_archive).hexdigest() byte_count = len(receipt.raw_archive) @@ -107,7 +187,12 @@ def record(self, receipt: BackfillArtifactReceipt) -> BackfillArtifactResult: _ITEM_LOCK_SQL, (claim.run_id, claim.feed, claim.report_date), ) - cursor.fetchone() + current_item = cursor.fetchone() + expected_item = (claim.source_url, "running", claim.attempt_number) + if current_item != expected_item: + raise BackfillArtifactConflictError( + "receipt does not match the current backfill claim" + ) cursor.execute( _ARTIFACT_INSERT_SQL, ( @@ -120,7 +205,27 @@ def record(self, receipt: BackfillArtifactReceipt) -> BackfillArtifactResult: receipt.raw_archive, ), ) - cursor.fetchone() + artifact_inserted = cursor.fetchone() is not None + if not artifact_inserted: + cursor.execute(_ARTIFACT_SELECT_SQL, (digest,)) + stored_artifact = cursor.fetchone() + expected_artifact = ( + claim.feed, + claim.report_date, + claim.source_url, + filename, + byte_count, + receipt.raw_archive, + ) + if stored_artifact is None: + raise BackfillArtifactConflictError( + "conflicting historical source artifact" + ) + normalized_artifact = (*stored_artifact[:-1], bytes(stored_artifact[-1])) + if normalized_artifact != expected_artifact: + raise BackfillArtifactConflictError( + "conflicting historical source artifact" + ) cursor.execute( _LINK_INSERT_SQL, ( @@ -133,18 +238,40 @@ def record(self, receipt: BackfillArtifactReceipt) -> BackfillArtifactResult: receipt.source_last_modified, ), ) - cursor.fetchone() - cursor.execute( - _EVENT_INSERT_SQL, - ( - claim.run_id, - claim.feed, - claim.report_date, - "artifact_recorded", - claim.attempt_number, - ), - ) + link_inserted = cursor.fetchone() is not None + if not link_inserted: + cursor.execute( + _LINK_SELECT_SQL, + (claim.run_id, claim.feed, claim.report_date, claim.attempt_number), + ) + stored_link = cursor.fetchone() + expected_link = ( + digest, + receipt.downloaded_at, + receipt.source_last_modified, + ) + if stored_link != expected_link: + raise BackfillArtifactConflictError( + "conflicting historical backfill artifact link" + ) + else: + cursor.execute( + _EVENT_INSERT_SQL, + ( + claim.run_id, + claim.feed, + claim.report_date, + "artifact_recorded", + claim.attempt_number, + ), + ) self._connection.commit() - return BackfillArtifactResult(digest, byte_count, False) + return BackfillArtifactResult(digest, byte_count, not link_inserted) + except Exception: + try: + self._connection.rollback() + except Exception: + pass + raise finally: cursor.close() diff --git a/app/backend/tests/test_backfill_artifacts.py b/app/backend/tests/test_backfill_artifacts.py index 90f7413..d1a3599 100644 --- a/app/backend/tests/test_backfill_artifacts.py +++ b/app/backend/tests/test_backfill_artifacts.py @@ -1,10 +1,13 @@ """Behavior tests for immutable historical archive artifact registration.""" -from datetime import date, datetime, timezone +from dataclasses import replace +from datetime import date, datetime, timedelta, timezone import hashlib import unittest +from unittest.mock import patch from batterywatch_api.backfill_artifacts import ( + BackfillArtifactConflictError, BackfillArtifactReceipt, BackfillArtifactResult, PostgreSQLBackfillArtifactRegistrar, @@ -39,10 +42,18 @@ def close(self) -> None: class FakeConnection: - def __init__(self, *, fetchone_results=(), fail_on_execute=None, failure=None) -> None: + def __init__( + self, + *, + fetchone_results=(), + fail_on_execute=None, + failure=None, + rollback_failure=None, + ) -> None: self.fetchone_results = list(fetchone_results) self.fail_on_execute = fail_on_execute self.failure = failure + self.rollback_failure = rollback_failure self.executions = [] self.cursor_calls = self.closed_cursors = 0 self.commits = self.rollbacks = 0 @@ -56,6 +67,8 @@ def commit(self) -> None: def rollback(self) -> None: self.rollbacks += 1 + if self.rollback_failure is not None: + raise self.rollback_failure def receipt(raw_archive: bytes = b"official archive bytes") -> BackfillArtifactReceipt: @@ -142,6 +155,178 @@ def test_records_content_link_and_event_in_one_transaction(self) -> None: ("run-20260828", "dispatch_price", REPORT_DATE, "artifact_recorded", 3), ) + def test_exact_replay_compares_content_and_link_without_duplicate_event(self) -> None: + evidence = receipt() + digest = hashlib.sha256(evidence.raw_archive).hexdigest() + connection = FakeConnection( + fetchone_results=( + (SOURCE_URL, "running", 3), + None, + ( + "dispatch_price", + REPORT_DATE, + SOURCE_URL, + "PUBLIC_DISPATCHIS_20260828.zip", + len(evidence.raw_archive), + evidence.raw_archive, + ), + None, + (digest, evidence.downloaded_at, evidence.source_last_modified), + ) + ) + + result = PostgreSQLBackfillArtifactRegistrar(connection).record(evidence) + + self.assertEqual( + result, + BackfillArtifactResult(digest, len(evidence.raw_archive), True), + ) + self.assertEqual((connection.commits, connection.rollbacks), (1, 0)) + self.assertEqual(len(connection.executions), 5) + self.assertIn( + "SELECT feed, report_date, source_url, filename, byte_count, raw_archive", + connection.executions[2][0], + ) + self.assertEqual(connection.executions[2][1], (digest,)) + self.assertIn( + "SELECT artifact_sha256, downloaded_at, source_last_modified", + connection.executions[4][0], + ) + self.assertEqual( + connection.executions[4][1], + ("run-20260828", "dispatch_price", REPORT_DATE, 3), + ) + self.assertNotIn( + "INSERT INTO historical_backfill_events", + "\n".join(statement for statement, _ in connection.executions), + ) + + def test_conflicting_content_rolls_back_and_fails_closed(self) -> None: + evidence = receipt() + connection = FakeConnection( + fetchone_results=( + (SOURCE_URL, "running", 3), + None, + ( + "dispatch_price", + REPORT_DATE, + SOURCE_URL, + "PUBLIC_DISPATCHIS_20260828.zip", + len(evidence.raw_archive), + b"different bytes", + ), + ) + ) + + with self.assertRaisesRegex( + BackfillArtifactConflictError, + "source artifact", + ): + PostgreSQLBackfillArtifactRegistrar(connection).record(evidence) + + self.assertEqual((connection.commits, connection.rollbacks), (0, 1)) + self.assertEqual((connection.cursor_calls, connection.closed_cursors), (1, 1)) + + def test_conflicting_attempt_link_rolls_back_and_fails_closed(self) -> None: + evidence = receipt() + digest = hashlib.sha256(evidence.raw_archive).hexdigest() + connection = FakeConnection( + fetchone_results=( + (SOURCE_URL, "running", 3), + None, + ( + "dispatch_price", + REPORT_DATE, + SOURCE_URL, + "PUBLIC_DISPATCHIS_20260828.zip", + len(evidence.raw_archive), + evidence.raw_archive, + ), + None, + ( + digest, + evidence.downloaded_at + timedelta(minutes=1), + evidence.source_last_modified, + ), + ) + ) + + with self.assertRaisesRegex( + BackfillArtifactConflictError, + "artifact link", + ): + PostgreSQLBackfillArtifactRegistrar(connection).record(evidence) + + self.assertEqual((connection.commits, connection.rollbacks), (0, 1)) + + def test_stale_claim_rolls_back_before_artifact_sql(self) -> None: + connection = FakeConnection( + fetchone_results=((SOURCE_URL, "pending", 2),) + ) + + with self.assertRaisesRegex( + BackfillArtifactConflictError, + "current backfill claim", + ): + PostgreSQLBackfillArtifactRegistrar(connection).record(receipt()) + + self.assertEqual((connection.commits, connection.rollbacks), (0, 1)) + self.assertEqual(len(connection.executions), 1) + + def test_invalid_receipts_fail_before_opening_a_cursor(self) -> None: + valid = receipt() + invalid_receipts = ( + object(), + replace(valid, claim=replace(valid.claim, run_id="bad run id")), + replace(valid, claim=replace(valid.claim, feed="regional_soc")), + replace(valid, claim=replace(valid.claim, report_date=datetime(2026, 8, 28))), + replace(valid, claim=replace(valid.claim, source_url=SOURCE_URL + ".wrong")), + replace(valid, claim=replace(valid.claim, attempt_number=0)), + replace(valid, claim=replace(valid.claim, attempt_number=True)), + replace(valid, downloaded_at=datetime(2026, 8, 29, 2)), + replace(valid, source_last_modified=datetime(2026, 8, 29, 1, 30)), + replace( + valid, + source_last_modified=valid.downloaded_at + timedelta(minutes=1), + ), + replace(valid, raw_archive=b""), + replace(valid, raw_archive=bytearray(b"mutable")), + ) + + for invalid in invalid_receipts: + with self.subTest(invalid=invalid): + connection = FakeConnection() + with self.assertRaises((TypeError, ValueError)): + PostgreSQLBackfillArtifactRegistrar(connection).record( + invalid # type: ignore[arg-type] + ) + self.assertEqual(connection.cursor_calls, 0) + + connection = FakeConnection() + with patch( + "batterywatch_api.backfill_artifacts.MAX_ARCHIVE_BYTES", + len(valid.raw_archive) - 1, + ): + with self.assertRaisesRegex(ValueError, "raw_archive"): + PostgreSQLBackfillArtifactRegistrar(connection).record(valid) + self.assertEqual(connection.cursor_calls, 0) + + def test_database_failure_preserves_original_when_rollback_also_fails(self) -> None: + database_failure = RuntimeError("database write failed") + connection = FakeConnection( + fetchone_results=((SOURCE_URL, "running", 3),), + fail_on_execute=2, + failure=database_failure, + rollback_failure=RuntimeError("rollback failed"), + ) + + with self.assertRaises(RuntimeError) as raised: + PostgreSQLBackfillArtifactRegistrar(connection).record(receipt()) + + self.assertIs(raised.exception, database_failure) + self.assertEqual((connection.commits, connection.rollbacks), (0, 1)) + self.assertEqual((connection.cursor_calls, connection.closed_cursors), (1, 1)) + if __name__ == "__main__": unittest.main() From 82657a04bcb1a6ffbb99ad2016054890dc40730f Mon Sep 17 00:00:00 2001 From: djurcola-agent Date: Sun, 30 Aug 2026 19:06:51 +1000 Subject: [PATCH 07/30] fix: allow historical artifact receipt events --- app/backend/tests/test_backfill_ledger.py | 2 +- .../tests/test_dispatch_price_ingestion.py | 2 +- .../test_historical_artifact_migration.py | 38 ++++++++++++++++++- app/deploy/migrate.sh | 2 + .../006_artifact_recorded_event.sql | 21 ++++++++++ 5 files changed, 62 insertions(+), 3 deletions(-) create mode 100644 app/migrations/006_artifact_recorded_event.sql diff --git a/app/backend/tests/test_backfill_ledger.py b/app/backend/tests/test_backfill_ledger.py index 467ac54..6da3588 100644 --- a/app/backend/tests/test_backfill_ledger.py +++ b/app/backend/tests/test_backfill_ledger.py @@ -84,7 +84,7 @@ def test_deploys_additive_run_item_and_event_schema(self) -> None: self.assertIn("004_historical_backfill_ledger.sql", migrate_script) self.assertIn("SELECT count(*) = 12", migrate_script) - self.assertEqual(migrate_script.count('--dbname="$database_url"'), 6) + self.assertEqual(migrate_script.count('--dbname="$database_url"'), 7) self.assertIn("ON DELETE RESTRICT", migration) self.assertIn("event_seq BIGSERIAL PRIMARY KEY", migration) self.assertIn("historical_backfill_items_claim_idx", migration) diff --git a/app/backend/tests/test_dispatch_price_ingestion.py b/app/backend/tests/test_dispatch_price_ingestion.py index c1c5c31..82cf8cc 100644 --- a/app/backend/tests/test_dispatch_price_ingestion.py +++ b/app/backend/tests/test_dispatch_price_ingestion.py @@ -227,7 +227,7 @@ def test_deploy_migration_applies_and_verifies_price_artifact_table(self) -> Non self.assertIn("'dispatch_price_artifacts'", migrate_script) self.assertIn("004_historical_backfill_ledger.sql", migrate_script) self.assertIn("database_url=$BATTERYWATCH_DATABASE_URL", migrate_script) - self.assertEqual(migrate_script.count('--dbname="$database_url"'), 6) + self.assertEqual(migrate_script.count('--dbname="$database_url"'), 7) self.assertNotIn("export PGDATABASE=", migrate_script) diff --git a/app/backend/tests/test_historical_artifact_migration.py b/app/backend/tests/test_historical_artifact_migration.py index cd30f69..40a9fcf 100644 --- a/app/backend/tests/test_historical_artifact_migration.py +++ b/app/backend/tests/test_historical_artifact_migration.py @@ -49,13 +49,49 @@ def test_migration_defines_immutable_artifacts_and_deploys_last(self) -> None: self.assertIn("SELECT count(*) = 12", migrate_script) for table in ("historical_source_artifacts", "historical_backfill_item_artifacts"): self.assertIn(f"'{table}'", migrate_script) - self.assertEqual(migrate_script.count('--dbname="$database_url"'), 6) + self.assertEqual(migrate_script.count('--dbname="$database_url"'), 7) upper = migration.upper() for forbidden in ("DROP ", "ALTER ", "TRUNCATE "): self.assertNotIn(forbidden, upper) self.assertNotRegex(upper, r"\bDELETE\s+FROM\b") + def test_expands_event_constraint_for_artifact_recorded_atomically(self) -> None: + app_root = Path(__file__).resolve().parents[2] + migration_path = app_root / "migrations" / "006_artifact_recorded_event.sql" + self.assertTrue(migration_path.exists(), "006 event migration is missing") + + migration = migration_path.read_text(encoding="utf-8") + migrate_script = (app_root / "deploy" / "migrate.sh").read_text( + encoding="utf-8" + ) + upper = migration.upper() + + self.assertIn("BEGIN;", upper) + self.assertIn("COMMIT;", upper) + self.assertIn("ALTER TABLE historical_backfill_events", migration) + self.assertIn( + "DROP CONSTRAINT IF EXISTS historical_backfill_events_event_type_check", + migration, + ) + self.assertIn( + "ADD CONSTRAINT historical_backfill_events_event_type_check", + migration, + ) + for event_type in ( + "planned", + "recovered", + "claimed", + "artifact_recorded", + "completed", + "failed", + ): + self.assertIn(f"'{event_type}'", migration) + self.assertIn("006_artifact_recorded_event.sql", migrate_script) + self.assertEqual(migrate_script.count('--dbname="$database_url"'), 7) + self.assertNotIn("TRUNCATE ", upper) + self.assertNotRegex(upper, r"\bDELETE\s+FROM\b") + if __name__ == "__main__": unittest.main() diff --git a/app/deploy/migrate.sh b/app/deploy/migrate.sh index 33b291e..a9b47c9 100755 --- a/app/deploy/migrate.sh +++ b/app/deploy/migrate.sh @@ -31,6 +31,8 @@ psql --dbname="$database_url" --no-psqlrc --set=ON_ERROR_STOP=1 \ --file="$app_dir/migrations/004_historical_backfill_ledger.sql" psql --dbname="$database_url" --no-psqlrc --set=ON_ERROR_STOP=1 \ --file="$app_dir/migrations/005_historical_source_artifacts.sql" +psql --dbname="$database_url" --no-psqlrc --set=ON_ERROR_STOP=1 \ + --file="$app_dir/migrations/006_artifact_recorded_event.sql" psql --dbname="$database_url" --no-psqlrc --set=ON_ERROR_STOP=1 --tuples-only --no-align \ <<'SQL' SELECT count(*) = 12 diff --git a/app/migrations/006_artifact_recorded_event.sql b/app/migrations/006_artifact_recorded_event.sql new file mode 100644 index 0000000..62221a9 --- /dev/null +++ b/app/migrations/006_artifact_recorded_event.sql @@ -0,0 +1,21 @@ +-- Allow immutable artifact receipts to append replay-history events. + +BEGIN; + +ALTER TABLE historical_backfill_events + DROP CONSTRAINT IF EXISTS historical_backfill_events_event_type_check; + +ALTER TABLE historical_backfill_events + ADD CONSTRAINT historical_backfill_events_event_type_check + CHECK ( + event_type IN ( + 'planned', + 'recovered', + 'claimed', + 'artifact_recorded', + 'completed', + 'failed' + ) + ); + +COMMIT; From 6286ee54eddd718af7a404a18d0ec4ce01af77f5 Mon Sep 17 00:00:00 2001 From: djurcola-agent Date: Sun, 30 Aug 2026 19:19:00 +1000 Subject: [PATCH 08/30] feat: record backfill item completion --- .../batterywatch_api/backfill_ledger.py | 96 +++++++++++++++++++ app/backend/tests/test_backfill_ledger.py | 44 +++++++++ 2 files changed, 140 insertions(+) diff --git a/app/backend/batterywatch_api/backfill_ledger.py b/app/backend/batterywatch_api/backfill_ledger.py index f2ea869..792dce5 100644 --- a/app/backend/batterywatch_api/backfill_ledger.py +++ b/app/backend/batterywatch_api/backfill_ledger.py @@ -33,6 +33,12 @@ class BackfillClaim: attempt_number: int +@dataclass(frozen=True, slots=True) +class BackfillItemCompletion: + replayed: bool + records_imported: int + + @dataclass(frozen=True, slots=True) class BackfillEnsureResult: created: bool @@ -121,6 +127,32 @@ def _managed_cursor(connection: _Connection) -> Iterator[_Cursor]: RETURNING feed, report_date, source_url, attempt_count """ +_COMPLETE_LOCK_SQL = """ +SELECT source_url, status, attempt_count +FROM historical_backfill_items +WHERE run_id = %s AND feed = %s AND report_date = %s +FOR UPDATE +""" + +_COMPLETE_UPDATE_SQL = """ +UPDATE historical_backfill_items +SET status = 'completed', completed_at = CURRENT_TIMESTAMP, + updated_at = CURRENT_TIMESTAMP, last_error = NULL +WHERE run_id = %s AND feed = %s AND report_date = %s + AND status = 'running' AND attempt_count = %s +RETURNING 1 +""" + +_COMPLETE_EVENT_SQL = """ +INSERT INTO historical_backfill_events ( + run_id, feed, report_date, event_type, attempt_number, details +) +VALUES ( + %s, %s, %s, %s, %s, + jsonb_build_object('records_imported', %s) +) +""" + _RECOVER_ITEMS_SQL = """ UPDATE historical_backfill_items SET status = 'pending', updated_at = CURRENT_TIMESTAMP, started_at = NULL @@ -185,12 +217,75 @@ def _validate_items(items: tuple[BackfillPlanItem, ...]) -> None: raise ValueError("invalid backfill archive URL") +def _validate_claim(claim: BackfillClaim) -> None: + if type(claim) is not BackfillClaim: + raise ValueError("invalid backfill claim") + _validate_run_id(claim.run_id) + _validate_items( + (BackfillPlanItem(claim.feed, claim.report_date, claim.source_url),) + ) + if ( + type(claim.attempt_number) is not int + or not 1 <= claim.attempt_number <= _MAX_BIGINT + ): + raise ValueError("invalid backfill attempt number") + + class PostgreSQLBackfillLedger: """Own the transaction that creates or exactly resumes a backfill run.""" def __init__(self, connection: _Connection): self._connection = connection + def complete( + self, claim: BackfillClaim, *, records_imported: int + ) -> BackfillItemCompletion: + _validate_claim(claim) + if ( + type(records_imported) is not int + or not 0 <= records_imported <= _MAX_BIGINT + ): + raise ValueError("invalid imported record count") + try: + with _managed_cursor(self._connection) as cursor: + item_key = (claim.run_id, claim.feed, claim.report_date) + cursor.execute(_COMPLETE_LOCK_SQL, item_key) + current = cursor.fetchone() + expected = ( + claim.source_url, + "running", + claim.attempt_number, + ) + if current != expected: + raise BackfillRunConflictError( + "backfill claim no longer owns the running item" + ) + cursor.execute( + _COMPLETE_UPDATE_SQL, + (*item_key, claim.attempt_number), + ) + if cursor.fetchone() is None: + raise BackfillRunConflictError( + "backfill item completion was not applied" + ) + cursor.execute( + _COMPLETE_EVENT_SQL, + ( + *item_key, + "completed", + claim.attempt_number, + records_imported, + ), + ) + self._connection.commit() + return BackfillItemCompletion(False, records_imported) + except Exception: + try: + self._connection.rollback() + except Exception: + pass + raise + def claim_next(self, run_id: str) -> BackfillClaim | None: _validate_run_id(run_id) try: @@ -314,6 +409,7 @@ def ensure_run( __all__ = [ "BackfillClaim", "BackfillEnsureResult", + "BackfillItemCompletion", "BackfillPlanItem", "BackfillRunConflictError", "BackfillRunSpec", diff --git a/app/backend/tests/test_backfill_ledger.py b/app/backend/tests/test_backfill_ledger.py index 6da3588..738d2c3 100644 --- a/app/backend/tests/test_backfill_ledger.py +++ b/app/backend/tests/test_backfill_ledger.py @@ -8,6 +8,7 @@ from batterywatch_api.backfill_ledger import ( BackfillClaim, BackfillEnsureResult, + BackfillItemCompletion, BackfillPlanItem, BackfillRunConflictError, BackfillRunSpec, @@ -95,6 +96,49 @@ def test_deploys_additive_run_item_and_event_schema(self) -> None: class PostgreSQLBackfillLedgerTests(unittest.TestCase): + def test_complete_records_guarded_item_transition_and_event(self) -> None: + claim = BackfillClaim( + "run-20260828", + "dispatch_price", + date(2026, 8, 28), + "https://www.nemweb.com.au/REPORTS/ARCHIVE/DispatchIS_Reports/" + "PUBLIC_DISPATCHIS_20260828.zip", + 4, + ) + connection = FakeConnection( + fetchone_results=( + (claim.source_url, "running", claim.attempt_number), + (1,), + ) + ) + + result = PostgreSQLBackfillLedger(connection).complete( + claim, records_imported=145 + ) + + self.assertEqual(result, BackfillItemCompletion(False, 145)) + self.assertEqual((connection.commits, connection.rollbacks), (1, 0)) + self.assertEqual((connection.cursor_calls, connection.closed_cursors), (1, 1)) + self.assertEqual(len(connection.executions), 3) + self.assertIn("FOR UPDATE", connection.executions[0][0]) + self.assertIn("status = 'completed'", connection.executions[1][0]) + self.assertIn("status = 'running'", connection.executions[1][0]) + self.assertEqual( + connection.executions[1][1], + (claim.run_id, claim.feed, claim.report_date, claim.attempt_number), + ) + self.assertEqual( + connection.executions[2][1], + ( + claim.run_id, + claim.feed, + claim.report_date, + "completed", + claim.attempt_number, + 145, + ), + ) + def test_claim_next_claims_deterministic_item_and_appends_event(self) -> None: run_id = "run-20260828" report_date = date(2026, 8, 28) From dd8dbc6f20f906d7a8dfb07065bfc24304e87f95 Mon Sep 17 00:00:00 2001 From: djurcola-agent Date: Sun, 30 Aug 2026 19:27:32 +1000 Subject: [PATCH 09/30] feat: record backfill item failures --- .../batterywatch_api/backfill_ledger.py | 77 +++++++++++++++ app/backend/tests/test_backfill_ledger.py | 95 +++++++++++++++++++ 2 files changed, 172 insertions(+) diff --git a/app/backend/batterywatch_api/backfill_ledger.py b/app/backend/batterywatch_api/backfill_ledger.py index 792dce5..0bf6b9e 100644 --- a/app/backend/batterywatch_api/backfill_ledger.py +++ b/app/backend/batterywatch_api/backfill_ledger.py @@ -39,6 +39,12 @@ class BackfillItemCompletion: records_imported: int +@dataclass(frozen=True, slots=True) +class BackfillItemFailure: + replayed: bool + error_summary: str + + @dataclass(frozen=True, slots=True) class BackfillEnsureResult: created: bool @@ -153,6 +159,25 @@ def _managed_cursor(connection: _Connection) -> Iterator[_Cursor]: ) """ +_FAIL_UPDATE_SQL = """ +UPDATE historical_backfill_items +SET status = 'failed', completed_at = CURRENT_TIMESTAMP, + updated_at = CURRENT_TIMESTAMP, last_error = %s +WHERE run_id = %s AND feed = %s AND report_date = %s + AND status = 'running' AND attempt_count = %s +RETURNING 1 +""" + +_FAIL_EVENT_SQL = """ +INSERT INTO historical_backfill_events ( + run_id, feed, report_date, event_type, attempt_number, details +) +VALUES ( + %s, %s, %s, %s, %s, + jsonb_build_object('error_summary', %s) +) +""" + _RECOVER_ITEMS_SQL = """ UPDATE historical_backfill_items SET status = 'pending', updated_at = CURRENT_TIMESTAMP, started_at = NULL @@ -237,6 +262,57 @@ class PostgreSQLBackfillLedger: def __init__(self, connection: _Connection): self._connection = connection + def fail( + self, claim: BackfillClaim, *, error_summary: str + ) -> BackfillItemFailure: + _validate_claim(claim) + if ( + type(error_summary) is not str + or not 1 <= len(error_summary) <= 2048 + or error_summary != error_summary.strip() + or any(character in "\r\n\0" for character in error_summary) + ): + raise ValueError("invalid backfill error summary") + try: + with _managed_cursor(self._connection) as cursor: + item_key = (claim.run_id, claim.feed, claim.report_date) + cursor.execute(_COMPLETE_LOCK_SQL, item_key) + current = cursor.fetchone() + expected = ( + claim.source_url, + "running", + claim.attempt_number, + ) + if current != expected: + raise BackfillRunConflictError( + "backfill claim no longer owns the running item" + ) + cursor.execute( + _FAIL_UPDATE_SQL, + (error_summary, *item_key, claim.attempt_number), + ) + if cursor.fetchone() is None: + raise BackfillRunConflictError( + "backfill item failure was not applied" + ) + cursor.execute( + _FAIL_EVENT_SQL, + ( + *item_key, + "failed", + claim.attempt_number, + error_summary, + ), + ) + self._connection.commit() + return BackfillItemFailure(False, error_summary) + except Exception: + try: + self._connection.rollback() + except Exception: + pass + raise + def complete( self, claim: BackfillClaim, *, records_imported: int ) -> BackfillItemCompletion: @@ -410,6 +486,7 @@ def ensure_run( "BackfillClaim", "BackfillEnsureResult", "BackfillItemCompletion", + "BackfillItemFailure", "BackfillPlanItem", "BackfillRunConflictError", "BackfillRunSpec", diff --git a/app/backend/tests/test_backfill_ledger.py b/app/backend/tests/test_backfill_ledger.py index 738d2c3..925de01 100644 --- a/app/backend/tests/test_backfill_ledger.py +++ b/app/backend/tests/test_backfill_ledger.py @@ -9,6 +9,7 @@ BackfillClaim, BackfillEnsureResult, BackfillItemCompletion, + BackfillItemFailure, BackfillPlanItem, BackfillRunConflictError, BackfillRunSpec, @@ -96,6 +97,77 @@ def test_deploys_additive_run_item_and_event_schema(self) -> None: class PostgreSQLBackfillLedgerTests(unittest.TestCase): + def test_fail_records_guarded_item_transition_and_event(self) -> None: + claim = BackfillClaim( + "run-20260828", + "dispatch_scada", + date(2026, 8, 28), + "https://www.nemweb.com.au/REPORTS/ARCHIVE/Dispatch_SCADA/" + "PUBLIC_DISPATCHSCADA_20260828.zip", + 2, + ) + connection = FakeConnection( + fetchone_results=( + (claim.source_url, "running", claim.attempt_number), + (1,), + ) + ) + + result = PostgreSQLBackfillLedger(connection).fail( + claim, error_summary="archive checksum mismatch" + ) + + self.assertEqual( + result, BackfillItemFailure(False, "archive checksum mismatch") + ) + self.assertEqual((connection.commits, connection.rollbacks), (1, 0)) + self.assertEqual((connection.cursor_calls, connection.closed_cursors), (1, 1)) + self.assertEqual(len(connection.executions), 3) + self.assertIn("FOR UPDATE", connection.executions[0][0]) + self.assertIn("status = 'failed'", connection.executions[1][0]) + self.assertIn("status = 'running'", connection.executions[1][0]) + self.assertEqual( + connection.executions[1][1], + ( + "archive checksum mismatch", + claim.run_id, + claim.feed, + claim.report_date, + claim.attempt_number, + ), + ) + self.assertEqual( + connection.executions[2][1], + ( + claim.run_id, + claim.feed, + claim.report_date, + "failed", + claim.attempt_number, + "archive checksum mismatch", + ), + ) + + def test_fail_rejects_invalid_error_summary_before_sql(self) -> None: + claim = BackfillClaim( + "run-20260828", + "dispatch_scada", + date(2026, 8, 28), + "https://www.nemweb.com.au/REPORTS/ARCHIVE/Dispatch_SCADA/" + "PUBLIC_DISPATCHSCADA_20260828.zip", + 1, + ) + connection = FakeConnection() + + with self.assertRaises(ValueError): + PostgreSQLBackfillLedger(connection).fail(claim, error_summary="") + + self.assertEqual( + (connection.executions, connection.cursor_calls, + connection.commits, connection.rollbacks), + ([], 0, 0, 0), + ) + def test_complete_records_guarded_item_transition_and_event(self) -> None: claim = BackfillClaim( "run-20260828", @@ -139,6 +211,29 @@ def test_complete_records_guarded_item_transition_and_event(self) -> None: ), ) + def test_complete_rejects_invalid_claim_before_sql(self) -> None: + claim = BackfillClaim( + "run-20260828", + "dispatch_price", + date(2026, 8, 28), + "https://www.nemweb.com.au/REPORTS/ARCHIVE/DispatchIS_Reports/" + "PUBLIC_DISPATCHIS_20260828.zip", + 1, + ) + connection = FakeConnection() + + with self.assertRaises(ValueError): + PostgreSQLBackfillLedger(connection).complete( + replace(claim, source_url="https://example.invalid/archive.zip"), + records_imported=0, + ) + + self.assertEqual( + (connection.executions, connection.cursor_calls, + connection.commits, connection.rollbacks), + ([], 0, 0, 0), + ) + def test_claim_next_claims_deterministic_item_and_appends_event(self) -> None: run_id = "run-20260828" report_date = date(2026, 8, 28) From 57b5baf23e4136717e67b5bbe95c5d586d9247a1 Mon Sep 17 00:00:00 2001 From: djurcola-agent Date: Sun, 30 Aug 2026 20:02:54 +1000 Subject: [PATCH 10/30] feat: summarize historical backfill progress --- .../batterywatch_api/backfill_ledger.py | 79 +++++++++++++++++++ app/backend/tests/test_backfill_ledger.py | 74 ++++++++++++++--- 2 files changed, 140 insertions(+), 13 deletions(-) diff --git a/app/backend/batterywatch_api/backfill_ledger.py b/app/backend/batterywatch_api/backfill_ledger.py index 0bf6b9e..49263fe 100644 --- a/app/backend/batterywatch_api/backfill_ledger.py +++ b/app/backend/batterywatch_api/backfill_ledger.py @@ -17,6 +17,18 @@ class BackfillRunSpec: ingestion_version: int +@dataclass(frozen=True, slots=True) +class BackfillRunProgress: + run_id: str + status: str + total: int + pending: int + running: int + completed: int + failed: int + total_attempts: int + + @dataclass(frozen=True, slots=True) class BackfillPlanItem: feed: str @@ -94,6 +106,21 @@ def _managed_cursor(connection: _Connection) -> Iterator[_Cursor]: FOR UPDATE """ +_RUN_PROGRESS_SQL = """ +SELECT + r.status, + COUNT(i.feed), + COUNT(*) FILTER (WHERE i.status = 'pending'), + COUNT(*) FILTER (WHERE i.status = 'running'), + COUNT(*) FILTER (WHERE i.status = 'completed'), + COUNT(*) FILTER (WHERE i.status = 'failed'), + COALESCE(SUM(i.attempt_count), 0)::bigint +FROM historical_backfill_runs AS r +LEFT JOIN historical_backfill_items AS i ON i.run_id = r.run_id +WHERE r.run_id = %s +GROUP BY r.status +""" + _ITEM_INSERT_SQL = """ INSERT INTO historical_backfill_items ( run_id, feed, report_date, source_url, status @@ -124,6 +151,12 @@ def _managed_cursor(connection: _Connection) -> Iterator[_Cursor]: FOR UPDATE SKIP LOCKED """ +_CLAIM_RUN_SELECT_SQL = """ +SELECT status +FROM historical_backfill_runs +WHERE run_id = %s +""" + _CLAIM_UPDATE_SQL = """ UPDATE historical_backfill_items SET status = 'running', attempt_count = attempt_count + 1, @@ -256,12 +289,51 @@ def _validate_claim(claim: BackfillClaim) -> None: raise ValueError("invalid backfill attempt number") +def _progress_from_row( + run_id: str, row: tuple[Any, ...] +) -> BackfillRunProgress: + if len(row) != 7 or row[0] not in {"running", "completed", "failed"}: + raise BackfillRunConflictError("invalid backfill progress row") + counts = row[1:] + if any( + type(value) is not int or not 0 <= value <= _MAX_BIGINT + for value in counts + ): + raise BackfillRunConflictError("invalid backfill progress counts") + total, pending, running, completed, failed, total_attempts = counts + if ( + total <= 0 + or pending + running + completed + failed != total + or total_attempts < running + completed + failed + ): + raise BackfillRunConflictError("inconsistent backfill progress counts") + return BackfillRunProgress(run_id, row[0], *counts) + + class PostgreSQLBackfillLedger: """Own the transaction that creates or exactly resumes a backfill run.""" def __init__(self, connection: _Connection): self._connection = connection + def progress(self, run_id: str) -> BackfillRunProgress: + _validate_run_id(run_id) + try: + with _managed_cursor(self._connection) as cursor: + cursor.execute(_RUN_PROGRESS_SQL, (run_id,)) + row = cursor.fetchone() + if row is None: + raise BackfillRunConflictError("backfill run does not exist") + result = _progress_from_row(run_id, row) + self._connection.commit() + return result + except Exception: + try: + self._connection.rollback() + except Exception: + pass + raise + def fail( self, claim: BackfillClaim, *, error_summary: str ) -> BackfillItemFailure: @@ -366,6 +438,12 @@ def claim_next(self, run_id: str) -> BackfillClaim | None: _validate_run_id(run_id) try: with _managed_cursor(self._connection) as cursor: + cursor.execute(_CLAIM_RUN_SELECT_SQL, (run_id,)) + run_row = cursor.fetchone() + if run_row is None: + raise BackfillRunConflictError("backfill run does not exist") + if run_row not in {("running",), ("completed",), ("failed",)}: + raise BackfillRunConflictError("invalid backfill run status") cursor.execute(_CLAIM_SELECT_SQL, (run_id,)) selected = cursor.fetchone() if selected is None: @@ -489,6 +567,7 @@ def ensure_run( "BackfillItemFailure", "BackfillPlanItem", "BackfillRunConflictError", + "BackfillRunProgress", "BackfillRunSpec", "PostgreSQLBackfillLedger", ] diff --git a/app/backend/tests/test_backfill_ledger.py b/app/backend/tests/test_backfill_ledger.py index 925de01..4fa95d7 100644 --- a/app/backend/tests/test_backfill_ledger.py +++ b/app/backend/tests/test_backfill_ledger.py @@ -12,6 +12,7 @@ BackfillItemFailure, BackfillPlanItem, BackfillRunConflictError, + BackfillRunProgress, BackfillRunSpec, PostgreSQLBackfillLedger, ) @@ -97,6 +98,40 @@ def test_deploys_additive_run_item_and_event_schema(self) -> None: class PostgreSQLBackfillLedgerTests(unittest.TestCase): + def test_progress_returns_deterministic_status_and_attempt_counts(self) -> None: + connection = FakeConnection( + fetchone_results=(("running", 62, 2, 1, 58, 1, 64),) + ) + + progress = PostgreSQLBackfillLedger(connection).progress("run-20260828") + + self.assertEqual( + progress, + BackfillRunProgress("run-20260828", "running", 62, 2, 1, 58, 1, 64), + ) + self.assertEqual((connection.commits, connection.rollbacks), (1, 0)) + self.assertEqual((connection.cursor_calls, connection.closed_cursors), (1, 1)) + self.assertEqual(len(connection.executions), 1) + statement, parameters = connection.executions[0] + self.assertIn("FILTER (WHERE i.status = 'pending')", statement) + self.assertIn("FILTER (WHERE i.status = 'running')", statement) + self.assertIn("FILTER (WHERE i.status = 'completed')", statement) + self.assertIn("FILTER (WHERE i.status = 'failed')", statement) + self.assertIn("SUM(i.attempt_count)", statement) + self.assertIn("COALESCE(SUM(i.attempt_count), 0)::bigint", statement) + self.assertEqual(parameters, ("run-20260828",)) + + def test_progress_rejects_inconsistent_database_counts(self) -> None: + connection = FakeConnection( + fetchone_results=(("running", 62, 2, 1, 58, 0, 64),) + ) + + with self.assertRaises(BackfillRunConflictError): + PostgreSQLBackfillLedger(connection).progress("run-20260828") + + self.assertEqual((connection.commits, connection.rollbacks), (0, 1)) + self.assertEqual((connection.cursor_calls, connection.closed_cursors), (1, 1)) + def test_fail_records_guarded_item_transition_and_event(self) -> None: claim = BackfillClaim( "run-20260828", @@ -243,6 +278,7 @@ def test_claim_next_claims_deterministic_item_and_appends_event(self) -> None: ) connection = FakeConnection( fetchone_results=( + ("running",), ("dispatch_price", report_date, source_url), ("dispatch_price", report_date, source_url, 4), ) @@ -256,16 +292,17 @@ def test_claim_next_claims_deterministic_item_and_appends_event(self) -> None: ) self.assertEqual((connection.commits, connection.rollbacks), (1, 0)) self.assertEqual((connection.cursor_calls, connection.closed_cursors), (1, 1)) - self.assertEqual(len(connection.executions), 3) - self.assertIn("status IN ('pending', 'failed')", connection.executions[0][0]) - self.assertIn("ORDER BY feed, report_date", connection.executions[0][0]) - self.assertIn("FOR UPDATE SKIP LOCKED", connection.executions[0][0]) - self.assertIn("UPDATE historical_backfill_items", connection.executions[1][0]) + self.assertEqual(len(connection.executions), 4) + self.assertIn("FROM historical_backfill_runs", connection.executions[0][0]) + self.assertIn("status IN ('pending', 'failed')", connection.executions[1][0]) + self.assertIn("ORDER BY feed, report_date", connection.executions[1][0]) + self.assertIn("FOR UPDATE SKIP LOCKED", connection.executions[1][0]) + self.assertIn("UPDATE historical_backfill_items", connection.executions[2][0]) self.assertEqual( - connection.executions[1][1], (run_id, "dispatch_price", report_date) + connection.executions[2][1], (run_id, "dispatch_price", report_date) ) self.assertEqual( - connection.executions[2][1], + connection.executions[3][1], (run_id, "dispatch_price", report_date, "claimed", 4), ) @@ -277,7 +314,7 @@ def test_claim_next_database_failure_rolls_back_and_reraises_same_error(self) -> ) failure = RuntimeError("injected database failure") connection = FakeConnection( - fetchone_results=(("dispatch_price", report_date, source_url),), + fetchone_results=(("running",),), fail_on_execute=2, failure=failure, ) @@ -302,19 +339,30 @@ def test_claim_next_rejects_invalid_run_id_before_sql(self) -> None: ) def test_claim_next_returns_none_when_no_item_is_eligible(self) -> None: - connection = FakeConnection(fetchone_results=(None,)) + connection = FakeConnection(fetchone_results=(("running",), None)) claim = PostgreSQLBackfillLedger(connection).claim_next("run-20260828") self.assertIsNone(claim) self.assertEqual((connection.commits, connection.rollbacks), (1, 0)) self.assertEqual((connection.cursor_calls, connection.closed_cursors), (1, 1)) - self.assertEqual(len(connection.executions), 1) - self.assertNotIn( - "INSERT INTO historical_backfill_events", - connection.executions[0][0], + self.assertEqual(len(connection.executions), 2) + self.assertFalse( + any( + "INSERT INTO historical_backfill_events" in statement + for statement, _ in connection.executions + ) ) + def test_claim_next_rejects_absent_run(self) -> None: + connection = FakeConnection(fetchone_results=(None,)) + + with self.assertRaises(BackfillRunConflictError): + PostgreSQLBackfillLedger(connection).claim_next("missing-run") + + self.assertEqual((connection.commits, connection.rollbacks), (0, 1)) + self.assertEqual((connection.cursor_calls, connection.closed_cursors), (1, 1)) + def test_ensure_new_run_plans_items_and_events_in_deterministic_order(self) -> None: spec = BackfillRunSpec( "run-20260828", From 0ed928d062ec62cf032f3502e0b29419da6e8eac Mon Sep 17 00:00:00 2001 From: djurcola-agent Date: Sun, 30 Aug 2026 20:16:05 +1000 Subject: [PATCH 11/30] feat: allow bounded NEMWeb daily archive fetches --- app/backend/batterywatch_api/nemweb_http.py | 23 ++++++++++--- app/backend/tests/test_nemweb_http.py | 38 ++++++++++++++++++++- 2 files changed, 56 insertions(+), 5 deletions(-) diff --git a/app/backend/batterywatch_api/nemweb_http.py b/app/backend/batterywatch_api/nemweb_http.py index 9b1c1d1..e327cc1 100644 --- a/app/backend/batterywatch_api/nemweb_http.py +++ b/app/backend/batterywatch_api/nemweb_http.py @@ -16,8 +16,9 @@ "https://www.nemweb.com.au/REPORTS/CURRENT/Dispatch_SCADA/", "https://www.nemweb.com.au/REPORTS/CURRENT/DispatchIS_Reports/", )) -_MAX_RESOURCE_BYTES = 16 * 1024 * 1024 -_ARTIFACT_PATH_RE = re.compile( +_CURRENT_RESOURCE_MAX_BYTES = 16 * 1024 * 1024 +_ARCHIVE_RESOURCE_MAX_BYTES = 128 * 1024 * 1024 +_CURRENT_ARTIFACT_PATH_RE = re.compile( r"(?:" r"/REPORTS/CURRENT/Dispatch_SCADA/" r"PUBLIC_DISPATCHSCADA_[0-9]{12}_[0-9]{1,32}\.zip" @@ -26,6 +27,15 @@ r"PUBLIC_DISPATCHIS_[0-9]{12}_[0-9]{1,32}\.zip" r")" ) +_ARCHIVE_ARTIFACT_PATH_RE = re.compile( + r"(?:" + r"/REPORTS/ARCHIVE/Dispatch_SCADA/" + r"PUBLIC_DISPATCHSCADA_[0-9]{8}\.zip" + r"|" + r"/REPORTS/ARCHIVE/DispatchIS_Reports/" + r"PUBLIC_DISPATCHIS_[0-9]{8}\.zip" + r")" +) class NemwebHttpError(ValueError): @@ -81,20 +91,25 @@ def fetch_nemweb_resource( """Fetch one NEMWeb resource through an injectable HTTPS opener.""" valid_url = False + resource_max_bytes = _CURRENT_RESOURCE_MAX_BYTES if type(url) is str: try: parts = urlsplit(url) except ValueError as error: raise NemwebHttpError("invalid NEMWeb request") from error + current_artifact = _CURRENT_ARTIFACT_PATH_RE.fullmatch(parts.path) is not None + archive_artifact = _ARCHIVE_ARTIFACT_PATH_RE.fullmatch(parts.path) is not None valid_url = url in _INDEX_URLS or ( parts.scheme == "https" and parts.netloc == "www.nemweb.com.au" and not parts.query and not parts.fragment - and _ARTIFACT_PATH_RE.fullmatch(parts.path) is not None + and (current_artifact or archive_artifact) ) + if archive_artifact: + resource_max_bytes = _ARCHIVE_RESOURCE_MAX_BYTES valid_limit = ( - type(max_bytes) is int and 0 < max_bytes <= _MAX_RESOURCE_BYTES + type(max_bytes) is int and 0 < max_bytes <= resource_max_bytes ) valid_timeout = ( (type(timeout_seconds) is int and 0 < timeout_seconds <= 60) diff --git a/app/backend/tests/test_nemweb_http.py b/app/backend/tests/test_nemweb_http.py index 92e0a20..1d88dc6 100644 --- a/app/backend/tests/test_nemweb_http.py +++ b/app/backend/tests/test_nemweb_http.py @@ -18,6 +18,14 @@ PRICE_ARTIFACT_URL = ( PRICE_INDEX_URL + "PUBLIC_DISPATCHIS_202608301205_0000000535164870.zip" ) +SCADA_ARCHIVE_URL = ( + "https://www.nemweb.com.au/REPORTS/ARCHIVE/Dispatch_SCADA/" + "PUBLIC_DISPATCHSCADA_20260829.zip" +) +PRICE_ARCHIVE_URL = ( + "https://www.nemweb.com.au/REPORTS/ARCHIVE/DispatchIS_Reports/" + "PUBLIC_DISPATCHIS_20260829.zip" +) class FakeResponse: @@ -79,6 +87,34 @@ def read(self, size: int = -1) -> bytes: class FetchNemwebResourceTests(unittest.TestCase): + def test_current_resources_retain_sixteen_mib_limit(self) -> None: + opener = FakeOpener(FakeResponse(b"small current response")) + + with self.assertRaises(http.NemwebHttpError): + http.fetch_nemweb_resource( + INDEX_URL, + max_bytes=16 * 1024 * 1024 + 1, + opener=opener, + ) + + self.assertEqual(opener.calls, []) + + def test_fetches_canonical_daily_archives_with_outer_bound(self) -> None: + limit = 128 * 1024 * 1024 + for url in (SCADA_ARCHIVE_URL, PRICE_ARCHIVE_URL): + with self.subTest(url=url): + response = FakeResponse(b"PK daily archive", url=url) + opener = FakeOpener(response) + + result = http.fetch_nemweb_resource( + url, + max_bytes=limit, + opener=opener, + ) + + self.assertEqual((result.requested_url, result.body), (url, response._body)) + self.assertEqual(response.read_sizes, [limit + 1]) + def test_fetches_official_dispatch_price_index_and_artifact(self) -> None: for url, body in ( (PRICE_INDEX_URL, b"price index"), @@ -221,7 +257,7 @@ def test_normalizes_invalid_request_arguments_to_public_error(self) -> None: ("https://www.nemweb.com.au/REPORTS/CURRENT/Dispatch_SCADA/not-canonical.zip", 1024, 10.0), (INDEX_URL, True, 10.0), (INDEX_URL, 0, 10.0), - (INDEX_URL, 16 * 1024 * 1024 + 1, 10.0), + (INDEX_URL, 128 * 1024 * 1024 + 1, 10.0), (INDEX_URL, 1024, True), (INDEX_URL, 1024, 0.0), (INDEX_URL, 1024, 61.0), From 7c94f15b6452029a049091fd8c5be46b9f6a6c08 Mon Sep 17 00:00:00 2001 From: djurcola-agent Date: Sun, 30 Aug 2026 20:31:09 +1000 Subject: [PATCH 12/30] fix: align historical backfill runtime schema --- .../batterywatch_api/backfill_artifacts.py | 4 +- app/backend/tests/test_backfill_artifacts.py | 4 +- app/backend/tests/test_backfill_ledger.py | 2 +- .../tests/test_dispatch_price_ingestion.py | 2 +- .../test_historical_artifact_migration.py | 31 +++++++++++- app/deploy/migrate.sh | 2 + .../007_backfill_runtime_details.sql | 49 +++++++++++++++++++ 7 files changed, 87 insertions(+), 7 deletions(-) create mode 100644 app/migrations/007_backfill_runtime_details.sql diff --git a/app/backend/batterywatch_api/backfill_artifacts.py b/app/backend/batterywatch_api/backfill_artifacts.py index 9005c47..79b3312 100644 --- a/app/backend/batterywatch_api/backfill_artifacts.py +++ b/app/backend/batterywatch_api/backfill_artifacts.py @@ -122,14 +122,14 @@ def _validate_receipt(receipt: object) -> BackfillArtifactReceipt: source_url, filename, byte_count, - raw_archive + raw_bytes ) VALUES (%s, %s, %s, %s, %s, %s, %s) ON CONFLICT DO NOTHING RETURNING 1 """ _ARTIFACT_SELECT_SQL = """ -SELECT feed, report_date, source_url, filename, byte_count, raw_archive +SELECT feed, report_date, source_url, filename, byte_count, raw_bytes FROM historical_source_artifacts WHERE artifact_sha256 = %s """ diff --git a/app/backend/tests/test_backfill_artifacts.py b/app/backend/tests/test_backfill_artifacts.py index d1a3599..420ff6f 100644 --- a/app/backend/tests/test_backfill_artifacts.py +++ b/app/backend/tests/test_backfill_artifacts.py @@ -118,6 +118,8 @@ def test_records_content_link_and_event_in_one_transaction(self) -> None: artifact_sql, artifact_parameters = connection.executions[1] self.assertIn("INSERT INTO historical_source_artifacts", artifact_sql) + self.assertIn("raw_bytes", artifact_sql) + self.assertNotIn("raw_archive", artifact_sql) self.assertIn("ON CONFLICT DO NOTHING RETURNING 1", artifact_sql) self.assertEqual( artifact_parameters, @@ -184,7 +186,7 @@ def test_exact_replay_compares_content_and_link_without_duplicate_event(self) -> self.assertEqual((connection.commits, connection.rollbacks), (1, 0)) self.assertEqual(len(connection.executions), 5) self.assertIn( - "SELECT feed, report_date, source_url, filename, byte_count, raw_archive", + "SELECT feed, report_date, source_url, filename, byte_count, raw_bytes", connection.executions[2][0], ) self.assertEqual(connection.executions[2][1], (digest,)) diff --git a/app/backend/tests/test_backfill_ledger.py b/app/backend/tests/test_backfill_ledger.py index 4fa95d7..c8ce656 100644 --- a/app/backend/tests/test_backfill_ledger.py +++ b/app/backend/tests/test_backfill_ledger.py @@ -87,7 +87,7 @@ def test_deploys_additive_run_item_and_event_schema(self) -> None: self.assertIn("004_historical_backfill_ledger.sql", migrate_script) self.assertIn("SELECT count(*) = 12", migrate_script) - self.assertEqual(migrate_script.count('--dbname="$database_url"'), 7) + self.assertEqual(migrate_script.count('--dbname="$database_url"'), 8) self.assertIn("ON DELETE RESTRICT", migration) self.assertIn("event_seq BIGSERIAL PRIMARY KEY", migration) self.assertIn("historical_backfill_items_claim_idx", migration) diff --git a/app/backend/tests/test_dispatch_price_ingestion.py b/app/backend/tests/test_dispatch_price_ingestion.py index 82cf8cc..911c1d3 100644 --- a/app/backend/tests/test_dispatch_price_ingestion.py +++ b/app/backend/tests/test_dispatch_price_ingestion.py @@ -227,7 +227,7 @@ def test_deploy_migration_applies_and_verifies_price_artifact_table(self) -> Non self.assertIn("'dispatch_price_artifacts'", migrate_script) self.assertIn("004_historical_backfill_ledger.sql", migrate_script) self.assertIn("database_url=$BATTERYWATCH_DATABASE_URL", migrate_script) - self.assertEqual(migrate_script.count('--dbname="$database_url"'), 7) + self.assertEqual(migrate_script.count('--dbname="$database_url"'), 8) self.assertNotIn("export PGDATABASE=", migrate_script) diff --git a/app/backend/tests/test_historical_artifact_migration.py b/app/backend/tests/test_historical_artifact_migration.py index 40a9fcf..ef5fdfd 100644 --- a/app/backend/tests/test_historical_artifact_migration.py +++ b/app/backend/tests/test_historical_artifact_migration.py @@ -49,7 +49,7 @@ def test_migration_defines_immutable_artifacts_and_deploys_last(self) -> None: self.assertIn("SELECT count(*) = 12", migrate_script) for table in ("historical_source_artifacts", "historical_backfill_item_artifacts"): self.assertIn(f"'{table}'", migrate_script) - self.assertEqual(migrate_script.count('--dbname="$database_url"'), 7) + self.assertEqual(migrate_script.count('--dbname="$database_url"'), 8) upper = migration.upper() for forbidden in ("DROP ", "ALTER ", "TRUNCATE "): @@ -88,7 +88,34 @@ def test_expands_event_constraint_for_artifact_recorded_atomically(self) -> None ): self.assertIn(f"'{event_type}'", migration) self.assertIn("006_artifact_recorded_event.sql", migrate_script) - self.assertEqual(migrate_script.count('--dbname="$database_url"'), 7) + self.assertEqual(migrate_script.count('--dbname="$database_url"'), 8) + self.assertNotIn("TRUNCATE ", upper) + self.assertNotRegex(upper, r"\bDELETE\s+FROM\b") + + def test_adds_runtime_ledger_error_and_event_detail_columns(self) -> None: + app_root = Path(__file__).resolve().parents[2] + migration_path = app_root / "migrations" / "007_backfill_runtime_details.sql" + self.assertTrue(migration_path.exists(), "007 runtime-details migration is missing") + + migration = migration_path.read_text(encoding="utf-8") + migrate_script = (app_root / "deploy" / "migrate.sh").read_text( + encoding="utf-8" + ) + upper = migration.upper() + + self.assertIn("BEGIN;", upper) + self.assertIn("COMMIT;", upper) + self.assertIn("ALTER TABLE historical_backfill_items", migration) + self.assertIn("ADD COLUMN IF NOT EXISTS last_error TEXT", migration) + self.assertIn("historical_backfill_items_last_error_check", migration) + self.assertIn("char_length(last_error) BETWEEN 1 AND 2048", migration) + self.assertIn("ALTER TABLE historical_backfill_events", migration) + self.assertIn("ADD COLUMN IF NOT EXISTS details JSONB", migration) + self.assertIn("DEFAULT '{}'::jsonb", migration) + self.assertIn("historical_backfill_events_details_check", migration) + self.assertIn("jsonb_typeof(details) = 'object'", migration) + self.assertIn("007_backfill_runtime_details.sql", migrate_script) + self.assertEqual(migrate_script.count('--dbname="$database_url"'), 8) self.assertNotIn("TRUNCATE ", upper) self.assertNotRegex(upper, r"\bDELETE\s+FROM\b") diff --git a/app/deploy/migrate.sh b/app/deploy/migrate.sh index a9b47c9..62d93ce 100755 --- a/app/deploy/migrate.sh +++ b/app/deploy/migrate.sh @@ -33,6 +33,8 @@ psql --dbname="$database_url" --no-psqlrc --set=ON_ERROR_STOP=1 \ --file="$app_dir/migrations/005_historical_source_artifacts.sql" psql --dbname="$database_url" --no-psqlrc --set=ON_ERROR_STOP=1 \ --file="$app_dir/migrations/006_artifact_recorded_event.sql" +psql --dbname="$database_url" --no-psqlrc --set=ON_ERROR_STOP=1 \ + --file="$app_dir/migrations/007_backfill_runtime_details.sql" psql --dbname="$database_url" --no-psqlrc --set=ON_ERROR_STOP=1 --tuples-only --no-align \ <<'SQL' SELECT count(*) = 12 diff --git a/app/migrations/007_backfill_runtime_details.sql b/app/migrations/007_backfill_runtime_details.sql new file mode 100644 index 0000000..2d5672f --- /dev/null +++ b/app/migrations/007_backfill_runtime_details.sql @@ -0,0 +1,49 @@ +-- Add runtime columns already required by the historical backfill ledger code. + +BEGIN; + +ALTER TABLE historical_backfill_items + ADD COLUMN IF NOT EXISTS last_error TEXT; + +DO $migration$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'historical_backfill_items_last_error_check' + AND conrelid = 'historical_backfill_items'::regclass + ) THEN + ALTER TABLE historical_backfill_items + ADD CONSTRAINT historical_backfill_items_last_error_check + CHECK ( + last_error IS NULL + OR ( + char_length(last_error) BETWEEN 1 AND 2048 + AND last_error = btrim(last_error) + AND position(E'\r' IN last_error) = 0 + AND position(E'\n' IN last_error) = 0 + ) + ); + END IF; +END +$migration$; + +ALTER TABLE historical_backfill_events + ADD COLUMN IF NOT EXISTS details JSONB NOT NULL DEFAULT '{}'::jsonb; + +DO $migration$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'historical_backfill_events_details_check' + AND conrelid = 'historical_backfill_events'::regclass + ) THEN + ALTER TABLE historical_backfill_events + ADD CONSTRAINT historical_backfill_events_details_check + CHECK (jsonb_typeof(details) = 'object'); + END IF; +END +$migration$; + +COMMIT; From 2c8557908520be650c8fe9650f457c32e978fb30 Mon Sep 17 00:00:00 2001 From: djurcola-agent Date: Sun, 30 Aug 2026 20:44:13 +1000 Subject: [PATCH 13/30] feat: finalize completed backfill runs --- .../batterywatch_api/backfill_ledger.py | 78 +++++++++++++++++++ app/backend/tests/test_backfill_ledger.py | 75 ++++++++++++++++++ 2 files changed, 153 insertions(+) diff --git a/app/backend/batterywatch_api/backfill_ledger.py b/app/backend/batterywatch_api/backfill_ledger.py index 49263fe..6a7cb88 100644 --- a/app/backend/batterywatch_api/backfill_ledger.py +++ b/app/backend/batterywatch_api/backfill_ledger.py @@ -29,6 +29,12 @@ class BackfillRunProgress: total_attempts: int +@dataclass(frozen=True, slots=True) +class BackfillRunFinalization: + replayed: bool + progress: BackfillRunProgress + + @dataclass(frozen=True, slots=True) class BackfillPlanItem: feed: str @@ -121,6 +127,21 @@ def _managed_cursor(connection: _Connection) -> Iterator[_Cursor]: GROUP BY r.status """ +_FINALIZE_RUN_LOCK_SQL = """ +SELECT status +FROM historical_backfill_runs +WHERE run_id = %s +FOR UPDATE +""" + +_FINALIZE_RUN_UPDATE_SQL = """ +UPDATE historical_backfill_runs +SET status = 'completed', completed_at = CURRENT_TIMESTAMP, + updated_at = CURRENT_TIMESTAMP +WHERE run_id = %s AND status = 'running' +RETURNING 1 +""" + _ITEM_INSERT_SQL = """ INSERT INTO historical_backfill_items ( run_id, feed, report_date, source_url, status @@ -316,6 +337,62 @@ class PostgreSQLBackfillLedger: def __init__(self, connection: _Connection): self._connection = connection + def finalize(self, run_id: str) -> BackfillRunFinalization: + _validate_run_id(run_id) + try: + with _managed_cursor(self._connection) as cursor: + cursor.execute(_FINALIZE_RUN_LOCK_SQL, (run_id,)) + run_row = cursor.fetchone() + if run_row is None: + raise BackfillRunConflictError("backfill run does not exist") + if run_row not in {("running",), ("completed",), ("failed",)}: + raise BackfillRunConflictError("invalid backfill run status") + cursor.execute(_RUN_PROGRESS_SQL, (run_id,)) + progress_row = cursor.fetchone() + if progress_row is None: + raise BackfillRunConflictError("backfill run does not exist") + progress = _progress_from_row(run_id, progress_row) + if progress.status != run_row[0]: + raise BackfillRunConflictError("backfill run status changed") + if ( + progress.pending != 0 + or progress.running != 0 + or progress.failed != 0 + or progress.completed != progress.total + ): + raise BackfillRunConflictError("backfill run is incomplete") + if progress.status == "completed": + result = BackfillRunFinalization(True, progress) + elif progress.status == "running": + cursor.execute(_FINALIZE_RUN_UPDATE_SQL, (run_id,)) + if cursor.fetchone() is None: + raise BackfillRunConflictError( + "backfill run completion was not applied" + ) + result = BackfillRunFinalization( + False, + BackfillRunProgress( + progress.run_id, + "completed", + progress.total, + progress.pending, + progress.running, + progress.completed, + progress.failed, + progress.total_attempts, + ), + ) + else: + raise BackfillRunConflictError("backfill run cannot be finalized") + self._connection.commit() + return result + except Exception: + try: + self._connection.rollback() + except Exception: + pass + raise + def progress(self, run_id: str) -> BackfillRunProgress: _validate_run_id(run_id) try: @@ -567,6 +644,7 @@ def ensure_run( "BackfillItemFailure", "BackfillPlanItem", "BackfillRunConflictError", + "BackfillRunFinalization", "BackfillRunProgress", "BackfillRunSpec", "PostgreSQLBackfillLedger", diff --git a/app/backend/tests/test_backfill_ledger.py b/app/backend/tests/test_backfill_ledger.py index c8ce656..797267f 100644 --- a/app/backend/tests/test_backfill_ledger.py +++ b/app/backend/tests/test_backfill_ledger.py @@ -12,6 +12,7 @@ BackfillItemFailure, BackfillPlanItem, BackfillRunConflictError, + BackfillRunFinalization, BackfillRunProgress, BackfillRunSpec, PostgreSQLBackfillLedger, @@ -132,6 +133,80 @@ def test_progress_rejects_inconsistent_database_counts(self) -> None: self.assertEqual((connection.commits, connection.rollbacks), (0, 1)) self.assertEqual((connection.cursor_calls, connection.closed_cursors), (1, 1)) + def test_finalize_completes_run_with_deterministic_progress(self) -> None: + connection = FakeConnection( + fetchone_results=( + ("running",), + ("running", 2, 0, 0, 2, 0, 3), + (1,), + ) + ) + + result = PostgreSQLBackfillLedger(connection).finalize("run-20260828") + + self.assertEqual( + result, + BackfillRunFinalization( + False, + BackfillRunProgress( + "run-20260828", "completed", 2, 0, 0, 2, 0, 3 + ), + ), + ) + self.assertEqual((connection.commits, connection.rollbacks), (1, 0)) + self.assertEqual((connection.cursor_calls, connection.closed_cursors), (1, 1)) + self.assertEqual(len(connection.executions), 3) + self.assertIn("FOR UPDATE", connection.executions[0][0]) + self.assertIn("FILTER (WHERE i.status = 'completed')", connection.executions[1][0]) + self.assertIn("status = 'completed'", connection.executions[2][0]) + self.assertEqual(connection.executions[2][1], ("run-20260828",)) + + def test_finalize_rejects_incomplete_run_without_update(self) -> None: + connection = FakeConnection( + fetchone_results=( + ("running",), + ("running", 2, 1, 0, 1, 0, 2), + None, + ) + ) + + with self.assertRaisesRegex(BackfillRunConflictError, "incomplete"): + PostgreSQLBackfillLedger(connection).finalize("run-20260828") + + self.assertEqual((connection.commits, connection.rollbacks), (0, 1)) + self.assertEqual(len(connection.executions), 2) + + def test_finalize_exact_replay_is_idempotent_without_update(self) -> None: + connection = FakeConnection( + fetchone_results=( + ("completed",), + ("completed", 2, 0, 0, 2, 0, 3), + ) + ) + + result = PostgreSQLBackfillLedger(connection).finalize("run-20260828") + + self.assertTrue(result.replayed) + self.assertEqual(result.progress.status, "completed") + self.assertEqual(result.progress.completed, 2) + self.assertEqual((connection.commits, connection.rollbacks), (1, 0)) + self.assertEqual(len(connection.executions), 2) + + def test_finalize_rejects_missing_guarded_run_update(self) -> None: + connection = FakeConnection( + fetchone_results=( + ("running",), + ("running", 1, 0, 0, 1, 0, 1), + None, + ) + ) + + with self.assertRaisesRegex(BackfillRunConflictError, "not applied"): + PostgreSQLBackfillLedger(connection).finalize("run-20260828") + + self.assertEqual((connection.commits, connection.rollbacks), (0, 1)) + self.assertEqual(len(connection.executions), 3) + def test_fail_records_guarded_item_transition_and_event(self) -> None: claim = BackfillClaim( "run-20260828", From aa8e46d0d91e15b897f6806dd052a87e3fddc9a5 Mon Sep 17 00:00:00 2001 From: djurcola-agent Date: Sun, 30 Aug 2026 20:47:07 +1000 Subject: [PATCH 14/30] refactor: preserve historical SCADA source URL --- .../batterywatch_api/collector_service.py | 3 +- app/backend/tests/test_collector_service.py | 37 +++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/app/backend/batterywatch_api/collector_service.py b/app/backend/batterywatch_api/collector_service.py index 52e2a08..63bb141 100644 --- a/app/backend/batterywatch_api/collector_service.py +++ b/app/backend/batterywatch_api/collector_service.py @@ -102,6 +102,7 @@ def run_collection_cycle( *, collect: Callable[..., DispatchScadaCollection] = collect_latest_dispatch_scada, ingestor_factory: Callable[[Any], _Ingestor] = PostgreSQLDispatchScadaIngestor, + receipt_source_url: str | None = None, ) -> DispatchScadaIngestionResult: """Fetch, validate, map, and atomically persist one latest artifact.""" @@ -119,7 +120,7 @@ def run_collection_cycle( reference = artifact.reference receipt = DispatchScadaArtifactReceipt( source_artifact_id=reference.source_artifact_id, - source_url=reference.url, + source_url=(reference.url if receipt_source_url is None else receipt_source_url), zip_filename=reference.zip_filename, csv_member_name=artifact.csv_member_name, report_timestamp=reference.report_timestamp, diff --git a/app/backend/tests/test_collector_service.py b/app/backend/tests/test_collector_service.py index b8da199..51492bf 100644 --- a/app/backend/tests/test_collector_service.py +++ b/app/backend/tests/test_collector_service.py @@ -167,6 +167,43 @@ def ingestor_factory(connection): self.assertEqual(generators[0].source_id, "reviewed-registry") self.assertEqual(generators[0].ingestion_version, 1) + def test_cycle_preserves_explicit_historical_source_url(self) -> None: + captured: list[CapturingIngestor] = [] + + def ingestor_factory(connection): + ingestor = CapturingIngestor(connection) + captured.append(ingestor) + return ingestor + + run_collection_cycle( + object(), + ( + BatteryAsset( + "BAT1", + "Battery One", + "NSW1", + 10, + 20, + "reviewed-registry", + datetime(2025, 3, 31, tzinfo=UTC), + ), + ), + collect=lambda **kwargs: collection(), + ingestor_factory=ingestor_factory, + receipt_source_url=( + "https://www.nemweb.com.au/REPORTS/ARCHIVE/Dispatch_SCADA/" + "PUBLIC_DISPATCHSCADA_20260830.zip" + ), + ) + + assert captured[0].call is not None + receipt = captured[0].call[0] + self.assertEqual( + receipt.source_url, + "https://www.nemweb.com.au/REPORTS/ARCHIVE/Dispatch_SCADA/" + "PUBLIC_DISPATCHSCADA_20260830.zip", + ) + def test_price_cycle_persists_raw_artifact_and_five_regions(self) -> None: captured: list[CapturingPriceIngestor] = [] From 912936d63f434c47931326e4f4a6cf00df398c7b Mon Sep 17 00:00:00 2001 From: djurcola-agent Date: Sun, 30 Aug 2026 21:09:21 +1000 Subject: [PATCH 15/30] feat: add historical Dispatch SCADA claim runner --- .../historical_scada_backfill.py | 248 ++++++++++ .../tests/test_historical_scada_backfill.py | 455 ++++++++++++++++++ 2 files changed, 703 insertions(+) create mode 100644 app/backend/batterywatch_api/historical_scada_backfill.py create mode 100644 app/backend/tests/test_historical_scada_backfill.py diff --git a/app/backend/batterywatch_api/historical_scada_backfill.py b/app/backend/batterywatch_api/historical_scada_backfill.py new file mode 100644 index 0000000..449b2ae --- /dev/null +++ b/app/backend/batterywatch_api/historical_scada_backfill.py @@ -0,0 +1,248 @@ +"""One bounded historical Dispatch SCADA backfill claim.""" + +from __future__ import annotations + +from collections.abc import Callable, Iterable +from contextlib import contextmanager +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from email.utils import parsedate_to_datetime +from importlib import import_module +from typing import Any, Iterator + +from .backfill_artifacts import ( + BackfillArtifactReceipt, + PostgreSQLBackfillArtifactRegistrar, +) +from .backfill_ledger import BackfillClaim, PostgreSQLBackfillLedger +from .battery_assets import BatteryAsset +from .collector import DispatchScadaCollection +from .collector_service import run_collection_cycle +from .dispatch_scada import parse_dispatch_scada_csv +from .nemweb_archives import ( + ARCHIVE_SOURCE, + DISPATCH_SCADA_CURRENT_INDEX_URL, + DISPATCH_SCADA_FEED, + MAX_OUTER_ARCHIVE_BYTES, + ArchivePlanItem, + extract_nested_archive, +) +from .nemweb_dispatch_scada import ( + DispatchScadaArtifactRef, + extract_dispatch_scada_zip, +) +from .nemweb_http import fetch_nemweb_resource + + +UTC = timezone.utc +_NEM_TIMEZONE = timezone(timedelta(hours=10)) + + +@dataclass(frozen=True, slots=True) +class HistoricalScadaBackfillResult: + interval_artifact_count: int + raw_observation_count: int + mapped_power_count: int + replayed_interval_count: int + outer_artifact_replayed: bool + completion_replayed: bool + + +def _connect(database_url: str, *, connect_timeout: int) -> Any: + return import_module("psycopg").connect( + database_url, connect_timeout=connect_timeout + ) + + +@contextmanager +def _connection( + database_url: str, connect: Callable[..., Any] +) -> Iterator[Any]: + connection = connect(database_url, connect_timeout=10) + try: + yield connection + finally: + connection.close() + + +def _utc(value: datetime, name: str) -> datetime: + if type(value) is not datetime or value.tzinfo is None: + raise ValueError(f"invalid {name}") + try: + if value.utcoffset() is None: + raise ValueError(f"invalid {name}") + return value.astimezone(UTC) + except (AttributeError, OverflowError, TypeError, ValueError): + raise ValueError(f"invalid {name}") from None + + +def _last_modified(value: str | None) -> datetime | None: + if value is None: + return None + try: + parsed = parsedate_to_datetime(value) + except (TypeError, ValueError, OverflowError): + raise ValueError("invalid Last-Modified header") from None + if parsed.tzinfo is None or parsed.utcoffset() is None: + raise ValueError("invalid Last-Modified header") + return parsed.astimezone(UTC) + + +def _validate_inputs( + database_url: str, + claim: BackfillClaim, + start: datetime, + end: datetime, +) -> tuple[datetime, datetime]: + if type(database_url) is not str or not database_url: + raise ValueError("database URL is required") + if type(claim) is not BackfillClaim or claim.feed != DISPATCH_SCADA_FEED: + raise ValueError("invalid Dispatch SCADA backfill claim") + range_start = _utc(start, "backfill start") + range_end = _utc(end, "backfill end") + if range_end <= range_start: + raise ValueError("invalid backfill range") + return range_start, range_end + + +def _run_scada_backfill_claim( + database_url: str, + assets: Iterable[BatteryAsset], + claim: BackfillClaim, + start: datetime, + end: datetime, + *, + connect: Callable[..., Any] = _connect, + fetch: Callable[..., Any] = fetch_nemweb_resource, + clock: Callable[[], datetime] = lambda: datetime.now(UTC), + ledger_factory: Callable[[Any], Any] = PostgreSQLBackfillLedger, + registrar_factory: Callable[[Any], Any] = PostgreSQLBackfillArtifactRegistrar, + extract_archive: Callable[..., Any] = extract_nested_archive, + extract_zip: Callable[..., Any] = extract_dispatch_scada_zip, + parse_csv: Callable[..., Any] = parse_dispatch_scada_csv, + ingest_cycle: Callable[..., Any] = run_collection_cycle, +) -> HistoricalScadaBackfillResult: + range_start, range_end = _validate_inputs(database_url, claim, start, end) + materialized_assets = tuple(assets) + + resource = fetch(claim.source_url, max_bytes=MAX_OUTER_ARCHIVE_BYTES) + downloaded_at = _utc(clock(), "download time") + source_last_modified = _last_modified(resource.last_modified) + plan_item = ArchivePlanItem( + claim.feed, claim.report_date, ARCHIVE_SOURCE, claim.source_url + ) + + with _connection(database_url, connect) as connection: + artifact_result = registrar_factory(connection).record( + BackfillArtifactReceipt( + claim, + downloaded_at, + source_last_modified, + resource.body, + ) + ) + + extraction = extract_archive( + plan_item, + resource.body, + outer_url=claim.source_url, + start=range_start, + end=range_end, + ) + if not extraction.nested: + raise ValueError("archive contains no selected Dispatch SCADA intervals") + + raw_count = 0 + mapped_count = 0 + replayed_count = 0 + for nested in extraction.nested: + reference = DispatchScadaArtifactRef( + DISPATCH_SCADA_CURRENT_INDEX_URL + nested.member_name, + nested.member_name, + nested.source_artifact_id, + nested.interval_timestamp, + ) + artifact = extract_zip(reference, nested.raw_bytes) + records = parse_csv( + artifact.csv_payload, + source_artifact_id=nested.source_artifact_id, + ingestion_version=0, + correction_version=0, + naive_timezone=_NEM_TIMEZONE, + ) + collection = DispatchScadaCollection(artifact, tuple(records)) + with _connection(database_url, connect) as connection: + ingestion = ingest_cycle( + connection, + materialized_assets, + collect=lambda **unused: collection, + receipt_source_url=f"{extraction.outer.url}#{nested.member_name}", + ) + raw_count += ingestion.raw_observation_count + mapped_count += ingestion.mapped_power_count + replayed_count += int(ingestion.replayed) + + with _connection(database_url, connect) as connection: + completion = ledger_factory(connection).complete( + claim, records_imported=raw_count + ) + + return HistoricalScadaBackfillResult( + len(extraction.nested), + raw_count, + mapped_count, + replayed_count, + artifact_result.replayed, + completion.replayed, + ) + + +def run_scada_backfill_claim( + database_url: str, + assets: Iterable[BatteryAsset], + claim: BackfillClaim, + start: datetime, + end: datetime, + *, + connect: Callable[..., Any] = _connect, + fetch: Callable[..., Any] = fetch_nemweb_resource, + clock: Callable[[], datetime] = lambda: datetime.now(UTC), + ledger_factory: Callable[[Any], Any] = PostgreSQLBackfillLedger, + registrar_factory: Callable[[Any], Any] = PostgreSQLBackfillArtifactRegistrar, + extract_archive: Callable[..., Any] = extract_nested_archive, + extract_zip: Callable[..., Any] = extract_dispatch_scada_zip, + parse_csv: Callable[..., Any] = parse_dispatch_scada_csv, + ingest_cycle: Callable[..., Any] = run_collection_cycle, +) -> HistoricalScadaBackfillResult: + """Fetch, validate, and persist exactly one claimed daily SCADA item.""" + + _validate_inputs(database_url, claim, start, end) + try: + return _run_scada_backfill_claim( + database_url, + assets, + claim, + start, + end, + connect=connect, + fetch=fetch, + clock=clock, + ledger_factory=ledger_factory, + registrar_factory=registrar_factory, + extract_archive=extract_archive, + extract_zip=extract_zip, + parse_csv=parse_csv, + ingest_cycle=ingest_cycle, + ) + except Exception as error: + try: + with _connection(database_url, connect) as connection: + ledger_factory(connection).fail( + claim, error_summary=type(error).__name__ + ) + except Exception: + pass + raise + + +__all__ = ["HistoricalScadaBackfillResult", "run_scada_backfill_claim"] diff --git a/app/backend/tests/test_historical_scada_backfill.py b/app/backend/tests/test_historical_scada_backfill.py new file mode 100644 index 0000000..15e405a --- /dev/null +++ b/app/backend/tests/test_historical_scada_backfill.py @@ -0,0 +1,455 @@ +"""Tests for one claimed historical Dispatch SCADA archive item.""" + +from datetime import date, datetime, timezone +from typing import Any +import unittest + +from batterywatch_api.backfill_artifacts import BackfillArtifactResult +from batterywatch_api.backfill_ledger import ( + BackfillClaim, + BackfillItemCompletion, +) +from batterywatch_api.battery_assets import BatteryAsset +from batterywatch_api.collector import DispatchScadaCollection +from batterywatch_api.dispatch_scada_ingestion import DispatchScadaIngestionResult +from batterywatch_api.historical_scada_backfill import ( + HistoricalScadaBackfillResult, + run_scada_backfill_claim, +) +from batterywatch_api.nemweb_archives import ( + DISPATCH_SCADA_FEED, + NemwebArchiveExtraction, + NemwebNestedArchiveArtifact, + NemwebOuterArchiveArtifact, +) +from batterywatch_api.nemweb_dispatch_scada import DispatchScadaArtifact +from batterywatch_api.nemweb_http import NemwebHttpResource +from batterywatch_api.storage import GeneratorPower5m + + +UTC = timezone.utc +REPORT_DATE = date(2026, 8, 29) +START = datetime(2026, 8, 28, 14, 0, tzinfo=UTC) +END = datetime(2026, 8, 29, 14, 0, tzinfo=UTC) +OUTER_URL = ( + "https://www.nemweb.com.au/REPORTS/ARCHIVE/Dispatch_SCADA/" + "PUBLIC_DISPATCHSCADA_20260829.zip" +) +CLAIM = BackfillClaim("run-1", DISPATCH_SCADA_FEED, REPORT_DATE, OUTER_URL, 1) +ASSET = BatteryAsset( + "BAT1", + "Battery One", + "NSW1", + 10, + 20, + "reviewed-registry", + datetime(2025, 3, 31, tzinfo=UTC), +) + + +class FakeConnection: + def __init__(self, sequence: int) -> None: + self.sequence = sequence + self.close_calls = 0 + + def close(self) -> None: + self.close_calls += 1 + + +class HistoricalScadaBackfillTests(unittest.TestCase): + def test_ingests_two_ordered_intervals_with_fresh_connections(self) -> None: + connections: list[FakeConnection] = [] + operations: list[tuple[Any, ...]] = [] + + def connect(database_url: str, *, connect_timeout: int) -> FakeConnection: + self.assertEqual((database_url, connect_timeout), ("postgresql://private", 10)) + connection = FakeConnection(len(connections) + 1) + connections.append(connection) + return connection + + outer = NemwebOuterArchiveArtifact( + DISPATCH_SCADA_FEED, + REPORT_DATE, + OUTER_URL, + "PUBLIC_DISPATCHSCADA_20260829.zip", + "a" * 64, + b"outer", + ) + nested = tuple( + NemwebNestedArchiveArtifact( + outer, + f"PUBLIC_DISPATCHSCADA_20260829{hour:02d}05_{source_id}.zip", + source_id, + datetime(2026, 8, 28, hour, 5, tzinfo=UTC), + "b" * 64, + f"nested-{source_id}".encode(), + ) + for hour, source_id in ((14, "101"), (15, "102")) + ) + + def fetch(url: str, *, max_bytes: int) -> NemwebHttpResource: + self.assertEqual(url, OUTER_URL) + self.assertGreater(max_bytes, 0) + return NemwebHttpResource( + url, url, b"outer", "application/zip", None, + "Sat, 29 Aug 2026 01:02:03 GMT", + ) + + def extract_archive(*args: Any, **kwargs: Any) -> NemwebArchiveExtraction: + operations.append(("extract", args, kwargs)) + return NemwebArchiveExtraction(outer, nested) + + testcase = self + + class Registrar: + def __init__(self, connection: FakeConnection) -> None: + self.connection = connection + + def record(self, receipt: Any) -> BackfillArtifactResult: + testcase.assertEqual( + receipt.downloaded_at, + datetime(2026, 8, 30, 1, 2, 3, tzinfo=UTC), + ) + testcase.assertEqual( + receipt.source_last_modified, + datetime(2026, 8, 29, 1, 2, 3, tzinfo=UTC), + ) + testcase.assertEqual(receipt.raw_archive, b"outer") + operations.append(("record", self.connection.sequence, receipt)) + return BackfillArtifactResult("a" * 64, 5, True) + + class Ledger: + def __init__(self, connection: FakeConnection) -> None: + self.connection = connection + + def complete(self, claim: BackfillClaim, *, records_imported: int) -> BackfillItemCompletion: + operations.append( + ("complete", self.connection.sequence, claim, records_imported) + ) + return BackfillItemCompletion(True, records_imported) + + def fail(self, claim: BackfillClaim, *, error_summary: str) -> Any: + raise AssertionError("failure path must not run") + + def extract_zip(reference: Any, payload: bytes) -> DispatchScadaArtifact: + return DispatchScadaArtifact( + reference, + reference.zip_filename.removesuffix(".zip") + ".CSV", + "validated-csv", + "b" * 64, + payload, + ) + + def parse_csv(payload: str, **kwargs: Any) -> tuple[GeneratorPower5m, ...]: + source_id = kwargs["source_artifact_id"] + timestamp = next( + item.interval_timestamp for item in nested + if item.source_artifact_id == source_id + ) + return ( + GeneratorPower5m( + "BAT1", timestamp, 1.5, source_id, timestamp, 0, 0 + ), + GeneratorPower5m( + "UNMAPPED", timestamp, -2.0, source_id, timestamp, 0, 0 + ), + ) + + def ingest_cycle( + connection: FakeConnection, + assets: tuple[BatteryAsset, ...], + *, + collect: Any, + receipt_source_url: str, + ) -> DispatchScadaIngestionResult: + collection = collect(ingestion_version=0, correction_version=0) + operations.append( + ( + "ingest", + connection.sequence, + collection.artifact.reference.source_artifact_id, + receipt_source_url, + tuple(asset.duid for asset in assets), + ) + ) + return DispatchScadaIngestionResult(2, 1, True) + + result = run_scada_backfill_claim( + "postgresql://private", + (ASSET,), + CLAIM, + START, + END, + connect=connect, + fetch=fetch, + clock=lambda: datetime(2026, 8, 30, 1, 2, 3, tzinfo=UTC), + registrar_factory=Registrar, + ledger_factory=Ledger, + extract_archive=extract_archive, + extract_zip=extract_zip, + parse_csv=parse_csv, + ingest_cycle=ingest_cycle, + ) + + self.assertEqual( + result, + HistoricalScadaBackfillResult(2, 4, 2, 2, True, True), + ) + self.assertEqual([connection.close_calls for connection in connections], [1, 1, 1, 1]) + self.assertEqual( + [operation[0] for operation in operations], + ["record", "extract", "ingest", "ingest", "complete"], + ) + self.assertEqual( + [operation[3] for operation in operations if operation[0] == "ingest"], + [f"{OUTER_URL}#{nested[0].member_name}", f"{OUTER_URL}#{nested[1].member_name}"], + ) + self.assertEqual(operations[-1][-1], 4) + + def test_malformed_last_modified_records_sanitized_failure(self) -> None: + connections: list[FakeConnection] = [] + failures: list[tuple[BackfillClaim, str]] = [] + + def connect(database_url: str, *, connect_timeout: int) -> FakeConnection: + connection = FakeConnection(len(connections) + 1) + connections.append(connection) + return connection + + class Ledger: + def __init__(self, connection: FakeConnection) -> None: + self.connection = connection + + def fail(self, claim: BackfillClaim, *, error_summary: str) -> Any: + failures.append((claim, error_summary)) + return object() + + with self.assertRaisesRegex(ValueError, "Last-Modified") as raised: + run_scada_backfill_claim( + "postgresql://private", + (ASSET,), + CLAIM, + START, + END, + connect=connect, + fetch=lambda *args, **kwargs: NemwebHttpResource( + OUTER_URL, + OUTER_URL, + b"outer", + "application/zip", + None, + "not-an-http-date", + ), + clock=lambda: datetime(2026, 8, 30, 1, 2, 3, tzinfo=UTC), + ledger_factory=Ledger, + ) + + self.assertIsInstance(raised.exception, ValueError) + self.assertEqual(failures, [(CLAIM, "ValueError")]) + self.assertEqual(len(connections), 1) + self.assertEqual(connections[0].close_calls, 1) + + def test_nested_failure_after_success_records_failure_and_reraises_original(self) -> None: + connections: list[FakeConnection] = [] + failures: list[tuple[BackfillClaim, str, int]] = [] + failure = RuntimeError("secret-bearing database detail") + ingest_calls = 0 + + def connect(database_url: str, *, connect_timeout: int) -> FakeConnection: + connection = FakeConnection(len(connections) + 1) + connections.append(connection) + return connection + + outer = NemwebOuterArchiveArtifact( + DISPATCH_SCADA_FEED, + REPORT_DATE, + OUTER_URL, + "PUBLIC_DISPATCHSCADA_20260829.zip", + "a" * 64, + b"outer", + ) + nested = tuple( + NemwebNestedArchiveArtifact( + outer, + f"PUBLIC_DISPATCHSCADA_202608291{index}05_{100 + index}.zip", + str(100 + index), + datetime(2026, 8, 28, 14 + index, 5, tzinfo=UTC), + "b" * 64, + f"nested-{index}".encode(), + ) + for index in (0, 1) + ) + + class Registrar: + def __init__(self, connection: FakeConnection) -> None: + self.connection = connection + + def record(self, receipt: Any) -> BackfillArtifactResult: + return BackfillArtifactResult("a" * 64, 5, False) + + class Ledger: + def __init__(self, connection: FakeConnection) -> None: + self.connection = connection + + def fail(self, claim: BackfillClaim, *, error_summary: str) -> Any: + failures.append((claim, error_summary, self.connection.sequence)) + return object() + + def complete(self, claim: BackfillClaim, *, records_imported: int) -> Any: + raise AssertionError("completion must not run") + + def extract_zip(reference: Any, payload: bytes) -> DispatchScadaArtifact: + return DispatchScadaArtifact(reference, "member.CSV", "csv", "b" * 64, payload) + + def parse_csv(payload: str, **kwargs: Any) -> tuple[GeneratorPower5m, ...]: + source_id = kwargs["source_artifact_id"] + timestamp = next( + item.interval_timestamp for item in nested + if item.source_artifact_id == source_id + ) + return (GeneratorPower5m("BAT1", timestamp, 1, source_id, timestamp, 0, 0),) + + def ingest_cycle(*args: Any, **kwargs: Any) -> DispatchScadaIngestionResult: + nonlocal ingest_calls + ingest_calls += 1 + if ingest_calls == 2: + raise failure + return DispatchScadaIngestionResult(1, 1, False) + + with self.assertRaises(RuntimeError) as raised: + run_scada_backfill_claim( + "postgresql://private", + (ASSET,), + CLAIM, + START, + END, + connect=connect, + fetch=lambda *args, **kwargs: NemwebHttpResource( + OUTER_URL, OUTER_URL, b"outer", "application/zip", None, None + ), + clock=lambda: datetime(2026, 8, 30, tzinfo=UTC), + registrar_factory=Registrar, + ledger_factory=Ledger, + extract_archive=lambda *args, **kwargs: NemwebArchiveExtraction(outer, nested), + extract_zip=extract_zip, + parse_csv=parse_csv, + ingest_cycle=ingest_cycle, + ) + + self.assertIs(raised.exception, failure) + self.assertEqual(ingest_calls, 2) + self.assertEqual(failures, [(CLAIM, "RuntimeError", 4)]) + self.assertEqual([connection.close_calls for connection in connections], [1, 1, 1, 1]) + + def test_failure_recording_error_does_not_replace_original(self) -> None: + original = RuntimeError("original secret-bearing failure") + + def fetch(*args: Any, **kwargs: Any) -> Any: + raise original + + def connect(*args: Any, **kwargs: Any) -> Any: + raise OSError("secondary failure") + + with self.assertRaises(RuntimeError) as raised: + run_scada_backfill_claim( + "postgresql://private", + (ASSET,), + CLAIM, + START, + END, + connect=connect, + fetch=fetch, + ) + + self.assertIs(raised.exception, original) + + def test_invalid_inputs_fail_before_fetch_or_connect(self) -> None: + side_effects: list[str] = [] + + def fetch(*args: Any, **kwargs: Any) -> Any: + side_effects.append("fetch") + raise AssertionError("fetch must not run") + + def connect(*args: Any, **kwargs: Any) -> Any: + side_effects.append("connect") + raise AssertionError("connect must not run") + + invalid_claim = BackfillClaim("run-1", "dispatch_price", REPORT_DATE, OUTER_URL, 1) + cases = ( + ("", CLAIM, START, END), + ("postgresql://private", invalid_claim, START, END), + ("postgresql://private", CLAIM, START.replace(tzinfo=None), END), + ("postgresql://private", CLAIM, END, START), + ) + for database_url, claim, start, end in cases: + with self.subTest(database_url=database_url, claim=claim, start=start, end=end): + with self.assertRaises(ValueError): + run_scada_backfill_claim( + database_url, + (ASSET,), + claim, + start, + end, + connect=connect, + fetch=fetch, + ) + + self.assertEqual(side_effects, []) + + def test_zero_selected_intervals_fails_closed_without_completion(self) -> None: + connections: list[FakeConnection] = [] + failures: list[str] = [] + outer = NemwebOuterArchiveArtifact( + DISPATCH_SCADA_FEED, + REPORT_DATE, + OUTER_URL, + "PUBLIC_DISPATCHSCADA_20260829.zip", + "a" * 64, + b"outer", + ) + + def connect(*args: Any, **kwargs: Any) -> FakeConnection: + connection = FakeConnection(len(connections) + 1) + connections.append(connection) + return connection + + class Registrar: + def __init__(self, connection: FakeConnection) -> None: + self.connection = connection + + def record(self, receipt: Any) -> BackfillArtifactResult: + return BackfillArtifactResult("a" * 64, 5, False) + + class Ledger: + def __init__(self, connection: FakeConnection) -> None: + self.connection = connection + + def fail(self, claim: BackfillClaim, *, error_summary: str) -> Any: + failures.append(error_summary) + return object() + + def complete(self, claim: BackfillClaim, *, records_imported: int) -> Any: + raise AssertionError("completion must not run") + + with self.assertRaisesRegex(ValueError, "no selected"): + run_scada_backfill_claim( + "postgresql://private", + (ASSET,), + CLAIM, + START, + END, + connect=connect, + fetch=lambda *args, **kwargs: NemwebHttpResource( + OUTER_URL, OUTER_URL, b"outer", "application/zip", None, None + ), + clock=lambda: datetime(2026, 8, 30, tzinfo=UTC), + registrar_factory=Registrar, + ledger_factory=Ledger, + extract_archive=lambda *args, **kwargs: NemwebArchiveExtraction(outer, ()), + ) + + self.assertEqual(failures, ["ValueError"]) + self.assertEqual([connection.close_calls for connection in connections], [1, 1]) + + +if __name__ == "__main__": + unittest.main() From 77f825193890d89fb888fba2a94fe977407cdfd1 Mon Sep 17 00:00:00 2001 From: djurcola-agent Date: Sun, 30 Aug 2026 21:21:34 +1000 Subject: [PATCH 16/30] feat: add historical DispatchIS price claim runner --- .../batterywatch_api/collector_service.py | 3 +- .../historical_price_backfill.py | 247 +++++++++++++ app/backend/tests/test_collector_service.py | 26 ++ .../tests/test_historical_price_backfill.py | 332 ++++++++++++++++++ 4 files changed, 607 insertions(+), 1 deletion(-) create mode 100644 app/backend/batterywatch_api/historical_price_backfill.py create mode 100644 app/backend/tests/test_historical_price_backfill.py diff --git a/app/backend/batterywatch_api/collector_service.py b/app/backend/batterywatch_api/collector_service.py index 63bb141..a2d9e67 100644 --- a/app/backend/batterywatch_api/collector_service.py +++ b/app/backend/batterywatch_api/collector_service.py @@ -185,6 +185,7 @@ def run_price_collection_cycle( *, collect: Callable[..., DispatchPriceCollection] = collect_latest_dispatch_prices, ingestor_factory: Callable[[Any], _PriceIngestor] = PostgreSQLDispatchPriceIngestor, + receipt_source_url: str | None = None, ) -> DispatchPriceIngestionResult: """Fetch, validate, and atomically persist one official DispatchIS artifact.""" @@ -199,7 +200,7 @@ def run_price_collection_cycle( raise ValueError("invalid DispatchIS artifact version") receipt = DispatchPriceArtifactReceipt( source_artifact_id=reference.source_artifact_id, - source_url=reference.url, + source_url=(reference.url if receipt_source_url is None else receipt_source_url), zip_filename=reference.zip_filename, csv_member_name=artifact.csv_member_name, report_timestamp=reference.report_timestamp, diff --git a/app/backend/batterywatch_api/historical_price_backfill.py b/app/backend/batterywatch_api/historical_price_backfill.py new file mode 100644 index 0000000..f3ec1de --- /dev/null +++ b/app/backend/batterywatch_api/historical_price_backfill.py @@ -0,0 +1,247 @@ +"""One bounded historical DispatchIS regional-price backfill claim.""" + +from __future__ import annotations + +from collections.abc import Callable +from contextlib import contextmanager +from dataclasses import dataclass +from datetime import datetime, timezone +from email.utils import parsedate_to_datetime +from importlib import import_module +from typing import Any + +from .aemo import parse_dispatch_price_mms_csv +from .backfill_artifacts import ( + BackfillArtifactReceipt, + BackfillArtifactResult, + PostgreSQLBackfillArtifactRegistrar, +) +from .backfill_ledger import ( + BackfillClaim, + BackfillItemCompletion, + PostgreSQLBackfillLedger, +) +from .collector_service import run_price_collection_cycle +from .dispatch_price_ingestion import DispatchPriceIngestionResult +from .nemweb_archives import ( + ARCHIVE_SOURCE, + DISPATCHIS_PRICE_FEED, + MAX_OUTER_ARCHIVE_BYTES, + ArchivePlanItem, + NemwebArchiveExtraction, + extract_nested_archive, +) +from .nemweb_dispatch_prices import ( + DISPATCH_PRICE_INDEX_URL, + DispatchPriceArtifact, + DispatchPriceArtifactRef, + DispatchPriceCollection, + extract_dispatch_price_zip, +) +from .nemweb_http import NemwebHttpResource, fetch_nemweb_resource + +UTC = timezone.utc + + +@dataclass(frozen=True, slots=True) +class HistoricalPriceBackfillResult: + interval_artifact_count: int + price_count: int + applied_price_count: int + replayed_interval_count: int + outer_artifact_replayed: bool + completion_replayed: bool + + +def _connect(database_url: str, *, connect_timeout: int) -> Any: + psycopg = import_module("psycopg") + return psycopg.connect(database_url, connect_timeout=connect_timeout) + + +@contextmanager +def _connection(database_url: str, connect: Callable[..., Any]): + connection = connect(database_url, connect_timeout=10) + try: + yield connection + finally: + connection.close() + + +def _validated_range( + database_url: str, + claim: BackfillClaim, + start: datetime, + end: datetime, +) -> tuple[datetime, datetime]: + if not isinstance(database_url, str) or not database_url: + raise ValueError("database URL is required") + if type(claim) is not BackfillClaim or claim.feed != "dispatch_price": + raise ValueError("dispatch price backfill claim is required") + if ( + not isinstance(start, datetime) + or not isinstance(end, datetime) + or start.tzinfo is None + or start.utcoffset() is None + or end.tzinfo is None + or end.utcoffset() is None + ): + raise ValueError("backfill range must be timezone-aware") + start_utc = start.astimezone(UTC) + end_utc = end.astimezone(UTC) + if start_utc >= end_utc: + raise ValueError("backfill start must precede end") + return start_utc, end_utc + + +def _last_modified(value: str | None) -> datetime | None: + if value is None: + return None + try: + parsed = parsedate_to_datetime(value) + except (TypeError, ValueError, OverflowError) as error: + raise ValueError("invalid NEMWeb Last-Modified header") from error + if parsed.tzinfo is None or parsed.utcoffset() is None: + raise ValueError("invalid NEMWeb Last-Modified header") + return parsed.astimezone(UTC) + + +def _execute_price_claim( + database_url: str, + claim: BackfillClaim, + start: datetime, + end: datetime, + *, + connect: Callable[..., Any], + fetch: Callable[..., NemwebHttpResource], + clock: Callable[[], datetime], + ledger_factory: Callable[[Any], Any], + registrar_factory: Callable[[Any], Any], + extract_archive: Callable[..., NemwebArchiveExtraction], + extract_zip: Callable[[DispatchPriceArtifactRef, bytes], DispatchPriceArtifact], + parse_csv: Callable[..., tuple[Any, ...]], + ingest_cycle: Callable[..., DispatchPriceIngestionResult], +) -> HistoricalPriceBackfillResult: + start_utc, end_utc = _validated_range(database_url, claim, start, end) + resource = fetch(claim.source_url, max_bytes=MAX_OUTER_ARCHIVE_BYTES) + downloaded_at = clock() + if downloaded_at.tzinfo is None or downloaded_at.utcoffset() is None: + raise ValueError("download clock must be timezone-aware") + downloaded_at = downloaded_at.astimezone(UTC) + + outer_receipt = BackfillArtifactReceipt( + claim, + downloaded_at, + _last_modified(resource.last_modified), + resource.body, + ) + with _connection(database_url, connect) as connection: + artifact_result: BackfillArtifactResult = registrar_factory(connection).record( + outer_receipt + ) + + extraction = extract_archive( + ArchivePlanItem( + DISPATCHIS_PRICE_FEED, + claim.report_date, + ARCHIVE_SOURCE, + claim.source_url, + ), + resource.body, + outer_url=claim.source_url, + start=start_utc, + end=end_utc, + ) + if not extraction.nested: + raise ValueError("daily DispatchIS archive selected no intervals") + + price_count = 0 + applied_price_count = 0 + replayed_intervals = 0 + for nested in extraction.nested: + reference = DispatchPriceArtifactRef( + url=DISPATCH_PRICE_INDEX_URL + nested.member_name, + zip_filename=nested.member_name, + source_artifact_id=nested.source_artifact_id, + report_timestamp=nested.interval_timestamp, + ) + artifact = extract_zip(reference, nested.raw_bytes) + records = tuple( + parse_csv( + artifact.csv_payload, + source_id=reference.source_artifact_id, + ingestion_version=0, + correction_version=0, + ) + ) + collection = DispatchPriceCollection(artifact, records) + with _connection(database_url, connect) as connection: + interval_result = ingest_cycle( + connection, + collect=lambda **unused: collection, + receipt_source_url=f"{claim.source_url}#{nested.member_name}", + ) + price_count += len(records) + applied_price_count += interval_result.price_count + replayed_intervals += int(interval_result.replayed) + + with _connection(database_url, connect) as connection: + completion: BackfillItemCompletion = ledger_factory(connection).complete( + claim, + records_imported=price_count, + ) + return HistoricalPriceBackfillResult( + len(extraction.nested), + price_count, + applied_price_count, + replayed_intervals, + artifact_result.replayed, + completion.replayed, + ) + + +def run_price_backfill_claim( + database_url: str, + claim: BackfillClaim, + start: datetime, + end: datetime, + *, + connect: Callable[..., Any] = _connect, + fetch: Callable[..., NemwebHttpResource] = fetch_nemweb_resource, + clock: Callable[[], datetime] = lambda: datetime.now(UTC), + ledger_factory: Callable[[Any], Any] = PostgreSQLBackfillLedger, + registrar_factory: Callable[[Any], Any] = PostgreSQLBackfillArtifactRegistrar, + extract_archive: Callable[..., NemwebArchiveExtraction] = extract_nested_archive, + extract_zip: Callable[[DispatchPriceArtifactRef, bytes], DispatchPriceArtifact] = extract_dispatch_price_zip, + parse_csv: Callable[..., tuple[Any, ...]] = parse_dispatch_price_mms_csv, + ingest_cycle: Callable[..., DispatchPriceIngestionResult] = run_price_collection_cycle, +) -> HistoricalPriceBackfillResult: + _validated_range(database_url, claim, start, end) + try: + return _execute_price_claim( + database_url, + claim, + start, + end, + connect=connect, + fetch=fetch, + clock=clock, + ledger_factory=ledger_factory, + registrar_factory=registrar_factory, + extract_archive=extract_archive, + extract_zip=extract_zip, + parse_csv=parse_csv, + ingest_cycle=ingest_cycle, + ) + except Exception as error: + try: + with _connection(database_url, connect) as connection: + ledger_factory(connection).fail( + claim, + error_summary=type(error).__name__, + ) + except Exception: + pass + raise + + +__all__ = ["HistoricalPriceBackfillResult", "run_price_backfill_claim"] diff --git a/app/backend/tests/test_collector_service.py b/app/backend/tests/test_collector_service.py index 51492bf..4552149 100644 --- a/app/backend/tests/test_collector_service.py +++ b/app/backend/tests/test_collector_service.py @@ -228,6 +228,32 @@ def ingestor_factory(connection): ("NSW1", "QLD1", "SA1", "TAS1", "VIC1"), ) + def test_price_cycle_preserves_explicit_historical_source_url(self) -> None: + captured: list[CapturingPriceIngestor] = [] + + def ingestor_factory(connection): + ingestor = CapturingPriceIngestor(connection) + captured.append(ingestor) + return ingestor + + run_price_collection_cycle( + object(), + collect=lambda **kwargs: price_collection(), + ingestor_factory=ingestor_factory, + receipt_source_url=( + "https://www.nemweb.com.au/REPORTS/ARCHIVE/DispatchIS_Reports/" + "PUBLIC_DISPATCHIS_20260829.zip#PUBLIC_DISPATCHIS_202608291405_123.zip" + ), + ) + + assert captured[0].call is not None + receipt, _ = captured[0].call + self.assertEqual( + receipt.source_url, + "https://www.nemweb.com.au/REPORTS/ARCHIVE/DispatchIS_Reports/" + "PUBLIC_DISPATCHIS_20260829.zip#PUBLIC_DISPATCHIS_202608291405_123.zip", + ) + def test_database_cycle_closes_connection_when_ingestion_fails(self) -> None: connection = ClosingConnection() failure = RuntimeError("database write failed") diff --git a/app/backend/tests/test_historical_price_backfill.py b/app/backend/tests/test_historical_price_backfill.py new file mode 100644 index 0000000..134826f --- /dev/null +++ b/app/backend/tests/test_historical_price_backfill.py @@ -0,0 +1,332 @@ +"""Tests for one claimed historical DispatchIS price archive item.""" + +from datetime import date, datetime, timezone +from typing import Any +import unittest + +from batterywatch_api.backfill_artifacts import BackfillArtifactResult +from batterywatch_api.backfill_ledger import BackfillClaim, BackfillItemCompletion +from batterywatch_api.dispatch_price_ingestion import DispatchPriceIngestionResult +from batterywatch_api.nemweb_archives import ( + DISPATCHIS_PRICE_FEED, + NemwebArchiveExtraction, + NemwebNestedArchiveArtifact, + NemwebOuterArchiveArtifact, +) +from batterywatch_api.nemweb_dispatch_prices import ( + DispatchPriceArtifact, + DispatchPriceArtifactRef, + DispatchPriceCollection, +) +from batterywatch_api.nemweb_http import NemwebHttpResource +from batterywatch_api.historical_price_backfill import run_price_backfill_claim +from batterywatch_api.storage import RegionalPrice5m + +UTC = timezone.utc + + +class Connection: + def __init__(self, number: int) -> None: + self.number = number + self.close_calls = 0 + + def close(self) -> None: + self.close_calls += 1 + + +class HistoricalPriceBackfillTests(unittest.TestCase): + def test_ingests_two_five_region_intervals_with_fresh_connections(self) -> None: + connections: list[Connection] = [] + operations: list[str] = [] + report_date = date(2026, 8, 29) + outer_url = ( + "https://www.nemweb.com.au/REPORTS/ARCHIVE/DispatchIS_Reports/" + "PUBLIC_DISPATCHIS_20260829.zip" + ) + outer = NemwebOuterArchiveArtifact( + DISPATCHIS_PRICE_FEED, + report_date, + outer_url, + "PUBLIC_DISPATCHIS_20260829.zip", + "a" * 64, + b"outer", + ) + nested = tuple( + NemwebNestedArchiveArtifact( + outer, + f"PUBLIC_DISPATCHIS_20260829{hour:02d}05_{100 + index}.zip", + str(100 + index), + datetime(2026, 8, 29, hour, 5, tzinfo=UTC), + chr(ord("b") + index) * 64, + f"nested-{index}".encode(), + ) + for index, hour in enumerate((14, 15)) + ) + extraction = NemwebArchiveExtraction(outer, nested) + claim = BackfillClaim( + "price-run", + "dispatch_price", + report_date, + outer_url, + 1, + ) + + def connect(database_url: str, *, connect_timeout: int) -> Connection: + self.assertEqual((database_url, connect_timeout), ("postgresql://private", 10)) + connection = Connection(len(connections) + 1) + connections.append(connection) + return connection + + def fetch(url: str, *, max_bytes: int) -> NemwebHttpResource: + self.assertEqual(url, outer_url) + self.assertEqual(max_bytes, 128 * 1024 * 1024) + operations.append("fetch") + return NemwebHttpResource(url, url, b"outer", "application/zip", None, None) + + class Registrar: + def __init__(self, connection: Connection) -> None: + self.connection = connection + + def record(self, receipt: Any) -> BackfillArtifactResult: + operations.append(f"record:{self.connection.number}") + return BackfillArtifactResult("a" * 64, len(receipt.raw_archive), True) + + def extract_archive(*args: Any, **kwargs: Any) -> NemwebArchiveExtraction: + self.assertEqual(args[0].feed, DISPATCHIS_PRICE_FEED) + operations.append("extract") + return extraction + + def extract_zip(reference: DispatchPriceArtifactRef, payload: bytes) -> DispatchPriceArtifact: + return DispatchPriceArtifact( + reference, + reference.zip_filename.removesuffix(".zip") + ".CSV", + "csv", + "d" * 64, + payload, + ) + + def parse_csv( + payload: str, + *, + source_id: str, + ingestion_version: int, + correction_version: int, + ) -> tuple[RegionalPrice5m, ...]: + del payload, ingestion_version, correction_version + timestamp = next(item.interval_timestamp for item in nested if item.source_artifact_id == source_id) + prices = (100.0, -25.0, 75.0, 0.0, 50.0) + return tuple( + RegionalPrice5m( + region, + timestamp, + price, + "negative" if price < 0 else "available", + source_id, + timestamp, + int(source_id), + ) + for region, price in zip(("NSW1", "QLD1", "SA1", "TAS1", "VIC1"), prices) + ) + + def ingest_cycle( + connection: Connection, + *, + collect: Any, + receipt_source_url: str, + ) -> DispatchPriceIngestionResult: + collection: DispatchPriceCollection = collect(ingestion_version=0, correction_version=0) + operations.append(f"ingest:{connection.number}:{collection.artifact.reference.source_artifact_id}") + self.assertEqual(receipt_source_url, f"{outer_url}#{collection.artifact.reference.zip_filename}") + self.assertIn(-25.0, tuple(record.price_aud_per_mwh for record in collection.records)) + return DispatchPriceIngestionResult(0, True) + + class Ledger: + def __init__(self, connection: Connection) -> None: + self.connection = connection + + def complete(self, actual_claim: BackfillClaim, *, records_imported: int) -> BackfillItemCompletion: + self.assert_claim(actual_claim) + operations.append(f"complete:{self.connection.number}:{records_imported}") + return BackfillItemCompletion(True, records_imported) + + def assert_claim(self, actual_claim: BackfillClaim) -> None: + if actual_claim != claim: + raise AssertionError("claim mismatch") + + def fail(self, actual_claim: BackfillClaim, *, error_summary: str) -> Any: + raise AssertionError(f"failure path must not run: {actual_claim} {error_summary}") + + result = run_price_backfill_claim( + "postgresql://private", + claim, + datetime(2026, 8, 29, 14, 0, tzinfo=UTC), + datetime(2026, 8, 29, 16, 0, tzinfo=UTC), + connect=connect, + fetch=fetch, + clock=lambda: datetime(2026, 8, 30, 3, 0, tzinfo=UTC), + ledger_factory=Ledger, + registrar_factory=Registrar, + extract_archive=extract_archive, + extract_zip=extract_zip, + parse_csv=parse_csv, + ingest_cycle=ingest_cycle, + ) + + self.assertEqual(result.interval_artifact_count, 2) + self.assertEqual(result.price_count, 10) + self.assertEqual(result.applied_price_count, 0) + self.assertEqual(result.replayed_interval_count, 2) + self.assertTrue(result.outer_artifact_replayed) + self.assertTrue(result.completion_replayed) + self.assertEqual( + operations, + ["fetch", "record:1", "extract", "ingest:2:100", "ingest:3:101", "complete:4:10"], + ) + self.assertEqual([connection.close_calls for connection in connections], [1, 1, 1, 1]) + + def test_malformed_last_modified_records_sanitized_failure(self) -> None: + connections: list[Connection] = [] + failures: list[tuple[BackfillClaim, str]] = [] + claim = BackfillClaim( + "price-run", + "dispatch_price", + date(2026, 8, 29), + "https://www.nemweb.com.au/REPORTS/ARCHIVE/DispatchIS_Reports/" + "PUBLIC_DISPATCHIS_20260829.zip", + 1, + ) + + def connect(unused_url: str, *, connect_timeout: int) -> Connection: + self.assertEqual(connect_timeout, 10) + connection = Connection(len(connections) + 1) + connections.append(connection) + return connection + + class Ledger: + def __init__(self, connection: Connection) -> None: + self.connection = connection + + def fail(self, actual_claim: BackfillClaim, *, error_summary: str) -> Any: + failures.append((actual_claim, error_summary)) + + with self.assertRaisesRegex(ValueError, "Last-Modified"): + run_price_backfill_claim( + "postgresql://private", + claim, + datetime(2026, 8, 29, 14, tzinfo=UTC), + datetime(2026, 8, 29, 15, tzinfo=UTC), + connect=connect, + fetch=lambda *args, **kwargs: NemwebHttpResource( + claim.source_url, + claim.source_url, + b"outer", + "application/zip", + None, + "not-a-date", + ), + ledger_factory=Ledger, + ) + + self.assertEqual(failures, [(claim, "ValueError")]) + self.assertEqual([connection.close_calls for connection in connections], [1]) + + def test_invalid_inputs_fail_before_fetch_or_connect(self) -> None: + claim = BackfillClaim( + "price-run", + "dispatch_price", + date(2026, 8, 29), + "https://www.nemweb.com.au/REPORTS/ARCHIVE/DispatchIS_Reports/" + "PUBLIC_DISPATCHIS_20260829.zip", + 1, + ) + calls: list[str] = [] + + def forbidden_connect(*args: Any, **kwargs: Any) -> Any: + calls.append("connect") + raise AssertionError("invalid input must not connect") + + def forbidden_fetch(*args: Any, **kwargs: Any) -> NemwebHttpResource: + calls.append("fetch") + raise AssertionError("invalid input must not fetch") + + for database_url, actual_claim, start, end in ( + ("", claim, datetime(2026, 8, 29, 14, tzinfo=UTC), datetime(2026, 8, 29, 15, tzinfo=UTC)), + ("postgresql://private", BackfillClaim(claim.run_id, "dispatch_scada", claim.report_date, claim.source_url, 1), datetime(2026, 8, 29, 14, tzinfo=UTC), datetime(2026, 8, 29, 15, tzinfo=UTC)), + ("postgresql://private", claim, datetime(2026, 8, 29, 14), datetime(2026, 8, 29, 15, tzinfo=UTC)), + ("postgresql://private", claim, datetime(2026, 8, 29, 15, tzinfo=UTC), datetime(2026, 8, 29, 15, tzinfo=UTC)), + ): + with self.subTest(database_url=database_url, feed=actual_claim.feed, start=start): + with self.assertRaises(ValueError): + run_price_backfill_claim( + database_url, + actual_claim, + start, + end, + connect=forbidden_connect, + fetch=forbidden_fetch, + ) + self.assertEqual(calls, []) + + def test_zero_selected_intervals_records_failure_without_completion(self) -> None: + connections: list[Connection] = [] + failures: list[str] = [] + claim = BackfillClaim( + "price-run", + "dispatch_price", + date(2026, 8, 29), + "https://www.nemweb.com.au/REPORTS/ARCHIVE/DispatchIS_Reports/" + "PUBLIC_DISPATCHIS_20260829.zip", + 1, + ) + outer = NemwebOuterArchiveArtifact( + DISPATCHIS_PRICE_FEED, + claim.report_date, + claim.source_url, + "PUBLIC_DISPATCHIS_20260829.zip", + "a" * 64, + b"outer", + ) + + def connect(unused_url: str, *, connect_timeout: int) -> Connection: + connection = Connection(len(connections) + 1) + connections.append(connection) + return connection + + class Registrar: + def __init__(self, connection: Connection) -> None: + self.connection = connection + + def record(self, receipt: Any) -> BackfillArtifactResult: + return BackfillArtifactResult("a" * 64, len(receipt.raw_archive), False) + + class Ledger: + def __init__(self, connection: Connection) -> None: + self.connection = connection + + def fail(self, actual_claim: BackfillClaim, *, error_summary: str) -> Any: + failures.append(error_summary) + + def complete(self, *args: Any, **kwargs: Any) -> Any: + raise AssertionError("empty archive must not complete") + + with self.assertRaisesRegex(ValueError, "selected no intervals"): + run_price_backfill_claim( + "postgresql://private", + claim, + datetime(2026, 8, 29, 14, tzinfo=UTC), + datetime(2026, 8, 29, 15, tzinfo=UTC), + connect=connect, + fetch=lambda *args, **kwargs: NemwebHttpResource( + claim.source_url, claim.source_url, b"outer", None, None, None + ), + registrar_factory=Registrar, + ledger_factory=Ledger, + extract_archive=lambda *args, **kwargs: NemwebArchiveExtraction(outer, ()), + ) + + self.assertEqual(failures, ["ValueError"]) + self.assertEqual([connection.close_calls for connection in connections], [1, 1]) + + +if __name__ == "__main__": + unittest.main() From c20c105a39392542ca92c785db9b95e91cffc8fe Mon Sep 17 00:00:00 2001 From: djurcola-agent Date: Sun, 30 Aug 2026 21:30:21 +1000 Subject: [PATCH 17/30] feat: orchestrate supplied historical backfill plans --- .../batterywatch_api/historical_backfill.py | 159 +++++++++++ app/backend/tests/test_historical_backfill.py | 263 ++++++++++++++++++ 2 files changed, 422 insertions(+) create mode 100644 app/backend/batterywatch_api/historical_backfill.py create mode 100644 app/backend/tests/test_historical_backfill.py diff --git a/app/backend/batterywatch_api/historical_backfill.py b/app/backend/batterywatch_api/historical_backfill.py new file mode 100644 index 0000000..8341c0a --- /dev/null +++ b/app/backend/batterywatch_api/historical_backfill.py @@ -0,0 +1,159 @@ +"""Deterministic orchestration over a supplied historical backfill plan.""" + +from __future__ import annotations + +from collections.abc import Callable, Iterable +from contextlib import contextmanager +from dataclasses import dataclass +from importlib import import_module +from typing import Any, Iterator + +from .backfill_ledger import ( + BackfillClaim, + BackfillEnsureResult, + BackfillPlanItem, + BackfillRunFinalization, + BackfillRunSpec, + PostgreSQLBackfillLedger, +) +from .battery_assets import BatteryAsset +from .historical_price_backfill import ( + HistoricalPriceBackfillResult, + run_price_backfill_claim, +) +from .historical_scada_backfill import ( + HistoricalScadaBackfillResult, + run_scada_backfill_claim, +) + + +@dataclass(frozen=True, slots=True) +class HistoricalBackfillResult: + ensure_result: BackfillEnsureResult + claims: tuple[BackfillClaim, ...] + scada_results: tuple[HistoricalScadaBackfillResult, ...] + price_results: tuple[HistoricalPriceBackfillResult, ...] + finalization: BackfillRunFinalization + + @property + def claimed_count(self) -> int: + return len(self.claims) + + @property + def scada_raw_observation_count(self) -> int: + return sum(result.raw_observation_count for result in self.scada_results) + + @property + def scada_mapped_power_count(self) -> int: + return sum(result.mapped_power_count for result in self.scada_results) + + @property + def price_source_record_count(self) -> int: + return sum(result.price_count for result in self.price_results) + + @property + def price_applied_record_count(self) -> int: + return sum(result.applied_price_count for result in self.price_results) + + @property + def replayed_interval_count(self) -> int: + return sum(result.replayed_interval_count for result in self.scada_results) + sum( + result.replayed_interval_count for result in self.price_results + ) + + @property + def replayed_outer_artifact_count(self) -> int: + return sum(result.outer_artifact_replayed for result in self.scada_results) + sum( + result.outer_artifact_replayed for result in self.price_results + ) + + +def _connect(database_url: str, *, connect_timeout: int) -> Any: + return import_module("psycopg").connect( + database_url, + connect_timeout=connect_timeout, + ) + + +@contextmanager +def _connection( + database_url: str, + connect: Callable[..., Any], +) -> Iterator[Any]: + connection = connect(database_url, connect_timeout=10) + try: + yield connection + finally: + connection.close() + + +def run_historical_backfill( + database_url: str, + assets: Iterable[BatteryAsset], + spec: BackfillRunSpec, + planned_items: Iterable[BackfillPlanItem], + *, + connect: Callable[..., Any] = _connect, + ledger_factory: Callable[[Any], Any] = PostgreSQLBackfillLedger, + run_scada: Callable[..., HistoricalScadaBackfillResult] = run_scada_backfill_claim, + run_price: Callable[..., HistoricalPriceBackfillResult] = run_price_backfill_claim, +) -> HistoricalBackfillResult: + """Ensure, drain, and finalize one bounded supplied backfill plan.""" + + if type(database_url) is not str or not database_url: + raise ValueError("database URL is required") + materialized_assets = tuple(assets) + materialized_items = tuple(planned_items) + + with _connection(database_url, connect) as connection: + ensure_result = ledger_factory(connection).ensure_run( + spec, + materialized_items, + ) + + claims: list[BackfillClaim] = [] + scada_results: list[HistoricalScadaBackfillResult] = [] + price_results: list[HistoricalPriceBackfillResult] = [] + while True: + with _connection(database_url, connect) as connection: + claim = ledger_factory(connection).claim_next(spec.run_id) + if claim is None: + break + if len(claims) >= len(materialized_items): + raise ValueError("historical backfill claim limit exceeded") + claims.append(claim) + if claim.feed == "dispatch_scada": + scada_results.append( + run_scada( + database_url, + materialized_assets, + claim, + spec.requested_start, + spec.requested_end, + ) + ) + elif claim.feed == "dispatch_price": + price_results.append( + run_price( + database_url, + claim, + spec.requested_start, + spec.requested_end, + ) + ) + else: + raise ValueError("unsupported historical backfill feed") + + with _connection(database_url, connect) as connection: + finalization = ledger_factory(connection).finalize(spec.run_id) + + return HistoricalBackfillResult( + ensure_result, + tuple(claims), + tuple(scada_results), + tuple(price_results), + finalization, + ) + + +__all__ = ["HistoricalBackfillResult", "run_historical_backfill"] diff --git a/app/backend/tests/test_historical_backfill.py b/app/backend/tests/test_historical_backfill.py new file mode 100644 index 0000000..bc35f5a --- /dev/null +++ b/app/backend/tests/test_historical_backfill.py @@ -0,0 +1,263 @@ +"""Tests for deterministic supplied-plan historical backfill orchestration.""" + +from datetime import date, datetime, timezone +from typing import Any +import unittest + +from batterywatch_api.backfill_ledger import ( + BackfillClaim, + BackfillEnsureResult, + BackfillPlanItem, + BackfillRunFinalization, + BackfillRunProgress, + BackfillRunSpec, +) +from batterywatch_api.battery_assets import BatteryAsset +from batterywatch_api.historical_price_backfill import HistoricalPriceBackfillResult +from batterywatch_api.historical_scada_backfill import HistoricalScadaBackfillResult +from batterywatch_api.historical_backfill import run_historical_backfill + +UTC = timezone.utc + + +class Connection: + def __init__(self, number: int) -> None: + self.number = number + self.close_calls = 0 + + def close(self) -> None: + self.close_calls += 1 + + +class HistoricalBackfillTests(unittest.TestCase): + def test_ensures_claims_dispatches_and_finalizes_supplied_plan(self) -> None: + connections: list[Connection] = [] + operations: list[str] = [] + start = datetime(2026, 8, 29, tzinfo=UTC) + end = datetime(2026, 8, 30, tzinfo=UTC) + spec = BackfillRunSpec("run-1", start, end, 1) + scada_item = BackfillPlanItem( + "dispatch_scada", + date(2026, 8, 29), + "https://www.nemweb.com.au/REPORTS/ARCHIVE/Dispatch_SCADA/" + "PUBLIC_DISPATCHSCADA_20260829.zip", + ) + price_item = BackfillPlanItem( + "dispatch_price", + date(2026, 8, 29), + "https://www.nemweb.com.au/REPORTS/ARCHIVE/DispatchIS_Reports/" + "PUBLIC_DISPATCHIS_20260829.zip", + ) + scada_claim = BackfillClaim( + spec.run_id, + scada_item.feed, + scada_item.report_date, + scada_item.source_url, + 1, + ) + price_claim = BackfillClaim( + spec.run_id, + price_item.feed, + price_item.report_date, + price_item.source_url, + 1, + ) + claims = [scada_claim, price_claim, None] + assets = ( + BatteryAsset( + "BAT1", + "Battery One", + "NSW1", + 10, + 20, + "reviewed-registry", + datetime(2025, 3, 31, tzinfo=UTC), + ), + ) + progress = BackfillRunProgress("run-1", "completed", 2, 0, 0, 2, 0, 2) + finalization = BackfillRunFinalization(False, progress) + + def connect(database_url: str, *, connect_timeout: int) -> Connection: + self.assertEqual((database_url, connect_timeout), ("postgresql://private", 10)) + connection = Connection(len(connections) + 1) + connections.append(connection) + return connection + + class Ledger: + def __init__(self, connection: Connection) -> None: + self.connection = connection + + def ensure_run(self, actual_spec: BackfillRunSpec, items: Any) -> BackfillEnsureResult: + self.assert_equal(actual_spec, spec) + self.assert_equal(tuple(items), (scada_item, price_item)) + operations.append(f"ensure:{self.connection.number}") + return BackfillEnsureResult(True, False, 2, 0) + + def claim_next(self, run_id: str) -> BackfillClaim | None: + self.assert_equal(run_id, spec.run_id) + operations.append(f"claim:{self.connection.number}") + return claims.pop(0) + + def finalize(self, run_id: str) -> BackfillRunFinalization: + self.assert_equal(run_id, spec.run_id) + operations.append(f"finalize:{self.connection.number}") + return finalization + + def assert_equal(self, actual: Any, expected: Any) -> None: + if actual != expected: + raise AssertionError(f"{actual!r} != {expected!r}") + + def run_scada( + database_url: str, + actual_assets: Any, + claim: BackfillClaim, + range_start: datetime, + range_end: datetime, + ) -> HistoricalScadaBackfillResult: + self.assertEqual((database_url, tuple(actual_assets), claim, range_start, range_end), ("postgresql://private", assets, scada_claim, start, end)) + operations.append("scada") + return HistoricalScadaBackfillResult(2, 20, 4, 1, True, False) + + def run_price( + database_url: str, + claim: BackfillClaim, + range_start: datetime, + range_end: datetime, + ) -> HistoricalPriceBackfillResult: + self.assertEqual((database_url, claim, range_start, range_end), ("postgresql://private", price_claim, start, end)) + operations.append("price") + return HistoricalPriceBackfillResult(2, 10, 5, 0, False, False) + + result = run_historical_backfill( + "postgresql://private", + assets, + spec, + (scada_item, price_item), + connect=connect, + ledger_factory=Ledger, + run_scada=run_scada, + run_price=run_price, + ) + + self.assertEqual(result.ensure_result, BackfillEnsureResult(True, False, 2, 0)) + self.assertEqual(result.claims, (scada_claim, price_claim)) + self.assertEqual(result.scada_results, (HistoricalScadaBackfillResult(2, 20, 4, 1, True, False),)) + self.assertEqual(result.price_results, (HistoricalPriceBackfillResult(2, 10, 5, 0, False, False),)) + self.assertEqual(result.finalization, finalization) + self.assertEqual(result.claimed_count, 2) + self.assertEqual(result.scada_raw_observation_count, 20) + self.assertEqual(result.scada_mapped_power_count, 4) + self.assertEqual(result.price_source_record_count, 10) + self.assertEqual(result.price_applied_record_count, 5) + self.assertEqual(result.replayed_interval_count, 1) + self.assertEqual(result.replayed_outer_artifact_count, 1) + self.assertEqual( + operations, + ["ensure:1", "claim:2", "scada", "claim:3", "price", "claim:4", "finalize:5"], + ) + self.assertEqual([connection.close_calls for connection in connections], [1, 1, 1, 1, 1]) + + def test_runner_failure_stops_without_finalization_and_preserves_error(self) -> None: + connections: list[Connection] = [] + failure = RuntimeError("private database detail") + start = datetime(2026, 8, 29, tzinfo=UTC) + end = datetime(2026, 8, 30, tzinfo=UTC) + spec = BackfillRunSpec("run-2", start, end, 1) + item = BackfillPlanItem( + "dispatch_scada", + date(2026, 8, 29), + "https://www.nemweb.com.au/REPORTS/ARCHIVE/Dispatch_SCADA/" + "PUBLIC_DISPATCHSCADA_20260829.zip", + ) + claim = BackfillClaim(spec.run_id, item.feed, item.report_date, item.source_url, 1) + claims = [claim] + + def connect(unused_url: str, *, connect_timeout: int) -> Connection: + connection = Connection(len(connections) + 1) + connections.append(connection) + return connection + + class Ledger: + def __init__(self, connection: Connection) -> None: + self.connection = connection + + def ensure_run(self, actual_spec: BackfillRunSpec, items: Any) -> BackfillEnsureResult: + return BackfillEnsureResult(True, False, 1, 0) + + def claim_next(self, run_id: str) -> BackfillClaim | None: + return claims.pop(0) + + def finalize(self, run_id: str) -> BackfillRunFinalization: + raise AssertionError("failed invocation must not finalize") + + def fail_scada(*args: Any, **kwargs: Any) -> HistoricalScadaBackfillResult: + raise failure + + with self.assertRaises(RuntimeError) as raised: + run_historical_backfill( + "postgresql://private", + (), + spec, + (item,), + connect=connect, + ledger_factory=Ledger, + run_scada=fail_scada, + ) + + self.assertIs(raised.exception, failure) + self.assertEqual([connection.close_calls for connection in connections], [1, 1]) + + def test_claim_count_is_bounded_by_supplied_plan(self) -> None: + start = datetime(2026, 8, 29, tzinfo=UTC) + end = datetime(2026, 8, 30, tzinfo=UTC) + spec = BackfillRunSpec("run-3", start, end, 1) + item = BackfillPlanItem( + "dispatch_scada", + date(2026, 8, 29), + "https://www.nemweb.com.au/REPORTS/ARCHIVE/Dispatch_SCADA/" + "PUBLIC_DISPATCHSCADA_20260829.zip", + ) + claim = BackfillClaim(spec.run_id, item.feed, item.report_date, item.source_url, 1) + duplicate_claims = [claim, claim, None] + run_calls: list[BackfillClaim] = [] + + class Ledger: + def __init__(self, connection: Connection) -> None: + self.connection = connection + + def ensure_run(self, actual_spec: BackfillRunSpec, items: Any) -> BackfillEnsureResult: + return BackfillEnsureResult(True, False, 1, 0) + + def claim_next(self, run_id: str) -> BackfillClaim | None: + return duplicate_claims.pop(0) + + def finalize(self, run_id: str) -> BackfillRunFinalization: + raise AssertionError("over-claimed invocation must not finalize") + + def run_scada(*args: Any, **kwargs: Any) -> HistoricalScadaBackfillResult: + run_calls.append(args[2]) + return HistoricalScadaBackfillResult(1, 1, 1, 0, False, False) + + next_connection = 0 + + def connect(unused_url: str, *, connect_timeout: int) -> Connection: + nonlocal next_connection + next_connection += 1 + return Connection(next_connection) + + with self.assertRaisesRegex(ValueError, "claim limit"): + run_historical_backfill( + "postgresql://private", + (), + spec, + (item,), + connect=connect, + ledger_factory=Ledger, + run_scada=run_scada, + ) + + self.assertEqual(run_calls, [claim]) + + +if __name__ == "__main__": + unittest.main() From e0090c637317e7cffa5702fb71cd7b62ee1ee32a Mon Sep 17 00:00:00 2001 From: djurcola-agent Date: Sun, 30 Aug 2026 21:43:47 +1000 Subject: [PATCH 18/30] feat: add bounded historical backfill command --- .../batterywatch_api/backfill_service.py | 201 ++++++++++++++++++ app/backend/tests/test_backfill_service.py | 169 +++++++++++++++ 2 files changed, 370 insertions(+) create mode 100644 app/backend/batterywatch_api/backfill_service.py create mode 100644 app/backend/tests/test_backfill_service.py diff --git a/app/backend/batterywatch_api/backfill_service.py b/app/backend/batterywatch_api/backfill_service.py new file mode 100644 index 0000000..77c3e83 --- /dev/null +++ b/app/backend/batterywatch_api/backfill_service.py @@ -0,0 +1,201 @@ +"""Bounded operator command for historical NEMWeb power and price backfills.""" + +from __future__ import annotations + +import argparse +from collections.abc import Callable, Iterable +from datetime import date, datetime, timedelta, timezone +import json +import os +from pathlib import Path +import sys +from typing import Any + +from .backfill_ledger import BackfillPlanItem, BackfillRunSpec +from .battery_assets import BatteryAsset, load_battery_assets +from .historical_backfill import HistoricalBackfillResult, run_historical_backfill +from .nemweb_archives import ( + DISPATCHIS_PRICE_FEED, + DISPATCH_SCADA_FEED, + plan_archive_range, +) + +UTC = timezone.utc +_NEM_TIMEZONE = timezone(timedelta(hours=10)) +_FEED_MAP = { + "power": DISPATCH_SCADA_FEED, + "price": DISPATCHIS_PRICE_FEED, +} +_LEDGER_FEED_MAP = { + DISPATCH_SCADA_FEED: "dispatch_scada", + DISPATCHIS_PRICE_FEED: "dispatch_price", +} + + +def _parse_utc(value: str) -> datetime: + if not isinstance(value, str) or not value: + raise ValueError("UTC timestamp is required") + normalized = value[:-1] + "+00:00" if value.endswith("Z") else value + try: + parsed = datetime.fromisoformat(normalized) + except ValueError: + raise ValueError("invalid UTC timestamp") from None + if parsed.tzinfo is None or parsed.utcoffset() != timedelta(0): + raise ValueError("timestamp must use UTC") + return parsed.astimezone(UTC) + + +def _candidate_dates(start: datetime, end: datetime) -> tuple[date, ...]: + first = start.astimezone(_NEM_TIMEZONE).date() - timedelta(days=1) + last = end.astimezone(_NEM_TIMEZONE).date() + return tuple(first + timedelta(days=index) for index in range((last - first).days + 1)) + + +def build_operator_plan( + run_id: str, + start: datetime, + end: datetime, + *, + feeds: Iterable[str] = ("power", "price"), + ingestion_version: int = 1, +) -> tuple[BackfillRunSpec, tuple[BackfillPlanItem, ...]]: + """Build canonical bounded ledger items from explicit operator inputs.""" + + try: + requested = tuple(feeds) + except (TypeError, ValueError): + raise ValueError("invalid backfill feeds") from None + if ( + not requested + or len(set(requested)) != len(requested) + or any(type(feed) is not str or feed not in _FEED_MAP for feed in requested) + ): + raise ValueError("invalid backfill feeds") + archive_feeds = tuple( + archive_feed + for name, archive_feed in _FEED_MAP.items() + if name in requested + ) + candidates = _candidate_dates(start, end) + archived_dates = {feed: candidates for feed in archive_feeds} + archive_plan = plan_archive_range( + start, + end, + feeds=archive_feeds, + archived_dates=archived_dates, + ) + items = tuple( + BackfillPlanItem( + _LEDGER_FEED_MAP[item.feed], + item.report_date, + item.url, + ) + for item in archive_plan.items + ) + return ( + BackfillRunSpec( + run_id, + archive_plan.start, + archive_plan.end, + ingestion_version, + ), + items, + ) + + +def _summary( + result: HistoricalBackfillResult, + planned_item_count: int, + spec: BackfillRunSpec, +) -> dict[str, Any]: + progress = result.finalization.progress + return { + "status": progress.status, + "run_id": progress.run_id, + "requested_start": spec.requested_start.isoformat().replace("+00:00", "Z"), + "requested_end": spec.requested_end.isoformat().replace("+00:00", "Z"), + "planned_item_count": planned_item_count, + "claimed_item_count": result.claimed_count, + "created": result.ensure_result.created, + "resumed": result.ensure_result.resumed, + "recovered_item_count": result.ensure_result.recovered_count, + "scada_raw_observation_count": result.scada_raw_observation_count, + "scada_mapped_power_count": result.scada_mapped_power_count, + "price_source_record_count": result.price_source_record_count, + "price_applied_record_count": result.price_applied_record_count, + "replayed_interval_count": result.replayed_interval_count, + "replayed_outer_artifact_count": result.replayed_outer_artifact_count, + "completion_replayed": result.finalization.replayed, + "total_items": progress.total, + "pending_items": progress.pending, + "running_items": progress.running, + "completed_items": progress.completed, + "failed_items": progress.failed, + "total_attempts": progress.total_attempts, + } + + +def main( + argv: list[str] | None = None, + *, + environ: dict[str, str] | None = None, + load_assets: Callable[[Path], Iterable[BatteryAsset]] = load_battery_assets, + run: Callable[..., HistoricalBackfillResult] = run_historical_backfill, +) -> int: + parser = argparse.ArgumentParser(prog="batterywatch-backfill") + parser.add_argument("--run-id", required=True) + parser.add_argument("--start", required=True) + parser.add_argument("--end", required=True) + parser.add_argument("--feeds", default="power,price") + parser.add_argument("--ingestion-version", type=int, default=1) + parser.add_argument("--assets-path", type=Path) + arguments = parser.parse_args(argv) + environment = os.environ if environ is None else environ + + try: + database_url = environment.get("BATTERYWATCH_DATABASE_URL", "") + if not database_url: + raise ValueError("database URL is required") + start = _parse_utc(arguments.start) + end = _parse_utc(arguments.end) + spec, items = build_operator_plan( + arguments.run_id, + start, + end, + feeds=tuple(arguments.feeds.split(",")), + ingestion_version=arguments.ingestion_version, + ) + assets_path = arguments.assets_path or Path( + environment.get( + "BATTERYWATCH_ASSETS_PATH", + str(Path(__file__).resolve().parents[2] / "config/battery_assets.json"), + ) + ) + result = run( + database_url, + tuple(load_assets(assets_path)), + spec, + items, + ) + print(json.dumps(_summary(result, len(items), spec), sort_keys=True)) + return 0 + except Exception as error: + print( + json.dumps( + { + "error_type": type(error).__name__, + "run_id": arguments.run_id, + "status": "error", + }, + sort_keys=True, + ), + file=sys.stderr, + ) + return 1 + + +__all__ = ["build_operator_plan", "main"] + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/app/backend/tests/test_backfill_service.py b/app/backend/tests/test_backfill_service.py new file mode 100644 index 0000000..48088d2 --- /dev/null +++ b/app/backend/tests/test_backfill_service.py @@ -0,0 +1,169 @@ +"""Tests for the bounded historical backfill operator command.""" + +from contextlib import redirect_stderr, redirect_stdout +from datetime import date, datetime, timezone +from io import StringIO +import json +from pathlib import Path +from typing import Any +import unittest + +from batterywatch_api.backfill_ledger import ( + BackfillClaim, + BackfillEnsureResult, + BackfillRunFinalization, + BackfillRunProgress, +) +from batterywatch_api.battery_assets import BatteryAsset +from batterywatch_api.historical_backfill import HistoricalBackfillResult +from batterywatch_api.historical_price_backfill import HistoricalPriceBackfillResult +from batterywatch_api.historical_scada_backfill import HistoricalScadaBackfillResult +from batterywatch_api.backfill_service import build_operator_plan, main + +UTC = timezone.utc + + +class BackfillServiceTests(unittest.TestCase): + def test_builds_canonical_ledger_plan_for_requested_feeds(self) -> None: + start = datetime(2026, 8, 29, 0, 0, tzinfo=UTC) + end = datetime(2026, 8, 29, 10, 0, tzinfo=UTC) + + spec, items = build_operator_plan( + "operator-run", + start, + end, + feeds=("price", "power"), + ingestion_version=7, + ) + + self.assertEqual((spec.run_id, spec.requested_start, spec.requested_end, spec.ingestion_version), ("operator-run", start, end, 7)) + self.assertEqual( + tuple((item.feed, item.report_date, item.source_url) for item in items), + ( + ( + "dispatch_scada", + date(2026, 8, 29), + "https://www.nemweb.com.au/REPORTS/ARCHIVE/Dispatch_SCADA/" + "PUBLIC_DISPATCHSCADA_20260829.zip", + ), + ( + "dispatch_price", + date(2026, 8, 29), + "https://www.nemweb.com.au/REPORTS/ARCHIVE/DispatchIS_Reports/" + "PUBLIC_DISPATCHIS_20260829.zip", + ), + ), + ) + + def test_main_outputs_deterministic_success_summary_without_database_url(self) -> None: + stdout = StringIO() + stderr = StringIO() + captured: dict[str, Any] = {} + start = datetime(2026, 8, 29, 0, 0, tzinfo=UTC) + end = datetime(2026, 8, 29, 10, 0, tzinfo=UTC) + scada_claim = BackfillClaim( + "operator-run", + "dispatch_scada", + date(2026, 8, 29), + "https://www.nemweb.com.au/REPORTS/ARCHIVE/Dispatch_SCADA/" + "PUBLIC_DISPATCHSCADA_20260829.zip", + 1, + ) + price_claim = BackfillClaim( + "operator-run", + "dispatch_price", + date(2026, 8, 29), + "https://www.nemweb.com.au/REPORTS/ARCHIVE/DispatchIS_Reports/" + "PUBLIC_DISPATCHIS_20260829.zip", + 1, + ) + progress = BackfillRunProgress("operator-run", "completed", 2, 0, 0, 2, 0, 2) + result = HistoricalBackfillResult( + BackfillEnsureResult(True, False, 2, 0), + (scada_claim, price_claim), + (HistoricalScadaBackfillResult(1, 100, 12, 0, False, False),), + (HistoricalPriceBackfillResult(1, 5, 5, 1, True, False),), + BackfillRunFinalization(False, progress), + ) + + def load_assets(path: Path) -> tuple[BatteryAsset, ...]: + captured["assets_path"] = path + return ( + BatteryAsset( + "BAT1", + "Battery One", + "NSW1", + 10, + 20, + "reviewed-registry", + datetime(2025, 3, 31, tzinfo=UTC), + ), + ) + + def run(database_url: str, assets: Any, spec: Any, items: Any) -> HistoricalBackfillResult: + captured.update(database_url=database_url, assets=tuple(assets), spec=spec, items=tuple(items)) + return result + + with redirect_stdout(stdout), redirect_stderr(stderr): + exit_code = main( + [ + "--run-id", "operator-run", + "--start", "2026-08-29T00:00:00Z", + "--end", "2026-08-29T10:00:00Z", + "--feeds", "power,price", + "--assets-path", "/tmp/reviewed-assets.json", + ], + environ={"BATTERYWATCH_DATABASE_URL": "postgresql://secret-value"}, + load_assets=load_assets, + run=run, + ) + + self.assertEqual(exit_code, 0) + self.assertEqual(stderr.getvalue(), "") + payload = json.loads(stdout.getvalue()) + self.assertEqual(payload["status"], "completed") + self.assertEqual(payload["run_id"], "operator-run") + self.assertEqual(payload["requested_start"], "2026-08-29T00:00:00Z") + self.assertEqual(payload["requested_end"], "2026-08-29T10:00:00Z") + self.assertEqual(payload["planned_item_count"], 2) + self.assertEqual(payload["claimed_item_count"], 2) + self.assertEqual(payload["scada_raw_observation_count"], 100) + self.assertEqual(payload["scada_mapped_power_count"], 12) + self.assertEqual(payload["price_source_record_count"], 5) + self.assertEqual(payload["price_applied_record_count"], 5) + self.assertEqual(payload["replayed_interval_count"], 1) + self.assertEqual(payload["replayed_outer_artifact_count"], 1) + self.assertEqual(payload["total_attempts"], 2) + self.assertNotIn("secret-value", stdout.getvalue()) + self.assertEqual(captured["assets_path"], Path("/tmp/reviewed-assets.json")) + + def test_main_failure_reports_only_error_type_and_run_id(self) -> None: + stdout = StringIO() + stderr = StringIO() + + def fail_run(*args: Any, **kwargs: Any) -> HistoricalBackfillResult: + raise RuntimeError("postgresql://secret-value private row") + + with redirect_stdout(stdout), redirect_stderr(stderr): + exit_code = main( + [ + "--run-id", "operator-run", + "--start", "2026-08-29T00:00:00Z", + "--end", "2026-08-29T10:00:00Z", + ], + environ={"BATTERYWATCH_DATABASE_URL": "postgresql://secret-value"}, + load_assets=lambda path: (), + run=fail_run, + ) + + self.assertEqual(exit_code, 1) + self.assertEqual(stdout.getvalue(), "") + self.assertEqual( + json.loads(stderr.getvalue()), + {"error_type": "RuntimeError", "run_id": "operator-run", "status": "error"}, + ) + self.assertNotIn("secret-value", stderr.getvalue()) + + +if __name__ == "__main__": + unittest.main() From d72c0cb3ffd95eb7ed0757c9eacbe65206ab9b53 Mon Sep 17 00:00:00 2001 From: djurcola-agent Date: Sun, 30 Aug 2026 22:00:07 +1000 Subject: [PATCH 19/30] feat: parse authoritative next-day battery SOC --- app/backend/batterywatch_api/nextday_soc.py | 256 ++++++++++++++++++ .../tests/fixtures/historical/README.md | 11 + ...egionsum-bdu-soc-20260830-2145-reduced.csv | 8 + ...day-unit-solution-soc-20260829-reduced.csv | 6 + app/backend/tests/test_nextday_soc.py | 109 ++++++++ 5 files changed, 390 insertions(+) create mode 100644 app/backend/batterywatch_api/nextday_soc.py create mode 100644 app/backend/tests/fixtures/historical/dispatch-regionsum-bdu-soc-20260830-2145-reduced.csv create mode 100644 app/backend/tests/fixtures/historical/nextday-unit-solution-soc-20260829-reduced.csv create mode 100644 app/backend/tests/test_nextday_soc.py diff --git a/app/backend/batterywatch_api/nextday_soc.py b/app/backend/batterywatch_api/nextday_soc.py new file mode 100644 index 0000000..21c7972 --- /dev/null +++ b/app/backend/batterywatch_api/nextday_soc.py @@ -0,0 +1,256 @@ +"""Strict parser for authoritative Next Day Dispatch UnitSolution SOC.""" + +from __future__ import annotations + +import csv +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from io import StringIO +from math import isfinite +import re +from typing import Literal + +_NEM_TIMEZONE = timezone(timedelta(hours=10)) +_METADATA_PREFIX = ("C", "NEMP.WORLD", "NEXT_DAY_DISPATCH", "AEMO", "PUBLIC") +_TABLE_PREFIX = ("DISPATCH", "UNIT_SOLUTION", "6") +_REQUIRED_COLUMNS = frozenset( + ( + "SETTLEMENTDATE", + "RUNNO", + "DUID", + "DISPATCHINTERVAL", + "INTERVENTION", + "LASTCHANGED", + "INITIAL_ENERGY_STORAGE", + ) +) +_DUID_RE = re.compile(r"^[A-Z0-9_]{1,32}$") +_ARTIFACT_RE = re.compile(r"^[0-9a-f]{64}$") +_DISPATCH_INTERVAL_RE = re.compile(r"^(\d{8})(\d{3})$") + + +class NextDaySocParseError(ValueError): + """Raised when Next Day SOC source data cannot be normalized safely.""" + + +def _timestamp(value: str, *, field: str, aligned: bool) -> datetime: + normalized = value.strip().replace("/", "-").replace(" ", "T", 1) + try: + parsed = datetime.fromisoformat(normalized) + except (AttributeError, TypeError, ValueError) as exc: + raise NextDaySocParseError(f"invalid {field}") from exc + if parsed.tzinfo is None or parsed.utcoffset() is None: + parsed = parsed.replace(tzinfo=_NEM_TIMEZONE) + result = parsed.astimezone(timezone.utc) + if aligned and (result.minute % 5 or result.second or result.microsecond): + raise NextDaySocParseError(f"invalid {field} alignment") + return result + + +def _report_timestamp(row: list[str]) -> datetime: + if len(row) != 10 or tuple(row[:5]) != _METADATA_PREFIX: + raise NextDaySocParseError("invalid Next Day metadata") + return _timestamp(f"{row[5]} {row[6]}", field="report timestamp", aligned=False) + + +def _version(value: str, *, field: str, maximum: int) -> int: + if not value or not value.isascii() or not value.isdecimal(): + raise NextDaySocParseError(f"invalid {field}") + parsed = int(value) + if parsed > maximum: + raise NextDaySocParseError(f"invalid {field}") + return parsed + + +def _soc_mwh(value: str) -> float | None: + if value == "": + return None + try: + parsed = float(value) + except (TypeError, ValueError) as exc: + raise NextDaySocParseError("invalid INITIAL_ENERGY_STORAGE") from exc + if not isfinite(parsed) or parsed < 0: + raise NextDaySocParseError("invalid INITIAL_ENERGY_STORAGE") + return parsed + + +def _dispatch_interval(value: str, interval_start: datetime) -> str: + match = _DISPATCH_INTERVAL_RE.fullmatch(value) + if match is None: + raise NextDaySocParseError("invalid DISPATCHINTERVAL") + period = int(match.group(2)) + if not 1 <= period <= 288: + raise NextDaySocParseError("invalid DISPATCHINTERVAL") + try: + market_date = datetime.strptime(match.group(1), "%Y%m%d").replace( + tzinfo=_NEM_TIMEZONE + ) + except ValueError as exc: + raise NextDaySocParseError("invalid DISPATCHINTERVAL") from exc + expected = (market_date + timedelta(hours=4, minutes=period * 5)).astimezone( + timezone.utc + ) + if expected != interval_start: + raise NextDaySocParseError("DISPATCHINTERVAL does not match SETTLEMENTDATE") + return value + + +def _validated_downloaded_at(value: datetime, report_timestamp: datetime) -> datetime: + if not isinstance(value, datetime) or value.tzinfo is None or value.utcoffset() is None: + raise NextDaySocParseError("invalid downloaded_at") + result = value.astimezone(timezone.utc) + if result < report_timestamp: + raise NextDaySocParseError("downloaded_at precedes report timestamp") + return result + + +@dataclass(frozen=True, slots=True) +class NextDaySocObservation: + """One authoritative per-DUID initial energy storage observation.""" + + duid: str + interval_start: datetime + soc_mwh: float | None + intervention: int + run_number: int + dispatch_interval: str + last_changed: datetime + source_artifact_id: str + report_timestamp: datetime + downloaded_at: datetime + ingestion_version: int + correction_version: int + + @property + def publication_latency_seconds(self) -> int: + return int((self.report_timestamp - self.interval_start).total_seconds()) + + @property + def publication_status(self) -> Literal["next_day"]: + return "next_day" + + +def parse_nextday_unit_solution_soc( + payload: str, + *, + duids: frozenset[str], + source_artifact_id: str, + downloaded_at: datetime, + ingestion_version: int, + correction_version: int = 0, +) -> tuple[NextDaySocObservation, ...]: + """Parse current public Next Day UnitSolution v6 rows for reviewed DUIDs.""" + + if type(payload) is not str or not payload: + raise NextDaySocParseError("invalid payload") + if type(duids) is not frozenset or not duids or any( + type(duid) is not str or _DUID_RE.fullmatch(duid) is None for duid in duids + ): + raise NextDaySocParseError("invalid reviewed DUID set") + if type(source_artifact_id) is not str or _ARTIFACT_RE.fullmatch(source_artifact_id) is None: + raise NextDaySocParseError("invalid source artifact identity") + ingestion = _version(str(ingestion_version), field="ingestion_version", maximum=2**63 - 1) + correction = _version(str(correction_version), field="correction_version", maximum=2**63 - 1) + + reader = csv.reader(StringIO(payload)) + try: + metadata = next(reader) + except (StopIteration, csv.Error, TypeError, UnicodeError) as exc: + raise NextDaySocParseError("incomplete Next Day report") from exc + report_timestamp = _report_timestamp(metadata) + downloaded = _validated_downloaded_at(downloaded_at, report_timestamp) + + header: list[str] | None = None + observations: list[NextDaySocObservation] = [] + seen: set[tuple[str, datetime, int, int]] = set() + report_record_count = 1 + expected_count: int | None = None + try: + for row in reader: + report_record_count += 1 + if expected_count is not None: + raise NextDaySocParseError("data follows report trailer") + if row[:2] == ["C", "END OF REPORT"]: + if len(row) != 3: + raise NextDaySocParseError("invalid report trailer") + expected_count = _version( + row[2], field="report row count", maximum=10_000_000 + ) + continue + + if row[:4] == ["I", *_TABLE_PREFIX]: + if header is not None or len(row) != len(set(row)): + raise NextDaySocParseError("invalid UnitSolution header") + if not _REQUIRED_COLUMNS.issubset(row[4:]): + raise NextDaySocParseError("missing UnitSolution columns") + header = row + continue + if row[:4] != ["D", *_TABLE_PREFIX]: + continue + if header is None or len(row) != len(header): + raise NextDaySocParseError("malformed UnitSolution row") + values = dict(zip(header[4:], row[4:])) + duid = values["DUID"] + if duid not in duids: + continue + interval_start = _timestamp( + values["SETTLEMENTDATE"], field="SETTLEMENTDATE", aligned=True + ) + run_number = _version(values["RUNNO"], field="RUNNO", maximum=999) + intervention = _version( + values["INTERVENTION"], field="INTERVENTION", maximum=1 + ) + dispatch_interval = _dispatch_interval( + values["DISPATCHINTERVAL"], interval_start + ) + last_changed = _timestamp( + values["LASTCHANGED"], field="LASTCHANGED", aligned=False + ) + if last_changed > report_timestamp or report_timestamp <= interval_start: + raise NextDaySocParseError("invalid source timestamp ordering") + key = (duid, interval_start, intervention, run_number) + if key in seen: + raise NextDaySocParseError("duplicate UnitSolution observation") + seen.add(key) + observations.append( + NextDaySocObservation( + duid=duid, + interval_start=interval_start, + soc_mwh=_soc_mwh(values["INITIAL_ENERGY_STORAGE"]), + intervention=intervention, + run_number=run_number, + dispatch_interval=dispatch_interval, + last_changed=last_changed, + source_artifact_id=source_artifact_id, + report_timestamp=report_timestamp, + downloaded_at=downloaded, + ingestion_version=ingestion, + correction_version=correction, + ) + ) + except (csv.Error, TypeError, UnicodeError) as exc: + raise NextDaySocParseError("invalid CSV") from exc + if expected_count is None: + raise NextDaySocParseError("missing report trailer") + if expected_count != report_record_count: + raise NextDaySocParseError("report row count mismatch") + if header is None: + raise NextDaySocParseError("missing UnitSolution v6 table") + return tuple( + sorted( + observations, + key=lambda item: ( + item.duid, + item.interval_start, + item.intervention, + item.run_number, + ), + ) + ) + + +__all__ = [ + "NextDaySocObservation", + "NextDaySocParseError", + "parse_nextday_unit_solution_soc", +] diff --git a/app/backend/tests/fixtures/historical/README.md b/app/backend/tests/fixtures/historical/README.md index 7c56e0c..d101a5d 100644 --- a/app/backend/tests/fixtures/historical/README.md +++ b/app/backend/tests/fixtures/historical/README.md @@ -13,5 +13,16 @@ present. `fb22ab052a31020fcfcc574b5c2d29c43a2ee4caba903a634ce0980091ac7381`. It retains the authentic PRICE v5 envelope and all five regions, including negative prices. +- `nextday-unit-solution-soc-20260829-reduced.csv` derives from + `PUBLIC_NEXT_DAY_DISPATCH_20260829_0000000535139038.zip`, source SHA-256 + `d7a2abdd2947ed4b222166b9f60e3a8052838190027dd9ce03cb291ba2d29bc4`. + It retains the authentic Next Day `UNIT_SOLUTION` v6 envelope, two ADPBA1 + observations with `INITIAL_ENERGY_STORAGE`, and one reviewed KEPBG1 battery + row where the authoritative field is blank. +- `dispatch-regionsum-bdu-soc-20260830-2145-reduced.csv` derives from + `PUBLIC_DISPATCHIS_202608302145_0000000535256209.zip`, source SHA-256 + `a95e8ea8a19aa279377327f20a6a3ad083d4c135de487fe8bec4e5477cc78c0e`. + It retains the authentic `REGIONSUM` v9 envelope and all five regional BDU + values, including the published blank for TAS1. The reduced fixture trailer row count is recomputed for the retained records. diff --git a/app/backend/tests/fixtures/historical/dispatch-regionsum-bdu-soc-20260830-2145-reduced.csv b/app/backend/tests/fixtures/historical/dispatch-regionsum-bdu-soc-20260830-2145-reduced.csv new file mode 100644 index 0000000..2d9a6b2 --- /dev/null +++ b/app/backend/tests/fixtures/historical/dispatch-regionsum-bdu-soc-20260830-2145-reduced.csv @@ -0,0 +1,8 @@ +C,NEMP.WORLD,DISPATCHIS,AEMO,PUBLIC,2026/08/30,21:40:09,0000000535256209,DISPATCHIS,0000000535256208 +I,DISPATCH,REGIONSUM,9,SETTLEMENTDATE,RUNNO,REGIONID,DISPATCHINTERVAL,INTERVENTION,TOTALDEMAND,AVAILABLEGENERATION,AVAILABLELOAD,DEMANDFORECAST,DISPATCHABLEGENERATION,DISPATCHABLELOAD,NETINTERCHANGE,EXCESSGENERATION,LOWER5MINDISPATCH,LOWER5MINIMPORT,LOWER5MINLOCALDISPATCH,LOWER5MINLOCALPRICE,LOWER5MINLOCALREQ,LOWER5MINPRICE,LOWER5MINREQ,LOWER5MINSUPPLYPRICE,LOWER60SECDISPATCH,LOWER60SECIMPORT,LOWER60SECLOCALDISPATCH,LOWER60SECLOCALPRICE,LOWER60SECLOCALREQ,LOWER60SECPRICE,LOWER60SECREQ,LOWER60SECSUPPLYPRICE,LOWER6SECDISPATCH,LOWER6SECIMPORT,LOWER6SECLOCALDISPATCH,LOWER6SECLOCALPRICE,LOWER6SECLOCALREQ,LOWER6SECPRICE,LOWER6SECREQ,LOWER6SECSUPPLYPRICE,RAISE5MINDISPATCH,RAISE5MINIMPORT,RAISE5MINLOCALDISPATCH,RAISE5MINLOCALPRICE,RAISE5MINLOCALREQ,RAISE5MINPRICE,RAISE5MINREQ,RAISE5MINSUPPLYPRICE,RAISE60SECDISPATCH,RAISE60SECIMPORT,RAISE60SECLOCALDISPATCH,RAISE60SECLOCALPRICE,RAISE60SECLOCALREQ,RAISE60SECPRICE,RAISE60SECREQ,RAISE60SECSUPPLYPRICE,RAISE6SECDISPATCH,RAISE6SECIMPORT,RAISE6SECLOCALDISPATCH,RAISE6SECLOCALPRICE,RAISE6SECLOCALREQ,RAISE6SECPRICE,RAISE6SECREQ,RAISE6SECSUPPLYPRICE,AGGEGATEDISPATCHERROR,AGGREGATEDISPATCHERROR,LASTCHANGED,INITIALSUPPLY,CLEAREDSUPPLY,LOWERREGIMPORT,LOWERREGLOCALDISPATCH,LOWERREGLOCALREQ,LOWERREGREQ,RAISEREGIMPORT,RAISEREGLOCALDISPATCH,RAISEREGLOCALREQ,RAISEREGREQ,RAISE5MINLOCALVIOLATION,RAISEREGLOCALVIOLATION,RAISE60SECLOCALVIOLATION,RAISE6SECLOCALVIOLATION,LOWER5MINLOCALVIOLATION,LOWERREGLOCALVIOLATION,LOWER60SECLOCALVIOLATION,LOWER6SECLOCALVIOLATION,RAISE5MINVIOLATION,RAISEREGVIOLATION,RAISE60SECVIOLATION,RAISE6SECVIOLATION,LOWER5MINVIOLATION,LOWERREGVIOLATION,LOWER60SECVIOLATION,LOWER6SECVIOLATION,RAISE6SECACTUALAVAILABILITY,RAISE60SECACTUALAVAILABILITY,RAISE5MINACTUALAVAILABILITY,RAISEREGACTUALAVAILABILITY,LOWER6SECACTUALAVAILABILITY,LOWER60SECACTUALAVAILABILITY,LOWER5MINACTUALAVAILABILITY,LOWERREGACTUALAVAILABILITY,LORSURPLUS,LRCSURPLUS,TOTALINTERMITTENTGENERATION,DEMAND_AND_NONSCHEDGEN,UIGF,SEMISCHEDULE_CLEAREDMW,SEMISCHEDULE_COMPLIANCEMW,SS_SOLAR_UIGF,SS_WIND_UIGF,SS_SOLAR_CLEAREDMW,SS_WIND_CLEAREDMW,SS_SOLAR_COMPLIANCEMW,SS_WIND_COMPLIANCEMW,WDR_INITIALMW,WDR_AVAILABLE,WDR_DISPATCHED,RAISE1SECLOCALDISPATCH,LOWER1SECLOCALDISPATCH,RAISE1SECACTUALAVAILABILITY,LOWER1SECACTUALAVAILABILITY,SS_SOLAR_AVAILABILITY,SS_WIND_AVAILABILITY,BDU_ENERGY_STORAGE,BDU_MIN_AVAIL,BDU_MAX_AVAIL,BDU_CLEAREDMW_GEN,BDU_CLEAREDMW_LOAD,BDU_INITIAL_ENERGY_STORAGE +D,DISPATCH,REGIONSUM,9,"2026/08/30 21:45:00",1,NSW1,20260830213,0,8305.72,12523.16911,2091,-26,7391.26,0,-914.46,0,,,88,,,,,,,,78,,,,,,,,78,,,,,,,,41.91,,,,,,,,41.91,,,,,,,,41.91,,,,,,,11.97805,"2026/08/30 21:40:04",8363.80496,8339.65,,20,,,,31,,,,,,,,,,,,,,,,,,,648.9079,719.9079,956.9079,504.308720,836,957,889,821.015296,,,89.29649,8428.946490,391.16911,391.16911,0,0,391.16911,0,391.16911,0,0,0,0,0,37,0,602,592,0,391.16911,2307.310380,2091,1667,354,0,2343.893640 +D,DISPATCH,REGIONSUM,9,"2026/08/30 21:45:00",1,QLD1,20260830213,0,6551.26,11209.66097,2453,-27,7637.36,0,1086.1,0,,,45,,,,,,,,115,,,,,,,,86.91,,,,,,,,71.23,,,,,,,,250.65,,,,,,,,262,,,,,,,12.54008,"2026/08/30 21:40:04",6615.53698,6594.02,,16,,,,73,,,,,,,,,,,,,,,,,,,1252.530165,1254.491659,1119.495181,1423.345180,1364.805925,1462.128296,1337.128296,2047.615309,,,95.702,6689.7220,1499.66097,1499.66097,0,0.17,1499.49097,0.17,1499.49097,0,0,0,0,0,70,0,1144,1191,0.17,1499.49097,1384.975960,2453,2071,210,0,1408.241420 +D,DISPATCH,REGIONSUM,9,"2026/08/30 21:45:00",1,SA1,20260830213,0,1532.22,2977.5306,916,-8,1421.53,0,-110.69,0,,,109.72,,,,,,,,106,,,,,,,,105,,,,,,,,130,,,,,,,,172,,,,,,,,172,,,,,,,0,"2026/08/30 21:40:04",1542.97975,1532.43,,74,,,,34,,,,,,,,,,,,,,,,,,,438,440,440,542,438,457,447,816.276420,,,0,1532.43,518.5306,518.5306,0,0,518.5306,0,518.5306,0,0,0,0,0,91,0,415,406,0,518.5306,724.287990,916,927,154,0,737.806270 +D,DISPATCH,REGIONSUM,9,"2026/08/30 21:45:00",1,TAS1,20260830213,0,1050.7,2066.36496,0,-4,1472.46,0,421.76,0,,,0,,,,,,,,0,,,,,,,,0,,,,,,,,0,,,,,,,,0,,,,,,,,0,,,,,,,0.21674,"2026/08/30 21:40:04",1054.48486,1050.7,,50,,,,50,,,,,,,,,,,,,,,,,,,226.778216,291.831033,321.011513,303,105.629019,478.233593,721.291022,522.971014,,,141.52466,1192.224660,266.36496,266.36496,0,0,266.36496,0,266.36496,0,0,0,0,0,0,0,38,0,0,266.36496,,,,,, +D,DISPATCH,REGIONSUM,9,"2026/08/30 21:45:00",1,VIC1,20260830213,0,6059.53,9600.00861,1350,-31,5680.01,0,-379.52,0,,,88,,,,,,,,163.15,,,,,,,,80,,,,,,,,233,,,,,,,,153,,,,,,,,141.65,,,,,,,3.75198,"2026/08/30 21:40:04",6121.86648,6085.82,,50,,,,32,,,,,,,,,,,,,,,,,,,1131,1136,1114,974,876,896,885,1148.694190,,,65.33605,6151.156050,408.00861,408.00861,0,0,408.00861,0,408.00861,0,0,0,0,0,132.89,0,641,656,0,408.00861,1405.252610,1350,1387,0,0,1407.424930 +C,"END OF REPORT",8 diff --git a/app/backend/tests/fixtures/historical/nextday-unit-solution-soc-20260829-reduced.csv b/app/backend/tests/fixtures/historical/nextday-unit-solution-soc-20260829-reduced.csv new file mode 100644 index 0000000..988384d --- /dev/null +++ b/app/backend/tests/fixtures/historical/nextday-unit-solution-soc-20260829-reduced.csv @@ -0,0 +1,6 @@ +C,NEMP.WORLD,NEXT_DAY_DISPATCH,AEMO,PUBLIC,2026/08/30,04:10:00,0000000535139038,NEXT_DAY_DISPATCH,0000000535139034 +I,DISPATCH,UNIT_SOLUTION,6,SETTLEMENTDATE,RUNNO,DUID,TRADETYPE,DISPATCHINTERVAL,INTERVENTION,CONNECTIONPOINTID,DISPATCHMODE,AGCSTATUS,INITIALMW,TOTALCLEARED,RAMPDOWNRATE,RAMPUPRATE,LOWER5MIN,LOWER60SEC,LOWER6SEC,RAISE5MIN,RAISE60SEC,RAISE6SEC,DOWNEPF,UPEPF,MARGINAL5MINVALUE,MARGINAL60SECVALUE,MARGINAL6SECVALUE,MARGINALVALUE,VIOLATION5MINDEGREE,VIOLATION60SECDEGREE,VIOLATION6SECDEGREE,VIOLATIONDEGREE,LASTCHANGED,LOWERREG,RAISEREG,AVAILABILITY,RAISE6SECFLAGS,RAISE60SECFLAGS,RAISE5MINFLAGS,RAISEREGFLAGS,LOWER6SECFLAGS,LOWER60SECFLAGS,LOWER5MINFLAGS,LOWERREGFLAGS,RAISEREGAVAILABILITY,RAISEREGENABLEMENTMAX,RAISEREGENABLEMENTMIN,LOWERREGAVAILABILITY,LOWERREGENABLEMENTMAX,LOWERREGENABLEMENTMIN,RAISE6SECACTUALAVAILABILITY,RAISE60SECACTUALAVAILABILITY,RAISE5MINACTUALAVAILABILITY,RAISEREGACTUALAVAILABILITY,LOWER6SECACTUALAVAILABILITY,LOWER60SECACTUALAVAILABILITY,LOWER5MINACTUALAVAILABILITY,LOWERREGACTUALAVAILABILITY,SEMIDISPATCHCAP,DISPATCHMODETIME,LOWER1SEC,RAISE1SEC,RAISE1SECFLAGS,LOWER1SECFLAGS,RAISE1SECACTUALAVAILABILITY,LOWER1SECACTUALAVAILABILITY,CONFORMANCE_MODE,UIGF,INITIAL_ENERGY_STORAGE,ENERGY_STORAGE,MIN_AVAILABILITY,ELEMENT_CAP +D,DISPATCH,UNIT_SOLUTION,6,"2026/08/29 04:05:00",1,ADPBA1,0,20260829001,0,SMVE14,0,1,0.063,0,93.12,93.12,3,3,3,3,3,3,,,,,,,,,,,"2026/08/29 04:00:06",0,0,6,1,1,1,1,1,1,1,1,7.75999,6,-6,7.75999,6,-6,3,3,3,3,3,3,3,3,0,0,0,0,0,0,0,0,,0,3.786,3.78315,6, +D,DISPATCH,UNIT_SOLUTION,6,"2026/08/29 04:05:00",1,KEPBG1,0,20260829001,0,QROW3K,0,0,0,0,0,0,0,0,0,0,0,0,,,,,,,,,,,"2026/08/29 04:00:06",0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,,0,,,0, +D,DISPATCH,UNIT_SOLUTION,6,"2026/08/29 04:10:00",1,ADPBA1,0,20260829002,0,SMVE14,0,1,0.1,0,93.12,93.12,3,3,3,3,3,3,,,,,,,,,,,"2026/08/29 04:05:05",0,0,6,1,1,1,1,1,1,1,1,7.75999,6,-6,7.75999,6,-6,3,3,3,3,3,3,3,3,0,0,0,0,0,0,0,0,,0,3.765,3.76047,6, +C,"END OF REPORT",6 diff --git a/app/backend/tests/test_nextday_soc.py b/app/backend/tests/test_nextday_soc.py new file mode 100644 index 0000000..751c46c --- /dev/null +++ b/app/backend/tests/test_nextday_soc.py @@ -0,0 +1,109 @@ +"""Tests for authoritative Next Day UnitSolution SOC parsing.""" + +from datetime import datetime, timezone +from pathlib import Path +from typing import Any +import unittest + +from batterywatch_api.nextday_soc import NextDaySocParseError, parse_nextday_unit_solution_soc + +UTC = timezone.utc +FIXTURE = Path(__file__).parent / "fixtures" / "historical" / "nextday-unit-solution-soc-20260829-reduced.csv" + + +class NextDaySocParserTests(unittest.TestCase): + def setUp(self) -> None: + self.payload = FIXTURE.read_text(encoding="utf-8") + self.downloaded_at = datetime(2026, 8, 30, 0, 0, tzinfo=UTC) + + def parse(self, payload: str | None = None, *, duids: frozenset[str] = frozenset(("ADPBA1", "KEPBG1"))): + return parse_nextday_unit_solution_soc( + self.payload if payload is None else payload, + duids=duids, + source_artifact_id="d7a2abdd2947ed4b222166b9f60e3a8052838190027dd9ce03cb291ba2d29bc4", + downloaded_at=self.downloaded_at, + ingestion_version=8, + correction_version=2, + ) + + def test_parses_real_derived_mwh_null_and_publication_latency(self) -> None: + observations = self.parse() + + self.assertEqual(len(observations), 3) + first = observations[0] + self.assertEqual(first.duid, "ADPBA1") + self.assertEqual(first.interval_start, datetime(2026, 8, 28, 18, 5, tzinfo=UTC)) + self.assertEqual(first.soc_mwh, 3.786) + self.assertEqual(first.intervention, 0) + self.assertEqual(first.run_number, 1) + self.assertEqual(first.dispatch_interval, "20260829001") + self.assertEqual(first.last_changed, datetime(2026, 8, 28, 18, 0, 6, tzinfo=UTC)) + self.assertEqual(first.report_timestamp, datetime(2026, 8, 29, 18, 10, tzinfo=UTC)) + self.assertEqual(first.downloaded_at, self.downloaded_at) + self.assertEqual(first.publication_latency_seconds, 86_700) + self.assertEqual(first.publication_status, "next_day") + self.assertEqual(first.ingestion_version, 8) + self.assertEqual(first.correction_version, 2) + missing = next(item for item in observations if item.duid == "KEPBG1") + self.assertIsNone(missing.soc_mwh) + self.assertEqual( + [(item.duid, item.interval_start) for item in observations], + sorted((item.duid, item.interval_start) for item in observations), + ) + + def test_filters_to_reviewed_duids_without_inventing_missing_rows(self) -> None: + observations = self.parse(duids=frozenset(("ADPBA1",))) + self.assertEqual(len(observations), 2) + self.assertEqual({item.duid for item in observations}, {"ADPBA1"}) + + def test_rejects_wrong_version_duplicate_rows_and_bad_trailer_count(self) -> None: + duplicate = self.payload.replace( + 'C,"END OF REPORT",6', + self.payload.splitlines()[2] + '\nC,"END OF REPORT",7', + ) + cases = ( + self.payload.replace("I,DISPATCH,UNIT_SOLUTION,6,", "I,DISPATCH,UNIT_SOLUTION,5,", 1), + duplicate, + self.payload.replace('C,"END OF REPORT",6', 'C,"END OF REPORT",7'), + ) + for payload in cases: + with self.subTest(payload=payload[-80:]): + with self.assertRaises(NextDaySocParseError): + self.parse(payload) + + def test_rejects_negative_nonfinite_or_misaligned_authoritative_values(self) -> None: + cases = ( + self.payload.replace(",3.786,3.78315,", ",-1,3.78315,", 1), + self.payload.replace(",3.786,3.78315,", ",NaN,3.78315,", 1), + self.payload.replace("2026/08/29 04:05:00", "2026/08/29 04:06:00", 1), + ) + for payload in cases: + with self.subTest(payload=payload[0:80]): + with self.assertRaises(NextDaySocParseError): + self.parse(payload) + + def test_rejects_invalid_provenance_inputs_before_parsing(self) -> None: + valid = { + "payload": self.payload, + "duids": frozenset(("ADPBA1",)), + "source_artifact_id": "d7a2abdd2947ed4b222166b9f60e3a8052838190027dd9ce03cb291ba2d29bc4", + "downloaded_at": self.downloaded_at, + "ingestion_version": 8, + "correction_version": 2, + } + invalid: tuple[dict[str, Any], ...] = ( + {"duids": {"ADPBA1"}}, + {"source_artifact_id": "not-a-digest"}, + {"downloaded_at": datetime(2026, 8, 30)}, + {"ingestion_version": True}, + {"correction_version": -1}, + ) + for override in invalid: + with self.subTest(override=override): + arguments = valid | override + with self.assertRaises(NextDaySocParseError): + parse_nextday_unit_solution_soc(**arguments) + + +if __name__ == "__main__": + unittest.main() From ec421fbeda7663bfaa250582845f4915cc9c8963 Mon Sep 17 00:00:00 2001 From: djurcola-agent Date: Sun, 30 Aug 2026 23:04:29 +1000 Subject: [PATCH 20/30] feat: add authoritative SOC schema --- app/backend/tests/test_backfill_ledger.py | 4 +- .../tests/test_dispatch_price_ingestion.py | 4 +- .../test_historical_artifact_migration.py | 89 ++++++- app/deploy/migrate.sh | 7 +- app/migrations/008_authoritative_soc.sql | 230 ++++++++++++++++++ 5 files changed, 324 insertions(+), 10 deletions(-) create mode 100644 app/migrations/008_authoritative_soc.sql diff --git a/app/backend/tests/test_backfill_ledger.py b/app/backend/tests/test_backfill_ledger.py index 797267f..45ab628 100644 --- a/app/backend/tests/test_backfill_ledger.py +++ b/app/backend/tests/test_backfill_ledger.py @@ -87,8 +87,8 @@ def test_deploys_additive_run_item_and_event_schema(self) -> None: self.assertIn(f"'{table}'", migrate_script) self.assertIn("004_historical_backfill_ledger.sql", migrate_script) - self.assertIn("SELECT count(*) = 12", migrate_script) - self.assertEqual(migrate_script.count('--dbname="$database_url"'), 8) + self.assertIn("SELECT count(*) = 13", migrate_script) + self.assertEqual(migrate_script.count('--dbname="$database_url"'), 9) self.assertIn("ON DELETE RESTRICT", migration) self.assertIn("event_seq BIGSERIAL PRIMARY KEY", migration) self.assertIn("historical_backfill_items_claim_idx", migration) diff --git a/app/backend/tests/test_dispatch_price_ingestion.py b/app/backend/tests/test_dispatch_price_ingestion.py index 911c1d3..0c590ec 100644 --- a/app/backend/tests/test_dispatch_price_ingestion.py +++ b/app/backend/tests/test_dispatch_price_ingestion.py @@ -223,11 +223,11 @@ def test_deploy_migration_applies_and_verifies_price_artifact_table(self) -> Non self.assertIn("raw_zip BYTEA NOT NULL", migration) self.assertIn("PUBLIC_DISPATCHIS_", migration) self.assertIn("003_dispatch_price_artifacts.sql", migrate_script) - self.assertIn("SELECT count(*) = 12", migrate_script) + self.assertIn("SELECT count(*) = 13", migrate_script) self.assertIn("'dispatch_price_artifacts'", migrate_script) self.assertIn("004_historical_backfill_ledger.sql", migrate_script) self.assertIn("database_url=$BATTERYWATCH_DATABASE_URL", migrate_script) - self.assertEqual(migrate_script.count('--dbname="$database_url"'), 8) + self.assertEqual(migrate_script.count('--dbname="$database_url"'), 9) self.assertNotIn("export PGDATABASE=", migrate_script) diff --git a/app/backend/tests/test_historical_artifact_migration.py b/app/backend/tests/test_historical_artifact_migration.py index ef5fdfd..3b299f1 100644 --- a/app/backend/tests/test_historical_artifact_migration.py +++ b/app/backend/tests/test_historical_artifact_migration.py @@ -46,10 +46,10 @@ def test_migration_defines_immutable_artifacts_and_deploys_last(self) -> None: self.assertIn("004_historical_backfill_ledger.sql", migrate_script) self.assertIn("005_historical_source_artifacts.sql", migrate_script) - self.assertIn("SELECT count(*) = 12", migrate_script) + self.assertIn("SELECT count(*) = 13", migrate_script) for table in ("historical_source_artifacts", "historical_backfill_item_artifacts"): self.assertIn(f"'{table}'", migrate_script) - self.assertEqual(migrate_script.count('--dbname="$database_url"'), 8) + self.assertEqual(migrate_script.count('--dbname="$database_url"'), 9) upper = migration.upper() for forbidden in ("DROP ", "ALTER ", "TRUNCATE "): @@ -88,7 +88,7 @@ def test_expands_event_constraint_for_artifact_recorded_atomically(self) -> None ): self.assertIn(f"'{event_type}'", migration) self.assertIn("006_artifact_recorded_event.sql", migrate_script) - self.assertEqual(migrate_script.count('--dbname="$database_url"'), 8) + self.assertEqual(migrate_script.count('--dbname="$database_url"'), 9) self.assertNotIn("TRUNCATE ", upper) self.assertNotRegex(upper, r"\bDELETE\s+FROM\b") @@ -115,10 +115,91 @@ def test_adds_runtime_ledger_error_and_event_detail_columns(self) -> None: self.assertIn("historical_backfill_events_details_check", migration) self.assertIn("jsonb_typeof(details) = 'object'", migration) self.assertIn("007_backfill_runtime_details.sql", migrate_script) - self.assertEqual(migrate_script.count('--dbname="$database_url"'), 8) + self.assertEqual(migrate_script.count('--dbname="$database_url"'), 9) self.assertNotIn("TRUNCATE ", upper) self.assertNotRegex(upper, r"\bDELETE\s+FROM\b") + def test_adds_authoritative_individual_soc_schema_after_runtime_details(self) -> None: + app_root = Path(__file__).resolve().parents[2] + migration_path = app_root / "migrations" / "008_authoritative_soc.sql" + self.assertTrue(migration_path.exists(), "008 authoritative-SOC migration is missing") + + migration = migration_path.read_text(encoding="utf-8") + migrate_script = (app_root / "deploy" / "migrate.sh").read_text(encoding="utf-8") + upper = migration.upper() + + self.assertIn("BEGIN;", upper) + self.assertIn("COMMIT;", upper) + self.assertIn("CREATE TABLE IF NOT EXISTS raw_nextday_soc_observations", migration) + for column in ( + "artifact_sha256", + "generator_id", + "interval_start", + "soc_mwh", + "intervention", + "run_number", + "dispatch_interval", + "last_changed", + "report_timestamp", + "downloaded_at", + "ingestion_version", + "correction_version", + ): + self.assertIn(column, migration) + self.assertIn("REFERENCES historical_source_artifacts", migration) + self.assertIn("REFERENCES generators", migration) + self.assertIn("ON DELETE RESTRICT", migration) + self.assertIn("raw_nextday_soc_observations_artifact_time_idx", migration) + self.assertIn("raw_nextday_soc_observations_generator_time_idx", migration) + + self.assertIn("ALTER TABLE generator_soc_5m", migration) + for column in ( + "soc_mwh", + "capacity_mwh", + "capacity_effective_from", + "capacity_effective_to", + "capacity_source_id", + "capacity_source_timestamp", + "report_timestamp", + "downloaded_at", + "intervention", + "run_number", + "dispatch_interval", + "source_artifact_sha256", + ): + self.assertIn(f"ADD COLUMN IF NOT EXISTS {column}", migration) + self.assertIn("generator_soc_5m_authoritative_soc_check", migration) + self.assertIn("generator_soc_5m_capacity_provenance_check", migration) + self.assertIn("generator_soc_5m_authoritative_metadata_check", migration) + self.assertGreaterEqual( + migration.count("num_nonnulls("), + 4, + "all-or-none SOC provenance checks must not pass through SQL NULL", + ) + self.assertIn("'nextday_soc'", migration) + self.assertIn("PUBLIC_NEXT_DAY_DISPATCH_", migration) + self.assertIn("Next_Day_Dispatch/", migration) + for constraint in ( + "historical_backfill_items_feed_check", + "historical_backfill_events_feed_check", + "historical_source_artifacts_feed_check", + "historical_source_artifacts_archive_identity_ck", + "historical_backfill_item_artifacts_feed_check", + ): + self.assertIn(f"DROP CONSTRAINT IF EXISTS {constraint}", migration) + self.assertIn(f"ADD CONSTRAINT {constraint}", migration) + + self.assertIn("008_authoritative_soc.sql", migrate_script) + self.assertLess( + migrate_script.index("007_backfill_runtime_details.sql"), + migrate_script.index("008_authoritative_soc.sql"), + ) + self.assertIn("SELECT count(*) = 13", migrate_script) + self.assertIn("'raw_nextday_soc_observations'", migrate_script) + self.assertEqual(migrate_script.count('--dbname="$database_url"'), 9) + for forbidden in ("DROP TABLE", "DROP COLUMN", "TRUNCATE ", "DELETE FROM"): + self.assertNotIn(forbidden, upper) + if __name__ == "__main__": unittest.main() diff --git a/app/deploy/migrate.sh b/app/deploy/migrate.sh index 62d93ce..7361ff7 100755 --- a/app/deploy/migrate.sh +++ b/app/deploy/migrate.sh @@ -35,9 +35,11 @@ psql --dbname="$database_url" --no-psqlrc --set=ON_ERROR_STOP=1 \ --file="$app_dir/migrations/006_artifact_recorded_event.sql" psql --dbname="$database_url" --no-psqlrc --set=ON_ERROR_STOP=1 \ --file="$app_dir/migrations/007_backfill_runtime_details.sql" +psql --dbname="$database_url" --no-psqlrc --set=ON_ERROR_STOP=1 \ + --file="$app_dir/migrations/008_authoritative_soc.sql" psql --dbname="$database_url" --no-psqlrc --set=ON_ERROR_STOP=1 --tuples-only --no-align \ <<'SQL' -SELECT count(*) = 12 +SELECT count(*) = 13 FROM information_schema.tables WHERE table_schema = 'public' AND table_name IN ( @@ -46,7 +48,8 @@ WHERE table_schema = 'public' 'dispatch_price_artifacts', 'historical_backfill_runs', 'historical_backfill_items', 'historical_backfill_events', - 'historical_source_artifacts', 'historical_backfill_item_artifacts' + 'historical_source_artifacts', 'historical_backfill_item_artifacts', + 'raw_nextday_soc_observations' ); SQL unset database_url diff --git a/app/migrations/008_authoritative_soc.sql b/app/migrations/008_authoritative_soc.sql new file mode 100644 index 0000000..736e966 --- /dev/null +++ b/app/migrations/008_authoritative_soc.sql @@ -0,0 +1,230 @@ +-- Add authoritative individual Next Day SOC observations and provenance. + +BEGIN; + +-- Widen the historical backfill/artifact feed contracts without rewriting +-- existing SCADA or price rows. The monthly Next Day archive is keyed by the +-- first day of its content month. +ALTER TABLE historical_backfill_items + DROP CONSTRAINT IF EXISTS historical_backfill_items_feed_check, + ADD CONSTRAINT historical_backfill_items_feed_check + CHECK (feed IN ('dispatch_price', 'dispatch_scada', 'nextday_soc')); + +ALTER TABLE historical_backfill_events + DROP CONSTRAINT IF EXISTS historical_backfill_events_feed_check, + ADD CONSTRAINT historical_backfill_events_feed_check + CHECK (feed IN ('dispatch_price', 'dispatch_scada', 'nextday_soc')); + +ALTER TABLE historical_source_artifacts + DROP CONSTRAINT IF EXISTS historical_source_artifacts_feed_check, + ADD CONSTRAINT historical_source_artifacts_feed_check + CHECK (feed IN ('dispatch_price', 'dispatch_scada', 'nextday_soc')), + DROP CONSTRAINT IF EXISTS historical_source_artifacts_archive_identity_ck, + ADD CONSTRAINT historical_source_artifacts_archive_identity_ck CHECK ( + ( + feed = 'dispatch_price' + AND source_url = + 'https://www.nemweb.com.au/REPORTS/ARCHIVE/DispatchIS_Reports/' || filename + AND filename = 'PUBLIC_DISPATCHIS_' || to_char(report_date, 'YYYYMMDD') || '.zip' + ) + OR ( + feed = 'dispatch_scada' + AND source_url = + 'https://www.nemweb.com.au/REPORTS/ARCHIVE/Dispatch_SCADA/' || filename + AND filename = 'PUBLIC_DISPATCHSCADA_' || to_char(report_date, 'YYYYMMDD') || '.zip' + ) + OR ( + feed = 'nextday_soc' + AND report_date = date_trunc('month', report_date)::date + AND source_url = + 'https://www.nemweb.com.au/REPORTS/ARCHIVE/Next_Day_Dispatch/' || filename + AND filename = + 'PUBLIC_NEXT_DAY_DISPATCH_' || to_char(report_date, 'YYYYMM') || '01.zip' + ) + ); + +ALTER TABLE historical_backfill_item_artifacts + DROP CONSTRAINT IF EXISTS historical_backfill_item_artifacts_feed_check, + ADD CONSTRAINT historical_backfill_item_artifacts_feed_check + CHECK (feed IN ('dispatch_price', 'dispatch_scada', 'nextday_soc')); + +CREATE TABLE IF NOT EXISTS raw_nextday_soc_observations ( + artifact_sha256 TEXT NOT NULL + REFERENCES historical_source_artifacts (artifact_sha256) ON DELETE RESTRICT, + generator_id TEXT NOT NULL + REFERENCES generators (generator_id) ON DELETE RESTRICT, + interval_start TIMESTAMPTZ NOT NULL, + soc_mwh DOUBLE PRECISION CHECK ( + soc_mwh IS NULL + OR ( + soc_mwh >= 0 + AND soc_mwh::TEXT NOT IN ('NaN', 'Infinity', '-Infinity') + ) + ), + intervention SMALLINT NOT NULL CHECK (intervention IN (0, 1)), + run_number INTEGER NOT NULL CHECK (run_number > 0), + dispatch_interval TEXT NOT NULL CHECK (dispatch_interval ~ '^[0-9]{11}$'), + last_changed TIMESTAMPTZ NOT NULL, + report_timestamp TIMESTAMPTZ NOT NULL, + downloaded_at TIMESTAMPTZ NOT NULL, + ingestion_version BIGINT NOT NULL CHECK (ingestion_version >= 0), + correction_version BIGINT NOT NULL CHECK (correction_version >= 0), + PRIMARY KEY ( + artifact_sha256, + generator_id, + interval_start, + intervention, + run_number, + ingestion_version, + correction_version + ), + CHECK (date_trunc('minute', interval_start) = interval_start), + CHECK (EXTRACT(MINUTE FROM interval_start)::INTEGER % 5 = 0), + CHECK (last_changed <= report_timestamp), + CHECK (report_timestamp <= downloaded_at) +); + +CREATE INDEX IF NOT EXISTS raw_nextday_soc_observations_artifact_time_idx + ON raw_nextday_soc_observations (artifact_sha256, interval_start ASC); +CREATE INDEX IF NOT EXISTS raw_nextday_soc_observations_generator_time_idx + ON raw_nextday_soc_observations (generator_id, interval_start DESC); + +ALTER TABLE generator_soc_5m + ADD COLUMN IF NOT EXISTS soc_mwh DOUBLE PRECISION, + ADD COLUMN IF NOT EXISTS capacity_mwh DOUBLE PRECISION, + ADD COLUMN IF NOT EXISTS capacity_effective_from TIMESTAMPTZ, + ADD COLUMN IF NOT EXISTS capacity_effective_to TIMESTAMPTZ, + ADD COLUMN IF NOT EXISTS capacity_source_id TEXT, + ADD COLUMN IF NOT EXISTS capacity_source_timestamp TIMESTAMPTZ, + ADD COLUMN IF NOT EXISTS report_timestamp TIMESTAMPTZ, + ADD COLUMN IF NOT EXISTS downloaded_at TIMESTAMPTZ, + ADD COLUMN IF NOT EXISTS intervention SMALLINT, + ADD COLUMN IF NOT EXISTS run_number INTEGER, + ADD COLUMN IF NOT EXISTS dispatch_interval TEXT, + ADD COLUMN IF NOT EXISTS source_artifact_sha256 TEXT + REFERENCES historical_source_artifacts (artifact_sha256) ON DELETE RESTRICT; + +DO $migration$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'generator_soc_5m_authoritative_soc_check' + AND conrelid = 'generator_soc_5m'::regclass + ) THEN + ALTER TABLE generator_soc_5m + ADD CONSTRAINT generator_soc_5m_authoritative_soc_check + CHECK ( + soc_mwh IS NULL + OR ( + soc_mwh >= 0 + AND soc_mwh::TEXT NOT IN ('NaN', 'Infinity', '-Infinity') + ) + ); + END IF; +END +$migration$; + +DO $migration$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'generator_soc_5m_capacity_provenance_check' + AND conrelid = 'generator_soc_5m'::regclass + ) THEN + ALTER TABLE generator_soc_5m + ADD CONSTRAINT generator_soc_5m_capacity_provenance_check + CHECK ( + source_artifact_sha256 IS NULL + OR ( + ( + soc_percent IS NULL + AND num_nonnulls( + capacity_mwh, + capacity_effective_from, + capacity_effective_to, + capacity_source_id, + capacity_source_timestamp + ) = 0 + ) + OR ( + soc_percent IS NOT NULL + AND soc_mwh IS NOT NULL + AND num_nonnulls( + capacity_mwh, + capacity_effective_from, + capacity_source_id, + capacity_source_timestamp + ) = 4 + AND capacity_mwh > 0 + AND capacity_mwh::TEXT NOT IN ('NaN', 'Infinity', '-Infinity') + AND soc_mwh <= capacity_mwh + AND abs(soc_percent - (100.0 * soc_mwh / capacity_mwh)) <= 0.000001 + AND capacity_effective_from <= interval_start + AND ( + capacity_effective_to IS NULL + OR ( + capacity_effective_from < capacity_effective_to + AND interval_start < capacity_effective_to + ) + ) + AND char_length(capacity_source_id) BETWEEN 1 AND 255 + AND capacity_source_id = btrim(capacity_source_id) + AND capacity_source_timestamp <= interval_start + ) + ) + ); + END IF; +END +$migration$; + +DO $migration$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'generator_soc_5m_authoritative_metadata_check' + AND conrelid = 'generator_soc_5m'::regclass + ) THEN + ALTER TABLE generator_soc_5m + ADD CONSTRAINT generator_soc_5m_authoritative_metadata_check + CHECK ( + ( + num_nonnulls( + source_artifact_sha256, + report_timestamp, + downloaded_at, + intervention, + run_number, + dispatch_interval, + soc_mwh, + capacity_mwh, + capacity_effective_from, + capacity_effective_to, + capacity_source_id, + capacity_source_timestamp + ) = 0 + ) + OR ( + num_nonnulls( + source_artifact_sha256, + report_timestamp, + downloaded_at, + intervention, + run_number, + dispatch_interval + ) = 6 + AND + source_artifact_sha256 ~ '^[0-9a-f]{64}$' + AND report_timestamp <= downloaded_at + AND intervention IN (0, 1) + AND run_number > 0 + AND dispatch_interval ~ '^[0-9]{11}$' + ) + ); + END IF; +END +$migration$; + +COMMIT; From cb1fd8be11cda224585bfb4d520ffa756aa3131c Mon Sep 17 00:00:00 2001 From: djurcola-agent Date: Sun, 30 Aug 2026 23:01:31 +1000 Subject: [PATCH 21/30] feat: parse regional BDU energy storage --- .../batterywatch_api/regional_bdu_soc.py | 272 ++++++++++++++++++ app/backend/tests/test_regional_bdu_soc.py | 131 +++++++++ 2 files changed, 403 insertions(+) create mode 100644 app/backend/batterywatch_api/regional_bdu_soc.py create mode 100644 app/backend/tests/test_regional_bdu_soc.py diff --git a/app/backend/batterywatch_api/regional_bdu_soc.py b/app/backend/batterywatch_api/regional_bdu_soc.py new file mode 100644 index 0000000..e4aa52e --- /dev/null +++ b/app/backend/batterywatch_api/regional_bdu_soc.py @@ -0,0 +1,272 @@ +"""Strict parser for regional aggregate BDU initial energy storage.""" + +from __future__ import annotations + +import csv +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from io import StringIO +from math import isfinite +import re +from typing import Literal + +_NEM_TIMEZONE = timezone(timedelta(hours=10)) +_METADATA_PREFIX = ("C", "NEMP.WORLD", "DISPATCHIS", "AEMO", "PUBLIC") +_TABLE_PREFIX = ("DISPATCH", "REGIONSUM", "9") +_REQUIRED_COLUMNS = frozenset( + ( + "SETTLEMENTDATE", + "RUNNO", + "REGIONID", + "DISPATCHINTERVAL", + "INTERVENTION", + "LASTCHANGED", + "BDU_INITIAL_ENERGY_STORAGE", + ) +) +_REGIONS = frozenset(("NSW1", "QLD1", "SA1", "TAS1", "VIC1")) +_ARTIFACT_RE = re.compile(r"^[0-9a-f]{64}$") +_DISPATCH_INTERVAL_RE = re.compile(r"^(\d{8})(\d{3})$") + + +class RegionalBduSocParseError(ValueError): + """Raised when regional aggregate BDU source data is unsafe to publish.""" + + +def _timestamp(value: str, *, field: str, aligned: bool) -> datetime: + normalized = value.strip().replace("/", "-").replace(" ", "T", 1) + try: + parsed = datetime.fromisoformat(normalized) + except (AttributeError, TypeError, ValueError) as exc: + raise RegionalBduSocParseError(f"invalid {field}") from exc + if parsed.tzinfo is None or parsed.utcoffset() is None: + parsed = parsed.replace(tzinfo=_NEM_TIMEZONE) + result = parsed.astimezone(timezone.utc) + if aligned and (result.minute % 5 or result.second or result.microsecond): + raise RegionalBduSocParseError(f"invalid {field} alignment") + return result + + +def _report_timestamp(row: list[str]) -> datetime: + if len(row) != 10 or tuple(row[:5]) != _METADATA_PREFIX: + raise RegionalBduSocParseError("invalid DispatchIS metadata") + return _timestamp(f"{row[5]} {row[6]}", field="report timestamp", aligned=False) + + +def _version(value: str, *, field: str, maximum: int) -> int: + if not value or not value.isascii() or not value.isdecimal(): + raise RegionalBduSocParseError(f"invalid {field}") + parsed = int(value) + if parsed > maximum: + raise RegionalBduSocParseError(f"invalid {field}") + return parsed + + +def _soc_mwh(value: str) -> float | None: + if value == "": + return None + try: + parsed = float(value) + except (TypeError, ValueError) as exc: + raise RegionalBduSocParseError("invalid BDU_INITIAL_ENERGY_STORAGE") from exc + if not isfinite(parsed) or parsed < 0: + raise RegionalBduSocParseError("invalid BDU_INITIAL_ENERGY_STORAGE") + return parsed + + +def _dispatch_interval(value: str, interval_start: datetime) -> str: + match = _DISPATCH_INTERVAL_RE.fullmatch(value) + if match is None: + raise RegionalBduSocParseError("invalid DISPATCHINTERVAL") + period = int(match.group(2)) + if not 1 <= period <= 288: + raise RegionalBduSocParseError("invalid DISPATCHINTERVAL") + try: + market_date = datetime.strptime(match.group(1), "%Y%m%d").replace( + tzinfo=_NEM_TIMEZONE + ) + except ValueError as exc: + raise RegionalBduSocParseError("invalid DISPATCHINTERVAL") from exc + expected = (market_date + timedelta(hours=4, minutes=period * 5)).astimezone( + timezone.utc + ) + if expected != interval_start: + raise RegionalBduSocParseError( + "DISPATCHINTERVAL does not match SETTLEMENTDATE" + ) + return value + + +def _validated_downloaded_at(value: datetime, report_timestamp: datetime) -> datetime: + if not isinstance(value, datetime) or value.tzinfo is None or value.utcoffset() is None: + raise RegionalBduSocParseError("invalid downloaded_at") + result = value.astimezone(timezone.utc) + if result < report_timestamp: + raise RegionalBduSocParseError("downloaded_at precedes report timestamp") + return result + + +@dataclass(frozen=True, slots=True) +class RegionalBduSocObservation: + """One region-level aggregate BDU initial-energy-storage observation.""" + + region_id: str + interval_start: datetime + soc_mwh: float | None + intervention: int + run_number: int + dispatch_interval: str + last_changed: datetime + source_artifact_id: str + report_timestamp: datetime + downloaded_at: datetime + ingestion_version: int + correction_version: int + + @property + def scope(self) -> Literal["regional_aggregate"]: + return "regional_aggregate" + + @property + def publication_status(self) -> Literal["near_real_time_regional"]: + return "near_real_time_regional" + + +def parse_dispatch_regionsum_bdu_soc( + payload: str, + *, + source_artifact_id: str, + downloaded_at: datetime, + ingestion_version: int, + correction_version: int = 0, +) -> tuple[RegionalBduSocObservation, ...]: + """Parse public DispatchIS REGIONSUM v9 regional BDU storage rows.""" + + if type(payload) is not str or not payload: + raise RegionalBduSocParseError("invalid payload") + if ( + type(source_artifact_id) is not str + or _ARTIFACT_RE.fullmatch(source_artifact_id) is None + ): + raise RegionalBduSocParseError("invalid source artifact identity") + ingestion = _version( + str(ingestion_version), field="ingestion_version", maximum=2**63 - 1 + ) + correction = _version( + str(correction_version), field="correction_version", maximum=2**63 - 1 + ) + + reader = csv.reader(StringIO(payload)) + try: + metadata = next(reader) + except (StopIteration, csv.Error, TypeError, UnicodeError) as exc: + raise RegionalBduSocParseError("incomplete DispatchIS report") from exc + report_timestamp = _report_timestamp(metadata) + downloaded = _validated_downloaded_at(downloaded_at, report_timestamp) + + header: list[str] | None = None + observations: list[RegionalBduSocObservation] = [] + seen: set[tuple[str, datetime, int, int]] = set() + report_record_count = 1 + expected_count: int | None = None + try: + for row in reader: + report_record_count += 1 + if expected_count is not None: + raise RegionalBduSocParseError("data follows report trailer") + if row[:2] == ["C", "END OF REPORT"]: + if len(row) != 3: + raise RegionalBduSocParseError("invalid report trailer") + expected_count = _version( + row[2], field="report row count", maximum=10_000_000 + ) + continue + + if row[:4] == ["I", *_TABLE_PREFIX]: + if header is not None or len(row) != len(set(row)): + raise RegionalBduSocParseError("invalid REGIONSUM header") + if not _REQUIRED_COLUMNS.issubset(row[4:]): + raise RegionalBduSocParseError("missing REGIONSUM columns") + header = row + continue + if row[:4] != ["D", *_TABLE_PREFIX]: + continue + if header is None or len(row) != len(header): + raise RegionalBduSocParseError("malformed REGIONSUM row") + values = dict(zip(header[4:], row[4:])) + region_id = values["REGIONID"] + if region_id not in _REGIONS: + raise RegionalBduSocParseError("invalid REGIONID") + interval_start = _timestamp( + values["SETTLEMENTDATE"], field="SETTLEMENTDATE", aligned=True + ) + run_number = _version(values["RUNNO"], field="RUNNO", maximum=999) + intervention = _version( + values["INTERVENTION"], field="INTERVENTION", maximum=1 + ) + dispatch_interval = _dispatch_interval( + values["DISPATCHINTERVAL"], interval_start + ) + last_changed = _timestamp( + values["LASTCHANGED"], field="LASTCHANGED", aligned=False + ) + if last_changed > report_timestamp: + raise RegionalBduSocParseError("invalid source timestamp ordering") + key = (region_id, interval_start, intervention, run_number) + if key in seen: + raise RegionalBduSocParseError("duplicate REGIONSUM observation") + seen.add(key) + observations.append( + RegionalBduSocObservation( + region_id=region_id, + interval_start=interval_start, + soc_mwh=_soc_mwh(values["BDU_INITIAL_ENERGY_STORAGE"]), + intervention=intervention, + run_number=run_number, + dispatch_interval=dispatch_interval, + last_changed=last_changed, + source_artifact_id=source_artifact_id, + report_timestamp=report_timestamp, + downloaded_at=downloaded, + ingestion_version=ingestion, + correction_version=correction, + ) + ) + except (csv.Error, TypeError, UnicodeError) as exc: + raise RegionalBduSocParseError("invalid CSV") from exc + if expected_count is None: + raise RegionalBduSocParseError("missing report trailer") + if expected_count != report_record_count: + raise RegionalBduSocParseError("report row count mismatch") + if header is None: + raise RegionalBduSocParseError("missing REGIONSUM v9 table") + regional_groups: dict[tuple[datetime, int, int], set[str]] = {} + for observation in observations: + group_key = ( + observation.interval_start, + observation.intervention, + observation.run_number, + ) + regional_groups.setdefault(group_key, set()).add(observation.region_id) + if not regional_groups or any( + regions != _REGIONS for regions in regional_groups.values() + ): + raise RegionalBduSocParseError("incomplete regional aggregate coverage") + return tuple( + sorted( + observations, + key=lambda item: ( + item.interval_start, + item.region_id, + item.intervention, + item.run_number, + ), + ) + ) + + +__all__ = [ + "RegionalBduSocObservation", + "RegionalBduSocParseError", + "parse_dispatch_regionsum_bdu_soc", +] diff --git a/app/backend/tests/test_regional_bdu_soc.py b/app/backend/tests/test_regional_bdu_soc.py new file mode 100644 index 0000000..22db062 --- /dev/null +++ b/app/backend/tests/test_regional_bdu_soc.py @@ -0,0 +1,131 @@ +"""Tests for regional aggregate BDU energy-storage parsing.""" + +from datetime import datetime, timezone +from pathlib import Path +from typing import Any +import unittest + +from batterywatch_api.regional_bdu_soc import ( + RegionalBduSocParseError, + parse_dispatch_regionsum_bdu_soc, +) + +UTC = timezone.utc +FIXTURE = ( + Path(__file__).parent + / "fixtures" + / "historical" + / "dispatch-regionsum-bdu-soc-20260830-2145-reduced.csv" +) + + +class RegionalBduSocParserTests(unittest.TestCase): + def setUp(self) -> None: + self.payload = FIXTURE.read_text(encoding="utf-8") + self.downloaded_at = datetime(2026, 8, 30, 11, 45, tzinfo=UTC) + + def parse(self, payload: str | None = None): + return parse_dispatch_regionsum_bdu_soc( + self.payload if payload is None else payload, + source_artifact_id=( + "e83dd01b41bd3a5eef355e16d72d79f176a652ee27471615971c55a4c4f42561" + ), + downloaded_at=self.downloaded_at, + ingestion_version=8, + correction_version=2, + ) + + def test_parses_real_derived_regional_values_and_null(self) -> None: + observations = self.parse() + + self.assertEqual(len(observations), 5) + self.assertEqual( + [item.region_id for item in observations], + ["NSW1", "QLD1", "SA1", "TAS1", "VIC1"], + ) + nsw = observations[0] + self.assertEqual(nsw.interval_start, datetime(2026, 8, 30, 11, 45, tzinfo=UTC)) + self.assertEqual(nsw.soc_mwh, 2343.89364) + self.assertEqual(nsw.intervention, 0) + self.assertEqual(nsw.run_number, 1) + self.assertEqual(nsw.dispatch_interval, "20260830213") + self.assertEqual(nsw.last_changed, datetime(2026, 8, 30, 11, 40, 4, tzinfo=UTC)) + self.assertEqual( + nsw.report_timestamp, + datetime(2026, 8, 30, 11, 40, 9, tzinfo=UTC), + ) + self.assertEqual(nsw.downloaded_at, self.downloaded_at) + self.assertEqual(nsw.scope, "regional_aggregate") + self.assertEqual(nsw.publication_status, "near_real_time_regional") + self.assertEqual(nsw.ingestion_version, 8) + self.assertEqual(nsw.correction_version, 2) + tas = next(item for item in observations if item.region_id == "TAS1") + self.assertIsNone(tas.soc_mwh) + + def test_never_exposes_regional_aggregate_as_duid_data(self) -> None: + observations = self.parse() + + self.assertTrue(all(not hasattr(item, "duid") for item in observations)) + self.assertEqual( + {item.region_id for item in observations}, + {"NSW1", "QLD1", "SA1", "TAS1", "VIC1"}, + ) + + def test_rejects_wrong_version_duplicate_rows_and_bad_trailer_count(self) -> None: + data_row = self.payload.splitlines()[2] + duplicate = self.payload.replace( + 'C,"END OF REPORT",8', + data_row + '\nC,"END OF REPORT",9', + ) + missing_region = "\n".join( + line for line in self.payload.splitlines() if ",TAS1," not in line + ).replace('C,"END OF REPORT",8', 'C,"END OF REPORT",7') + cases = ( + self.payload.replace("I,DISPATCH,REGIONSUM,9,", "I,DISPATCH,REGIONSUM,8,", 1), + duplicate, + missing_region, + self.payload.replace('C,"END OF REPORT",8', 'C,"END OF REPORT",9'), + ) + for payload in cases: + with self.subTest(payload=payload[-100:]): + with self.assertRaises(RegionalBduSocParseError): + self.parse(payload) + + def test_rejects_invalid_region_negative_nonfinite_and_misaligned_values(self) -> None: + cases = ( + self.payload.replace(",NSW1,", ",XX1,", 1), + self.payload.replace(",2343.893640\n", ",-1\n", 1), + self.payload.replace(",2343.893640\n", ",NaN\n", 1), + self.payload.replace("2026/08/30 21:45:00", "2026/08/30 21:46:00", 1), + ) + for payload in cases: + with self.subTest(payload=payload[:100]): + with self.assertRaises(RegionalBduSocParseError): + self.parse(payload) + + def test_rejects_invalid_provenance_inputs_before_parsing(self) -> None: + valid: dict[str, Any] = { + "payload": self.payload, + "source_artifact_id": ( + "e83dd01b41bd3a5eef355e16d72d79f176a652ee27471615971c55a4c4f42561" + ), + "downloaded_at": self.downloaded_at, + "ingestion_version": 8, + "correction_version": 2, + } + invalid: tuple[dict[str, Any], ...] = ( + {"payload": b"not text"}, + {"source_artifact_id": "not-a-digest"}, + {"downloaded_at": datetime(2026, 8, 30)}, + {"ingestion_version": True}, + {"correction_version": -1}, + ) + for override in invalid: + with self.subTest(override=override): + arguments = valid | override + with self.assertRaises(RegionalBduSocParseError): + parse_dispatch_regionsum_bdu_soc(**arguments) + + +if __name__ == "__main__": + unittest.main() From ed847c2789f1314d41d1bf0445844e1bbf580838 Mon Sep 17 00:00:00 2001 From: djurcola-agent Date: Sun, 30 Aug 2026 23:22:53 +1000 Subject: [PATCH 22/30] feat: persist authoritative next-day battery SOC --- .../batterywatch_api/nextday_soc_ingestion.py | 503 ++++++++++++++++++ .../tests/test_nextday_soc_ingestion.py | 375 +++++++++++++ 2 files changed, 878 insertions(+) create mode 100644 app/backend/batterywatch_api/nextday_soc_ingestion.py create mode 100644 app/backend/tests/test_nextday_soc_ingestion.py diff --git a/app/backend/batterywatch_api/nextday_soc_ingestion.py b/app/backend/batterywatch_api/nextday_soc_ingestion.py new file mode 100644 index 0000000..39cd11e --- /dev/null +++ b/app/backend/batterywatch_api/nextday_soc_ingestion.py @@ -0,0 +1,503 @@ +"""Atomic persistence for authoritative individual Next Day SOC.""" + +from __future__ import annotations + +from contextlib import contextmanager +from dataclasses import dataclass +from datetime import datetime +from typing import Any, Iterable, Iterator, Protocol + +from .battery_assets import BatteryAsset +from .nextday_soc import NextDaySocObservation + + +@dataclass(frozen=True, slots=True) +class NextDaySocIngestionResult: + source_rows: int + raw_inserted: int + raw_replayed: int + effective_candidates: int + effective_applied: int + effective_replayed: int + source_null_count: int + percentage_count: int + + +class NextDaySocConflictError(Exception): + """Raised when stored SOC evidence conflicts at the same precedence.""" + + +class _Cursor(Protocol): + rowcount: int + + def execute(self, statement: str, parameters: tuple[Any, ...]) -> None: ... + def executemany( + self, + statement: str, + parameters: Iterable[tuple[Any, ...]], + ) -> None: ... + def fetchone(self) -> tuple[Any, ...] | None: ... + def fetchall(self) -> list[tuple[Any, ...]]: ... + def close(self) -> None: ... + + +class _Connection(Protocol): + def cursor(self) -> _Cursor: ... + def commit(self) -> None: ... + def rollback(self) -> None: ... + + +@contextmanager +def _managed_cursor(connection: _Connection) -> Iterator[_Cursor]: + cursor = connection.cursor() + try: + yield cursor + finally: + cursor.close() + + +_ARTIFACT_SELECT_SQL = """ +SELECT feed +FROM historical_source_artifacts +WHERE artifact_sha256 = %s +FOR SHARE +""" + +_RAW_SELECT_SQL = """ +SELECT artifact_sha256, generator_id, interval_start, soc_mwh, + intervention, run_number, dispatch_interval, last_changed, + report_timestamp, downloaded_at, ingestion_version, correction_version +FROM raw_nextday_soc_observations +WHERE artifact_sha256 = %s + AND ingestion_version = %s + AND correction_version = %s +FOR UPDATE +""" + +_RAW_INSERT_SQL = """ +INSERT INTO raw_nextday_soc_observations ( + artifact_sha256, generator_id, interval_start, soc_mwh, + intervention, run_number, dispatch_interval, last_changed, + report_timestamp, downloaded_at, ingestion_version, correction_version +) +VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) +""" + +_EFFECTIVE_SELECT_SQL = """ +SELECT generator_id, interval_start, soc_percent, + source_id, source_timestamp, ingestion_version, correction_version, + quality_flags, soc_mwh, capacity_mwh, capacity_effective_from, + capacity_effective_to, capacity_source_id, capacity_source_timestamp, + report_timestamp, downloaded_at, intervention, run_number, + dispatch_interval, source_artifact_sha256 +FROM generator_soc_5m +WHERE generator_id = ANY(%s) + AND interval_start >= %s + AND interval_start <= %s +FOR UPDATE +""" + +_EFFECTIVE_UPSERT_SQL = """ +INSERT INTO generator_soc_5m ( + generator_id, interval_start, soc_percent, + source_id, source_timestamp, ingestion_version, correction_version, + quality_flags, soc_mwh, capacity_mwh, capacity_effective_from, + capacity_effective_to, capacity_source_id, capacity_source_timestamp, + report_timestamp, downloaded_at, intervention, run_number, + dispatch_interval, source_artifact_sha256 +) +VALUES ( + %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, + %s, %s, %s, %s, %s, %s, %s, %s, %s, %s +) +ON CONFLICT (generator_id, interval_start) DO UPDATE +SET soc_percent = EXCLUDED.soc_percent, + source_id = EXCLUDED.source_id, + source_timestamp = EXCLUDED.source_timestamp, + ingestion_version = EXCLUDED.ingestion_version, + correction_version = EXCLUDED.correction_version, + quality_flags = EXCLUDED.quality_flags, + soc_mwh = EXCLUDED.soc_mwh, + capacity_mwh = EXCLUDED.capacity_mwh, + capacity_effective_from = EXCLUDED.capacity_effective_from, + capacity_effective_to = EXCLUDED.capacity_effective_to, + capacity_source_id = EXCLUDED.capacity_source_id, + capacity_source_timestamp = EXCLUDED.capacity_source_timestamp, + report_timestamp = EXCLUDED.report_timestamp, + downloaded_at = EXCLUDED.downloaded_at, + intervention = EXCLUDED.intervention, + run_number = EXCLUDED.run_number, + dispatch_interval = EXCLUDED.dispatch_interval, + source_artifact_sha256 = EXCLUDED.source_artifact_sha256 +WHERE generator_soc_5m.source_artifact_sha256 IS NULL + OR ( + ( + EXCLUDED.intervention, + EXCLUDED.correction_version, + EXCLUDED.ingestion_version, + EXCLUDED.report_timestamp, + EXCLUDED.source_artifact_sha256 +) > ( + generator_soc_5m.intervention, + generator_soc_5m.correction_version, + generator_soc_5m.ingestion_version, + generator_soc_5m.report_timestamp, + generator_soc_5m.source_artifact_sha256 + ) +) +""" + + +def _raw_parameters(observation: NextDaySocObservation) -> tuple[Any, ...]: + return ( + observation.source_artifact_id, + observation.duid, + observation.interval_start, + observation.soc_mwh, + observation.intervention, + observation.run_number, + observation.dispatch_interval, + observation.last_changed, + observation.report_timestamp, + observation.downloaded_at, + observation.ingestion_version, + observation.correction_version, + ) + + +def _precedence(observation: NextDaySocObservation) -> tuple[Any, ...]: + return ( + observation.intervention, + observation.correction_version, + observation.ingestion_version, + observation.report_timestamp, + observation.source_artifact_id, + ) + + +def _effective_parameters( + observation: NextDaySocObservation, + asset: BatteryAsset, +) -> tuple[Any, ...]: + quality_flags: list[str] + capacity_mwh: float | None + capacity_effective_from: datetime | None + capacity_source_id: str | None + capacity_source_timestamp: datetime | None + soc_percent: float | None + if observation.soc_mwh is None: + soc_percent = None + capacity_mwh = None + capacity_effective_from = None + capacity_source_id = None + capacity_source_timestamp = None + quality_flags = ["authoritative_soc_missing"] + elif asset.source_timestamp > observation.interval_start: + soc_percent = None + capacity_mwh = None + capacity_effective_from = None + capacity_source_id = None + capacity_source_timestamp = None + quality_flags = ["capacity_not_effective"] + elif observation.soc_mwh > asset.storage_capacity_mwh: + soc_percent = None + capacity_mwh = None + capacity_effective_from = None + capacity_source_id = None + capacity_source_timestamp = None + quality_flags = ["soc_exceeds_capacity"] + else: + capacity_mwh = asset.storage_capacity_mwh + capacity_effective_from = asset.source_timestamp + capacity_source_id = asset.source_id + capacity_source_timestamp = asset.source_timestamp + soc_percent = 100.0 * observation.soc_mwh / capacity_mwh + quality_flags = [] + return ( + observation.duid, + observation.interval_start, + soc_percent, + observation.source_artifact_id, + observation.last_changed, + observation.ingestion_version, + observation.correction_version, + quality_flags, + observation.soc_mwh, + capacity_mwh, + capacity_effective_from, + None, + capacity_source_id, + capacity_source_timestamp, + observation.report_timestamp, + observation.downloaded_at, + observation.intervention, + observation.run_number, + observation.dispatch_interval, + observation.source_artifact_id, + ) + + +def _effective_precedence(parameters: tuple[Any, ...]) -> tuple[Any, ...]: + return ( + parameters[16], + parameters[6], + parameters[5], + parameters[14], + parameters[19], + ) + + +def _effective_matches( + stored: tuple[Any, ...], + expected: tuple[Any, ...], +) -> bool: + if len(stored) != len(expected): + return False + try: + stored_flags = tuple(stored[7]) + expected_flags = tuple(expected[7]) + except TypeError: + return False + return ( + stored[:7] == expected[:7] + and stored_flags == expected_flags + and stored[8:] == expected[8:] + ) + + +def _validate_inputs( + observations: tuple[NextDaySocObservation, ...], + assets: tuple[BatteryAsset, ...], +) -> dict[str, BatteryAsset]: + if not observations or any( + not isinstance(item, NextDaySocObservation) for item in observations + ): + raise ValueError("invalid Next Day SOC observations") + if not assets or any(not isinstance(item, BatteryAsset) for item in assets): + raise ValueError("invalid reviewed battery assets") + asset_by_duid = {asset.duid: asset for asset in assets} + if len(asset_by_duid) != len(assets): + raise ValueError("duplicate reviewed battery asset") + if any(item.duid not in asset_by_duid for item in observations): + raise ValueError("unreviewed Next Day SOC DUID") + + first = observations[0] + if any( + item.source_artifact_id != first.source_artifact_id + or item.ingestion_version != first.ingestion_version + or item.correction_version != first.correction_version + or item.report_timestamp != first.report_timestamp + or item.downloaded_at != first.downloaded_at + for item in observations + ): + raise ValueError("mixed Next Day SOC source identity") + seen: set[tuple[Any, ...]] = set() + for item in observations: + key = ( + item.source_artifact_id, + item.duid, + item.interval_start, + item.intervention, + item.run_number, + item.ingestion_version, + item.correction_version, + ) + if key in seen: + raise ValueError("duplicate Next Day SOC natural key") + seen.add(key) + return asset_by_duid + + +def _select_effective_candidates( + observations: tuple[NextDaySocObservation, ...], +) -> tuple[NextDaySocObservation, ...]: + grouped: dict[tuple[str, datetime], list[NextDaySocObservation]] = {} + for item in observations: + grouped.setdefault((item.duid, item.interval_start), []).append(item) + selected: list[NextDaySocObservation] = [] + for key, candidates in grouped.items(): + winner_precedence = max(_precedence(item) for item in candidates) + winners = [item for item in candidates if _precedence(item) == winner_precedence] + winner_payloads = {_raw_parameters(item) for item in winners} + if len(winner_payloads) != 1: + raise NextDaySocConflictError( + f"conflicting effective Next Day SOC candidates for {key[0]}" + ) + selected.append(winners[0]) + return tuple(sorted(selected, key=lambda item: (item.duid, item.interval_start))) + + +class PostgreSQLNextDaySocIngestor: + """Persist one authoritative Next Day SOC artifact transactionally.""" + + def __init__(self, connection: _Connection): + self._connection = connection + + def ingest( + self, + observations: Iterable[NextDaySocObservation], + assets: Iterable[BatteryAsset], + ) -> NextDaySocIngestionResult: + try: + materialized = tuple(observations) + materialized_assets = tuple(assets) + asset_by_duid = _validate_inputs(materialized, materialized_assets) + effective_candidates = _select_effective_candidates(materialized) + first = materialized[0] + + with _managed_cursor(self._connection) as cursor: + cursor.execute(_ARTIFACT_SELECT_SQL, (first.source_artifact_id,)) + artifact = cursor.fetchone() + if artifact != ("nextday_soc",): + raise ValueError("artifact is not registered as nextday_soc") + + cursor.execute( + _RAW_SELECT_SQL, + ( + first.source_artifact_id, + first.ingestion_version, + first.correction_version, + ), + ) + stored_raw = { + ( + row[0], + row[1], + row[2], + row[4], + row[5], + row[10], + row[11], + ): tuple(row) + for row in cursor.fetchall() + } + raw_missing: list[tuple[Any, ...]] = [] + raw_replayed = 0 + for observation in materialized: + parameters = _raw_parameters(observation) + key = ( + parameters[0], + parameters[1], + parameters[2], + parameters[4], + parameters[5], + parameters[10], + parameters[11], + ) + stored = stored_raw.get(key) + if stored is None: + raw_missing.append(parameters) + elif stored == parameters: + raw_replayed += 1 + else: + raise NextDaySocConflictError( + "stored raw Next Day SOC conflicts with source" + ) + + duids = sorted({item.duid for item in effective_candidates}) + intervals = [item.interval_start for item in effective_candidates] + cursor.execute( + _EFFECTIVE_SELECT_SQL, + (duids, min(intervals), max(intervals)), + ) + stored_effective = { + (row[0], row[1]): tuple(row) for row in cursor.fetchall() + } + + effective_apply: list[tuple[Any, ...]] = [] + effective_replayed = 0 + percentage_count = 0 + for observation in effective_candidates: + parameters = _effective_parameters( + observation, + asset_by_duid[observation.duid], + ) + if parameters[2] is not None: + percentage_count += 1 + stored = stored_effective.get((parameters[0], parameters[1])) + if stored is None: + effective_apply.append(parameters) + continue + if stored[19] is None: + effective_apply.append(parameters) + continue + candidate_precedence = _effective_precedence(parameters) + stored_precedence = _effective_precedence(stored) + if candidate_precedence > stored_precedence: + effective_apply.append(parameters) + elif candidate_precedence == stored_precedence: + if not _effective_matches(stored, parameters): + raise NextDaySocConflictError( + "stored effective Next Day SOC conflicts at equal precedence" + ) + effective_replayed += 1 + else: + effective_replayed += 1 + + if raw_missing: + cursor.executemany(_RAW_INSERT_SQL, raw_missing) + raw_inserted = cursor.rowcount + if raw_inserted != len(raw_missing): + raise NextDaySocConflictError( + "raw Next Day SOC bulk write was incomplete" + ) + else: + raw_inserted = 0 + if effective_apply: + cursor.executemany(_EFFECTIVE_UPSERT_SQL, effective_apply) + effective_applied = cursor.rowcount + if not 0 <= effective_applied <= len(effective_apply): + raise NextDaySocConflictError( + "invalid effective Next Day SOC bulk result" + ) + if effective_applied < len(effective_apply): + cursor.execute( + _EFFECTIVE_SELECT_SQL, + (duids, min(intervals), max(intervals)), + ) + final_effective = { + (row[0], row[1]): tuple(row) for row in cursor.fetchall() + } + for parameters in effective_apply: + stored = final_effective.get((parameters[0], parameters[1])) + if stored is None or stored[19] is None: + raise NextDaySocConflictError( + "guarded effective Next Day SOC write is missing" + ) + candidate_precedence = _effective_precedence(parameters) + stored_precedence = _effective_precedence(stored) + if stored_precedence < candidate_precedence or ( + stored_precedence == candidate_precedence + and not _effective_matches(stored, parameters) + ): + raise NextDaySocConflictError( + "guarded effective Next Day SOC write conflicts" + ) + effective_replayed += len(effective_apply) - effective_applied + else: + effective_applied = 0 + + self._connection.commit() + return NextDaySocIngestionResult( + source_rows=len(materialized), + raw_inserted=raw_inserted, + raw_replayed=raw_replayed, + effective_candidates=len(effective_candidates), + effective_applied=effective_applied, + effective_replayed=effective_replayed, + source_null_count=sum(item.soc_mwh is None for item in materialized), + percentage_count=percentage_count, + ) + except Exception: + try: + self._connection.rollback() + except Exception: + pass + raise + + +__all__ = [ + "NextDaySocConflictError", + "NextDaySocIngestionResult", + "PostgreSQLNextDaySocIngestor", +] diff --git a/app/backend/tests/test_nextday_soc_ingestion.py b/app/backend/tests/test_nextday_soc_ingestion.py new file mode 100644 index 0000000..e7a766a --- /dev/null +++ b/app/backend/tests/test_nextday_soc_ingestion.py @@ -0,0 +1,375 @@ +"""Tests for atomic authoritative individual Next Day SOC ingestion.""" + +from dataclasses import replace +from datetime import datetime, timedelta, timezone +from pathlib import Path +import unittest + +from batterywatch_api.battery_assets import load_battery_assets +from batterywatch_api.nextday_soc import parse_nextday_unit_solution_soc +from batterywatch_api.nextday_soc_ingestion import ( + NextDaySocConflictError, + NextDaySocIngestionResult, + PostgreSQLNextDaySocIngestor, +) + +UTC = timezone.utc +BACKEND = Path(__file__).resolve().parents[1] +FIXTURE = BACKEND / "tests" / "fixtures" / "historical" / "nextday-unit-solution-soc-20260829-reduced.csv" +ASSETS = BACKEND.parent / "config" / "battery_assets.json" +ARTIFACT_SHA = "d7a2abdd2947ed4b222166b9f60e3a8052838190027dd9ce03cb291ba2d29bc4" +DOWNLOADED_AT = datetime(2026, 8, 30, 0, 0, tzinfo=UTC) + + +class FakeCursor: + def __init__(self, connection) -> None: + self.connection = connection + self.query_kind = "" + self.rowcount = -1 + + def execute(self, statement, parameters) -> None: + self.connection.operation_count += 1 + if self.connection.operation_count == self.connection.fail_on_operation: + raise self.connection.failure + self.connection.executions.append((statement, tuple(parameters))) + if "FROM historical_source_artifacts" in statement: + self.query_kind = "artifact" + elif "FROM raw_nextday_soc_observations" in statement: + self.query_kind = "raw" + elif "FROM generator_soc_5m" in statement: + self.query_kind = "effective" + else: + self.query_kind = "" + + def executemany(self, statement, parameters) -> None: + self.connection.operation_count += 1 + if self.connection.operation_count == self.connection.fail_on_operation: + raise self.connection.failure + materialized = tuple(tuple(item) for item in parameters) + self.connection.bulk_executions.append((statement, materialized)) + if self.connection.bulk_rowcounts: + self.rowcount = self.connection.bulk_rowcounts.pop(0) + else: + self.rowcount = len(materialized) + + def fetchone(self): + if self.query_kind == "artifact": + return self.connection.artifact_row + return None + + def fetchall(self): + if self.query_kind == "raw": + return list(self.connection.raw_rows) + if self.query_kind == "effective": + if self.connection.effective_fetch_results: + return list(self.connection.effective_fetch_results.pop(0)) + return list(self.connection.effective_rows) + return [] + + def close(self) -> None: + self.connection.closed_cursors += 1 + + +class FakeConnection: + def __init__( + self, + *, + artifact_row=("nextday_soc",), + raw_rows=(), + effective_rows=(), + fail_on_operation=None, + failure=None, + rollback_failure=None, + bulk_rowcounts=(), + effective_fetch_results=(), + ) -> None: + self.artifact_row = artifact_row + self.raw_rows = tuple(raw_rows) + self.effective_rows = tuple(effective_rows) + self.executions = [] + self.bulk_executions = [] + self.fail_on_operation = fail_on_operation + self.failure = failure + self.rollback_failure = rollback_failure + self.operation_count = 0 + self.bulk_rowcounts = list(bulk_rowcounts) + self.effective_fetch_results = list(effective_fetch_results) + self.cursor_calls = self.closed_cursors = 0 + self.commits = self.rollbacks = 0 + + def cursor(self): + self.cursor_calls += 1 + return FakeCursor(self) + + def commit(self) -> None: + self.commits += 1 + + def rollback(self) -> None: + self.rollbacks += 1 + if self.rollback_failure is not None: + raise self.rollback_failure + + +def observations(): + return parse_nextday_unit_solution_soc( + FIXTURE.read_text(encoding="utf-8"), + duids=frozenset(("ADPBA1", "KEPBG1")), + source_artifact_id=ARTIFACT_SHA, + downloaded_at=DOWNLOADED_AT, + ingestion_version=8, + correction_version=2, + ) + + +def assets(): + return tuple( + asset + for asset in load_battery_assets(ASSETS) + if asset.duid in {"ADPBA1", "KEPBG1"} + ) + + +class PostgreSQLNextDaySocIngestorTests(unittest.TestCase): + def test_ingests_real_fixture_raw_and_capacity_qualified_effective_rows(self) -> None: + connection = FakeConnection() + + result = PostgreSQLNextDaySocIngestor(connection).ingest( + observations(), + assets(), + ) + + self.assertEqual( + result, + NextDaySocIngestionResult( + source_rows=3, + raw_inserted=3, + raw_replayed=0, + effective_candidates=3, + effective_applied=3, + effective_replayed=0, + source_null_count=1, + percentage_count=2, + ), + ) + self.assertEqual( + (connection.cursor_calls, connection.closed_cursors), + (1, 1), + ) + self.assertEqual((connection.commits, connection.rollbacks), (1, 0)) + self.assertEqual(len(connection.executions), 3) + self.assertEqual(len(connection.bulk_executions), 2) + + raw_sql, raw_parameters = connection.bulk_executions[0] + self.assertIn("INSERT INTO raw_nextday_soc_observations", raw_sql) + self.assertEqual(len(raw_parameters), 3) + self.assertEqual({item[1] for item in raw_parameters}, {"ADPBA1", "KEPBG1"}) + self.assertIn(None, {item[3] for item in raw_parameters}) + + effective_sql, effective_parameters = connection.bulk_executions[1] + self.assertIn("INSERT INTO generator_soc_5m", effective_sql) + self.assertEqual(len(effective_parameters), 3) + adp_rows = [item for item in effective_parameters if item[0] == "ADPBA1"] + kep_row = next(item for item in effective_parameters if item[0] == "KEPBG1") + self.assertEqual(len(adp_rows), 2) + for row in adp_rows: + self.assertAlmostEqual(row[2], 100.0 * row[8] / 12.6) + self.assertEqual(row[9], 12.6) + self.assertEqual(row[12], "aemo-generation-information-2025") + self.assertEqual(row[19], ARTIFACT_SHA) + self.assertIsNone(kep_row[2]) + self.assertIsNone(kep_row[8]) + self.assertIsNone(kep_row[9]) + self.assertEqual(kep_row[7], ["authoritative_soc_missing"]) + + def test_exact_raw_and_effective_replay_is_a_noop(self) -> None: + source = observations() + reviewed_assets = assets() + primer = FakeConnection() + PostgreSQLNextDaySocIngestor(primer).ingest(source, reviewed_assets) + raw_rows = primer.bulk_executions[0][1] + effective_rows = primer.bulk_executions[1][1] + connection = FakeConnection(raw_rows=raw_rows, effective_rows=effective_rows) + + result = PostgreSQLNextDaySocIngestor(connection).ingest( + source, + reviewed_assets, + ) + + self.assertEqual(result.raw_inserted, 0) + self.assertEqual(result.raw_replayed, 3) + self.assertEqual(result.effective_applied, 0) + self.assertEqual(result.effective_replayed, 3) + self.assertEqual(connection.bulk_executions, []) + self.assertEqual((connection.commits, connection.rollbacks), (1, 0)) + + def test_conflicting_raw_replay_rolls_back(self) -> None: + source = observations() + primer = FakeConnection() + PostgreSQLNextDaySocIngestor(primer).ingest(source, assets()) + stored = list(primer.bulk_executions[0][1]) + stored[0] = stored[0][:3] + (999.0,) + stored[0][4:] + connection = FakeConnection(raw_rows=stored) + + with self.assertRaises(NextDaySocConflictError): + PostgreSQLNextDaySocIngestor(connection).ingest(source, assets()) + + self.assertEqual(connection.bulk_executions, []) + self.assertEqual((connection.commits, connection.rollbacks), (0, 1)) + + def test_intervention_wins_and_equal_precedence_conflict_fails_closed(self) -> None: + base = observations()[0] + normal = replace(base, soc_mwh=1.0, intervention=0) + intervened = replace(base, soc_mwh=2.0, intervention=1) + connection = FakeConnection() + + result = PostgreSQLNextDaySocIngestor(connection).ingest( + (normal, intervened), + assets(), + ) + + self.assertEqual(result.effective_candidates, 1) + self.assertEqual(connection.bulk_executions[1][1][0][8], 2.0) + + conflict = replace(normal, run_number=2, soc_mwh=3.0) + invalid_connection = FakeConnection() + with self.assertRaises(NextDaySocConflictError): + PostgreSQLNextDaySocIngestor(invalid_connection).ingest( + (normal, conflict), + assets(), + ) + self.assertEqual(invalid_connection.executions, []) + + def test_future_capacity_and_source_above_capacity_preserve_mwh_without_percent(self) -> None: + source = observations()[0] + asset = next(item for item in assets() if item.duid == source.duid) + cases = ( + ( + source, + replace( + asset, + source_timestamp=source.interval_start + timedelta(seconds=1), + ), + "capacity_not_effective", + ), + ( + replace(source, soc_mwh=asset.storage_capacity_mwh + 1.0), + asset, + "soc_exceeds_capacity", + ), + ) + for observation, reviewed_asset, flag in cases: + with self.subTest(flag=flag): + connection = FakeConnection() + PostgreSQLNextDaySocIngestor(connection).ingest( + (observation,), + (reviewed_asset,), + ) + row = connection.bulk_executions[1][1][0] + self.assertIsNone(row[2]) + self.assertEqual(row[8], observation.soc_mwh) + self.assertIsNone(row[9]) + self.assertEqual(row[7], [flag]) + + def test_stale_effective_revision_cannot_regress(self) -> None: + source = (observations()[0],) + primer = FakeConnection() + PostgreSQLNextDaySocIngestor(primer).ingest(source, assets()) + stored = list(primer.bulk_executions[1][1][0]) + stored[6] += 1 + connection = FakeConnection(effective_rows=(tuple(stored),)) + + result = PostgreSQLNextDaySocIngestor(connection).ingest(source, assets()) + + self.assertEqual(result.effective_applied, 0) + self.assertEqual(result.effective_replayed, 1) + self.assertEqual(len(connection.bulk_executions), 1) + self.assertIn("raw_nextday_soc_observations", connection.bulk_executions[0][0]) + + def test_authoritative_row_replaces_legacy_row_with_null_revision_metadata(self) -> None: + source = (observations()[0],) + observation = source[0] + legacy_row = ( + observation.duid, + observation.interval_start, + 50.0, + "legacy-fixture", + observation.last_changed, + 0, + 0, + ["legacy"], + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + ) + connection = FakeConnection(effective_rows=(legacy_row,)) + + result = PostgreSQLNextDaySocIngestor(connection).ingest(source, assets()) + + self.assertEqual(result.effective_applied, 1) + effective_sql = connection.bulk_executions[1][0] + self.assertIn("generator_soc_5m.source_artifact_sha256 IS NULL", effective_sql) + + def test_post_prefetch_equal_precedence_conflict_fails_closed(self) -> None: + source = (observations()[0],) + primer = FakeConnection() + PostgreSQLNextDaySocIngestor(primer).ingest(source, assets()) + conflicting = list(primer.bulk_executions[1][1][0]) + conflicting[8] = 999.0 + connection = FakeConnection( + bulk_rowcounts=(1, 0), + effective_fetch_results=((), (tuple(conflicting),)), + ) + + with self.assertRaises(NextDaySocConflictError): + PostgreSQLNextDaySocIngestor(connection).ingest(source, assets()) + + self.assertEqual((connection.commits, connection.rollbacks), (0, 1)) + + def test_invalid_inputs_fail_before_sql(self) -> None: + source = observations() + cases = ( + (source + (source[0],), assets()), + ((replace(source[0], source_artifact_id="b" * 64), source[1]), assets()), + ((replace(source[0], duid="UNKNOWN1"),), assets()), + ) + for invalid_source, reviewed_assets in cases: + with self.subTest(duids=[item.duid for item in invalid_source]): + connection = FakeConnection() + with self.assertRaises(ValueError): + PostgreSQLNextDaySocIngestor(connection).ingest( + invalid_source, + reviewed_assets, + ) + self.assertEqual(connection.executions, []) + + def test_sql_failure_rolls_back_and_preserves_original_error(self) -> None: + failure = RuntimeError("raw bulk write failed") + connection = FakeConnection( + fail_on_operation=4, + failure=failure, + rollback_failure=RuntimeError("rollback also failed"), + ) + same_error = False + + try: + PostgreSQLNextDaySocIngestor(connection).ingest(observations(), assets()) + except RuntimeError as raised: + same_error = raised is failure + else: + self.fail("expected raw bulk write failure") + + self.assertTrue(same_error) + self.assertEqual((connection.commits, connection.rollbacks), (0, 1)) + + +if __name__ == "__main__": + unittest.main() From 5001f5474f9d2d8b3a8cdb0aeb938b727cf932ec Mon Sep 17 00:00:00 2001 From: djurcola-agent Date: Sun, 30 Aug 2026 23:33:55 +1000 Subject: [PATCH 23/30] feat: plan bounded Next Day monthly archives --- app/backend/batterywatch_api/nemweb_http.py | 15 +- .../batterywatch_api/nextday_archives.py | 263 ++++++++++++++++++ app/backend/tests/test_nemweb_http.py | 38 +++ app/backend/tests/test_nextday_archives.py | 130 +++++++++ 4 files changed, 444 insertions(+), 2 deletions(-) create mode 100644 app/backend/batterywatch_api/nextday_archives.py create mode 100644 app/backend/tests/test_nextday_archives.py diff --git a/app/backend/batterywatch_api/nemweb_http.py b/app/backend/batterywatch_api/nemweb_http.py index e327cc1..4691265 100644 --- a/app/backend/batterywatch_api/nemweb_http.py +++ b/app/backend/batterywatch_api/nemweb_http.py @@ -15,9 +15,11 @@ _INDEX_URLS = frozenset(( "https://www.nemweb.com.au/REPORTS/CURRENT/Dispatch_SCADA/", "https://www.nemweb.com.au/REPORTS/CURRENT/DispatchIS_Reports/", + "https://www.nemweb.com.au/REPORTS/ARCHIVE/Next_Day_Dispatch/", )) _CURRENT_RESOURCE_MAX_BYTES = 16 * 1024 * 1024 _ARCHIVE_RESOURCE_MAX_BYTES = 128 * 1024 * 1024 +_MONTHLY_ARCHIVE_RESOURCE_MAX_BYTES = 256 * 1024 * 1024 _CURRENT_ARTIFACT_PATH_RE = re.compile( r"(?:" r"/REPORTS/CURRENT/Dispatch_SCADA/" @@ -36,6 +38,10 @@ r"PUBLIC_DISPATCHIS_[0-9]{8}\.zip" r")" ) +_MONTHLY_ARCHIVE_ARTIFACT_PATH_RE = re.compile( + r"/REPORTS/ARCHIVE/Next_Day_Dispatch/" + r"PUBLIC_NEXT_DAY_DISPATCH_[0-9]{6}01\.zip" +) class NemwebHttpError(ValueError): @@ -99,14 +105,19 @@ def fetch_nemweb_resource( raise NemwebHttpError("invalid NEMWeb request") from error current_artifact = _CURRENT_ARTIFACT_PATH_RE.fullmatch(parts.path) is not None archive_artifact = _ARCHIVE_ARTIFACT_PATH_RE.fullmatch(parts.path) is not None + monthly_archive_artifact = ( + _MONTHLY_ARCHIVE_ARTIFACT_PATH_RE.fullmatch(parts.path) is not None + ) valid_url = url in _INDEX_URLS or ( parts.scheme == "https" and parts.netloc == "www.nemweb.com.au" and not parts.query and not parts.fragment - and (current_artifact or archive_artifact) + and (current_artifact or archive_artifact or monthly_archive_artifact) ) - if archive_artifact: + if monthly_archive_artifact: + resource_max_bytes = _MONTHLY_ARCHIVE_RESOURCE_MAX_BYTES + elif archive_artifact: resource_max_bytes = _ARCHIVE_RESOURCE_MAX_BYTES valid_limit = ( type(max_bytes) is int and 0 < max_bytes <= resource_max_bytes diff --git a/app/backend/batterywatch_api/nextday_archives.py b/app/backend/batterywatch_api/nextday_archives.py new file mode 100644 index 0000000..1beb6c9 --- /dev/null +++ b/app/backend/batterywatch_api/nextday_archives.py @@ -0,0 +1,263 @@ +"""Bounded discovery and planning for Next Day Dispatch monthly archives.""" + +from __future__ import annotations + +from dataclasses import dataclass +from collections.abc import Iterable +from datetime import date, datetime, timedelta, timezone +from html.parser import HTMLParser +import re +from typing import Final + +NEXTDAY_ARCHIVE_INDEX_URL: Final = ( + "https://www.nemweb.com.au/REPORTS/ARCHIVE/Next_Day_Dispatch/" +) +NEXTDAY_MONTHLY_ARCHIVE_MAX_BYTES: Final = 256 * 1024 * 1024 +NEXTDAY_ARCHIVE_INDEX_MAX_BYTES: Final = 64 * 1024 +MAX_NEXTDAY_ARCHIVE_RANGE_DAYS: Final = 366 +MAX_NEXTDAY_ARCHIVE_MONTHS: Final = 13 +_NEM_TIMEZONE: Final = timezone(timedelta(hours=10)) +_MONTHLY_NAME_RE: Final = re.compile( + r"PUBLIC_NEXT_DAY_DISPATCH_([0-9]{4})([0-9]{2})01\.zip" +) +_LISTING_PREFIX_RE: Final = re.compile( + r"\s*([A-Z][a-z]+, [A-Z][a-z]+ [0-9]{1,2}, [0-9]{4} " + r"[0-9]{2}:[0-9]{2} [AP]M)\s+([0-9]+)\s*$" +) + + +class NextDayArchiveError(ValueError): + """Raised when the monthly archive listing or request is invalid.""" + + +@dataclass(frozen=True, slots=True) +class NextDayMonthlyArchiveRef: + report_month: date + filename: str + url: str + size_bytes: int + listing_timestamp: datetime + + +@dataclass(frozen=True, slots=True) +class NextDayMonthlyArchivePlan: + start: datetime + end: datetime + items: tuple[NextDayMonthlyArchiveRef, ...] + missing_months: tuple[date, ...] + + +class _ListingCollector(HTMLParser): + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self._text: list[str] = [] + self._href: str | None = None + self._prefix = "" + self._anchor_text: list[str] = [] + self.links: list[tuple[str, str, str]] = [] + + def handle_starttag( + self, + tag: str, + attrs: list[tuple[str, str | None]], + ) -> None: + lowered = tag.lower() + if lowered == "br": + self._text.clear() + return + if lowered != "a" or self._href is not None: + return + hrefs = [value for name, value in attrs if name.lower() == "href"] + self._href = hrefs[0] if len(hrefs) == 1 else "" + self._prefix = "".join(self._text) + self._anchor_text = [] + + def handle_data(self, data: str) -> None: + if self._href is None: + self._text.append(data) + else: + self._anchor_text.append(data) + + def handle_endtag(self, tag: str) -> None: + if tag.lower() != "a" or self._href is None: + return + self.links.append( + (self._href, "".join(self._anchor_text), self._prefix) + ) + self._href = None + self._prefix = "" + self._anchor_text = [] + self._text.clear() + + +def _listing_timestamp(value: str) -> datetime: + try: + parsed = datetime.strptime(value, "%A, %B %d, %Y %I:%M %p") + except (OverflowError, ValueError): + raise NextDayArchiveError("invalid Next Day archive listing") from None + weekday = value.split(",", 1)[0] + if parsed.strftime("%A") != weekday: + raise NextDayArchiveError("invalid Next Day archive listing") + return parsed.replace(tzinfo=_NEM_TIMEZONE).astimezone(timezone.utc) + + +def discover_nextday_monthly_archives( + index_html: str, + *, + index_url: str, +) -> tuple[NextDayMonthlyArchiveRef, ...]: + """Return canonical available monthly references in month order.""" + + if index_url != NEXTDAY_ARCHIVE_INDEX_URL or type(index_html) is not str: + raise NextDayArchiveError("invalid Next Day archive listing") + try: + encoded_size = len(index_html.encode("utf-8")) + except UnicodeEncodeError: + raise NextDayArchiveError("invalid Next Day archive listing") from None + if not 0 < encoded_size <= NEXTDAY_ARCHIVE_INDEX_MAX_BYTES: + raise NextDayArchiveError("invalid Next Day archive listing") + + collector = _ListingCollector() + try: + collector.feed(index_html) + collector.close() + except (TypeError, ValueError): + raise NextDayArchiveError("invalid Next Day archive listing") from None + + references: dict[date, NextDayMonthlyArchiveRef] = {} + for href, anchor_text, prefix in collector.links: + if "PUBLIC_NEXT_DAY_DISPATCH_" not in href and ( + "PUBLIC_NEXT_DAY_DISPATCH_" not in anchor_text + ): + continue + filename_match = _MONTHLY_NAME_RE.fullmatch(anchor_text) + if filename_match is None: + raise NextDayArchiveError("invalid Next Day archive listing") + filename = anchor_text + expected_href = f"/Reports/ARCHIVE/Next_Day_Dispatch/{filename}" + if href != expected_href: + raise NextDayArchiveError("invalid Next Day archive listing") + prefix_match = _LISTING_PREFIX_RE.fullmatch(prefix) + if prefix_match is None: + raise NextDayArchiveError("invalid Next Day archive listing") + try: + report_month = date( + int(filename_match.group(1)), + int(filename_match.group(2)), + 1, + ) + size_bytes = int(prefix_match.group(2)) + except (OverflowError, ValueError): + raise NextDayArchiveError("invalid Next Day archive listing") from None + if not 0 < size_bytes <= NEXTDAY_MONTHLY_ARCHIVE_MAX_BYTES: + raise NextDayArchiveError("invalid Next Day archive listing") + reference = NextDayMonthlyArchiveRef( + report_month, + filename, + f"{NEXTDAY_ARCHIVE_INDEX_URL}{filename}", + size_bytes, + _listing_timestamp(prefix_match.group(1)), + ) + existing = references.get(report_month) + if existing is not None and existing != reference: + raise NextDayArchiveError("conflicting Next Day archive listing") + if existing is not None: + raise NextDayArchiveError("duplicate Next Day archive listing") + references[report_month] = reference + + if not references: + raise NextDayArchiveError("empty Next Day archive listing") + return tuple(references[month] for month in sorted(references)) + + +def _strict_utc(value: datetime, name: str) -> datetime: + if type(value) is not datetime or value.tzinfo is None: + raise NextDayArchiveError(f"invalid Next Day archive {name}") + try: + offset = value.utcoffset() + except (OverflowError, TypeError, ValueError): + raise NextDayArchiveError(f"invalid Next Day archive {name}") from None + if offset != timedelta(0): + raise NextDayArchiveError(f"Next Day archive {name} must use UTC") + return value.astimezone(timezone.utc) + + +def _next_month(value: date) -> date: + return ( + date(value.year + 1, 1, 1) + if value.month == 12 + else date(value.year, value.month + 1, 1) + ) + + +def _intersecting_months(start: datetime, end: datetime) -> tuple[date, ...]: + first_local = start.astimezone(_NEM_TIMEZONE) + last_local = (end - timedelta(microseconds=1)).astimezone(_NEM_TIMEZONE) + current = date(first_local.year, first_local.month, 1) + last = date(last_local.year, last_local.month, 1) + months: list[date] = [] + while current <= last: + months.append(current) + current = _next_month(current) + return tuple(months) + + +def plan_nextday_monthly_archives( + start: datetime, + end: datetime, + references: Iterable[NextDayMonthlyArchiveRef], + *, + max_months: int = MAX_NEXTDAY_ARCHIVE_MONTHS, +) -> NextDayMonthlyArchivePlan: + """Plan bounded available monthly artifacts for a strict UTC range.""" + + start_utc = _strict_utc(start, "start") + end_utc = _strict_utc(end, "end") + if end_utc <= start_utc: + raise NextDayArchiveError("Next Day archive range must be increasing") + if end_utc - start_utc > timedelta(days=MAX_NEXTDAY_ARCHIVE_RANGE_DAYS): + raise NextDayArchiveError("Next Day archive range exceeds maximum") + if type(max_months) is not int or not 0 < max_months <= MAX_NEXTDAY_ARCHIVE_MONTHS: + raise NextDayArchiveError("invalid Next Day archive month limit") + try: + available = tuple(references) + except (TypeError, ValueError): + raise NextDayArchiveError("invalid Next Day archive references") from None + if any(type(item) is not NextDayMonthlyArchiveRef for item in available): + raise NextDayArchiveError("invalid Next Day archive references") + by_month: dict[date, NextDayMonthlyArchiveRef] = {} + for item in available: + if ( + item.report_month.day != 1 + or item.filename != f"PUBLIC_NEXT_DAY_DISPATCH_{item.report_month:%Y%m}01.zip" + or item.url != f"{NEXTDAY_ARCHIVE_INDEX_URL}{item.filename}" + or not 0 < item.size_bytes <= NEXTDAY_MONTHLY_ARCHIVE_MAX_BYTES + or item.listing_timestamp.tzinfo is None + or item.listing_timestamp.utcoffset() != timedelta(0) + or item.report_month in by_month + ): + raise NextDayArchiveError("invalid Next Day archive references") + by_month[item.report_month] = item + months = _intersecting_months(start_utc, end_utc) + if not months or len(months) > max_months: + raise NextDayArchiveError("Next Day archive plan exceeds month limit") + return NextDayMonthlyArchivePlan( + start_utc, + end_utc, + tuple(by_month[month] for month in months if month in by_month), + tuple(month for month in months if month not in by_month), + ) + + +__all__ = [ + "MAX_NEXTDAY_ARCHIVE_MONTHS", + "MAX_NEXTDAY_ARCHIVE_RANGE_DAYS", + "NEXTDAY_ARCHIVE_INDEX_MAX_BYTES", + "NEXTDAY_ARCHIVE_INDEX_URL", + "NEXTDAY_MONTHLY_ARCHIVE_MAX_BYTES", + "NextDayArchiveError", + "NextDayMonthlyArchivePlan", + "NextDayMonthlyArchiveRef", + "discover_nextday_monthly_archives", + "plan_nextday_monthly_archives", +] diff --git a/app/backend/tests/test_nemweb_http.py b/app/backend/tests/test_nemweb_http.py index 1d88dc6..b67a257 100644 --- a/app/backend/tests/test_nemweb_http.py +++ b/app/backend/tests/test_nemweb_http.py @@ -26,6 +26,12 @@ "https://www.nemweb.com.au/REPORTS/ARCHIVE/DispatchIS_Reports/" "PUBLIC_DISPATCHIS_20260829.zip" ) +NEXTDAY_INDEX_URL = ( + "https://www.nemweb.com.au/REPORTS/ARCHIVE/Next_Day_Dispatch/" +) +NEXTDAY_MONTHLY_ARCHIVE_URL = ( + NEXTDAY_INDEX_URL + "PUBLIC_NEXT_DAY_DISPATCH_20250701.zip" +) class FakeResponse: @@ -115,6 +121,38 @@ def test_fetches_canonical_daily_archives_with_outer_bound(self) -> None: self.assertEqual((result.requested_url, result.body), (url, response._body)) self.assertEqual(response.read_sizes, [limit + 1]) + def test_fetches_nextday_index_and_monthly_archive_with_dedicated_bound(self) -> None: + index_opener = FakeOpener( + FakeResponse(b"next day index", url=NEXTDAY_INDEX_URL) + ) + index = http.fetch_nemweb_resource( + NEXTDAY_INDEX_URL, + max_bytes=64 * 1024, + opener=index_opener, + ) + self.assertEqual(index.body, b"next day index") + + limit = 256 * 1024 * 1024 + response = FakeResponse( + b"PK monthly Next Day archive", + url=NEXTDAY_MONTHLY_ARCHIVE_URL, + ) + opener = FakeOpener(response) + result = http.fetch_nemweb_resource( + NEXTDAY_MONTHLY_ARCHIVE_URL, + max_bytes=limit, + opener=opener, + ) + self.assertEqual(result.body, response._body) + self.assertEqual(response.read_sizes, [limit + 1]) + + with self.assertRaises(http.NemwebHttpError): + http.fetch_nemweb_resource( + NEXTDAY_MONTHLY_ARCHIVE_URL, + max_bytes=limit + 1, + opener=opener, + ) + def test_fetches_official_dispatch_price_index_and_artifact(self) -> None: for url, body in ( (PRICE_INDEX_URL, b"price index"), diff --git a/app/backend/tests/test_nextday_archives.py b/app/backend/tests/test_nextday_archives.py new file mode 100644 index 0000000..05ffe3f --- /dev/null +++ b/app/backend/tests/test_nextday_archives.py @@ -0,0 +1,130 @@ +"""Tests for bounded Next Day monthly archive discovery and planning.""" + +from datetime import date, datetime, timedelta, timezone +import unittest + +from batterywatch_api.nextday_archives import ( + NEXTDAY_ARCHIVE_INDEX_URL, + NextDayArchiveError, + discover_nextday_monthly_archives, + plan_nextday_monthly_archives, +) + +UTC = timezone.utc +NEM_TIME = timezone(timedelta(hours=10)) + +INDEX_HTML = """
+[To Parent Directory]


+Monday, September 1, 2025 01:10 AM    213422082 PUBLIC_NEXT_DAY_DISPATCH_20250701.zip

+Wednesday, October 1, 2025 01:03 AM    217741574 PUBLIC_NEXT_DAY_DISPATCH_20250801.zip

+
""" + + +class NextDayArchiveDiscoveryTests(unittest.TestCase): + def test_discovers_canonical_months_with_listing_metadata(self) -> None: + references = discover_nextday_monthly_archives( + INDEX_HTML, + index_url=NEXTDAY_ARCHIVE_INDEX_URL, + ) + + self.assertEqual( + tuple( + ( + item.report_month, + item.filename, + item.url, + item.size_bytes, + item.listing_timestamp, + ) + for item in references + ), + ( + ( + date(2025, 7, 1), + "PUBLIC_NEXT_DAY_DISPATCH_20250701.zip", + NEXTDAY_ARCHIVE_INDEX_URL + + "PUBLIC_NEXT_DAY_DISPATCH_20250701.zip", + 213_422_082, + datetime(2025, 9, 1, 1, 10, tzinfo=NEM_TIME).astimezone(UTC), + ), + ( + date(2025, 8, 1), + "PUBLIC_NEXT_DAY_DISPATCH_20250801.zip", + NEXTDAY_ARCHIVE_INDEX_URL + + "PUBLIC_NEXT_DAY_DISPATCH_20250801.zip", + 217_741_574, + datetime(2025, 10, 1, 1, 3, tzinfo=NEM_TIME).astimezone(UTC), + ), + ), + ) + + def test_plans_intersecting_nem_months_and_reports_missing_archives(self) -> None: + references = discover_nextday_monthly_archives( + INDEX_HTML, + index_url=NEXTDAY_ARCHIVE_INDEX_URL, + ) + start = datetime(2025, 7, 15, 0, 0, tzinfo=UTC) + end = datetime(2025, 9, 15, 0, 0, tzinfo=UTC) + + plan = plan_nextday_monthly_archives(start, end, references) + + self.assertEqual((plan.start, plan.end), (start, end)) + self.assertEqual( + tuple(item.report_month for item in plan.items), + (date(2025, 7, 1), date(2025, 8, 1)), + ) + self.assertEqual(plan.missing_months, (date(2025, 9, 1),)) + + def test_rejects_malformed_duplicate_and_oversized_listing_rows(self) -> None: + malformed_cases = ( + INDEX_HTML.replace( + "/Reports/ARCHIVE/Next_Day_Dispatch/", + "/Reports/ARCHIVE/Other/", + 1, + ), + INDEX_HTML.replace("213422082", str(256 * 1024 * 1024 + 1)), + INDEX_HTML.replace("Monday, September", "Tuesday, September"), + INDEX_HTML.replace( + "", + "Monday, September 1, 2025 01:10 AM 213422082 " + '' + "PUBLIC_NEXT_DAY_DISPATCH_20250701.zip
", + ), + ) + for payload in malformed_cases: + with self.subTest(payload_size=len(payload)): + with self.assertRaises(NextDayArchiveError): + discover_nextday_monthly_archives( + payload, + index_url=NEXTDAY_ARCHIVE_INDEX_URL, + ) + + def test_rejects_non_utc_and_unbounded_plans(self) -> None: + references = discover_nextday_monthly_archives( + INDEX_HTML, + index_url=NEXTDAY_ARCHIVE_INDEX_URL, + ) + with self.assertRaises(NextDayArchiveError): + plan_nextday_monthly_archives( + datetime(2025, 7, 1, tzinfo=NEM_TIME), + datetime(2025, 8, 1, tzinfo=NEM_TIME), + references, + ) + with self.assertRaises(NextDayArchiveError): + plan_nextday_monthly_archives( + datetime(2025, 1, 1, tzinfo=UTC), + datetime(2026, 1, 3, tzinfo=UTC), + references, + ) + with self.assertRaises(NextDayArchiveError): + plan_nextday_monthly_archives( + datetime(2025, 7, 1, tzinfo=UTC), + datetime(2025, 9, 1, tzinfo=UTC), + references, + max_months=2, + ) + + +if __name__ == "__main__": + unittest.main() From a42fab36a579e85a4703467dbba47a0b5ac79b18 Mon Sep 17 00:00:00 2001 From: djurcola-agent Date: Sun, 30 Aug 2026 23:40:06 +1000 Subject: [PATCH 24/30] feat: validate bounded Next Day monthly archives --- .../nextday_monthly_extraction.py | 278 ++++++++++++++++++ .../tests/test_nextday_monthly_extraction.py | 130 ++++++++ 2 files changed, 408 insertions(+) create mode 100644 app/backend/batterywatch_api/nextday_monthly_extraction.py create mode 100644 app/backend/tests/test_nextday_monthly_extraction.py diff --git a/app/backend/batterywatch_api/nextday_monthly_extraction.py b/app/backend/batterywatch_api/nextday_monthly_extraction.py new file mode 100644 index 0000000..0e77c8d --- /dev/null +++ b/app/backend/batterywatch_api/nextday_monthly_extraction.py @@ -0,0 +1,278 @@ +"""Bounded one-day-at-a-time extraction of Next Day monthly archives.""" + +from __future__ import annotations + +from calendar import monthrange +from dataclasses import dataclass +from datetime import date +from hashlib import sha256 +from io import BytesIO +import re +import zlib +from zipfile import ( + ZIP_DEFLATED, + ZIP_STORED, + BadZipFile, + LargeZipFile, + ZipFile, + ZipInfo, +) + +from .nextday_archives import ( + NEXTDAY_ARCHIVE_INDEX_URL, + NEXTDAY_MONTHLY_ARCHIVE_MAX_BYTES, + NextDayArchiveError, + NextDayMonthlyArchiveRef, +) + +MAX_NEXTDAY_DAILY_ZIP_BYTES = 16 * 1024 * 1024 +MAX_NEXTDAY_DAILY_CSV_BYTES = 128 * 1024 * 1024 +MAX_NEXTDAY_MONTH_EXPANDED_ZIP_BYTES = 512 * 1024 * 1024 +MAX_NEXTDAY_COMPRESSION_RATIO = 100 +_MEMBER_NAME_RE = re.compile( + r"PUBLIC_NEXT_DAY_DISPATCH_([0-9]{4})([0-9]{2})([0-9]{2})_([0-9]{1,32})\.zip" +) +_FORMAT_ERRORS = ( + BadZipFile, + EOFError, + KeyError, + LargeZipFile, + NotImplementedError, + OSError, + OverflowError, + RuntimeError, + UnicodeError, + ValueError, + zlib.error, +) + + +@dataclass(frozen=True, slots=True) +class NextDayDailyMemberRef: + report_date: date + filename: str + publication_id: str + compressed_size: int + expanded_size: int + crc32: int + + +@dataclass(frozen=True, slots=True) +class NextDayMonthlyArchiveManifest: + reference: NextDayMonthlyArchiveRef + sha256: str + members: tuple[NextDayDailyMemberRef, ...] + + +@dataclass(frozen=True, slots=True) +class NextDayDailyArtifact: + member: NextDayDailyMemberRef + sha256: str + raw_zip_bytes: bytes + csv_filename: str + csv_bytes: bytes + + +def _reference_valid(reference: NextDayMonthlyArchiveRef, raw_bytes: bytes) -> bool: + return ( + type(reference) is NextDayMonthlyArchiveRef + and reference.report_month.day == 1 + and reference.filename + == f"PUBLIC_NEXT_DAY_DISPATCH_{reference.report_month:%Y%m}01.zip" + and reference.url == f"{NEXTDAY_ARCHIVE_INDEX_URL}{reference.filename}" + and type(reference.size_bytes) is int + and reference.size_bytes == len(raw_bytes) + and 0 < len(raw_bytes) <= NEXTDAY_MONTHLY_ARCHIVE_MAX_BYTES + ) + + +def _safe_member_sizes( + member: ZipInfo, + *, + max_compressed: int, + max_expanded: int, +) -> tuple[int, int]: + if ( + type(member.filename) is not str + or type(member.orig_filename) is not str + or member.filename != member.orig_filename + or member.is_dir() + or member.flag_bits & 1 + or member.compress_type not in (ZIP_STORED, ZIP_DEFLATED) + or member.comment + or member.extra + or type(member.compress_size) is not int + or type(member.file_size) is not int + or not 0 < member.compress_size <= max_compressed + or not 0 < member.file_size <= max_expanded + or member.file_size > member.compress_size * MAX_NEXTDAY_COMPRESSION_RATIO + ): + raise NextDayArchiveError("invalid Next Day archive member") + return member.compress_size, member.file_size + + +def _member_identity( + filename: str, + report_month: date, +) -> tuple[date, str]: + match = _MEMBER_NAME_RE.fullmatch(filename) + if match is None or "/" in filename or "\\" in filename or "\x00" in filename: + raise NextDayArchiveError("invalid Next Day daily member identity") + try: + report_date = date( + int(match.group(1)), + int(match.group(2)), + int(match.group(3)), + ) + except (OverflowError, ValueError): + raise NextDayArchiveError("invalid Next Day daily member identity") from None + if (report_date.year, report_date.month) != ( + report_month.year, + report_month.month, + ): + raise NextDayArchiveError("Next Day daily member is outside report month") + return report_date, match.group(4) + + +def validate_nextday_monthly_archive( + reference: NextDayMonthlyArchiveRef, + raw_bytes: bytes, +) -> NextDayMonthlyArchiveManifest: + """Validate complete outer structure before any daily publication.""" + + if type(raw_bytes) is not bytes or not _reference_valid(reference, raw_bytes): + raise NextDayArchiveError("invalid Next Day monthly archive") + try: + with ZipFile(BytesIO(raw_bytes)) as archive: + if archive.comment: + raise NextDayArchiveError("invalid Next Day monthly archive") + infos = archive.infolist() + expected_count = monthrange( + reference.report_month.year, + reference.report_month.month, + )[1] + if len(infos) != expected_count: + raise NextDayArchiveError("incomplete Next Day monthly archive") + if len({item.filename for item in infos}) != len(infos): + raise NextDayArchiveError("duplicate Next Day daily member") + members: list[NextDayDailyMemberRef] = [] + total_expanded = 0 + for info in infos: + compressed, expanded = _safe_member_sizes( + info, + max_compressed=MAX_NEXTDAY_DAILY_ZIP_BYTES, + max_expanded=MAX_NEXTDAY_DAILY_ZIP_BYTES, + ) + report_date, publication_id = _member_identity( + info.filename, + reference.report_month, + ) + total_expanded += expanded + if total_expanded > MAX_NEXTDAY_MONTH_EXPANDED_ZIP_BYTES: + raise NextDayArchiveError("Next Day monthly archive expands too large") + members.append( + NextDayDailyMemberRef( + report_date, + info.filename, + publication_id, + compressed, + expanded, + info.CRC, + ) + ) + members.sort(key=lambda item: item.report_date) + expected_dates = tuple( + date(reference.report_month.year, reference.report_month.month, day) + for day in range(1, expected_count + 1) + ) + if tuple(item.report_date for item in members) != expected_dates: + raise NextDayArchiveError("incomplete Next Day monthly archive") + except NextDayArchiveError: + raise + except _FORMAT_ERRORS as error: + raise NextDayArchiveError("invalid Next Day monthly archive") from error + return NextDayMonthlyArchiveManifest( + reference, + sha256(raw_bytes).hexdigest(), + tuple(members), + ) + + +def _member_info_matches(info: ZipInfo, member: NextDayDailyMemberRef) -> bool: + return ( + info.filename == member.filename + and info.compress_size == member.compressed_size + and info.file_size == member.expanded_size + and info.CRC == member.crc32 + ) + + +def read_nextday_daily_artifact( + manifest: NextDayMonthlyArchiveManifest, + raw_bytes: bytes, + member: NextDayDailyMemberRef, +) -> NextDayDailyArtifact: + """Read and verify exactly one selected daily ZIP and its single CSV.""" + + if ( + type(manifest) is not NextDayMonthlyArchiveManifest + or type(raw_bytes) is not bytes + or type(member) is not NextDayDailyMemberRef + or len(raw_bytes) != manifest.reference.size_bytes + or sha256(raw_bytes).hexdigest() != manifest.sha256 + or member not in manifest.members + ): + raise NextDayArchiveError("invalid Next Day daily artifact request") + try: + with ZipFile(BytesIO(raw_bytes)) as outer: + info = outer.getinfo(member.filename) + if not _member_info_matches(info, member): + raise NextDayArchiveError("changed Next Day daily member") + with outer.open(info) as stream: + daily_zip = stream.read(MAX_NEXTDAY_DAILY_ZIP_BYTES + 1) + if type(daily_zip) is not bytes or len(daily_zip) != member.expanded_size: + raise NextDayArchiveError("invalid Next Day daily member") + + with ZipFile(BytesIO(daily_zip)) as daily: + if daily.comment: + raise NextDayArchiveError("invalid Next Day daily artifact") + csv_infos = daily.infolist() + if len(csv_infos) != 1: + raise NextDayArchiveError("invalid Next Day daily artifact") + csv_info = csv_infos[0] + _safe_member_sizes( + csv_info, + max_compressed=MAX_NEXTDAY_DAILY_ZIP_BYTES, + max_expanded=MAX_NEXTDAY_DAILY_CSV_BYTES, + ) + expected_csv = member.filename.removesuffix(".zip") + ".CSV" + if csv_info.filename != expected_csv: + raise NextDayArchiveError("invalid Next Day daily CSV identity") + with daily.open(csv_info) as stream: + csv_bytes = stream.read(MAX_NEXTDAY_DAILY_CSV_BYTES + 1) + if type(csv_bytes) is not bytes or len(csv_bytes) != csv_info.file_size: + raise NextDayArchiveError("invalid Next Day daily CSV") + except NextDayArchiveError: + raise + except _FORMAT_ERRORS as error: + raise NextDayArchiveError("invalid Next Day daily artifact") from error + return NextDayDailyArtifact( + member, + sha256(daily_zip).hexdigest(), + daily_zip, + expected_csv, + csv_bytes, + ) + + +__all__ = [ + "MAX_NEXTDAY_COMPRESSION_RATIO", + "MAX_NEXTDAY_DAILY_CSV_BYTES", + "MAX_NEXTDAY_DAILY_ZIP_BYTES", + "MAX_NEXTDAY_MONTH_EXPANDED_ZIP_BYTES", + "NextDayDailyArtifact", + "NextDayDailyMemberRef", + "NextDayMonthlyArchiveManifest", + "read_nextday_daily_artifact", + "validate_nextday_monthly_archive", +] diff --git a/app/backend/tests/test_nextday_monthly_extraction.py b/app/backend/tests/test_nextday_monthly_extraction.py new file mode 100644 index 0000000..b79f344 --- /dev/null +++ b/app/backend/tests/test_nextday_monthly_extraction.py @@ -0,0 +1,130 @@ +"""Tests for bounded Next Day monthly archive extraction.""" + +from datetime import date, datetime, timezone +from io import BytesIO +from pathlib import Path +import struct +import unittest +from zipfile import ZIP_DEFLATED, ZipFile + +from batterywatch_api.nextday_archives import ( + NEXTDAY_ARCHIVE_INDEX_URL, + NextDayArchiveError, + NextDayMonthlyArchiveRef, +) +from batterywatch_api.nextday_monthly_extraction import ( + read_nextday_daily_artifact, + validate_nextday_monthly_archive, +) + +UTC = timezone.utc +FIXTURE = ( + Path(__file__).parent + / "fixtures" + / "historical" + / "nextday-unit-solution-soc-20260829-reduced.csv" +) + + +def _zip_bytes(filename: str, payload: bytes) -> bytes: + output = BytesIO() + with ZipFile(output, "w", compression=ZIP_DEFLATED) as archive: + archive.writestr(filename, payload) + return output.getvalue() + + +def _monthly_archive( + *, + day_numbers: tuple[int, ...] = tuple(range(1, 32)), + first_outer_name: str | None = None, + first_csv_name: str | None = None, +) -> tuple[NextDayMonthlyArchiveRef, bytes, bytes]: + csv_payload = FIXTURE.read_bytes() + output = BytesIO() + with ZipFile(output, "w", compression=ZIP_DEFLATED) as outer: + for index, day in enumerate(day_numbers): + publication_id = f"{47_000_000_000_000 + index + 1:016d}" + stem = f"PUBLIC_NEXT_DAY_DISPATCH_202507{day:02d}_{publication_id}" + outer_name = first_outer_name if index == 0 and first_outer_name else f"{stem}.zip" + csv_name = first_csv_name if index == 0 and first_csv_name else f"{stem}.CSV" + outer.writestr( + outer_name, + _zip_bytes(csv_name, csv_payload), + ) + raw_bytes = output.getvalue() + reference = NextDayMonthlyArchiveRef( + date(2025, 7, 1), + "PUBLIC_NEXT_DAY_DISPATCH_20250701.zip", + NEXTDAY_ARCHIVE_INDEX_URL + "PUBLIC_NEXT_DAY_DISPATCH_20250701.zip", + len(raw_bytes), + datetime(2025, 9, 1, tzinfo=UTC), + ) + return reference, raw_bytes, csv_payload + + +class NextDayMonthlyExtractionTests(unittest.TestCase): + def test_validates_complete_month_then_reads_one_daily_artifact(self) -> None: + reference, raw_bytes, csv_payload = _monthly_archive() + + manifest = validate_nextday_monthly_archive(reference, raw_bytes) + daily = read_nextday_daily_artifact(manifest, raw_bytes, manifest.members[0]) + + self.assertEqual(manifest.reference, reference) + self.assertEqual(len(manifest.members), 31) + self.assertEqual( + (manifest.members[0].report_date, manifest.members[-1].report_date), + (date(2025, 7, 1), date(2025, 7, 31)), + ) + self.assertEqual(daily.member, manifest.members[0]) + self.assertEqual(daily.csv_bytes, csv_payload) + self.assertEqual( + daily.csv_filename, + manifest.members[0].filename.removesuffix(".zip") + ".CSV", + ) + self.assertEqual(len(daily.sha256), 64) + self.assertEqual(daily.raw_zip_bytes[:2], b"PK") + + def test_rejects_incomplete_duplicate_and_unsafe_month_members(self) -> None: + invalid_archives = ( + _monthly_archive(day_numbers=tuple(range(1, 31)))[:2], + _monthly_archive(day_numbers=(1, 1, *range(3, 32)))[:2], + _monthly_archive(first_outer_name="../unsafe.zip")[:2], + ) + for reference, raw_bytes in invalid_archives: + with self.subTest(size=len(raw_bytes)): + with self.assertRaises(NextDayArchiveError): + validate_nextday_monthly_archive(reference, raw_bytes) + + def test_rejects_declared_oversized_daily_member(self) -> None: + reference, raw_bytes, _ = _monthly_archive() + corrupted = bytearray(raw_bytes) + central = corrupted.find(b"PK\x01\x02") + self.assertGreaterEqual(central, 0) + struct.pack_into(" None: + reference, raw_bytes, _ = _monthly_archive(first_csv_name="wrong.CSV") + manifest = validate_nextday_monthly_archive(reference, raw_bytes) + with self.assertRaises(NextDayArchiveError): + read_nextday_daily_artifact(manifest, raw_bytes, manifest.members[0]) + + reference, raw_bytes, _ = _monthly_archive() + manifest = validate_nextday_monthly_archive(reference, raw_bytes) + changed = raw_bytes[:-1] + bytes((raw_bytes[-1] ^ 1,)) + with self.assertRaises(NextDayArchiveError): + read_nextday_daily_artifact(manifest, changed, manifest.members[0]) + + +if __name__ == "__main__": + unittest.main() From e0025fdb1cf70ff417536071a1d5c577d11c42f4 Mon Sep 17 00:00:00 2001 From: djurcola-agent Date: Sun, 30 Aug 2026 23:58:05 +1000 Subject: [PATCH 25/30] Add monthly Next Day backfill planning --- .../batterywatch_api/backfill_artifacts.py | 27 ++++- .../batterywatch_api/backfill_ledger.py | 18 ++- .../batterywatch_api/backfill_service.py | 114 +++++++++++++----- app/backend/tests/test_backfill_artifacts.py | 58 +++++++++ app/backend/tests/test_backfill_ledger.py | 48 ++++++++ app/backend/tests/test_backfill_service.py | 32 +++++ 6 files changed, 262 insertions(+), 35 deletions(-) diff --git a/app/backend/batterywatch_api/backfill_artifacts.py b/app/backend/batterywatch_api/backfill_artifacts.py index 79b3312..31e0edd 100644 --- a/app/backend/batterywatch_api/backfill_artifacts.py +++ b/app/backend/batterywatch_api/backfill_artifacts.py @@ -12,14 +12,21 @@ MAX_ARCHIVE_BYTES = 512 * 1024 * 1024 _MAX_ATTEMPT_NUMBER = 2_147_483_647 _RUN_ID_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:-]{0,127}") -_FEED_URL_PREFIXES = { +_FEED_URL_RULES = { "dispatch_scada": ( "https://www.nemweb.com.au/REPORTS/ARCHIVE/Dispatch_SCADA/" - "PUBLIC_DISPATCHSCADA_" + "PUBLIC_DISPATCHSCADA_", + False, ), "dispatch_price": ( "https://www.nemweb.com.au/REPORTS/ARCHIVE/DispatchIS_Reports/" - "PUBLIC_DISPATCHIS_" + "PUBLIC_DISPATCHIS_", + False, + ), + "nextday_soc": ( + "https://www.nemweb.com.au/REPORTS/ARCHIVE/Next_Day_Dispatch/" + "PUBLIC_NEXT_DAY_DISPATCH_", + True, ), } @@ -71,12 +78,20 @@ def _validate_receipt(receipt: object) -> BackfillArtifactReceipt: raise TypeError("receipt claim must be BackfillClaim") if not isinstance(claim.run_id, str) or _RUN_ID_RE.fullmatch(claim.run_id) is None: raise ValueError("invalid run_id") - prefix = _FEED_URL_PREFIXES.get(claim.feed) - if prefix is None: + rule = _FEED_URL_RULES.get(claim.feed) + if rule is None: raise ValueError("invalid feed") if type(claim.report_date) is not date: raise TypeError("report_date must be a date") - expected_url = f"{prefix}{claim.report_date:%Y%m%d}.zip" + prefix, monthly = rule + if monthly and claim.report_date.day != 1: + raise ValueError("monthly report_date must be the first of the month") + stamp = ( + f"{claim.report_date:%Y%m}01" + if monthly + else f"{claim.report_date:%Y%m%d}" + ) + expected_url = f"{prefix}{stamp}.zip" if claim.source_url != expected_url: raise ValueError("source_url does not match feed and report_date") if ( diff --git a/app/backend/batterywatch_api/backfill_ledger.py b/app/backend/batterywatch_api/backfill_ledger.py index 6a7cb88..c6fd6cd 100644 --- a/app/backend/batterywatch_api/backfill_ledger.py +++ b/app/backend/batterywatch_api/backfill_ledger.py @@ -251,10 +251,17 @@ def _managed_cursor(connection: _Connection) -> Iterator[_Cursor]: "dispatch_price": ( "https://www.nemweb.com.au/REPORTS/ARCHIVE/DispatchIS_Reports/", "PUBLIC_DISPATCHIS_", + False, ), "dispatch_scada": ( "https://www.nemweb.com.au/REPORTS/ARCHIVE/Dispatch_SCADA/", "PUBLIC_DISPATCHSCADA_", + False, + ), + "nextday_soc": ( + "https://www.nemweb.com.au/REPORTS/ARCHIVE/Next_Day_Dispatch/", + "PUBLIC_NEXT_DAY_DISPATCH_", + True, ), } @@ -290,8 +297,15 @@ def _validate_items(items: tuple[BackfillPlanItem, ...]) -> None: if key in keys: raise ValueError("duplicate backfill plan item") keys.add(key) - base, prefix = _FEED_URLS[item.feed] - expected_url = f"{base}{prefix}{item.report_date:%Y%m%d}.zip" + base, prefix, monthly = _FEED_URLS[item.feed] + if monthly and item.report_date.day != 1: + raise ValueError("invalid monthly backfill report date") + stamp = ( + f"{item.report_date:%Y%m}01" + if monthly + else f"{item.report_date:%Y%m%d}" + ) + expected_url = f"{base}{prefix}{stamp}.zip" if item.source_url != expected_url: raise ValueError("invalid backfill archive URL") diff --git a/app/backend/batterywatch_api/backfill_service.py b/app/backend/batterywatch_api/backfill_service.py index 77c3e83..56297ff 100644 --- a/app/backend/batterywatch_api/backfill_service.py +++ b/app/backend/batterywatch_api/backfill_service.py @@ -4,6 +4,7 @@ import argparse from collections.abc import Callable, Iterable +from dataclasses import dataclass from datetime import date, datetime, timedelta, timezone import json import os @@ -19,6 +20,10 @@ DISPATCH_SCADA_FEED, plan_archive_range, ) +from .nextday_archives import ( + NextDayMonthlyArchiveRef, + plan_nextday_monthly_archives, +) UTC = timezone.utc _NEM_TIMEZONE = timezone(timedelta(hours=10)) @@ -32,6 +37,13 @@ } +@dataclass(frozen=True, slots=True) +class OperatorBackfillPlan: + spec: BackfillRunSpec + items: tuple[BackfillPlanItem, ...] + missing_nextday_months: tuple[date, ...] + + def _parse_utc(value: str) -> datetime: if not isinstance(value, str) or not value: raise ValueError("UTC timestamp is required") @@ -51,56 +63,99 @@ def _candidate_dates(start: datetime, end: datetime) -> tuple[date, ...]: return tuple(first + timedelta(days=index) for index in range((last - first).days + 1)) -def build_operator_plan( +def build_operator_plan_details( run_id: str, start: datetime, end: datetime, *, feeds: Iterable[str] = ("power", "price"), ingestion_version: int = 1, -) -> tuple[BackfillRunSpec, tuple[BackfillPlanItem, ...]]: - """Build canonical bounded ledger items from explicit operator inputs.""" + nextday_archives: Iterable[NextDayMonthlyArchiveRef] = (), +) -> OperatorBackfillPlan: + """Build a bounded plan and retain explicit unavailable SOC months.""" try: requested = tuple(feeds) except (TypeError, ValueError): raise ValueError("invalid backfill feeds") from None + allowed = {*_FEED_MAP, "soc"} if ( not requested or len(set(requested)) != len(requested) - or any(type(feed) is not str or feed not in _FEED_MAP for feed in requested) + or any(type(feed) is not str or feed not in allowed for feed in requested) ): raise ValueError("invalid backfill feeds") + archive_feeds = tuple( archive_feed for name, archive_feed in _FEED_MAP.items() if name in requested ) - candidates = _candidate_dates(start, end) - archived_dates = {feed: candidates for feed in archive_feeds} - archive_plan = plan_archive_range( - start, - end, - feeds=archive_feeds, - archived_dates=archived_dates, - ) - items = tuple( - BackfillPlanItem( - _LEDGER_FEED_MAP[item.feed], - item.report_date, - item.url, + daily_items: tuple[BackfillPlanItem, ...] = () + normalized_start: datetime | None = None + normalized_end: datetime | None = None + if archive_feeds: + candidates = _candidate_dates(start, end) + archived_dates = {feed: candidates for feed in archive_feeds} + archive_plan = plan_archive_range( + start, + end, + feeds=archive_feeds, + archived_dates=archived_dates, + ) + normalized_start = archive_plan.start + normalized_end = archive_plan.end + daily_items = tuple( + BackfillPlanItem( + _LEDGER_FEED_MAP[item.feed], + item.report_date, + item.url, + ) + for item in archive_plan.items + ) + + soc_items: tuple[BackfillPlanItem, ...] = () + missing_months: tuple[date, ...] = () + if "soc" in requested: + soc_plan = plan_nextday_monthly_archives(start, end, nextday_archives) + if normalized_start is None: + normalized_start = soc_plan.start + normalized_end = soc_plan.end + soc_items = tuple( + BackfillPlanItem("nextday_soc", item.report_month, item.url) + for item in soc_plan.items ) - for item in archive_plan.items + missing_months = soc_plan.missing_months + + if normalized_start is None or normalized_end is None: + raise ValueError("invalid backfill feeds") + return OperatorBackfillPlan( + BackfillRunSpec(run_id, normalized_start, normalized_end, ingestion_version), + daily_items + soc_items, + missing_months, ) - return ( - BackfillRunSpec( - run_id, - archive_plan.start, - archive_plan.end, - ingestion_version, - ), - items, + + +def build_operator_plan( + run_id: str, + start: datetime, + end: datetime, + *, + feeds: Iterable[str] = ("power", "price"), + ingestion_version: int = 1, + nextday_archives: Iterable[NextDayMonthlyArchiveRef] = (), +) -> tuple[BackfillRunSpec, tuple[BackfillPlanItem, ...]]: + """Build canonical bounded ledger items from explicit operator inputs.""" + + details = build_operator_plan_details( + run_id, + start, + end, + feeds=feeds, + ingestion_version=ingestion_version, + nextday_archives=nextday_archives, ) + return details.spec, details.items def _summary( @@ -194,7 +249,12 @@ def main( return 1 -__all__ = ["build_operator_plan", "main"] +__all__ = [ + "OperatorBackfillPlan", + "build_operator_plan", + "build_operator_plan_details", + "main", +] if __name__ == "__main__": diff --git a/app/backend/tests/test_backfill_artifacts.py b/app/backend/tests/test_backfill_artifacts.py index 420ff6f..874d902 100644 --- a/app/backend/tests/test_backfill_artifacts.py +++ b/app/backend/tests/test_backfill_artifacts.py @@ -87,6 +87,64 @@ def receipt(raw_archive: bytes = b"official archive bytes") -> BackfillArtifactR class PostgreSQLBackfillArtifactRegistrarTests(unittest.TestCase): + def test_records_canonical_monthly_nextday_soc_artifact(self) -> None: + report_date = date(2025, 7, 1) + source_url = ( + "https://www.nemweb.com.au/REPORTS/ARCHIVE/Next_Day_Dispatch/" + "PUBLIC_NEXT_DAY_DISPATCH_20250701.zip" + ) + evidence = BackfillArtifactReceipt( + BackfillClaim( + "soc-run-202507", + "nextday_soc", + report_date, + source_url, + 1, + ), + datetime(2026, 8, 30, tzinfo=UTC), + None, + b"official monthly outer zip", + ) + digest = hashlib.sha256(evidence.raw_archive).hexdigest() + connection = FakeConnection( + fetchone_results=((source_url, "running", 1), (1,), (1,)) + ) + + result = PostgreSQLBackfillArtifactRegistrar(connection).record(evidence) + + self.assertEqual( + result, + BackfillArtifactResult(digest, len(evidence.raw_archive), False), + ) + artifact_parameters = connection.executions[1][1] + self.assertEqual(artifact_parameters[1:5], ( + "nextday_soc", + report_date, + source_url, + "PUBLIC_NEXT_DAY_DISPATCH_20250701.zip", + )) + + def test_monthly_nextday_receipt_requires_first_day_identity(self) -> None: + evidence = BackfillArtifactReceipt( + BackfillClaim( + "soc-run-202507-bad", + "nextday_soc", + date(2025, 7, 2), + "https://www.nemweb.com.au/REPORTS/ARCHIVE/Next_Day_Dispatch/" + "PUBLIC_NEXT_DAY_DISPATCH_20250701.zip", + 1, + ), + datetime(2026, 8, 30, tzinfo=UTC), + None, + b"official monthly outer zip", + ) + connection = FakeConnection() + + with self.assertRaisesRegex(ValueError, "monthly"): + PostgreSQLBackfillArtifactRegistrar(connection).record(evidence) + + self.assertEqual(connection.cursor_calls, 0) + def test_records_content_link_and_event_in_one_transaction(self) -> None: evidence = receipt() digest = hashlib.sha256(evidence.raw_archive).hexdigest() diff --git a/app/backend/tests/test_backfill_ledger.py b/app/backend/tests/test_backfill_ledger.py index 45ab628..edc44f9 100644 --- a/app/backend/tests/test_backfill_ledger.py +++ b/app/backend/tests/test_backfill_ledger.py @@ -483,6 +483,54 @@ def test_ensure_new_run_plans_items_and_events_in_deterministic_order(self) -> N ) self.assertTrue(all("%s" in statement for statement, _ in connection.executions)) + def test_ensure_new_run_accepts_canonical_monthly_nextday_soc_item(self) -> None: + spec = BackfillRunSpec( + "soc-run-202507", + datetime(2025, 7, 1, tzinfo=UTC), + datetime(2025, 8, 1, tzinfo=UTC), + 3, + ) + item = BackfillPlanItem( + "nextday_soc", + date(2025, 7, 1), + "https://www.nemweb.com.au/REPORTS/ARCHIVE/Next_Day_Dispatch/" + "PUBLIC_NEXT_DAY_DISPATCH_20250701.zip", + ) + connection = FakeConnection(fetchone_results=((1,),)) + + result = PostgreSQLBackfillLedger(connection).ensure_run(spec, (item,)) + + self.assertEqual(result, BackfillEnsureResult(True, False, 1, 0)) + item_inserts = tuple( + parameters + for statement, parameters in connection.executions + if "INSERT INTO historical_backfill_items" in statement + ) + self.assertEqual( + item_inserts, + ((spec.run_id, item.feed, item.report_date, item.source_url),), + ) + + def test_monthly_nextday_soc_item_requires_first_day_identity(self) -> None: + spec = BackfillRunSpec( + "soc-run-202507-bad", + datetime(2025, 7, 1, tzinfo=UTC), + datetime(2025, 8, 1, tzinfo=UTC), + 3, + ) + item = BackfillPlanItem( + "nextday_soc", + date(2025, 7, 2), + "https://www.nemweb.com.au/REPORTS/ARCHIVE/Next_Day_Dispatch/" + "PUBLIC_NEXT_DAY_DISPATCH_20250701.zip", + ) + connection = FakeConnection() + + with self.assertRaisesRegex(ValueError, "monthly"): + PostgreSQLBackfillLedger(connection).ensure_run(spec, (item,)) + + self.assertEqual(connection.cursor_calls, 0) + def test_exact_resume_recovers_interrupted_item_and_preserves_attempt(self) -> None: start = datetime(2026, 8, 27, 14, tzinfo=UTC) end = datetime(2026, 8, 29, 14, tzinfo=UTC) diff --git a/app/backend/tests/test_backfill_service.py b/app/backend/tests/test_backfill_service.py index 48088d2..a24661d 100644 --- a/app/backend/tests/test_backfill_service.py +++ b/app/backend/tests/test_backfill_service.py @@ -19,11 +19,43 @@ from batterywatch_api.historical_price_backfill import HistoricalPriceBackfillResult from batterywatch_api.historical_scada_backfill import HistoricalScadaBackfillResult from batterywatch_api.backfill_service import build_operator_plan, main +from batterywatch_api.backfill_service import build_operator_plan_details +from batterywatch_api.nextday_archives import ( + NEXTDAY_ARCHIVE_INDEX_URL, + NextDayMonthlyArchiveRef, +) UTC = timezone.utc class BackfillServiceTests(unittest.TestCase): + def test_builds_soc_plan_with_explicit_missing_official_months(self) -> None: + def archive(month: int, size: int) -> NextDayMonthlyArchiveRef: + filename = f"PUBLIC_NEXT_DAY_DISPATCH_2025{month:02d}01.zip" + return NextDayMonthlyArchiveRef( + date(2025, month, 1), + filename, + NEXTDAY_ARCHIVE_INDEX_URL + filename, + size, + datetime(2025, month + 1, 1, tzinfo=UTC), + ) + + details = build_operator_plan_details( + "soc-operator-run", + datetime(2025, 7, 15, tzinfo=UTC), + datetime(2025, 9, 15, tzinfo=UTC), + feeds=("soc",), + ingestion_version=3, + nextday_archives=(archive(7, 210_000_000), archive(8, 220_000_000)), + ) + + self.assertEqual(details.spec.run_id, "soc-operator-run") + self.assertEqual( + tuple((item.feed, item.report_date) for item in details.items), + (("nextday_soc", date(2025, 7, 1)), ("nextday_soc", date(2025, 8, 1))), + ) + self.assertEqual(details.missing_nextday_months, (date(2025, 9, 1),)) + def test_builds_canonical_ledger_plan_for_requested_feeds(self) -> None: start = datetime(2026, 8, 29, 0, 0, tzinfo=UTC) end = datetime(2026, 8, 29, 10, 0, tzinfo=UTC) From 5cdf3117f86fb74b63b79e6600938a24044684ae Mon Sep 17 00:00:00 2001 From: djurcola-agent Date: Mon, 31 Aug 2026 00:02:28 +1000 Subject: [PATCH 26/30] Preserve Next Day daily publication timestamps --- .../nextday_monthly_extraction.py | 15 ++++++++++++++- .../tests/test_nextday_monthly_extraction.py | 15 +++++++++++---- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/app/backend/batterywatch_api/nextday_monthly_extraction.py b/app/backend/batterywatch_api/nextday_monthly_extraction.py index 0e77c8d..9f25fa5 100644 --- a/app/backend/batterywatch_api/nextday_monthly_extraction.py +++ b/app/backend/batterywatch_api/nextday_monthly_extraction.py @@ -4,7 +4,7 @@ from calendar import monthrange from dataclasses import dataclass -from datetime import date +from datetime import date, datetime, timedelta, timezone from hashlib import sha256 from io import BytesIO import re @@ -29,6 +29,8 @@ MAX_NEXTDAY_DAILY_CSV_BYTES = 128 * 1024 * 1024 MAX_NEXTDAY_MONTH_EXPANDED_ZIP_BYTES = 512 * 1024 * 1024 MAX_NEXTDAY_COMPRESSION_RATIO = 100 +_NEM_TIMEZONE = timezone(timedelta(hours=10)) +_UTC = timezone.utc _MEMBER_NAME_RE = re.compile( r"PUBLIC_NEXT_DAY_DISPATCH_([0-9]{4})([0-9]{2})([0-9]{2})_([0-9]{1,32})\.zip" ) @@ -52,6 +54,7 @@ class NextDayDailyMemberRef: report_date: date filename: str publication_id: str + artifact_published_at: datetime compressed_size: int expanded_size: int crc32: int @@ -134,6 +137,14 @@ def _member_identity( return report_date, match.group(4) +def _artifact_published_at(member: ZipInfo) -> datetime: + try: + published = datetime(*member.date_time, tzinfo=_NEM_TIMEZONE) + except (OverflowError, TypeError, ValueError): + raise NextDayArchiveError("invalid Next Day member timestamp") from None + return published.astimezone(_UTC) + + def validate_nextday_monthly_archive( reference: NextDayMonthlyArchiveRef, raw_bytes: bytes, @@ -175,6 +186,7 @@ def validate_nextday_monthly_archive( report_date, info.filename, publication_id, + _artifact_published_at(info), compressed, expanded, info.CRC, @@ -201,6 +213,7 @@ def validate_nextday_monthly_archive( def _member_info_matches(info: ZipInfo, member: NextDayDailyMemberRef) -> bool: return ( info.filename == member.filename + and _artifact_published_at(info) == member.artifact_published_at and info.compress_size == member.compressed_size and info.file_size == member.expanded_size and info.CRC == member.crc32 diff --git a/app/backend/tests/test_nextday_monthly_extraction.py b/app/backend/tests/test_nextday_monthly_extraction.py index b79f344..7d4ed9a 100644 --- a/app/backend/tests/test_nextday_monthly_extraction.py +++ b/app/backend/tests/test_nextday_monthly_extraction.py @@ -1,11 +1,11 @@ """Tests for bounded Next Day monthly archive extraction.""" -from datetime import date, datetime, timezone +from datetime import date, datetime, timedelta, timezone from io import BytesIO from pathlib import Path import struct import unittest -from zipfile import ZIP_DEFLATED, ZipFile +from zipfile import ZIP_DEFLATED, ZipFile, ZipInfo from batterywatch_api.nextday_archives import ( NEXTDAY_ARCHIVE_INDEX_URL, @@ -47,10 +47,13 @@ def _monthly_archive( stem = f"PUBLIC_NEXT_DAY_DISPATCH_202507{day:02d}_{publication_id}" outer_name = first_outer_name if index == 0 and first_outer_name else f"{stem}.zip" csv_name = first_csv_name if index == 0 and first_csv_name else f"{stem}.CSV" - outer.writestr( + published_nem = datetime(2025, 7, day) + timedelta(days=1) + member = ZipInfo( outer_name, - _zip_bytes(csv_name, csv_payload), + date_time=(*published_nem.date().timetuple()[:3], 4, 11, 26), ) + member.compress_type = ZIP_DEFLATED + outer.writestr(member, _zip_bytes(csv_name, csv_payload)) raw_bytes = output.getvalue() reference = NextDayMonthlyArchiveRef( date(2025, 7, 1), @@ -75,6 +78,10 @@ def test_validates_complete_month_then_reads_one_daily_artifact(self) -> None: (manifest.members[0].report_date, manifest.members[-1].report_date), (date(2025, 7, 1), date(2025, 7, 31)), ) + self.assertEqual( + manifest.members[0].artifact_published_at, + datetime(2025, 7, 1, 18, 11, 26, tzinfo=UTC), + ) self.assertEqual(daily.member, manifest.members[0]) self.assertEqual(daily.csv_bytes, csv_payload) self.assertEqual( From 09c292ff7a108e7eafd2612074dfc7cde057d743 Mon Sep 17 00:00:00 2001 From: djurcola-agent Date: Mon, 31 Aug 2026 00:12:09 +1000 Subject: [PATCH 27/30] Preserve nested Next Day source artifacts --- .../nested_source_artifacts.py | 234 ++++++++++++++++++ .../test_historical_artifact_migration.py | 20 ++ .../tests/test_nested_source_artifacts.py | 201 +++++++++++++++ app/migrations/008_authoritative_soc.sql | 61 +++++ 4 files changed, 516 insertions(+) create mode 100644 app/backend/batterywatch_api/nested_source_artifacts.py create mode 100644 app/backend/tests/test_nested_source_artifacts.py diff --git a/app/backend/batterywatch_api/nested_source_artifacts.py b/app/backend/batterywatch_api/nested_source_artifacts.py new file mode 100644 index 0000000..35add20 --- /dev/null +++ b/app/backend/batterywatch_api/nested_source_artifacts.py @@ -0,0 +1,234 @@ +"""Immutable registration of nested Next Day daily source artifacts.""" + +from __future__ import annotations + +from contextlib import contextmanager +from dataclasses import dataclass +from datetime import date, datetime, timedelta +from hashlib import sha256 +import re +from typing import Any, Iterator, Protocol + + +_MAX_DAILY_ZIP_BYTES = 16 * 1024 * 1024 +_SHA_RE = re.compile(r"[0-9a-f]{64}") +_PUBLICATION_RE = re.compile(r"[0-9]{1,32}") +_FILENAME_RE = re.compile( + r"PUBLIC_NEXT_DAY_DISPATCH_([0-9]{4})([0-9]{2})([0-9]{2})_([0-9]{1,32})\.zip" +) +_BASE_URL = "https://www.nemweb.com.au/REPORTS/ARCHIVE/Next_Day_Dispatch/" + + +@dataclass(frozen=True, slots=True) +class NestedSourceArtifactReceipt: + parent_artifact_sha256: str + report_date: date + outer_source_url: str + filename: str + publication_id: str + artifact_published_at: datetime + downloaded_at: datetime + raw_bytes: bytes + + +@dataclass(frozen=True, slots=True) +class NestedSourceArtifactResult: + artifact_sha256: str + byte_count: int + replayed: bool + + +class NestedSourceArtifactConflictError(ValueError): + """Stored parent or nested artifact conflicts with supplied evidence.""" + + +class _Cursor(Protocol): + def execute(self, statement: str, parameters: tuple[object, ...]) -> None: ... + + def fetchone(self) -> Any: ... + + def close(self) -> None: ... + + +class _Connection(Protocol): + def cursor(self) -> _Cursor: ... + + def commit(self) -> None: ... + + def rollback(self) -> None: ... + + +@contextmanager +def _managed_cursor(connection: _Connection) -> Iterator[_Cursor]: + cursor = connection.cursor() + try: + yield cursor + finally: + cursor.close() + + +_PARENT_SELECT_SQL = """ +SELECT feed, report_date, source_url, parent_artifact_sha256 +FROM historical_source_artifacts +WHERE artifact_sha256 = %s +FOR SHARE +""" + +_NESTED_INSERT_SQL = """ +INSERT INTO historical_source_artifacts ( + artifact_sha256, feed, report_date, source_url, filename, + byte_count, raw_bytes, parent_artifact_sha256, + artifact_published_at, artifact_downloaded_at, publication_id +) +VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) +ON CONFLICT DO NOTHING RETURNING 1 +""" + +_NESTED_SELECT_SQL = """ +SELECT feed, report_date, source_url, filename, byte_count, raw_bytes, + parent_artifact_sha256, artifact_published_at, + artifact_downloaded_at, publication_id +FROM historical_source_artifacts +WHERE artifact_sha256 = %s +""" + + +def _validate_receipt(receipt: object) -> NestedSourceArtifactReceipt: + if type(receipt) is not NestedSourceArtifactReceipt: + raise TypeError("receipt must be NestedSourceArtifactReceipt") + if ( + type(receipt.parent_artifact_sha256) is not str + or _SHA_RE.fullmatch(receipt.parent_artifact_sha256) is None + ): + raise ValueError("invalid parent artifact SHA") + if type(receipt.report_date) is not date: + raise TypeError("report_date must be a date") + expected_outer_url = ( + f"{_BASE_URL}PUBLIC_NEXT_DAY_DISPATCH_{receipt.report_date:%Y%m}01.zip" + ) + if receipt.outer_source_url != expected_outer_url: + raise ValueError("invalid outer source URL") + if type(receipt.filename) is not str: + raise TypeError("filename must be a string") + match = _FILENAME_RE.fullmatch(receipt.filename) + if match is None: + raise ValueError("invalid nested artifact filename") + try: + filename_date = date( + int(match.group(1)), + int(match.group(2)), + int(match.group(3)), + ) + except (OverflowError, ValueError): + raise ValueError("invalid nested artifact filename") from None + if filename_date != receipt.report_date: + raise ValueError("nested filename does not match report_date") + if ( + type(receipt.publication_id) is not str + or _PUBLICATION_RE.fullmatch(receipt.publication_id) is None + or match.group(4) != receipt.publication_id + ): + raise ValueError("invalid publication identity") + for value, name in ( + (receipt.artifact_published_at, "artifact_published_at"), + (receipt.downloaded_at, "downloaded_at"), + ): + if type(value) is not datetime or value.utcoffset() != timedelta(0): + raise ValueError(f"{name} must be UTC-aware") + if receipt.artifact_published_at > receipt.downloaded_at: + raise ValueError("artifact publication cannot follow download") + if type(receipt.raw_bytes) is not bytes: + raise TypeError("raw_bytes must be immutable bytes") + if not 1 <= len(receipt.raw_bytes) <= _MAX_DAILY_ZIP_BYTES: + raise ValueError("nested artifact size is outside accepted bounds") + if sha256(receipt.raw_bytes).hexdigest() == receipt.parent_artifact_sha256: + raise ValueError("nested artifact cannot equal its parent") + return receipt + + +class PostgreSQLNestedSourceArtifactRegistrar: + """Insert one nested daily artifact after verifying its monthly parent.""" + + def __init__(self, connection: _Connection): + self._connection = connection + + def record( + self, receipt: NestedSourceArtifactReceipt + ) -> NestedSourceArtifactResult: + receipt = _validate_receipt(receipt) + digest = sha256(receipt.raw_bytes).hexdigest() + byte_count = len(receipt.raw_bytes) + source_url = f"{receipt.outer_source_url}#{receipt.filename}" + expected_parent = ( + "nextday_soc", + date(receipt.report_date.year, receipt.report_date.month, 1), + receipt.outer_source_url, + None, + ) + try: + with _managed_cursor(self._connection) as cursor: + cursor.execute( + _PARENT_SELECT_SQL, + (receipt.parent_artifact_sha256,), + ) + if cursor.fetchone() != expected_parent: + raise NestedSourceArtifactConflictError( + "nested artifact parent conflicts" + ) + cursor.execute( + _NESTED_INSERT_SQL, + ( + digest, + "nextday_soc", + receipt.report_date, + source_url, + receipt.filename, + byte_count, + receipt.raw_bytes, + receipt.parent_artifact_sha256, + receipt.artifact_published_at, + receipt.downloaded_at, + receipt.publication_id, + ), + ) + inserted = cursor.fetchone() is not None + if not inserted: + cursor.execute(_NESTED_SELECT_SQL, (digest,)) + stored = cursor.fetchone() + if stored is None or len(stored) != 10: + raise NestedSourceArtifactConflictError( + "conflicting nested source artifact" + ) + expected = ( + "nextday_soc", + receipt.report_date, + source_url, + receipt.filename, + byte_count, + receipt.raw_bytes, + receipt.parent_artifact_sha256, + receipt.artifact_published_at, + receipt.downloaded_at, + receipt.publication_id, + ) + normalized = (*stored[:5], bytes(stored[5]), *stored[6:]) + if normalized != expected: + raise NestedSourceArtifactConflictError( + "conflicting nested source artifact" + ) + self._connection.commit() + return NestedSourceArtifactResult(digest, byte_count, not inserted) + except Exception: + try: + self._connection.rollback() + except Exception: + pass + raise + + +__all__ = [ + "NestedSourceArtifactConflictError", + "NestedSourceArtifactReceipt", + "NestedSourceArtifactResult", + "PostgreSQLNestedSourceArtifactRegistrar", +] diff --git a/app/backend/tests/test_historical_artifact_migration.py b/app/backend/tests/test_historical_artifact_migration.py index 3b299f1..2eccffe 100644 --- a/app/backend/tests/test_historical_artifact_migration.py +++ b/app/backend/tests/test_historical_artifact_migration.py @@ -179,6 +179,26 @@ def test_adds_authoritative_individual_soc_schema_after_runtime_details(self) -> self.assertIn("'nextday_soc'", migration) self.assertIn("PUBLIC_NEXT_DAY_DISPATCH_", migration) self.assertIn("Next_Day_Dispatch/", migration) + for column in ( + "parent_artifact_sha256", + "artifact_published_at", + "artifact_downloaded_at", + "publication_id", + ): + self.assertIn(f"ADD COLUMN IF NOT EXISTS {column}", migration) + self.assertIn("historical_source_artifacts_parent_fk", migration) + self.assertIn("REFERENCES historical_source_artifacts (artifact_sha256)", migration) + self.assertIn("historical_source_artifacts_nested_metadata_ck", migration) + self.assertRegex( + migration, + r"num_nonnulls\(\s*parent_artifact_sha256,\s*" + r"artifact_published_at,\s*artifact_downloaded_at,\s*" + r"publication_id\s*\)", + ) + self.assertIn("artifact_published_at <= artifact_downloaded_at", migration) + self.assertIn("source_url =", migration) + self.assertIn("|| '#' || filename", migration) + self.assertIn("historical_source_artifacts_parent_idx", migration) for constraint in ( "historical_backfill_items_feed_check", "historical_backfill_events_feed_check", diff --git a/app/backend/tests/test_nested_source_artifacts.py b/app/backend/tests/test_nested_source_artifacts.py new file mode 100644 index 0000000..6fc8a57 --- /dev/null +++ b/app/backend/tests/test_nested_source_artifacts.py @@ -0,0 +1,201 @@ +"""Tests for immutable nested Next Day source-artifact registration.""" + +from datetime import date, datetime, timedelta, timezone +from dataclasses import replace +import hashlib +import unittest + +from batterywatch_api.nested_source_artifacts import ( + NestedSourceArtifactConflictError, + NestedSourceArtifactReceipt, + NestedSourceArtifactResult, + PostgreSQLNestedSourceArtifactRegistrar, +) + +UTC = timezone.utc +PARENT_SHA = "a" * 64 +OUTER_URL = ( + "https://www.nemweb.com.au/REPORTS/ARCHIVE/Next_Day_Dispatch/" + "PUBLIC_NEXT_DAY_DISPATCH_20250701.zip" +) +DAILY_FILENAME = "PUBLIC_NEXT_DAY_DISPATCH_20250701_0000000470129643.zip" + + +class FakeCursor: + def __init__(self, connection) -> None: + self.connection = connection + + def execute(self, statement, parameters) -> None: + self.connection.executions.append((statement, tuple(parameters))) + + def fetchone(self): + if self.connection.fetchone_results: + return self.connection.fetchone_results.pop(0) + return None + + def close(self) -> None: + self.connection.closed_cursors += 1 + + +class FakeConnection: + def __init__(self, fetchone_results=()) -> None: + self.fetchone_results = list(fetchone_results) + self.executions = [] + self.cursor_calls = self.closed_cursors = 0 + self.commits = self.rollbacks = 0 + + def cursor(self): + self.cursor_calls += 1 + return FakeCursor(self) + + def commit(self) -> None: + self.commits += 1 + + def rollback(self) -> None: + self.rollbacks += 1 + + +def receipt(raw_bytes: bytes = b"official nested daily zip") -> NestedSourceArtifactReceipt: + return NestedSourceArtifactReceipt( + PARENT_SHA, + date(2025, 7, 1), + OUTER_URL, + DAILY_FILENAME, + "0000000470129643", + datetime(2025, 7, 1, 18, 11, 26, tzinfo=UTC), + datetime(2026, 8, 30, tzinfo=UTC), + raw_bytes, + ) + + +class PostgreSQLNestedSourceArtifactRegistrarTests(unittest.TestCase): + def test_registers_daily_artifact_under_canonical_monthly_parent(self) -> None: + evidence = receipt() + digest = hashlib.sha256(evidence.raw_bytes).hexdigest() + connection = FakeConnection( + (("nextday_soc", date(2025, 7, 1), OUTER_URL, None), (1,)) + ) + + result = PostgreSQLNestedSourceArtifactRegistrar(connection).record(evidence) + + self.assertEqual( + result, + NestedSourceArtifactResult(digest, len(evidence.raw_bytes), False), + ) + self.assertEqual((connection.commits, connection.rollbacks), (1, 0)) + self.assertEqual((connection.cursor_calls, connection.closed_cursors), (1, 1)) + self.assertEqual(len(connection.executions), 2) + self.assertIn("FOR SHARE", connection.executions[0][0]) + self.assertEqual(connection.executions[0][1], (PARENT_SHA,)) + insert_sql, parameters = connection.executions[1] + self.assertIn("INSERT INTO historical_source_artifacts", insert_sql) + self.assertIn("parent_artifact_sha256", insert_sql) + self.assertIn("artifact_published_at", insert_sql) + self.assertIn("artifact_downloaded_at", insert_sql) + self.assertEqual( + parameters, + ( + digest, + "nextday_soc", + evidence.report_date, + f"{OUTER_URL}#{DAILY_FILENAME}", + DAILY_FILENAME, + len(evidence.raw_bytes), + evidence.raw_bytes, + PARENT_SHA, + evidence.artifact_published_at, + evidence.downloaded_at, + evidence.publication_id, + ), + ) + + def test_exact_replay_verifies_stored_daily_artifact(self) -> None: + evidence = receipt() + digest = hashlib.sha256(evidence.raw_bytes).hexdigest() + source_url = f"{OUTER_URL}#{DAILY_FILENAME}" + connection = FakeConnection( + ( + ("nextday_soc", date(2025, 7, 1), OUTER_URL, None), + None, + ( + "nextday_soc", + evidence.report_date, + source_url, + evidence.filename, + len(evidence.raw_bytes), + evidence.raw_bytes, + evidence.parent_artifact_sha256, + evidence.artifact_published_at, + evidence.downloaded_at, + evidence.publication_id, + ), + ) + ) + + result = PostgreSQLNestedSourceArtifactRegistrar(connection).record(evidence) + + self.assertEqual( + result, + NestedSourceArtifactResult(digest, len(evidence.raw_bytes), True), + ) + self.assertEqual((connection.commits, connection.rollbacks), (1, 0)) + self.assertEqual(len(connection.executions), 3) + self.assertIn("SELECT feed, report_date", connection.executions[2][0]) + self.assertEqual(connection.executions[2][1], (digest,)) + + def test_same_sha_with_changed_publication_metadata_fails_closed(self) -> None: + evidence = receipt() + source_url = f"{OUTER_URL}#{DAILY_FILENAME}" + connection = FakeConnection( + ( + ("nextday_soc", date(2025, 7, 1), OUTER_URL, None), + None, + ( + "nextday_soc", + evidence.report_date, + source_url, + evidence.filename, + len(evidence.raw_bytes), + evidence.raw_bytes, + evidence.parent_artifact_sha256, + evidence.artifact_published_at + timedelta(seconds=2), + evidence.downloaded_at, + evidence.publication_id, + ), + ) + ) + + with self.assertRaisesRegex( + NestedSourceArtifactConflictError, + "conflicting nested", + ): + PostgreSQLNestedSourceArtifactRegistrar(connection).record(evidence) + + self.assertEqual((connection.commits, connection.rollbacks), (0, 1)) + + def test_wrong_parent_artifact_fails_closed(self) -> None: + connection = FakeConnection( + (("dispatch_price", date(2025, 7, 1), OUTER_URL, None),) + ) + + with self.assertRaisesRegex( + NestedSourceArtifactConflictError, + "parent conflicts", + ): + PostgreSQLNestedSourceArtifactRegistrar(connection).record(receipt()) + + self.assertEqual((connection.commits, connection.rollbacks), (0, 1)) + self.assertEqual(len(connection.executions), 1) + + def test_invalid_publication_identity_is_rejected_before_db_access(self) -> None: + invalid = replace(receipt(), publication_id="guessed") + connection = FakeConnection() + + with self.assertRaisesRegex(ValueError, "publication"): + PostgreSQLNestedSourceArtifactRegistrar(connection).record(invalid) + + self.assertEqual(connection.cursor_calls, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/migrations/008_authoritative_soc.sql b/app/migrations/008_authoritative_soc.sql index 736e966..16cd515 100644 --- a/app/migrations/008_authoritative_soc.sql +++ b/app/migrations/008_authoritative_soc.sql @@ -16,33 +16,94 @@ ALTER TABLE historical_backfill_events CHECK (feed IN ('dispatch_price', 'dispatch_scada', 'nextday_soc')); ALTER TABLE historical_source_artifacts + ADD COLUMN IF NOT EXISTS parent_artifact_sha256 TEXT, + ADD COLUMN IF NOT EXISTS artifact_published_at TIMESTAMPTZ, + ADD COLUMN IF NOT EXISTS artifact_downloaded_at TIMESTAMPTZ, + ADD COLUMN IF NOT EXISTS publication_id TEXT, DROP CONSTRAINT IF EXISTS historical_source_artifacts_feed_check, ADD CONSTRAINT historical_source_artifacts_feed_check CHECK (feed IN ('dispatch_price', 'dispatch_scada', 'nextday_soc')), + DROP CONSTRAINT IF EXISTS historical_source_artifacts_nested_metadata_ck, + ADD CONSTRAINT historical_source_artifacts_nested_metadata_ck CHECK ( + num_nonnulls( + parent_artifact_sha256, + artifact_published_at, + artifact_downloaded_at, + publication_id + ) IN (0, 4) + AND ( + parent_artifact_sha256 IS NULL + OR parent_artifact_sha256 <> artifact_sha256 + ) + AND ( + publication_id IS NULL + OR publication_id ~ '^[0-9]{1,32}$' + ) + AND ( + artifact_published_at IS NULL + OR artifact_published_at <= artifact_downloaded_at + ) + ), DROP CONSTRAINT IF EXISTS historical_source_artifacts_archive_identity_ck, ADD CONSTRAINT historical_source_artifacts_archive_identity_ck CHECK ( ( feed = 'dispatch_price' + AND parent_artifact_sha256 IS NULL AND source_url = 'https://www.nemweb.com.au/REPORTS/ARCHIVE/DispatchIS_Reports/' || filename AND filename = 'PUBLIC_DISPATCHIS_' || to_char(report_date, 'YYYYMMDD') || '.zip' ) OR ( feed = 'dispatch_scada' + AND parent_artifact_sha256 IS NULL AND source_url = 'https://www.nemweb.com.au/REPORTS/ARCHIVE/Dispatch_SCADA/' || filename AND filename = 'PUBLIC_DISPATCHSCADA_' || to_char(report_date, 'YYYYMMDD') || '.zip' ) OR ( feed = 'nextday_soc' + AND parent_artifact_sha256 IS NULL AND report_date = date_trunc('month', report_date)::date AND source_url = 'https://www.nemweb.com.au/REPORTS/ARCHIVE/Next_Day_Dispatch/' || filename AND filename = 'PUBLIC_NEXT_DAY_DISPATCH_' || to_char(report_date, 'YYYYMM') || '01.zip' ) + OR ( + feed = 'nextday_soc' + AND parent_artifact_sha256 IS NOT NULL + AND source_url = + 'https://www.nemweb.com.au/REPORTS/ARCHIVE/Next_Day_Dispatch/' + || 'PUBLIC_NEXT_DAY_DISPATCH_' + || to_char(report_date, 'YYYYMM') || '01.zip' + || '#' || filename + AND filename = + 'PUBLIC_NEXT_DAY_DISPATCH_' || to_char(report_date, 'YYYYMMDD') + || '_' || publication_id || '.zip' + ) ); +DO $migration$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'historical_source_artifacts_parent_fk' + AND conrelid = 'historical_source_artifacts'::regclass + ) THEN + ALTER TABLE historical_source_artifacts + ADD CONSTRAINT historical_source_artifacts_parent_fk + FOREIGN KEY (parent_artifact_sha256) + REFERENCES historical_source_artifacts (artifact_sha256) + ON DELETE RESTRICT; + END IF; +END +$migration$; + +CREATE INDEX IF NOT EXISTS historical_source_artifacts_parent_idx + ON historical_source_artifacts (parent_artifact_sha256) + WHERE parent_artifact_sha256 IS NOT NULL; + ALTER TABLE historical_backfill_item_artifacts DROP CONSTRAINT IF EXISTS historical_backfill_item_artifacts_feed_check, ADD CONSTRAINT historical_backfill_item_artifacts_feed_check From 43a09130235cdf1b09c101ec4276223562546dc2 Mon Sep 17 00:00:00 2001 From: djurcola-agent Date: Mon, 31 Aug 2026 00:59:23 +1000 Subject: [PATCH 28/30] feat: backfill authoritative next-day SOC --- .../historical_nextday_backfill.py | 330 +++++++++++ .../batterywatch_api/nextday_archives.py | 51 +- app/backend/batterywatch_api/nextday_soc.py | 22 +- ...-unit-solution-soc-v5-20250701-reduced.csv | 4 + .../tests/test_historical_nextday_backfill.py | 526 ++++++++++++++++++ app/backend/tests/test_nextday_archives.py | 27 + app/backend/tests/test_nextday_soc.py | 44 +- 7 files changed, 993 insertions(+), 11 deletions(-) create mode 100644 app/backend/batterywatch_api/historical_nextday_backfill.py create mode 100644 app/backend/tests/fixtures/historical/nextday-unit-solution-soc-v5-20250701-reduced.csv create mode 100644 app/backend/tests/test_historical_nextday_backfill.py diff --git a/app/backend/batterywatch_api/historical_nextday_backfill.py b/app/backend/batterywatch_api/historical_nextday_backfill.py new file mode 100644 index 0000000..f514e8b --- /dev/null +++ b/app/backend/batterywatch_api/historical_nextday_backfill.py @@ -0,0 +1,330 @@ +"""Two-pass monthly Next Day SOC backfill claim execution.""" + +from __future__ import annotations + +from collections.abc import Callable, Iterable +from contextlib import contextmanager +from dataclasses import dataclass +from datetime import date, datetime, timedelta, timezone +from email.utils import parsedate_to_datetime +from importlib import import_module +from typing import Any, Iterator + +from .backfill_artifacts import ( + BackfillArtifactReceipt, + PostgreSQLBackfillArtifactRegistrar, +) +from .backfill_ledger import ( + BackfillClaim, + BackfillItemCompletion, + PostgreSQLBackfillLedger, +) +from .battery_assets import BatteryAsset +from .nemweb_http import NemwebHttpResource, fetch_nemweb_resource +from .nested_source_artifacts import ( + NestedSourceArtifactReceipt, + PostgreSQLNestedSourceArtifactRegistrar, +) +from .nextday_archives import NextDayMonthlyArchiveRef, nextday_report_date_bounds +from .nextday_monthly_extraction import ( + NextDayDailyArtifact, + NextDayDailyMemberRef, + NextDayMonthlyArchiveManifest, + read_nextday_daily_artifact, + validate_nextday_monthly_archive, +) +from .nextday_soc import parse_nextday_unit_solution_soc +from .nextday_soc_ingestion import PostgreSQLNextDaySocIngestor + +UTC = timezone.utc +_MAX_MONTHLY_ARCHIVE_BYTES = 256 * 1024 * 1024 + + +@dataclass(frozen=True, slots=True) +class HistoricalNextDayBackfillResult: + artifact_sha256: str + outer_replayed: bool + daily_artifact_count: int + daily_artifacts_replayed: int + source_rows: int + raw_inserted: int + raw_replayed: int + effective_candidates: int + effective_applied: int + effective_replayed: int + source_null_count: int + percentage_count: int + + +@dataclass(frozen=True, slots=True) +class _ValidatedDaily: + member: NextDayDailyMemberRef + artifact_sha256: str + selected_rows: int + + +def _default_connect(database_url: str, **kwargs: object) -> Any: + return import_module("psycopg").connect(database_url, **kwargs) + + +@contextmanager +def _connection_scope( + connect: Callable[..., Any], + database_url: str, +) -> Iterator[Any]: + connection = connect(database_url) + try: + yield connection + finally: + connection.close() + + +def _require_utc(value: datetime, field: str) -> None: + if ( + type(value) is not datetime + or value.tzinfo is None + or value.utcoffset() != timedelta(0) + ): + raise ValueError(f"{field} must be UTC-aware") + + +def _source_last_modified(value: str | None) -> datetime | None: + if value is None: + return None + parsed = parsedate_to_datetime(value) + if parsed.tzinfo is None: + raise ValueError("source Last-Modified must include a timezone") + return parsed.astimezone(UTC) + + +def _candidate_report_dates(start: datetime, end: datetime) -> frozenset[date]: + bounds = nextday_report_date_bounds(start, end) + if bounds is None: + return frozenset() + first, last = bounds + dates: set[date] = set() + current = first + while current <= last: + dates.add(current) + current += timedelta(days=1) + return frozenset(dates) + + +def _validate_inputs( + assets: Iterable[BatteryAsset], + claim: BackfillClaim, + start: datetime, + end: datetime, + ingestion_version: int, +) -> tuple[BatteryAsset, ...]: + materialized_assets = tuple(assets) + if not materialized_assets or any(type(asset) is not BatteryAsset for asset in materialized_assets): + raise ValueError("assets must contain reviewed BatteryAsset records") + if type(claim) is not BackfillClaim or claim.feed != "nextday_soc": + raise ValueError("claim must be a nextday_soc BackfillClaim") + if claim.report_date.day != 1: + raise ValueError("nextday_soc claim must use the first day of the month") + _require_utc(start, "start") + _require_utc(end, "end") + if end <= start: + raise ValueError("end must be later than start") + if type(ingestion_version) is not int or ingestion_version < 1: + raise ValueError("ingestion_version must be a positive integer") + return materialized_assets + + +def _fail_claim( + database_url: str, + claim: BackfillClaim, + error_summary: str, + *, + connect: Callable[..., Any], + ledger_factory: Callable[[Any], Any], +) -> None: + with _connection_scope(connect, database_url) as connection: + ledger_factory(connection).fail(claim, error_summary=error_summary) + + +def run_nextday_soc_backfill_claim( + database_url: str, + assets: Iterable[BatteryAsset], + claim: BackfillClaim, + start: datetime, + end: datetime, + *, + ingestion_version: int, + connect: Callable[..., Any] = _default_connect, + fetch: Callable[..., NemwebHttpResource] = fetch_nemweb_resource, + clock: Callable[[], datetime] = lambda: datetime.now(UTC), + ledger_factory: Callable[[Any], Any] = PostgreSQLBackfillLedger, + registrar_factory: Callable[[Any], Any] = PostgreSQLBackfillArtifactRegistrar, + nested_registrar_factory: Callable[[Any], Any] = PostgreSQLNestedSourceArtifactRegistrar, + validate_archive: Callable[ + [NextDayMonthlyArchiveRef, bytes], NextDayMonthlyArchiveManifest + ] = validate_nextday_monthly_archive, + read_daily: Callable[ + [NextDayMonthlyArchiveManifest, bytes, NextDayDailyMemberRef], + NextDayDailyArtifact, + ] = read_nextday_daily_artifact, + parse_csv: Callable[..., tuple[Any, ...]] = parse_nextday_unit_solution_soc, + ingestor_factory: Callable[[Any], Any] = PostgreSQLNextDaySocIngestor, +) -> Any: + """Validate all selected daily reports before any SOC observation write.""" + + materialized_assets = _validate_inputs( + assets, claim, start, end, ingestion_version + ) + try: + downloaded_at = clock() + _require_utc(downloaded_at, "downloaded_at") + resource = fetch(claim.source_url, max_bytes=_MAX_MONTHLY_ARCHIVE_BYTES) + if resource.requested_url != claim.source_url or resource.resolved_url != claim.source_url: + raise ValueError("monthly archive response URL changed") + outer_receipt = BackfillArtifactReceipt( + claim, + downloaded_at, + _source_last_modified(resource.last_modified), + resource.body, + ) + with _connection_scope(connect, database_url) as connection: + outer_result = registrar_factory(connection).record(outer_receipt) + + reference = NextDayMonthlyArchiveRef( + claim.report_date, + claim.source_url.rsplit("/", 1)[-1], + claim.source_url, + len(resource.body), + downloaded_at, + ) + manifest = validate_archive(reference, resource.body) + if manifest.sha256 != outer_result.artifact_sha256: + raise ValueError("monthly manifest SHA does not match registered artifact") + candidate_dates = _candidate_report_dates(start, end) + selected_members = tuple( + member for member in manifest.members if member.report_date in candidate_dates + ) + if not selected_members: + raise ValueError("no daily Next Day artifact intersects the requested range") + duids = frozenset(asset.duid for asset in materialized_assets) + validated: list[_ValidatedDaily] = [] + daily_artifacts_replayed = 0 + for member in selected_members: + daily = read_daily(manifest, resource.body, member) + nested_receipt = NestedSourceArtifactReceipt( + outer_result.artifact_sha256, + member.report_date, + claim.source_url, + member.filename, + member.publication_id, + member.artifact_published_at, + downloaded_at, + daily.raw_zip_bytes, + ) + with _connection_scope(connect, database_url) as connection: + nested_result = nested_registrar_factory(connection).record( + nested_receipt + ) + if nested_result.artifact_sha256 != daily.sha256: + raise ValueError("daily artifact SHA does not match registration") + daily_artifacts_replayed += int(nested_result.replayed) + observations = tuple(parse_csv( + daily.csv_bytes.decode("utf-8-sig", errors="strict"), + duids=duids, + source_artifact_id=daily.sha256, + downloaded_at=downloaded_at, + ingestion_version=ingestion_version, + correction_version=int(member.publication_id), + )) + selected = tuple( + observation + for observation in observations + if start <= observation.interval_start < end + ) + validated.append( + _ValidatedDaily(member, daily.sha256, len(selected)) + ) + + source_rows = 0 + raw_inserted = 0 + raw_replayed = 0 + effective_candidates = 0 + effective_applied = 0 + effective_replayed = 0 + source_null_count = 0 + percentage_count = 0 + for validated_daily in validated: + daily = read_daily( + manifest, resource.body, validated_daily.member + ) + if daily.sha256 != validated_daily.artifact_sha256: + raise ValueError("daily artifact changed between validation passes") + observations = tuple(parse_csv( + daily.csv_bytes.decode("utf-8-sig", errors="strict"), + duids=duids, + source_artifact_id=daily.sha256, + downloaded_at=downloaded_at, + ingestion_version=ingestion_version, + correction_version=int(validated_daily.member.publication_id), + )) + selected = tuple( + observation + for observation in observations + if start <= observation.interval_start < end + ) + if len(selected) != validated_daily.selected_rows: + raise ValueError("selected daily rows changed between validation passes") + if not selected: + continue + with _connection_scope(connect, database_url) as connection: + ingestion = ingestor_factory(connection).ingest( + selected, materialized_assets + ) + source_rows += ingestion.source_rows + raw_inserted += ingestion.raw_inserted + raw_replayed += ingestion.raw_replayed + effective_candidates += ingestion.effective_candidates + effective_applied += ingestion.effective_applied + effective_replayed += ingestion.effective_replayed + source_null_count += ingestion.source_null_count + percentage_count += ingestion.percentage_count + + fully_replayed = ( + outer_result.replayed + and daily_artifacts_replayed == len(validated) + and raw_inserted == 0 + and raw_replayed == source_rows + and effective_applied == 0 + and effective_replayed == effective_candidates + ) + completion = BackfillItemCompletion(fully_replayed, source_rows) + with _connection_scope(connect, database_url) as connection: + ledger_factory(connection).complete(claim, completion) + return HistoricalNextDayBackfillResult( + outer_result.artifact_sha256, + outer_result.replayed, + len(validated), + daily_artifacts_replayed, + source_rows, + raw_inserted, + raw_replayed, + effective_candidates, + effective_applied, + effective_replayed, + source_null_count, + percentage_count, + ) + except Exception as exc: + _fail_claim( + database_url, + claim, + type(exc).__name__, + connect=connect, + ledger_factory=ledger_factory, + ) + raise + + +__all__ = [ + "HistoricalNextDayBackfillResult", + "run_nextday_soc_backfill_claim", +] diff --git a/app/backend/batterywatch_api/nextday_archives.py b/app/backend/batterywatch_api/nextday_archives.py index 1beb6c9..438e30d 100644 --- a/app/backend/batterywatch_api/nextday_archives.py +++ b/app/backend/batterywatch_api/nextday_archives.py @@ -190,11 +190,53 @@ def _next_month(value: date) -> date: ) +def _report_date_for_interval(value: datetime) -> date: + local = value.astimezone(_NEM_TIMEZONE) + return ( + local.date() - timedelta(days=1) + if (local.hour, local.minute) <= (4, 0) + else local.date() + ) + + +def nextday_report_date_bounds( + start: datetime, + end: datetime, +) -> tuple[date, date] | None: + """Return report-date bounds containing aligned five-minute intervals.""" + + start_utc = _strict_utc(start, "start") + end_utc = _strict_utc(end, "end") + if end_utc <= start_utc: + raise NextDayArchiveError("Next Day archive range must be increasing") + first_interval = start_utc.replace( + minute=start_utc.minute - (start_utc.minute % 5), + second=0, + microsecond=0, + ) + if first_interval < start_utc: + first_interval += timedelta(minutes=5) + end_probe = end_utc - timedelta(microseconds=1) + last_interval = end_probe.replace( + minute=end_probe.minute - (end_probe.minute % 5), + second=0, + microsecond=0, + ) + if first_interval > last_interval: + return None + return ( + _report_date_for_interval(first_interval), + _report_date_for_interval(last_interval), + ) + + def _intersecting_months(start: datetime, end: datetime) -> tuple[date, ...]: - first_local = start.astimezone(_NEM_TIMEZONE) - last_local = (end - timedelta(microseconds=1)).astimezone(_NEM_TIMEZONE) - current = date(first_local.year, first_local.month, 1) - last = date(last_local.year, last_local.month, 1) + bounds = nextday_report_date_bounds(start, end) + if bounds is None: + return () + first_report, last_report = bounds + current = date(first_report.year, first_report.month, 1) + last = date(last_report.year, last_report.month, 1) months: list[date] = [] while current <= last: months.append(current) @@ -259,5 +301,6 @@ def plan_nextday_monthly_archives( "NextDayMonthlyArchivePlan", "NextDayMonthlyArchiveRef", "discover_nextday_monthly_archives", + "nextday_report_date_bounds", "plan_nextday_monthly_archives", ] diff --git a/app/backend/batterywatch_api/nextday_soc.py b/app/backend/batterywatch_api/nextday_soc.py index 21c7972..995853d 100644 --- a/app/backend/batterywatch_api/nextday_soc.py +++ b/app/backend/batterywatch_api/nextday_soc.py @@ -12,7 +12,8 @@ _NEM_TIMEZONE = timezone(timedelta(hours=10)) _METADATA_PREFIX = ("C", "NEMP.WORLD", "NEXT_DAY_DISPATCH", "AEMO", "PUBLIC") -_TABLE_PREFIX = ("DISPATCH", "UNIT_SOLUTION", "6") +_TABLE_PREFIX = ("DISPATCH", "UNIT_SOLUTION") +_ACCEPTED_TABLE_VERSIONS = frozenset(("5", "6")) _REQUIRED_COLUMNS = frozenset( ( "SETTLEMENTDATE", @@ -139,7 +140,7 @@ def parse_nextday_unit_solution_soc( ingestion_version: int, correction_version: int = 0, ) -> tuple[NextDaySocObservation, ...]: - """Parse current public Next Day UnitSolution v6 rows for reviewed DUIDs.""" + """Parse public Next Day UnitSolution v5/v6 rows for reviewed DUIDs.""" if type(payload) is not str or not payload: raise NextDaySocParseError("invalid payload") @@ -161,6 +162,7 @@ def parse_nextday_unit_solution_soc( downloaded = _validated_downloaded_at(downloaded_at, report_timestamp) header: list[str] | None = None + table_identity: tuple[str, ...] | None = None observations: list[NextDaySocObservation] = [] seen: set[tuple[str, datetime, int, int]] = set() report_record_count = 1 @@ -178,16 +180,24 @@ def parse_nextday_unit_solution_soc( ) continue - if row[:4] == ["I", *_TABLE_PREFIX]: + if row[:3] == ["I", *_TABLE_PREFIX]: + if len(row) < 4 or row[3] not in _ACCEPTED_TABLE_VERSIONS: + raise NextDaySocParseError("unsupported UnitSolution version") if header is not None or len(row) != len(set(row)): raise NextDaySocParseError("invalid UnitSolution header") if not _REQUIRED_COLUMNS.issubset(row[4:]): raise NextDaySocParseError("missing UnitSolution columns") header = row + table_identity = tuple(row[1:4]) continue - if row[:4] != ["D", *_TABLE_PREFIX]: + if row[:3] != ["D", *_TABLE_PREFIX]: continue - if header is None or len(row) != len(header): + if ( + header is None + or table_identity is None + or tuple(row[1:4]) != table_identity + or len(row) != len(header) + ): raise NextDaySocParseError("malformed UnitSolution row") values = dict(zip(header[4:], row[4:])) duid = values["DUID"] @@ -235,7 +245,7 @@ def parse_nextday_unit_solution_soc( if expected_count != report_record_count: raise NextDaySocParseError("report row count mismatch") if header is None: - raise NextDaySocParseError("missing UnitSolution v6 table") + raise NextDaySocParseError("missing UnitSolution v5/v6 table") return tuple( sorted( observations, diff --git a/app/backend/tests/fixtures/historical/nextday-unit-solution-soc-v5-20250701-reduced.csv b/app/backend/tests/fixtures/historical/nextday-unit-solution-soc-v5-20250701-reduced.csv new file mode 100644 index 0000000..a5bd6d3 --- /dev/null +++ b/app/backend/tests/fixtures/historical/nextday-unit-solution-soc-v5-20250701-reduced.csv @@ -0,0 +1,4 @@ +C,NEMP.WORLD,NEXT_DAY_DISPATCH,AEMO,PUBLIC,2025/07/02,04:10:00,0000000470129643,NEXT_DAY_DISPATCH,0000000470129639 +I,DISPATCH,UNIT_SOLUTION,5,SETTLEMENTDATE,RUNNO,DUID,TRADETYPE,DISPATCHINTERVAL,INTERVENTION,CONNECTIONPOINTID,DISPATCHMODE,AGCSTATUS,INITIALMW,TOTALCLEARED,RAMPDOWNRATE,RAMPUPRATE,LOWER5MIN,LOWER60SEC,LOWER6SEC,RAISE5MIN,RAISE60SEC,RAISE6SEC,DOWNEPF,UPEPF,MARGINAL5MINVALUE,MARGINAL60SECVALUE,MARGINAL6SECVALUE,MARGINALVALUE,VIOLATION5MINDEGREE,VIOLATION60SECDEGREE,VIOLATION6SECDEGREE,VIOLATIONDEGREE,LASTCHANGED,LOWERREG,RAISEREG,AVAILABILITY,RAISE6SECFLAGS,RAISE60SECFLAGS,RAISE5MINFLAGS,RAISEREGFLAGS,LOWER6SECFLAGS,LOWER60SECFLAGS,LOWER5MINFLAGS,LOWERREGFLAGS,RAISEREGAVAILABILITY,RAISEREGENABLEMENTMAX,RAISEREGENABLEMENTMIN,LOWERREGAVAILABILITY,LOWERREGENABLEMENTMAX,LOWERREGENABLEMENTMIN,RAISE6SECACTUALAVAILABILITY,RAISE60SECACTUALAVAILABILITY,RAISE5MINACTUALAVAILABILITY,RAISEREGACTUALAVAILABILITY,LOWER6SECACTUALAVAILABILITY,LOWER60SECACTUALAVAILABILITY,LOWER5MINACTUALAVAILABILITY,LOWERREGACTUALAVAILABILITY,SEMIDISPATCHCAP,DISPATCHMODETIME,LOWER1SEC,RAISE1SEC,RAISE1SECFLAGS,LOWER1SECFLAGS,RAISE1SECACTUALAVAILABILITY,LOWER1SECACTUALAVAILABILITY,CONFORMANCE_MODE,UIGF,INITIAL_ENERGY_STORAGE,ENERGY_STORAGE,MIN_AVAILABILITY +D,DISPATCH,UNIT_SOLUTION,5,"2025/07/01 04:05:00",1,ADPBA1,0,20250701001,0,SMVE14,0,0,-0.001,0,93.12,93.12,0,0,0,0,0,0,,,,,,,,,,,"2025/07/01 04:00:03",0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,,0,0,4E-05,0 +C,END OF REPORT,4 diff --git a/app/backend/tests/test_historical_nextday_backfill.py b/app/backend/tests/test_historical_nextday_backfill.py new file mode 100644 index 0000000..cc151bb --- /dev/null +++ b/app/backend/tests/test_historical_nextday_backfill.py @@ -0,0 +1,526 @@ +"""Tests for two-pass authoritative Next Day monthly backfill claims.""" + +from datetime import date, datetime, timezone +import hashlib +from pathlib import Path +from types import SimpleNamespace +import unittest + +from batterywatch_api.backfill_artifacts import BackfillArtifactResult +from batterywatch_api.backfill_ledger import BackfillClaim, BackfillItemCompletion +from batterywatch_api.battery_assets import load_battery_assets +from batterywatch_api.historical_nextday_backfill import ( + HistoricalNextDayBackfillResult, + run_nextday_soc_backfill_claim, +) +from batterywatch_api.nemweb_http import NemwebHttpResource +from batterywatch_api.nested_source_artifacts import NestedSourceArtifactResult +from batterywatch_api.nextday_archives import ( + NEXTDAY_ARCHIVE_INDEX_URL, + NextDayMonthlyArchiveRef, +) +from batterywatch_api.nextday_monthly_extraction import ( + NextDayDailyArtifact, + NextDayDailyMemberRef, + NextDayMonthlyArchiveManifest, +) +from batterywatch_api.nextday_soc_ingestion import NextDaySocIngestionResult + +UTC = timezone.utc +OUTER_URL = ( + NEXTDAY_ARCHIVE_INDEX_URL + "PUBLIC_NEXT_DAY_DISPATCH_20250701.zip" +) +ASSETS = load_battery_assets( + Path(__file__).resolve().parents[2] / "config/battery_assets.json" +) + + +class FakeConnection: + def close(self) -> None: + pass + + +class FakeLedger: + failures: list[tuple[str, str]] = [] + completions: list[tuple[str, BackfillItemCompletion]] = [] + + def __init__(self, connection) -> None: + pass + + def fail(self, claim, *, error_summary): + self.failures.append((claim.run_id, error_summary)) + return SimpleNamespace(replayed=False) + + def complete(self, claim, completion): + self.completions.append((claim.run_id, completion)) + return SimpleNamespace(replayed=False) + + +class FakeOuterRegistrar: + replayed = False + + def __init__(self, connection) -> None: + pass + + def record(self, receipt): + return BackfillArtifactResult( + "a" * 64, len(receipt.raw_archive), self.replayed + ) + + +class FakeNestedRegistrar: + receipts = [] + replayed = False + + def __init__(self, connection) -> None: + pass + + def record(self, receipt): + self.receipts.append(receipt) + digest = hashlib.sha256(receipt.raw_bytes).hexdigest() + return NestedSourceArtifactResult( + digest, len(receipt.raw_bytes), self.replayed + ) + + +def _member(day: int) -> NextDayDailyMemberRef: + publication_id = f"{47_000_000 + day:016d}" + return NextDayDailyMemberRef( + date(2025, 7, day), + f"PUBLIC_NEXT_DAY_DISPATCH_202507{day:02d}_{publication_id}.zip", + publication_id, + datetime(2025, 7, day, 18, 11, tzinfo=UTC), + 10, + 20, + day, + ) + + +class HistoricalNextDayBackfillTests(unittest.TestCase): + def setUp(self) -> None: + FakeLedger.failures = [] + FakeLedger.completions = [] + FakeOuterRegistrar.replayed = False + FakeNestedRegistrar.receipts = [] + FakeNestedRegistrar.replayed = False + + def test_validates_all_selected_daily_reports_before_first_ingest(self) -> None: + raw_outer = b"monthly outer" + reference = NextDayMonthlyArchiveRef( + date(2025, 7, 1), + "PUBLIC_NEXT_DAY_DISPATCH_20250701.zip", + OUTER_URL, + len(raw_outer), + datetime(2026, 8, 30, tzinfo=UTC), + ) + members = (_member(1), _member(2), _member(3)) + manifest = NextDayMonthlyArchiveManifest(reference, "a" * 64, members) + parser_calls: list[str] = [] + ingestor_calls: list[object] = [] + + def fetch(url, *, max_bytes): + self.assertEqual(url, OUTER_URL) + return NemwebHttpResource(url, url, raw_outer, "application/zip", None, None) + + def validate(ref, payload): + self.assertEqual((ref, payload), (reference, raw_outer)) + return manifest + + def read_daily(received_manifest, payload, member): + self.assertEqual((received_manifest, payload), (manifest, raw_outer)) + raw = f"zip-{member.report_date.day}".encode() + csv_bytes = f"day-{member.report_date.day}".encode() + return NextDayDailyArtifact( + member, + hashlib.sha256(raw).hexdigest(), + raw, + member.filename.removesuffix(".zip") + ".CSV", + csv_bytes, + ) + + def parse(payload, **kwargs): + parser_calls.append(payload) + if payload == "day-3": + raise ValueError("bad final daily report") + return (SimpleNamespace(interval_start=datetime(2025, 7, 2, tzinfo=UTC)),) + + def ingestor_factory(connection): + ingestor_calls.append(connection) + return SimpleNamespace(ingest=lambda observations, assets: None) + + claim = BackfillClaim( + "soc-run-202507", + "nextday_soc", + date(2025, 7, 1), + OUTER_URL, + 1, + ) + with self.assertRaisesRegex(ValueError, "bad final daily report"): + run_nextday_soc_backfill_claim( + "postgresql://redacted", + ASSETS, + claim, + datetime(2025, 7, 1, tzinfo=UTC), + datetime(2025, 7, 3, tzinfo=UTC), + ingestion_version=3, + connect=lambda *args, **kwargs: FakeConnection(), + fetch=fetch, + clock=lambda: datetime(2026, 8, 30, tzinfo=UTC), + ledger_factory=FakeLedger, + registrar_factory=FakeOuterRegistrar, + nested_registrar_factory=FakeNestedRegistrar, + validate_archive=validate, + read_daily=read_daily, + parse_csv=parse, + ingestor_factory=ingestor_factory, + ) + + self.assertEqual(parser_calls, ["day-1", "day-2", "day-3"]) + self.assertEqual(len(FakeNestedRegistrar.receipts), 3) + self.assertEqual(ingestor_calls, []) + self.assertEqual(FakeLedger.failures, [(claim.run_id, "ValueError")]) + + def test_revalidates_then_ingests_each_day_and_completes_claim(self) -> None: + raw_outer = b"monthly outer" + downloaded_at = datetime(2026, 8, 30, tzinfo=UTC) + reference = NextDayMonthlyArchiveRef( + date(2025, 7, 1), + "PUBLIC_NEXT_DAY_DISPATCH_20250701.zip", + OUTER_URL, + len(raw_outer), + downloaded_at, + ) + members = (_member(1), _member(2)) + manifest = NextDayMonthlyArchiveManifest(reference, "a" * 64, members) + parse_calls: list[tuple[str, int]] = [] + read_calls: list[int] = [] + ingest_calls: list[tuple[object, ...]] = [] + + def fetch(url, *, max_bytes): + return NemwebHttpResource( + url, url, raw_outer, "application/zip", None, None + ) + + def read_daily(received_manifest, payload, member): + read_calls.append(member.report_date.day) + raw = f"zip-{member.report_date.day}".encode() + return NextDayDailyArtifact( + member, + hashlib.sha256(raw).hexdigest(), + raw, + member.filename.removesuffix(".zip") + ".CSV", + f"day-{member.report_date.day}".encode(), + ) + + def parse(payload, **kwargs): + parse_calls.append((payload, kwargs["correction_version"])) + day = int(payload[-1]) + return ( + SimpleNamespace( + interval_start=datetime(2025, 7, day, 12, tzinfo=UTC) + ), + ) + + class FakeIngestor: + def __init__(self, connection): + pass + + def ingest(self, observations, assets): + materialized = tuple(observations) + ingest_calls.append(materialized) + return NextDaySocIngestionResult(1, 1, 0, 1, 1, 0, 0, 1) + + claim = BackfillClaim( + "soc-run-202507", + "nextday_soc", + date(2025, 7, 1), + OUTER_URL, + 1, + ) + result = run_nextday_soc_backfill_claim( + "postgresql://redacted", + ASSETS, + claim, + datetime(2025, 7, 1, tzinfo=UTC), + datetime(2025, 7, 3, tzinfo=UTC), + ingestion_version=3, + connect=lambda *args, **kwargs: FakeConnection(), + fetch=fetch, + clock=lambda: downloaded_at, + ledger_factory=FakeLedger, + registrar_factory=FakeOuterRegistrar, + nested_registrar_factory=FakeNestedRegistrar, + validate_archive=lambda ref, payload: manifest, + read_daily=read_daily, + parse_csv=parse, + ingestor_factory=FakeIngestor, + ) + + self.assertEqual(read_calls, [1, 2, 1, 2]) + self.assertEqual( + parse_calls, + [ + ("day-1", 47_000_001), + ("day-2", 47_000_002), + ("day-1", 47_000_001), + ("day-2", 47_000_002), + ], + ) + self.assertEqual([len(call) for call in ingest_calls], [1, 1]) + self.assertEqual( + FakeLedger.completions, + [(claim.run_id, BackfillItemCompletion(False, 2))], + ) + self.assertEqual(FakeLedger.failures, []) + self.assertEqual( + result, + HistoricalNextDayBackfillResult( + "a" * 64, False, 2, 0, 2, 2, 0, 2, 2, 0, 0, 2 + ), + ) + + def test_exact_replay_is_classified_deterministically(self) -> None: + raw_outer = b"monthly outer" + downloaded_at = datetime(2026, 8, 30, tzinfo=UTC) + member = _member(1) + reference = NextDayMonthlyArchiveRef( + date(2025, 7, 1), + "PUBLIC_NEXT_DAY_DISPATCH_20250701.zip", + OUTER_URL, + len(raw_outer), + downloaded_at, + ) + manifest = NextDayMonthlyArchiveManifest( + reference, "a" * 64, (member,) + ) + FakeOuterRegistrar.replayed = True + FakeNestedRegistrar.replayed = True + + def read_daily(received_manifest, payload, received_member): + raw = b"zip-1" + return NextDayDailyArtifact( + received_member, + hashlib.sha256(raw).hexdigest(), + raw, + received_member.filename.removesuffix(".zip") + ".CSV", + b"day-1", + ) + + class ReplayIngestor: + def __init__(self, connection): + pass + + def ingest(self, observations, assets): + return NextDaySocIngestionResult(1, 0, 1, 1, 0, 1, 0, 1) + + claim = BackfillClaim( + "soc-replay-202507", + "nextday_soc", + date(2025, 7, 1), + OUTER_URL, + 2, + ) + result = run_nextday_soc_backfill_claim( + "postgresql://redacted", + ASSETS, + claim, + datetime(2025, 7, 1, tzinfo=UTC), + datetime(2025, 7, 1, 14, tzinfo=UTC), + ingestion_version=3, + connect=lambda *args, **kwargs: FakeConnection(), + fetch=lambda url, **kwargs: NemwebHttpResource( + url, url, raw_outer, "application/zip", None, None + ), + clock=lambda: downloaded_at, + ledger_factory=FakeLedger, + registrar_factory=FakeOuterRegistrar, + nested_registrar_factory=FakeNestedRegistrar, + validate_archive=lambda ref, payload: manifest, + read_daily=read_daily, + parse_csv=lambda payload, **kwargs: ( + SimpleNamespace( + interval_start=datetime(2025, 7, 1, 12, tzinfo=UTC) + ), + ), + ingestor_factory=ReplayIngestor, + ) + + self.assertEqual( + FakeLedger.completions, + [(claim.run_id, BackfillItemCompletion(True, 1))], + ) + self.assertEqual( + result, + HistoricalNextDayBackfillResult( + "a" * 64, True, 1, 1, 1, 0, 1, 1, 0, 1, 0, 1 + ), + ) + + class IncompleteReplayIngestor: + def __init__(self, connection): + pass + + def ingest(self, observations, assets): + return NextDaySocIngestionResult(1, 0, 0, 1, 0, 0, 0, 0) + + FakeLedger.completions = [] + partial_claim = BackfillClaim( + "soc-partial-replay-202507", + "nextday_soc", + date(2025, 7, 1), + OUTER_URL, + 3, + ) + run_nextday_soc_backfill_claim( + "postgresql://redacted", + ASSETS, + partial_claim, + datetime(2025, 7, 1, tzinfo=UTC), + datetime(2025, 7, 1, 14, tzinfo=UTC), + ingestion_version=3, + connect=lambda *args, **kwargs: FakeConnection(), + fetch=lambda url, **kwargs: NemwebHttpResource( + url, url, raw_outer, "application/zip", None, None + ), + clock=lambda: downloaded_at, + ledger_factory=FakeLedger, + registrar_factory=FakeOuterRegistrar, + nested_registrar_factory=FakeNestedRegistrar, + validate_archive=lambda ref, payload: manifest, + read_daily=read_daily, + parse_csv=lambda payload, **kwargs: ( + SimpleNamespace( + interval_start=datetime(2025, 7, 1, 12, tzinfo=UTC) + ), + ), + ingestor_factory=IncompleteReplayIngestor, + ) + self.assertEqual( + FakeLedger.completions, + [(partial_claim.run_id, BackfillItemCompletion(False, 1))], + ) + + def test_fails_closed_when_range_selects_no_daily_member(self) -> None: + raw_outer = b"monthly outer" + downloaded_at = datetime(2026, 8, 30, tzinfo=UTC) + reference = NextDayMonthlyArchiveRef( + date(2025, 7, 1), + "PUBLIC_NEXT_DAY_DISPATCH_20250701.zip", + OUTER_URL, + len(raw_outer), + downloaded_at, + ) + manifest = NextDayMonthlyArchiveManifest( + reference, "a" * 64, (_member(2),) + ) + claim = BackfillClaim( + "soc-empty-202507", + "nextday_soc", + date(2025, 7, 1), + OUTER_URL, + 1, + ) + + with self.assertRaisesRegex(ValueError, "no daily"): + run_nextday_soc_backfill_claim( + "postgresql://redacted", + ASSETS, + claim, + datetime(2025, 7, 1, tzinfo=UTC), + datetime(2025, 7, 1, 1, tzinfo=UTC), + ingestion_version=3, + connect=lambda *args, **kwargs: FakeConnection(), + fetch=lambda url, **kwargs: NemwebHttpResource( + url, url, raw_outer, "application/zip", None, None + ), + clock=lambda: downloaded_at, + ledger_factory=FakeLedger, + registrar_factory=FakeOuterRegistrar, + validate_archive=lambda ref, payload: manifest, + read_daily=lambda *args: self.fail("must not read an unselected day"), + ) + + self.assertEqual(FakeLedger.completions, []) + self.assertEqual(FakeLedger.failures, [(claim.run_id, "ValueError")]) + + def test_maps_nem_0400_interval_to_previous_report_date(self) -> None: + raw_outer = b"monthly outer" + downloaded_at = datetime(2026, 8, 30, tzinfo=UTC) + reference = NextDayMonthlyArchiveRef( + date(2025, 7, 1), + "PUBLIC_NEXT_DAY_DISPATCH_20250701.zip", + OUTER_URL, + len(raw_outer), + downloaded_at, + ) + members = (_member(1), _member(2)) + manifest = NextDayMonthlyArchiveManifest(reference, "a" * 64, members) + ingested: list[tuple[object, ...]] = [] + + def read_daily(received_manifest, payload, member): + raw = f"zip-{member.report_date.day}".encode() + return NextDayDailyArtifact( + member, + hashlib.sha256(raw).hexdigest(), + raw, + member.filename.removesuffix(".zip") + ".CSV", + f"day-{member.report_date.day}".encode(), + ) + + def parse(payload, **kwargs): + day = int(payload[-1]) + interval = datetime(2025, 7, 1, 18, 0, tzinfo=UTC) + if day == 2: + interval = datetime(2025, 7, 1, 18, 5, tzinfo=UTC) + return (SimpleNamespace(interval_start=interval),) + + class BoundaryIngestor: + def __init__(self, connection): + pass + + def ingest(self, observations, assets): + materialized = tuple(observations) + ingested.append(materialized) + return NextDaySocIngestionResult(1, 1, 0, 1, 1, 0, 0, 1) + + claim = BackfillClaim( + "soc-boundary-202507", + "nextday_soc", + date(2025, 7, 1), + OUTER_URL, + 1, + ) + run_nextday_soc_backfill_claim( + "postgresql://redacted", + ASSETS, + claim, + datetime(2025, 7, 1, 18, 0, tzinfo=UTC), + datetime(2025, 7, 1, 18, 1, tzinfo=UTC), + ingestion_version=3, + connect=lambda *args, **kwargs: FakeConnection(), + fetch=lambda url, **kwargs: NemwebHttpResource( + url, url, raw_outer, "application/zip", None, None + ), + clock=lambda: downloaded_at, + ledger_factory=FakeLedger, + registrar_factory=FakeOuterRegistrar, + nested_registrar_factory=FakeNestedRegistrar, + validate_archive=lambda ref, payload: manifest, + read_daily=read_daily, + parse_csv=parse, + ingestor_factory=BoundaryIngestor, + ) + + self.assertEqual( + [receipt.filename for receipt in FakeNestedRegistrar.receipts], + [members[0].filename], + ) + self.assertEqual([len(rows) for rows in ingested], [1]) + self.assertEqual( + FakeLedger.completions, + [(claim.run_id, BackfillItemCompletion(False, 1))], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/backend/tests/test_nextday_archives.py b/app/backend/tests/test_nextday_archives.py index 05ffe3f..5f0865f 100644 --- a/app/backend/tests/test_nextday_archives.py +++ b/app/backend/tests/test_nextday_archives.py @@ -75,6 +75,33 @@ def test_plans_intersecting_nem_months_and_reports_missing_archives(self) -> Non ) self.assertEqual(plan.missing_months, (date(2025, 9, 1),)) + def test_plans_months_using_the_nem_0405_report_day_cutoff(self) -> None: + references = discover_nextday_monthly_archives( + INDEX_HTML, + index_url=NEXTDAY_ARCHIVE_INDEX_URL, + ) + cases = ( + ( + datetime(2025, 7, 31, 18, 0, tzinfo=UTC), + datetime(2025, 7, 31, 18, 1, tzinfo=UTC), + date(2025, 7, 1), + ), + ( + datetime(2025, 7, 31, 18, 5, tzinfo=UTC), + datetime(2025, 7, 31, 18, 6, tzinfo=UTC), + date(2025, 8, 1), + ), + ) + + for start, end, expected_month in cases: + with self.subTest(start=start): + plan = plan_nextday_monthly_archives(start, end, references) + self.assertEqual( + tuple(item.report_month for item in plan.items), + (expected_month,), + ) + self.assertEqual(plan.missing_months, ()) + def test_rejects_malformed_duplicate_and_oversized_listing_rows(self) -> None: malformed_cases = ( INDEX_HTML.replace( diff --git a/app/backend/tests/test_nextday_soc.py b/app/backend/tests/test_nextday_soc.py index 751c46c..b296ca0 100644 --- a/app/backend/tests/test_nextday_soc.py +++ b/app/backend/tests/test_nextday_soc.py @@ -9,6 +9,12 @@ UTC = timezone.utc FIXTURE = Path(__file__).parent / "fixtures" / "historical" / "nextday-unit-solution-soc-20260829-reduced.csv" +V5_FIXTURE = ( + Path(__file__).parent + / "fixtures" + / "historical" + / "nextday-unit-solution-soc-v5-20250701-reduced.csv" +) class NextDaySocParserTests(unittest.TestCase): @@ -56,13 +62,40 @@ def test_filters_to_reviewed_duids_without_inventing_missing_rows(self) -> None: self.assertEqual(len(observations), 2) self.assertEqual({item.duid for item in observations}, {"ADPBA1"}) + def test_parses_real_derived_v5_initial_energy_storage(self) -> None: + observations = parse_nextday_unit_solution_soc( + V5_FIXTURE.read_text(encoding="utf-8"), + duids=frozenset(("ADPBA1",)), + source_artifact_id=( + "5c3653d787824f5250210de11f67a6ff" + "334433894016290e45952046d84576f4" + ), + downloaded_at=datetime(2026, 8, 30, tzinfo=UTC), + ingestion_version=8, + correction_version=47_012_9643, + ) + + self.assertEqual(len(observations), 1) + observation = observations[0] + self.assertEqual(observation.duid, "ADPBA1") + self.assertEqual(observation.soc_mwh, 0.0) + self.assertEqual( + observation.interval_start, + datetime(2025, 6, 30, 18, 5, tzinfo=UTC), + ) + self.assertEqual( + observation.report_timestamp, + datetime(2025, 7, 1, 18, 10, tzinfo=UTC), + ) + self.assertEqual(observation.publication_latency_seconds, 86_700) + def test_rejects_wrong_version_duplicate_rows_and_bad_trailer_count(self) -> None: duplicate = self.payload.replace( 'C,"END OF REPORT",6', self.payload.splitlines()[2] + '\nC,"END OF REPORT",7', ) cases = ( - self.payload.replace("I,DISPATCH,UNIT_SOLUTION,6,", "I,DISPATCH,UNIT_SOLUTION,5,", 1), + self.payload.replace("I,DISPATCH,UNIT_SOLUTION,6,", "I,DISPATCH,UNIT_SOLUTION,4,", 1), duplicate, self.payload.replace('C,"END OF REPORT",6', 'C,"END OF REPORT",7'), ) @@ -71,6 +104,15 @@ def test_rejects_wrong_version_duplicate_rows_and_bad_trailer_count(self) -> Non with self.assertRaises(NextDaySocParseError): self.parse(payload) + def test_rejects_mixed_accepted_unit_solution_versions(self) -> None: + mixed = self.payload.replace( + "I,DISPATCH,UNIT_SOLUTION,6,", + "I,DISPATCH,UNIT_SOLUTION,5,", + 1, + ) + with self.assertRaises(NextDaySocParseError): + self.parse(mixed) + def test_rejects_negative_nonfinite_or_misaligned_authoritative_values(self) -> None: cases = ( self.payload.replace(",3.786,3.78315,", ",-1,3.78315,", 1), From 3a3affd83ce9d1e8cccba1aac1f31d6635b0b208 Mon Sep 17 00:00:00 2001 From: djurcola-agent Date: Mon, 31 Aug 2026 01:09:28 +1000 Subject: [PATCH 29/30] feat: orchestrate next-day SOC backfills --- .../batterywatch_api/historical_backfill.py | 39 ++++++- app/backend/tests/test_historical_backfill.py | 108 ++++++++++++++++-- 2 files changed, 134 insertions(+), 13 deletions(-) diff --git a/app/backend/batterywatch_api/historical_backfill.py b/app/backend/batterywatch_api/historical_backfill.py index 8341c0a..ec0769e 100644 --- a/app/backend/batterywatch_api/historical_backfill.py +++ b/app/backend/batterywatch_api/historical_backfill.py @@ -21,6 +21,10 @@ HistoricalPriceBackfillResult, run_price_backfill_claim, ) +from .historical_nextday_backfill import ( + HistoricalNextDayBackfillResult, + run_nextday_soc_backfill_claim, +) from .historical_scada_backfill import ( HistoricalScadaBackfillResult, run_scada_backfill_claim, @@ -34,6 +38,7 @@ class HistoricalBackfillResult: scada_results: tuple[HistoricalScadaBackfillResult, ...] price_results: tuple[HistoricalPriceBackfillResult, ...] finalization: BackfillRunFinalization + nextday_results: tuple[HistoricalNextDayBackfillResult, ...] = () @property def claimed_count(self) -> int: @@ -55,17 +60,33 @@ def price_source_record_count(self) -> int: def price_applied_record_count(self) -> int: return sum(result.applied_price_count for result in self.price_results) + @property + def nextday_source_record_count(self) -> int: + return sum(result.source_rows for result in self.nextday_results) + + @property + def nextday_applied_record_count(self) -> int: + return sum(result.effective_applied for result in self.nextday_results) + + @property + def nextday_null_count(self) -> int: + return sum(result.source_null_count for result in self.nextday_results) + + @property + def nextday_percentage_count(self) -> int: + return sum(result.percentage_count for result in self.nextday_results) + @property def replayed_interval_count(self) -> int: return sum(result.replayed_interval_count for result in self.scada_results) + sum( result.replayed_interval_count for result in self.price_results - ) + ) + sum(result.raw_replayed for result in self.nextday_results) @property def replayed_outer_artifact_count(self) -> int: return sum(result.outer_artifact_replayed for result in self.scada_results) + sum( result.outer_artifact_replayed for result in self.price_results - ) + ) + sum(result.outer_replayed for result in self.nextday_results) def _connect(database_url: str, *, connect_timeout: int) -> Any: @@ -97,6 +118,7 @@ def run_historical_backfill( ledger_factory: Callable[[Any], Any] = PostgreSQLBackfillLedger, run_scada: Callable[..., HistoricalScadaBackfillResult] = run_scada_backfill_claim, run_price: Callable[..., HistoricalPriceBackfillResult] = run_price_backfill_claim, + run_nextday: Callable[..., HistoricalNextDayBackfillResult] = run_nextday_soc_backfill_claim, ) -> HistoricalBackfillResult: """Ensure, drain, and finalize one bounded supplied backfill plan.""" @@ -114,6 +136,7 @@ def run_historical_backfill( claims: list[BackfillClaim] = [] scada_results: list[HistoricalScadaBackfillResult] = [] price_results: list[HistoricalPriceBackfillResult] = [] + nextday_results: list[HistoricalNextDayBackfillResult] = [] while True: with _connection(database_url, connect) as connection: claim = ledger_factory(connection).claim_next(spec.run_id) @@ -141,6 +164,17 @@ def run_historical_backfill( spec.requested_end, ) ) + elif claim.feed == "nextday_soc": + nextday_results.append( + run_nextday( + database_url, + materialized_assets, + claim, + spec.requested_start, + spec.requested_end, + spec.ingestion_version, + ) + ) else: raise ValueError("unsupported historical backfill feed") @@ -153,6 +187,7 @@ def run_historical_backfill( tuple(scada_results), tuple(price_results), finalization, + tuple(nextday_results), ) diff --git a/app/backend/tests/test_historical_backfill.py b/app/backend/tests/test_historical_backfill.py index bc35f5a..47da517 100644 --- a/app/backend/tests/test_historical_backfill.py +++ b/app/backend/tests/test_historical_backfill.py @@ -14,6 +14,7 @@ ) from batterywatch_api.battery_assets import BatteryAsset from batterywatch_api.historical_price_backfill import HistoricalPriceBackfillResult +from batterywatch_api.historical_nextday_backfill import HistoricalNextDayBackfillResult from batterywatch_api.historical_scada_backfill import HistoricalScadaBackfillResult from batterywatch_api.historical_backfill import run_historical_backfill @@ -48,6 +49,12 @@ def test_ensures_claims_dispatches_and_finalizes_supplied_plan(self) -> None: "https://www.nemweb.com.au/REPORTS/ARCHIVE/DispatchIS_Reports/" "PUBLIC_DISPATCHIS_20260829.zip", ) + soc_item = BackfillPlanItem( + "nextday_soc", + date(2026, 8, 1), + "https://www.nemweb.com.au/REPORTS/ARCHIVE/Next_Day_Dispatch/" + "PUBLIC_NEXT_DAY_DISPATCH_20260801.zip", + ) scada_claim = BackfillClaim( spec.run_id, scada_item.feed, @@ -62,7 +69,14 @@ def test_ensures_claims_dispatches_and_finalizes_supplied_plan(self) -> None: price_item.source_url, 1, ) - claims = [scada_claim, price_claim, None] + soc_claim = BackfillClaim( + spec.run_id, + soc_item.feed, + soc_item.report_date, + soc_item.source_url, + 1, + ) + claims = [scada_claim, price_claim, soc_claim, None] assets = ( BatteryAsset( "BAT1", @@ -74,7 +88,7 @@ def test_ensures_claims_dispatches_and_finalizes_supplied_plan(self) -> None: datetime(2025, 3, 31, tzinfo=UTC), ), ) - progress = BackfillRunProgress("run-1", "completed", 2, 0, 0, 2, 0, 2) + progress = BackfillRunProgress("run-1", "completed", 3, 0, 0, 3, 0, 3) finalization = BackfillRunFinalization(False, progress) def connect(database_url: str, *, connect_timeout: int) -> Connection: @@ -89,9 +103,9 @@ def __init__(self, connection: Connection) -> None: def ensure_run(self, actual_spec: BackfillRunSpec, items: Any) -> BackfillEnsureResult: self.assert_equal(actual_spec, spec) - self.assert_equal(tuple(items), (scada_item, price_item)) + self.assert_equal(tuple(items), (scada_item, price_item, soc_item)) operations.append(f"ensure:{self.connection.number}") - return BackfillEnsureResult(True, False, 2, 0) + return BackfillEnsureResult(True, False, 3, 0) def claim_next(self, run_id: str) -> BackfillClaim | None: self.assert_equal(run_id, spec.run_id) @@ -128,34 +142,106 @@ def run_price( operations.append("price") return HistoricalPriceBackfillResult(2, 10, 5, 0, False, False) + def run_soc( + database_url: str, + actual_assets: Any, + claim: BackfillClaim, + range_start: datetime, + range_end: datetime, + ingestion_version: int, + ) -> HistoricalNextDayBackfillResult: + self.assertEqual( + ( + database_url, + tuple(actual_assets), + claim, + range_start, + range_end, + ingestion_version, + ), + ("postgresql://private", assets, soc_claim, start, end, 1), + ) + operations.append("soc") + return HistoricalNextDayBackfillResult( + "a" * 64, + False, + 2, + 0, + 20, + 18, + 2, + 20, + 18, + 2, + 3, + 15, + ) + result = run_historical_backfill( "postgresql://private", assets, spec, - (scada_item, price_item), + (scada_item, price_item, soc_item), connect=connect, ledger_factory=Ledger, run_scada=run_scada, run_price=run_price, + run_nextday=run_soc, ) - self.assertEqual(result.ensure_result, BackfillEnsureResult(True, False, 2, 0)) - self.assertEqual(result.claims, (scada_claim, price_claim)) + self.assertEqual(result.ensure_result, BackfillEnsureResult(True, False, 3, 0)) + self.assertEqual(result.claims, (scada_claim, price_claim, soc_claim)) self.assertEqual(result.scada_results, (HistoricalScadaBackfillResult(2, 20, 4, 1, True, False),)) self.assertEqual(result.price_results, (HistoricalPriceBackfillResult(2, 10, 5, 0, False, False),)) + self.assertEqual( + result.nextday_results, + ( + HistoricalNextDayBackfillResult( + "a" * 64, + False, + 2, + 0, + 20, + 18, + 2, + 20, + 18, + 2, + 3, + 15, + ), + ), + ) self.assertEqual(result.finalization, finalization) - self.assertEqual(result.claimed_count, 2) + self.assertEqual(result.claimed_count, 3) self.assertEqual(result.scada_raw_observation_count, 20) self.assertEqual(result.scada_mapped_power_count, 4) self.assertEqual(result.price_source_record_count, 10) self.assertEqual(result.price_applied_record_count, 5) - self.assertEqual(result.replayed_interval_count, 1) + self.assertEqual(result.nextday_source_record_count, 20) + self.assertEqual(result.nextday_applied_record_count, 18) + self.assertEqual(result.nextday_null_count, 3) + self.assertEqual(result.nextday_percentage_count, 15) + self.assertEqual(result.replayed_interval_count, 3) self.assertEqual(result.replayed_outer_artifact_count, 1) self.assertEqual( operations, - ["ensure:1", "claim:2", "scada", "claim:3", "price", "claim:4", "finalize:5"], + [ + "ensure:1", + "claim:2", + "scada", + "claim:3", + "price", + "claim:4", + "soc", + "claim:5", + "finalize:6", + ], + ) + self.assertEqual( + [connection.close_calls for connection in connections], + [1, 1, 1, 1, 1, 1], ) - self.assertEqual([connection.close_calls for connection in connections], [1, 1, 1, 1, 1]) def test_runner_failure_stops_without_finalization_and_preserves_error(self) -> None: connections: list[Connection] = [] From a5b2c635772fe1287ad66018139bbc80680c266d Mon Sep 17 00:00:00 2001 From: djurcola-agent Date: Mon, 31 Aug 2026 01:10:17 +1000 Subject: [PATCH 30/30] feat: discover SOC archives in backfill CLI --- .../batterywatch_api/backfill_service.py | 63 +++++++- app/backend/tests/test_backfill_service.py | 134 ++++++++++++++++-- 2 files changed, 186 insertions(+), 11 deletions(-) diff --git a/app/backend/batterywatch_api/backfill_service.py b/app/backend/batterywatch_api/backfill_service.py index 56297ff..8649fad 100644 --- a/app/backend/batterywatch_api/backfill_service.py +++ b/app/backend/batterywatch_api/backfill_service.py @@ -20,8 +20,12 @@ DISPATCH_SCADA_FEED, plan_archive_range, ) +from .nemweb_http import fetch_nemweb_resource from .nextday_archives import ( + NEXTDAY_ARCHIVE_INDEX_MAX_BYTES, + NEXTDAY_ARCHIVE_INDEX_URL, NextDayMonthlyArchiveRef, + discover_nextday_monthly_archives, plan_nextday_monthly_archives, ) @@ -162,6 +166,7 @@ def _summary( result: HistoricalBackfillResult, planned_item_count: int, spec: BackfillRunSpec, + missing_nextday_months: tuple[date, ...], ) -> dict[str, Any]: progress = result.finalization.progress return { @@ -178,6 +183,13 @@ def _summary( "scada_mapped_power_count": result.scada_mapped_power_count, "price_source_record_count": result.price_source_record_count, "price_applied_record_count": result.price_applied_record_count, + "nextday_source_record_count": result.nextday_source_record_count, + "nextday_applied_record_count": result.nextday_applied_record_count, + "nextday_null_count": result.nextday_null_count, + "nextday_percentage_count": result.nextday_percentage_count, + "missing_nextday_months": tuple( + month.isoformat() for month in missing_nextday_months + ), "replayed_interval_count": result.replayed_interval_count, "replayed_outer_artifact_count": result.replayed_outer_artifact_count, "completion_replayed": result.finalization.replayed, @@ -196,6 +208,10 @@ def main( environ: dict[str, str] | None = None, load_assets: Callable[[Path], Iterable[BatteryAsset]] = load_battery_assets, run: Callable[..., HistoricalBackfillResult] = run_historical_backfill, + fetch_index: Callable[..., Any] = fetch_nemweb_resource, + discover_nextday: Callable[..., tuple[NextDayMonthlyArchiveRef, ...]] = ( + discover_nextday_monthly_archives + ), ) -> int: parser = argparse.ArgumentParser(prog="batterywatch-backfill") parser.add_argument("--run-id", required=True) @@ -213,13 +229,49 @@ def main( raise ValueError("database URL is required") start = _parse_utc(arguments.start) end = _parse_utc(arguments.end) - spec, items = build_operator_plan( + requested_feeds = tuple(arguments.feeds.split(",")) + nextday_references: tuple[NextDayMonthlyArchiveRef, ...] = () + if "soc" in requested_feeds: + index_resource = fetch_index( + NEXTDAY_ARCHIVE_INDEX_URL, + max_bytes=NEXTDAY_ARCHIVE_INDEX_MAX_BYTES, + ) + index_payload = index_resource.body.decode("utf-8", errors="strict") + nextday_references = discover_nextday( + index_payload, + index_url=NEXTDAY_ARCHIVE_INDEX_URL, + ) + details = build_operator_plan_details( arguments.run_id, start, end, - feeds=tuple(arguments.feeds.split(",")), + feeds=requested_feeds, ingestion_version=arguments.ingestion_version, + nextday_archives=nextday_references, ) + spec, items = details.spec, details.items + if not items and details.missing_nextday_months: + print( + json.dumps( + { + "missing_nextday_months": [ + month.isoformat() + for month in details.missing_nextday_months + ], + "planned_item_count": 0, + "requested_end": spec.requested_end.isoformat().replace( + "+00:00", "Z" + ), + "requested_start": spec.requested_start.isoformat().replace( + "+00:00", "Z" + ), + "run_id": spec.run_id, + "status": "source_unavailable", + }, + sort_keys=True, + ) + ) + return 0 assets_path = arguments.assets_path or Path( environment.get( "BATTERYWATCH_ASSETS_PATH", @@ -232,7 +284,12 @@ def main( spec, items, ) - print(json.dumps(_summary(result, len(items), spec), sort_keys=True)) + print( + json.dumps( + _summary(result, len(items), spec, details.missing_nextday_months), + sort_keys=True, + ) + ) return 0 except Exception as error: print( diff --git a/app/backend/tests/test_backfill_service.py b/app/backend/tests/test_backfill_service.py index a24661d..7f4b62b 100644 --- a/app/backend/tests/test_backfill_service.py +++ b/app/backend/tests/test_backfill_service.py @@ -16,6 +16,7 @@ ) from batterywatch_api.battery_assets import BatteryAsset from batterywatch_api.historical_backfill import HistoricalBackfillResult +from batterywatch_api.historical_nextday_backfill import HistoricalNextDayBackfillResult from batterywatch_api.historical_price_backfill import HistoricalPriceBackfillResult from batterywatch_api.historical_scada_backfill import HistoricalScadaBackfillResult from batterywatch_api.backfill_service import build_operator_plan, main @@ -109,13 +110,36 @@ def test_main_outputs_deterministic_success_summary_without_database_url(self) - "PUBLIC_DISPATCHIS_20260829.zip", 1, ) - progress = BackfillRunProgress("operator-run", "completed", 2, 0, 0, 2, 0, 2) + soc_claim = BackfillClaim( + "operator-run", + "nextday_soc", + date(2026, 8, 1), + NEXTDAY_ARCHIVE_INDEX_URL + "PUBLIC_NEXT_DAY_DISPATCH_20260801.zip", + 1, + ) + progress = BackfillRunProgress("operator-run", "completed", 3, 0, 0, 3, 0, 3) result = HistoricalBackfillResult( - BackfillEnsureResult(True, False, 2, 0), - (scada_claim, price_claim), + BackfillEnsureResult(True, False, 3, 0), + (scada_claim, price_claim, soc_claim), (HistoricalScadaBackfillResult(1, 100, 12, 0, False, False),), (HistoricalPriceBackfillResult(1, 5, 5, 1, True, False),), BackfillRunFinalization(False, progress), + ( + HistoricalNextDayBackfillResult( + "a" * 64, + False, + 1, + 0, + 10, + 8, + 2, + 10, + 8, + 2, + 1, + 7, + ), + ), ) def load_assets(path: Path) -> tuple[BatteryAsset, ...]: @@ -136,18 +160,37 @@ def run(database_url: str, assets: Any, spec: Any, items: Any) -> HistoricalBack captured.update(database_url=database_url, assets=tuple(assets), spec=spec, items=tuple(items)) return result + filename = "PUBLIC_NEXT_DAY_DISPATCH_20260801.zip" + nextday_reference = NextDayMonthlyArchiveRef( + date(2026, 8, 1), + filename, + NEXTDAY_ARCHIVE_INDEX_URL + filename, + 220_000_000, + datetime(2026, 9, 1, tzinfo=UTC), + ) + + def fetch_index(url: str, *, max_bytes: int) -> Any: + captured.update(index_url=url, index_max_bytes=max_bytes) + return type("Resource", (), {"body": b"official-index"})() + + def discover_nextday(payload: str, *, index_url: str) -> tuple[NextDayMonthlyArchiveRef, ...]: + captured.update(index_payload=payload, discovery_index_url=index_url) + return (nextday_reference,) + with redirect_stdout(stdout), redirect_stderr(stderr): exit_code = main( [ "--run-id", "operator-run", "--start", "2026-08-29T00:00:00Z", "--end", "2026-08-29T10:00:00Z", - "--feeds", "power,price", + "--feeds", "power,price,soc", "--assets-path", "/tmp/reviewed-assets.json", ], environ={"BATTERYWATCH_DATABASE_URL": "postgresql://secret-value"}, load_assets=load_assets, run=run, + fetch_index=fetch_index, + discover_nextday=discover_nextday, ) self.assertEqual(exit_code, 0) @@ -157,17 +200,26 @@ def run(database_url: str, assets: Any, spec: Any, items: Any) -> HistoricalBack self.assertEqual(payload["run_id"], "operator-run") self.assertEqual(payload["requested_start"], "2026-08-29T00:00:00Z") self.assertEqual(payload["requested_end"], "2026-08-29T10:00:00Z") - self.assertEqual(payload["planned_item_count"], 2) - self.assertEqual(payload["claimed_item_count"], 2) + self.assertEqual(payload["planned_item_count"], 3) + self.assertEqual(payload["claimed_item_count"], 3) self.assertEqual(payload["scada_raw_observation_count"], 100) self.assertEqual(payload["scada_mapped_power_count"], 12) self.assertEqual(payload["price_source_record_count"], 5) self.assertEqual(payload["price_applied_record_count"], 5) - self.assertEqual(payload["replayed_interval_count"], 1) + self.assertEqual(payload["nextday_source_record_count"], 10) + self.assertEqual(payload["nextday_applied_record_count"], 8) + self.assertEqual(payload["nextday_null_count"], 1) + self.assertEqual(payload["nextday_percentage_count"], 7) + self.assertEqual(payload["missing_nextday_months"], []) + self.assertEqual(payload["replayed_interval_count"], 3) self.assertEqual(payload["replayed_outer_artifact_count"], 1) - self.assertEqual(payload["total_attempts"], 2) + self.assertEqual(payload["total_attempts"], 3) self.assertNotIn("secret-value", stdout.getvalue()) self.assertEqual(captured["assets_path"], Path("/tmp/reviewed-assets.json")) + self.assertEqual(captured["index_url"], NEXTDAY_ARCHIVE_INDEX_URL) + self.assertEqual(captured["index_payload"], "official-index") + self.assertEqual(captured["discovery_index_url"], NEXTDAY_ARCHIVE_INDEX_URL) + self.assertGreater(captured["index_max_bytes"], 0) def test_main_failure_reports_only_error_type_and_run_id(self) -> None: stdout = StringIO() @@ -196,6 +248,72 @@ def fail_run(*args: Any, **kwargs: Any) -> HistoricalBackfillResult: ) self.assertNotIn("secret-value", stderr.getvalue()) + def test_main_reports_soc_source_unavailable_without_running_database_work(self) -> None: + stdout = StringIO() + stderr = StringIO() + calls: list[str] = [] + filename = "PUBLIC_NEXT_DAY_DISPATCH_20260601.zip" + last_available = NextDayMonthlyArchiveRef( + date(2026, 6, 1), + filename, + NEXTDAY_ARCHIVE_INDEX_URL + filename, + 220_000_000, + datetime(2026, 7, 1, tzinfo=UTC), + ) + + def fetch_index(url: str, *, max_bytes: int) -> Any: + return type("Resource", (), {"body": b"official-index"})() + + def discover_nextday( + payload: str, + *, + index_url: str, + ) -> tuple[NextDayMonthlyArchiveRef, ...]: + return (last_available,) + + def fail_load(path: Path) -> tuple[BatteryAsset, ...]: + calls.append("load") + raise AssertionError("source-unavailable run must not load assets") + + def fail_run(*args: Any, **kwargs: Any) -> HistoricalBackfillResult: + calls.append("run") + raise AssertionError("source-unavailable run must not open database work") + + with redirect_stdout(stdout), redirect_stderr(stderr): + exit_code = main( + [ + "--run-id", + "soc-missing-run", + "--start", + "2026-07-31T15:05:00Z", + "--end", + "2026-08-30T15:05:00Z", + "--feeds", + "soc", + ], + environ={"BATTERYWATCH_DATABASE_URL": "postgresql://secret-value"}, + load_assets=fail_load, + run=fail_run, + fetch_index=fetch_index, + discover_nextday=discover_nextday, + ) + + self.assertEqual(exit_code, 0) + self.assertEqual(stderr.getvalue(), "") + self.assertEqual(calls, []) + self.assertEqual( + json.loads(stdout.getvalue()), + { + "missing_nextday_months": ["2026-07-01", "2026-08-01"], + "planned_item_count": 0, + "requested_end": "2026-08-30T15:05:00Z", + "requested_start": "2026-07-31T15:05:00Z", + "run_id": "soc-missing-run", + "status": "source_unavailable", + }, + ) + self.assertNotIn("secret-value", stdout.getvalue()) + if __name__ == "__main__": unittest.main()