diff --git a/.gitignore b/.gitignore index be6df2946..db4269fd0 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,8 @@ __pycache__/ *_metadata.json *_diffs.yaml !bedrock/utils/**/*.csv +# Published original vs EIA-anchored histogram panels (required for deck CI) +!bedrock/analysis/electricity/historical/original_vs_eia_anchored_deck/figures/*.png # asets stands for activity sets !bedrock/extract/**/*_asets.csv !bedrock/extract/external_data/*.csv @@ -52,8 +54,3 @@ scratch/ # Ignore analysis output bedrock/analysis/**/output/ -# Original electricity-disagg implementation freeze (must be committed) -!bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/ -!bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/** -!bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/**/*.parquet -!bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/**/*.json diff --git a/bedrock/analysis/electricity/README.md b/bedrock/analysis/electricity/README.md index 611ca5116..73942039b 100644 --- a/bedrock/analysis/electricity/README.md +++ b/bedrock/analysis/electricity/README.md @@ -5,19 +5,16 @@ Two eras, one tree. - **`current/`** — analyses of **today’s** EIA-anchored generation / transmission / distribution production (YAML flags `implement_electricity_disaggregation` / `implement_electricity_mixed_units`). -- **`historical/`** — immutable freeze of the **original** UGO / Table 8.3 / - Table 2.4 electricity-disagg implementation. Do not run historical Python - against current production. +- **`historical/`** — published original vs EIA-anchored (pre-MECS) comparison + tables and figures (not live production). - **`shared/`** — helpers used by both (eGRID vs national totals; 221100 reallocation matrix export). | Path | What it is | |---|---| -| `current/diagnostics/` | BLy / EF / full-trace / year-alignment / F01000 BLy attribution | -| `current/eia_gtd/` | Stub for results-deck G/T/D tables (`EIAPurchaserAllocation`) | -| `current/vs_original_elec_disagg/` | Live EIA path vs the original-implementation freeze | -| `current/eia_gtd_code_impl_f8f73b01.plan.md` | Design note for current production (internal names are stale) | -| `historical/original_elec_disagg_implementation/` | Read-only freeze (`q`, `x`, E/D/N/BLy, …) | +| `current/diagnostics/` | BLy / EF / full-trace / year-alignment / F01000 BLy attribution / comparison deck | +| `current/eia_gtd/` | Results-deck G/T/D tables (`EIAPurchaserAllocation`) | +| `historical/original_vs_eia_anchored_deck/` | PPTX extract: class MWh, D/N tables, vs-footing histogram PNGs | -Do not invent a snapshotter for the freeze. If freeze files are missing, they -are missing or not tracked. +Do not regenerate freeze parquets into this tree. Archived dumps live under Dropbox +`Documentation/Archive/`. diff --git a/bedrock/analysis/electricity/current/diagnostics/README.md b/bedrock/analysis/electricity/current/diagnostics/README.md index a289a754e..fa828308a 100644 --- a/bedrock/analysis/electricity/current/diagnostics/README.md +++ b/bedrock/analysis/electricity/current/diagnostics/README.md @@ -18,6 +18,7 @@ diagnostics/ year_alignment/ # BLy vs E under A/q year handling hh_vs_interindustry/ # F01000 BLy attribution vs interindustry (not D0 class MWh) probes/ # one-off sector probes + deck/ # five-slide PPTX (current vs pre-MECS vs original vs production) output/ # reports/figures (gitignored) __tests__/ ``` @@ -30,6 +31,7 @@ Run everything from the **repo root** with the project venv ## Shared prerequisites 1. **Configs** (in repo): + - `2025_usa_cornerstone_v0_3` — production (non-disagg, margins on) - `2025_usa_cornerstone_v0_3_electricity_footing` — footing - `2025_usa_cornerstone_v0_3_electricity_reallocation` - `2025_usa_cornerstone_v0_3_electricity_disaggregation` @@ -154,10 +156,12 @@ python -m bedrock.analysis.electricity.current.diagnostics.ef_comparison.analyze | `electricity_reallocation/` | Suite PNGs for that step vs v0.3.1 electricity footing | | `electricity_disaggregation/` | Suite PNGs for 3-way vs v0.3.1 electricity footing | | `electricity_mixed_units/` | Suite PNGs for mixed units vs v0.3.1 electricity footing | -| `panel/ef_panels_vs_v0_3_N.png` | 3-panel N % hist | -| `panel/ef_panels_vs_v0_3_D.png` | 3-panel D % hist | +| `panel/ef_panels_vs_v0_3_{N,D}.png` | Live `plot_ef` write path (absent until a new sheet run) | | `panel/n_variance_*.csv`, `n_variance_explained.md` | From `analyze_n_variance` | +Published original / pre-MECS EIA histogram panels and D/N tables live in +`historical/original_vs_eia_anchored_deck/`, not under `output/ef/panel/`. + Dropped sectors (e.g. mixed-units `221110` kg/MWh vs kg/USD) are footnoted on figures. --- @@ -235,6 +239,35 @@ Prints to stdout (no dedicated output file). --- +## 7. Comparison PPTX — `deck/` + +Five-slide decks matching the original-vs-EIA-anchored template: class MWh, +electricity-sector D/N by step, and 3-panel histograms. Writes ``.pptx`` under +``output/deck/``. Existing BLy / footing-plot / full-trace scripts stay as they are. + +| Pair | Histograms | +|---|---| +| `current_vs_eia_gtd` | Both rows vs v0.3.1 footing (current generated; pre-MECS published PNG). MECS vs dollar-weight D/N is in the tables (**same** when they match). | +| `current_vs_original` | Original row = published `v0.2_original_electricity_disagg_*`; bottom = current vs v0.3.1 | +| `eia_gtd_vs_original` | Published original PNGs over published `v0.3_eia_gtd_pre_mecs_*` | +| `current_vs_production` | Both rows vs Cornerstone v0.3 production (non-disagg, margins on). Top = current electricity-disagg steps; bottom = production vs itself (0% check). Production has no G/T/D or class MWh. | + +Identical D/N (or class MWh) cells are labeled **same**. Sectors not in the model +at a step stay **N/A**. Missing live cache shows **—**. Original and pre-MECS +EIA G/T/D tables come from +`historical/original_vs_eia_anchored_deck/tables.yaml`. + +```bash +python -m bedrock.analysis.electricity.current.diagnostics.deck --pair current_vs_eia_gtd +python -m bedrock.analysis.electricity.current.diagnostics.deck --pair current_vs_production +python -m bedrock.analysis.electricity.current.diagnostics.deck --all +python -m bedrock.analysis.electricity.current.diagnostics.deck --all --derive +``` + +``--derive`` live-runs missing **current** (post-MECS) electricity-disagg steps and the **production** (`2025_usa_cornerstone_v0_3`) cache. + +--- + ## Suggested order 1. Sheet cache + BLy waterfalls (`bly_dispersion`) and EF plots (`ef_comparison.plot_ef`) @@ -248,5 +281,6 @@ Prints to stdout (no dedicated output file). ```bash python -m pytest bedrock/analysis/electricity/current/diagnostics/__tests__ \ + bedrock/analysis/electricity/current/diagnostics/deck/__tests__ \ bedrock/analysis/electricity/current/eia_gtd/__tests__ -q ``` diff --git a/bedrock/analysis/electricity/current/diagnostics/__init__.py b/bedrock/analysis/electricity/current/diagnostics/__init__.py index ca4ca2fd4..0b71561c4 100644 --- a/bedrock/analysis/electricity/current/diagnostics/__init__.py +++ b/bedrock/analysis/electricity/current/diagnostics/__init__.py @@ -8,6 +8,7 @@ - ``year_alignment`` — BLy vs E under A/q year handling - ``hh_vs_interindustry`` — F01000 BLy attribution vs interindustry (not D0 class MWh) - ``probes`` — one-off sector probes (e.g. 221200) +- ``deck`` — five-slide PPTX comparisons (current vs pre-MECS vs original) Shared infrastructure (``paths``, ``manifest``, ``local_data``) stays at package root. Outputs remain under ``output/`` (layout unchanged by reorganization). diff --git a/bedrock/analysis/electricity/current/diagnostics/deck/__init__.py b/bedrock/analysis/electricity/current/diagnostics/deck/__init__.py new file mode 100644 index 000000000..631c00904 --- /dev/null +++ b/bedrock/analysis/electricity/current/diagnostics/deck/__init__.py @@ -0,0 +1,4 @@ +"""Five-slide PPTX comparing electricity-disaggregation implementations. + +``python -m bedrock.analysis.electricity.current.diagnostics.deck --all`` +""" diff --git a/bedrock/analysis/electricity/current/diagnostics/deck/__main__.py b/bedrock/analysis/electricity/current/diagnostics/deck/__main__.py new file mode 100644 index 000000000..afd43b8c1 --- /dev/null +++ b/bedrock/analysis/electricity/current/diagnostics/deck/__main__.py @@ -0,0 +1,6 @@ +"""``python -m bedrock.analysis.electricity.current.diagnostics.deck``.""" + +from bedrock.analysis.electricity.current.diagnostics.deck.build import main + +if __name__ == '__main__': + main() diff --git a/bedrock/analysis/electricity/current/diagnostics/deck/__tests__/__init__.py b/bedrock/analysis/electricity/current/diagnostics/deck/__tests__/__init__.py new file mode 100644 index 000000000..d7e390244 --- /dev/null +++ b/bedrock/analysis/electricity/current/diagnostics/deck/__tests__/__init__.py @@ -0,0 +1 @@ +"""Deck tests.""" diff --git a/bedrock/analysis/electricity/current/diagnostics/deck/__tests__/test_deck.py b/bedrock/analysis/electricity/current/diagnostics/deck/__tests__/test_deck.py new file mode 100644 index 000000000..52d78bc77 --- /dev/null +++ b/bedrock/analysis/electricity/current/diagnostics/deck/__tests__/test_deck.py @@ -0,0 +1,448 @@ +"""Synthetic tests for the five-slide electricity comparison deck.""" + +from __future__ import annotations + +import zipfile +from pathlib import Path + +import pandas as pd +import pytest +from PIL import Image +from pptx import Presentation as PptxPresentation +from pptx.util import Inches + +from bedrock.analysis.electricity.current.diagnostics.deck import histograms as hist +from bedrock.analysis.electricity.current.diagnostics.deck.data import ( + NA, + SAME, + ImplBundle, + StepSnapshot, + c_col_is_monetary, + ef_kg_per_usd, + fill_mixed_c_col, + format_twh, + sector_ef_usd, + star_aggregate, + values_match, +) +from bedrock.analysis.electricity.current.diagnostics.deck.histograms import ( + _panel_data, + frozen_panel_png, + pairwise_frame, + perc_frame, + stack_panel_pngs, + write_hist_png, +) +from bedrock.analysis.electricity.current.diagnostics.deck.pairs import ( + AGGREGATE_ONLY_NA, + PAIRS, + STEPS, + ImplId, + na_sectors_at_step, +) +from bedrock.analysis.electricity.current.diagnostics.deck.pptx_write import write_pptx +from bedrock.analysis.electricity.current.diagnostics.deck.tables import ( + class_mwh_grids, + ef_grids, + format_cell_pair, +) +from bedrock.analysis.electricity.historical.original_vs_eia_anchored_deck.published import ( + published_ef, +) + + +def _step( + d: dict[str, float], + n: dict[str, float] | None = None, + *, + mixed: bool = False, + c_col: float | None = None, + class_mwh: dict[str, float] | None = None, + target: dict[str, float] | None = None, + q: dict[str, float] | None = None, + x: dict[str, float] | None = None, +) -> StepSnapshot: + return StepSnapshot( + d=pd.Series(d, dtype=float), + n=pd.Series(n if n is not None else d, dtype=float), + q=pd.Series(q, dtype=float) if q is not None else None, + x=pd.Series(x, dtype=float) if x is not None else None, + mixed=mixed, + c_col=c_col, + class_mwh=pd.Series(class_mwh, dtype=float) if class_mwh is not None else None, + class_mwh_target=pd.Series(target, dtype=float) if target is not None else None, + ) + + +def _eia_class() -> tuple[dict[str, float], dict[str, float]]: + model = { + 'Residential': 1.5488e9, + 'Commercial': 1.0e9, + 'Industrial': 1.2e9, + 'Transportation': 0.1e9, + 'Exports': 0.4638e9, + } + return model, dict(model) + + +def test_ef_kg_per_usd_scales_generation_when_mixed() -> None: + raw = pd.Series({'221110': 100.0, '221121': 0.2}) + usd = ef_kg_per_usd(raw, mixed=True, c_col=0.05) + assert usd.loc['221110'] == pytest.approx(5.0) + assert usd.loc['221121'] == pytest.approx(0.2) + + +def test_identity_c_col_does_not_convert_generation() -> None: + raw = pd.Series({'221110': 398.866}) + usd = ef_kg_per_usd(raw, mixed=True, c_col=1.0) + assert usd.loc['221110'] == pytest.approx(398.866) + assert not c_col_is_monetary(1.0) + assert c_col_is_monetary(0.0216) + + +def test_fill_mixed_c_col_from_three_way_dollar_q() -> None: + q_mwh = 4.312557e9 + q_usd = 1.9951353935173175e11 + mixed = _step( + {'221110': 398.86642559883023, '221121': 0.36, '221122': 0.119}, + n={'221110': 398.86642559883023, '221121': 0.36, '221122': 0.119}, + mixed=True, + c_col=None, + q={'221110': q_mwh, '221121': 1e9, '221122': 1e10}, + ) + three = _step( + {'221110': 8.622, '221121': 0.36, '221122': 0.119}, + q={'221110': q_usd, '221121': 1e9, '221122': 1e10}, + ) + bundle = ImplBundle('eia_gtd', {'three_way': three, 'mixed_units': mixed}) + fill_mixed_c_col(bundle) + assert mixed.c_col == pytest.approx(q_mwh / q_usd) + assert sector_ef_usd(mixed, 'N', '221110') == pytest.approx(8.622, rel=1e-3) + + +def test_fill_mixed_c_col_replaces_identity() -> None: + q_mwh = 4.312557e9 + q_usd = 1.9951353935173175e11 + mixed = _step( + {'221110': 398.866}, + mixed=True, + c_col=1.0, + q={'221110': q_mwh}, + ) + three = _step({'221110': 8.622}, q={'221110': q_usd}) + fill_mixed_c_col(ImplBundle('eia_gtd', {'three_way': three, 'mixed_units': mixed})) + assert mixed.c_col == pytest.approx(q_mwh / q_usd) + + +def test_fill_mixed_c_col_keeps_original_freeze_c_col() -> None: + mixed = _step( + {'221110': 816.64}, + mixed=True, + c_col=0.013975279012407282, + q={'221110': 4.312557e9}, + ) + three = _step({'221110': 10.176}, q={'221110': 3.08e11}) + fill_mixed_c_col(ImplBundle('original', {'three_way': three, 'mixed_units': mixed})) + assert mixed.c_col == pytest.approx(0.013975279012407282) + + +def test_star_aggregate_x_weighted() -> None: + step = _step( + {'221110': 10.0, '221121': 1.0, '221122': 0.0}, + x={'221110': 2.0, '221121': 2.0, '221122': 6.0}, + ) + # (10*2 + 1*2 + 0*6) / 10 = 2.2 + assert star_aggregate(step, 'D') == pytest.approx(2.2) + + +def test_format_twh() -> None: + assert format_twh(1_548_800_000) == '1,548.8 TWh' + + +def test_na_and_same_cells() -> None: + left = ImplBundle( + 'current', + { + 'footing': _step({'221100': 2.409}), + 'reallocation': _step({'221100': 2.416}), + 'three_way': _step({'221110': 8.0, '221121': 0.226, '221122': 0.0}), + 'mixed_units': _step( + {'221110': 8.0, '221121': 0.226, '221122': 0.0}, + mixed=True, + c_col=1.0, + ), + }, + ) + right = ImplBundle( + 'eia_gtd', + { + 'footing': _step({'221100': 2.409}), + 'reallocation': _step({'221100': 2.416}), + 'three_way': _step({'221110': 8.0, '221121': 0.226, '221122': 0.0}), + 'mixed_units': _step( + {'221110': 8.0, '221121': 0.226, '221122': 0.0}, + mixed=True, + c_col=1.0, + ), + }, + ) + assert format_cell_pair(left, right, 'D', '221110', 'footing', False) == NA + assert format_cell_pair(left, right, 'D', '221100', 'footing', True) == SAME + assert format_cell_pair(right, left, 'D', '221110', 'three_way', True) != SAME + assert values_match(2.409, 2.409) + + +def test_class_mwh_same_when_totals_match() -> None: + model, target = _eia_class() + mixed = _step({'221100': 1.0}, class_mwh=model, target=target) + top = ImplBundle('current', {'mixed_units': mixed}) + bottom = ImplBundle('eia_gtd', {'mixed_units': mixed}) + pair = PAIRS['current_vs_eia_gtd'] + top_grid, bottom_grid = class_mwh_grids(pair, top, bottom) + assert bottom_grid.rows[0][1] == SAME + assert top_grid.rows[-1][1] == '4,312.6 TWh' + + +def test_pairwise_perc_formula() -> None: + left = _step({'1111A0': 1.1, '221121': 0.4}) + right = _step({'1111A0': 1.0, '221121': 0.4}) + frame, drops = pairwise_frame(left, right, 'D') + by_sector = frame.set_index('sector')['perc_diff'] + assert by_sector.loc['1111A0'] == pytest.approx(0.1) + assert by_sector.loc['221121'] == pytest.approx(0.0) + assert drops == [] + + +def test_perc_frame_inner_join() -> None: + frame = perc_frame( + pd.Series({'a': 2.0, 'b': 1.0}), + pd.Series({'a': 1.0, 'c': 9.0}), + ) + assert list(frame['sector']) == ['a'] + assert frame['perc_diff'].iloc[0] == pytest.approx(1.0) + + +def test_ef_grids_two_tables() -> None: + pair = PAIRS['current_vs_original'] + original = ImplBundle( + 'original', + { + 'footing': _step({'221100': 2.386}), + 'three_way': _step({'221110': 7.138, '221121': 0.225, '221122': 0.0}), + }, + ) + current = ImplBundle( + 'current', + { + 'footing': _step({'221100': 2.409}), + 'three_way': _step({'221110': 7.197, '221121': 0.226, '221122': 0.0}), + }, + ) + top, bottom = ef_grids(pair, original, current, 'D') + assert top.headers[2] == 'v0.2' + assert bottom.headers[2] == 'v0.3.1' + assert top.rows[2][1] == '221110 (G)' + n_top, _n_bottom = ef_grids(pair, original, current, 'N') + assert n_top.rows[2][4] == '9.213' + assert n_top.rows[2][5] == '10.070' + + +def test_stack_panel_pngs_preserves_pixels(tmp_path: Path) -> None: + a = tmp_path / 'a.png' + b = tmp_path / 'b.png' + Image.new('RGB', (10, 4), (255, 0, 0)).save(a) + Image.new('RGB', (10, 6), (0, 0, 255)).save(b) + out = tmp_path / 'stacked.png' + stack_panel_pngs([a, b], out) + im = Image.open(out) + assert im.size == (10, 10) + + +def test_write_hist_png_uses_frozen_vs_footing_panels( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + panel = tmp_path / 'panel' + panel.mkdir() + Image.new('RGB', (20, 8), (255, 0, 0)).save( + panel / hist.FROZEN_PANEL_PNG[('original', 'D')] + ) + Image.new('RGB', (20, 8), (0, 0, 255)).save( + panel / hist.FROZEN_PANEL_PNG[('eia_gtd', 'D')] + ) + monkeypatch.setattr(hist, 'FIGURES_DIR', panel) + out = tmp_path / 'hist.png' + write_hist_png( + PAIRS['eia_gtd_vs_original'], + ImplBundle('original', {}), + ImplBundle('eia_gtd', {}), + 'D', + out, + ) + im = Image.open(out) + assert im.size == (20, 16) + + +def test_published_original_n_matches_pptx() -> None: + assert published_ef('original', 'N', '221110', 'three_way') == pytest.approx(9.213) + assert published_ef('original', 'N', '221110', 'mixed_units') == pytest.approx( + 10.070 + ) + assert published_ef('eia_gtd', 'N', '221110', 'three_way') == pytest.approx(8.622) + + +def test_published_original_panel_is_on_disk() -> None: + path = frozen_panel_png('original', 'D') + assert path is not None + assert path.name == 'v0.2_original_electricity_disagg_D.png' + published: tuple[tuple[ImplId, str], ...] = ( + ('original', 'D'), + ('original', 'N'), + ('eia_gtd', 'D'), + ('eia_gtd', 'N'), + ) + for impl_id, kind in published: + found = frozen_panel_png(impl_id, kind) + assert found is not None, f'missing published panel for {impl_id} {kind}' + + +def test_write_pptx_five_slides( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(hist, 'FIGURES_DIR', tmp_path / 'no_frozen_panels') + model, target = _eia_class() + sectors = {f'{i:06d}': float(i) for i in range(10)} + sectors.update({'221110': 8.0, '221121': 0.3, '221122': 0.1, '221100': 2.4}) + step_3way = _step(sectors, class_mwh=model, target=target) + step_mixed = _step(sectors, mixed=True, c_col=0.02, class_mwh=model, target=target) + step_re = _step({'221100': 2.4, **{f'{i:06d}': float(i) for i in range(10)}}) + step_foot = _step( + {'221100': 2.39, **{f'{i:06d}': float(i) * 0.9 for i in range(10)}} + ) + bundle_a = ImplBundle( + 'current', + { + 'footing': step_foot, + 'reallocation': step_re, + 'three_way': step_3way, + 'mixed_units': step_mixed, + }, + ) + bundle_b = ImplBundle( + 'eia_gtd', + { + 'footing': step_foot, + 'reallocation': step_re, + 'three_way': step_3way, + 'mixed_units': step_mixed, + }, + ) + out = tmp_path / 'deck.pptx' + write_pptx(PAIRS['current_vs_eia_gtd'], bundle_a, bundle_b, out, png_dir=tmp_path) + assert out.is_file() + with zipfile.ZipFile(out) as z: + slides = [n for n in z.namelist() if n.startswith('ppt/slides/slide')] + media = [n for n in z.namelist() if n.startswith('ppt/media/')] + assert len(slides) == 5 + assert len(media) >= 2 + prs = PptxPresentation(str(out)) + d_tables = [s for s in prs.slides[1].shapes if s.has_table] + assert d_tables + table = d_tables[0].table + run = table.cell(1, 0).text_frame.paragraphs[0].runs[0] + assert run.font.size is not None + assert run.font.size.pt == 12 + assert d_tables[0].height < Inches(3.2) + assert d_tables[0].width < Inches(6.5) + + +def test_production_pair_schema_and_na() -> None: + pair = PAIRS['current_vs_production'] + assert pair.hist_baseline == 'peer' + assert pair.top == 'current' + assert pair.bottom == 'production' + for step in ('footing', 'reallocation', 'three_way', 'mixed_units'): + assert na_sectors_at_step('production', step) == AGGREGATE_ONLY_NA + assert '221110' not in na_sectors_at_step('current', 'three_way') + + +def test_production_tables_and_peer_histograms() -> None: + pair = PAIRS['current_vs_production'] + prod_step = _step({'221100': 2.5, '1111A0': 2.0}) + production = ImplBundle( + 'production', + { + 'footing': prod_step, + 'reallocation': prod_step, + 'three_way': prod_step, + 'mixed_units': prod_step, + }, + ) + current = ImplBundle( + 'current', + { + 'footing': _step({'221100': 2.4, '1111A0': 1.0}), + 'reallocation': _step({'221100': 2.41, '1111A0': 1.0}), + 'three_way': _step( + {'221110': 8.0, '221121': 0.2, '221122': 0.0, '1111A0': 1.0} + ), + 'mixed_units': _step( + {'221110': 8.0, '221121': 0.2, '221122': 0.0, '1111A0': 1.0}, + mixed=True, + c_col=0.02, + ), + }, + ) + assert ( + format_cell_pair(production, current, 'D', '221110', 'three_way', False) == NA + ) + assert format_cell_pair( + production, current, 'D', '221100', 'mixed_units', False + ) == ('2.500') + assert ( + format_cell_pair(current, production, 'D', '221100', 'three_way', False) == NA + ) + _top_class, bottom_class = class_mwh_grids(pair, current, production) + assert bottom_class.rows[0][0] == '(no class MWh)' + frame, _drops = _panel_data( + pair, current, 'current', current, production, 'reallocation', 'D' + ) + by_sector = frame.set_index('sector')['perc_diff'] + assert by_sector.loc['1111A0'] == pytest.approx(-0.5) + + +def test_write_pptx_current_vs_production(tmp_path: Path) -> None: + sectors = {f'{i:06d}': float(i) for i in range(10)} + sectors.update({'221110': 8.0, '221121': 0.3, '221122': 0.1, '221100': 2.4}) + prod_sectors = {f'{i:06d}': float(i) * 0.95 for i in range(10)} + prod_sectors['221100'] = 2.5 + current = ImplBundle( + 'current', + { + 'footing': _step( + {'221100': 2.39, **{f'{i:06d}': float(i) * 0.9 for i in range(10)}} + ), + 'reallocation': _step( + {'221100': 2.4, **{f'{i:06d}': float(i) for i in range(10)}} + ), + 'three_way': _step(sectors), + 'mixed_units': _step(sectors, mixed=True, c_col=0.02), + }, + ) + production = ImplBundle( + 'production', + {step: _step(prod_sectors) for step in STEPS}, + ) + out = tmp_path / 'current_vs_production.pptx' + write_pptx( + PAIRS['current_vs_production'], + current, + production, + out, + png_dir=tmp_path, + ) + assert out.is_file() + with zipfile.ZipFile(out) as z: + slides = [n for n in z.namelist() if n.startswith('ppt/slides/slide')] + media = [n for n in z.namelist() if n.startswith('ppt/media/')] + assert len(slides) == 5 + assert len(media) >= 2 diff --git a/bedrock/analysis/electricity/current/diagnostics/deck/build.py b/bedrock/analysis/electricity/current/diagnostics/deck/build.py new file mode 100644 index 000000000..b0c798016 --- /dev/null +++ b/bedrock/analysis/electricity/current/diagnostics/deck/build.py @@ -0,0 +1,77 @@ +"""Build one or all five-slide comparison PPTX files.""" + +from __future__ import annotations + +from pathlib import Path + +import click + +from bedrock.analysis.electricity.current.diagnostics.deck.pairs import ( + IMPLEMENTATIONS, + PAIRS, + Pair, +) +from bedrock.analysis.electricity.current.diagnostics.deck.paths import ( + DECK_OUT_DIR, + ensure_deck_dirs, +) +from bedrock.analysis.electricity.current.diagnostics.deck.pptx_write import write_pptx +from bedrock.analysis.electricity.current.diagnostics.deck.sources import ( + load_impl_bundle, +) + + +def build_pair( + pair: Pair, + *, + derive: bool = False, + out_dir: Path | None = None, +) -> Path: + ensure_deck_dirs() + dest = out_dir or DECK_OUT_DIR + dest.mkdir(parents=True, exist_ok=True) + vs_footing = pair.hist_mode == 'vs_footing' + top = load_impl_bundle( + IMPLEMENTATIONS[pair.top], + derive=derive, + load_snapshot_footing=vs_footing, + ) + bottom = load_impl_bundle( + IMPLEMENTATIONS[pair.bottom], + derive=derive, + load_snapshot_footing=vs_footing, + ) + png_dir = dest / 'png' + return write_pptx(pair, top, bottom, dest / pair.filename, png_dir=png_dir) + + +@click.command() +@click.option( + '--pair', + 'pair_key', + type=click.Choice(sorted(PAIRS)), + default=None, + help='Build one pair. Default with --all is every pair.', +) +@click.option('--all', 'build_all', is_flag=True, help='Build all four PPTX files.') +@click.option( + '--derive', + is_flag=True, + help='Live-derive missing current and production steps (slow).', +) +def main(pair_key: str | None, build_all: bool, derive: bool) -> None: + if build_all: + keys = list(PAIRS) + elif pair_key is None: + raise click.UsageError('pass --pair or --all') + else: + keys = [pair_key] + written: list[Path] = [] + for key in keys: + path = build_pair(PAIRS[key], derive=derive) + written.append(path) + click.echo(f'Wrote {path}') + + +if __name__ == '__main__': + main() diff --git a/bedrock/analysis/electricity/current/diagnostics/deck/data.py b/bedrock/analysis/electricity/current/diagnostics/deck/data.py new file mode 100644 index 000000000..b0c6b92ac --- /dev/null +++ b/bedrock/analysis/electricity/current/diagnostics/deck/data.py @@ -0,0 +1,237 @@ +"""Load and normalize D/N/q/class-MWh vectors for the comparison deck.""" + +from __future__ import annotations + +import math +from dataclasses import dataclass, field +from pathlib import Path +from typing import TypeGuard + +import pandas as pd + +from bedrock.analysis.electricity.current.diagnostics.deck.pairs import ( + CHILD_SECTORS, + CLASS_ORDER, + GENERATION_SECTOR, + STAR_SECTOR, + ImplId, + StepId, +) + +MISSING = '—' +NA = 'N/A' +SAME = 'same' + +# Display rounding used for "same" (matches the sample's three-decimal EFs). +EF_DECIMALS = 3 +SAME_ATOL = 5e-4 +# electricity_conversion_factors(mixed Aq) uses q already in MWh, so c_col ≈ 1. +_IDENTITY_C_COL = 1.0 +_IDENTITY_C_COL_ATOL = 1e-6 +# Dollar generation q is orders of magnitude above eGRID MWh (~4e9). +_Q_USD_OVER_MWH_MIN = 10.0 + + +@dataclass +class StepSnapshot: + """One YAML step's vectors. Missing pieces stay ``None``.""" + + d: pd.Series | None = None + n: pd.Series | None = None + q: pd.Series | None = None + x: pd.Series | None = None + mixed: bool = False + c_col: float | None = None + class_mwh: pd.Series | None = None + class_mwh_target: pd.Series | None = None + + +@dataclass +class ImplBundle: + impl_id: ImplId + steps: dict[StepId, StepSnapshot] = field(default_factory=dict) + + +def series_from_parquet(path: Path, value_col: str | None = None) -> pd.Series: + """Accept original (1 × sectors) or pre-MECS (sectors × 1) freeze layouts.""" + frame = pd.read_parquet(path) + if value_col is not None and value_col in frame.columns: + out = frame[value_col].astype(float) + out.index = out.index.map(str) + return out + if frame.shape[0] == 1 and frame.shape[1] > 1: + out = frame.iloc[0].astype(float) + out.index = out.index.map(str) + out.index.name = 'sector' + return out + if frame.shape[1] == 1: + out = frame.iloc[:, 0].astype(float) + out.index = out.index.map(str) + return out + squeezed = frame.squeeze() + if not isinstance(squeezed, pd.Series): + raise TypeError(f'expected a Series from {path}, got {type(squeezed)}') + out = squeezed.astype(float) + out.index = out.index.map(str) + return out + + +def class_mwh_from_parquet(path: Path) -> tuple[pd.Series, pd.Series | None]: + frame = pd.read_parquet(path) + if 'class' not in frame.columns or 'value' not in frame.columns: + raise ValueError(f'{path} needs class/value columns') + model = pd.Series( + frame['value'].astype(float).to_numpy(), + index=frame['class'].astype(str), + dtype=float, + ) + model = model[~model.index.isin(['HH'])] + target: pd.Series | None = None + if 'target' in frame.columns: + target = pd.Series( + frame['target'].astype(float).to_numpy(), + index=frame['class'].astype(str), + dtype=float, + ) + target = target[~target.index.isin(['HH'])] + return model, target + + +def c_col_is_monetary(c_col: float | None) -> TypeGuard[float]: + """True when ``c_col`` is MWh/$ from dollar generation ``q``, not ~1 from mixed ``q``.""" + if c_col is None or not math.isfinite(c_col) or c_col <= 0: + return False + return abs(float(c_col) - _IDENTITY_C_COL) > _IDENTITY_C_COL_ATOL + + +def c_col_from_q_usd_and_mwh(q_usd: float, mwh: float) -> float: + from bedrock.transform.eeio.electricity_disaggregation import ( # noqa: PLC0415 + electricity_output_factor, + ) + + return float(electricity_output_factor(q_usd, mwh)) + + +def fill_mixed_c_col(bundle: ImplBundle) -> None: + """Recover monetary ``c_col`` for mixed units from 3-way dollar ``q`` / mixed MWh ``q``. + + Needed when ``c_col`` was computed after generation ``q`` was already + converted to MWh (identity ~1). + """ + mixed = bundle.steps.get('mixed_units') + if mixed is None or not mixed.mixed: + return + if c_col_is_monetary(mixed.c_col): + return + three = bundle.steps.get('three_way') + if ( + three is None + or three.q is None + or mixed.q is None + or GENERATION_SECTOR not in three.q.index + or GENERATION_SECTOR not in mixed.q.index + ): + return + q_usd = float(three.q.loc[GENERATION_SECTOR]) + mwh = float(mixed.q.loc[GENERATION_SECTOR]) + if q_usd <= 0 or mwh <= 0 or q_usd < mwh * _Q_USD_OVER_MWH_MIN: + return + mixed.c_col = c_col_from_q_usd_and_mwh(q_usd, mwh) + + +def ef_kg_per_usd( + ef: pd.Series, + *, + mixed: bool, + c_col: float | None, +) -> pd.Series: + """Native D/N → kg/USD. Mixed-units generation is kg/MWh × monetary ``c_col``.""" + out = ef.astype(float).copy() + out.index = out.index.map(str) + if mixed and c_col_is_monetary(c_col) and GENERATION_SECTOR in out.index: + out.loc[GENERATION_SECTOR] = float(out.loc[GENERATION_SECTOR]) * float(c_col) + return out + + +def _usd_weights(step: StepSnapshot) -> pd.Series | None: + raw = step.x if step.x is not None else step.q + if raw is None: + return None + w = raw.astype(float).copy() + w.index = w.index.map(str) + if step.mixed and c_col_is_monetary(step.c_col) and GENERATION_SECTOR in w.index: + w.loc[GENERATION_SECTOR] = float(w.loc[GENERATION_SECTOR]) / float(step.c_col) + return w + + +def star_aggregate(step: StepSnapshot, kind: str) -> float | None: + """``221100*``: x-weighted (else q-weighted) mean of child kg/USD EFs.""" + src = step.d if kind == 'D' else step.n + if src is None: + return None + usd = ef_kg_per_usd(src, mixed=step.mixed, c_col=step.c_col) + weights = _usd_weights(step) + if weights is None: + present = [s for s in CHILD_SECTORS if s in usd.index] + if not present: + return None + return float(usd.reindex(present).astype(float).mean()) + num = 0.0 + den = 0.0 + for sector in CHILD_SECTORS: + if sector not in usd.index: + continue + w = float(weights.get(sector, 0.0) or 0.0) + num += float(usd.loc[sector]) * w + den += w + if den == 0.0: + return None + return num / den + + +def sector_ef_usd(step: StepSnapshot, kind: str, sector: str) -> float | None: + if sector == STAR_SECTOR: + return star_aggregate(step, kind) + src = step.d if kind == 'D' else step.n + if src is None: + return None + usd = ef_kg_per_usd(src, mixed=step.mixed, c_col=step.c_col) + if sector not in usd.index: + return None + return float(usd.loc[sector]) + + +def values_match(left: float | None, right: float | None) -> bool: + if left is None or right is None: + return False + return abs(left - right) <= SAME_ATOL + + +def format_ef(value: float | None) -> str: + if value is None: + return MISSING + if abs(value) < 1e-12: + return '0' + return f'{value:.{EF_DECIMALS}f}' + + +def format_twh(mwh: float) -> str: + return f'{mwh / 1e6:,.1f} TWh' + + +def format_ratio(model: float, target: float) -> str: + if target == 0.0: + return MISSING + return f'{model / target:.3f}' + + +def grouped_mwh(class_mwh: pd.Series, members: tuple[str, ...]) -> float: + total = 0.0 + for name in members: + if name in class_mwh.index: + total += float(class_mwh.loc[name]) + return total + + +def class_total_mwh(class_mwh: pd.Series) -> float: + return grouped_mwh(class_mwh, CLASS_ORDER) diff --git a/bedrock/analysis/electricity/current/diagnostics/deck/histograms.py b/bedrock/analysis/electricity/current/diagnostics/deck/histograms.py new file mode 100644 index 000000000..88c8c39f6 --- /dev/null +++ b/bedrock/analysis/electricity/current/diagnostics/deck/histograms.py @@ -0,0 +1,400 @@ +"""Two-row × three-panel D/N percent-diff histograms for slides 3 and 5.""" + +from __future__ import annotations + +from pathlib import Path + +import pandas as pd +from matplotlib import pyplot as plt +from matplotlib.axes import Axes +from matplotlib.figure import Figure +from PIL import Image + +from bedrock.analysis.electricity.current.diagnostics.deck.data import ( + ImplBundle, + StepSnapshot, + ef_kg_per_usd, +) +from bedrock.analysis.electricity.current.diagnostics.deck.pairs import ( + GENERATION_SECTOR, + HIST_PANEL_TITLE, + HIST_STEPS, + IMPLEMENTATIONS, + ImplId, + Pair, + StepId, +) +from bedrock.analysis.electricity.current.diagnostics.ef_comparison.vs_footing_frames import ( + DroppedSector, + format_drop_footnote, +) +from bedrock.analysis.electricity.historical.original_vs_eia_anchored_deck.paths import ( + FIGURES_DIR as _PACK_FIGURES_DIR, +) +from bedrock.analysis.electricity.historical.original_vs_eia_anchored_deck.paths import ( + PANEL_PNG as FROZEN_PANEL_PNG, +) +from bedrock.utils.validation.analysis.diagnostics_plots import _beyond_20_text +from bedrock.utils.validation.analysis.plotting import ( + DEFAULT_XLIM, + apply_axis_fonts, + percent_histogram, + save_and_close, + setup_mpl, +) + +MIXED_UNITS_DROP = 'mixed units are incompatible for plotting (kg/MWh vs kg/USD)' +NOT_IN_FOOTING = 'not present in footing (cannot compare on a shared sector code)' +ONLY_IN_FOOTING = 'present only in footing (aggregate electricity; not in this step)' + +FIGURES_DIR = _PACK_FIGURES_DIR + + +def _sector_names(index: pd.Index) -> pd.Series: + try: + from bedrock.utils.validation.diagnostics_helpers import ( # noqa: PLC0415 + get_aligned_sector_desc, + ) + + desc = get_aligned_sector_desc() + except Exception: + desc = {} + return pd.Series({str(s): desc.get(str(s), str(s)) for s in index}) + + +def perc_frame(step: pd.Series, base: pd.Series) -> pd.DataFrame: + """``(step − base) / |base|`` on the inner-joined sector index.""" + a = step.astype(float).copy() + a.index = a.index.map(str) + b = base.astype(float).copy() + b.index = b.index.map(str) + idx = a.index.intersection(b.index) + a = a.reindex(idx) + b = b.reindex(idx) + denom = b.abs() + perc = (a - b) / denom + perc = perc.where(denom != 0) + names = _sector_names(idx) + return pd.DataFrame( + { + 'sector': list(idx), + 'sector_name': names.reindex(idx).astype(str).to_numpy(), + 'perc_diff': perc.to_numpy(), + } + ) + + +def _apply_electricity_drops( + frame: pd.DataFrame, + drops: list[DroppedSector], +) -> pd.DataFrame: + drop_sectors = {d.sector for d in drops} + return frame[~frame['sector'].isin(drop_sectors)].reset_index(drop=True) + + +def footing_drops( + step: pd.Series, + footing: pd.Series, + *, + mixed_step: bool, +) -> list[DroppedSector]: + """Same electricity-sector drop rules as vs-footing diagnostics plots.""" + step_secs = set(step.index.map(str)) + foot_secs = set(footing.index.map(str)) + drops: list[DroppedSector] = [] + seen: set[str] = set() + candidates = {'221100', '221110', '221121', '221122'} + for sector in sorted(candidates): + if sector in seen: + continue + if mixed_step and sector == GENERATION_SECTOR and sector in step_secs: + drops.append(DroppedSector(sector, MIXED_UNITS_DROP)) + seen.add(sector) + continue + if sector in step_secs and sector not in foot_secs: + drops.append(DroppedSector(sector, NOT_IN_FOOTING)) + seen.add(sector) + continue + if sector in foot_secs and sector not in step_secs: + drops.append(DroppedSector(sector, ONLY_IN_FOOTING)) + seen.add(sector) + return drops + + +def _usd_or_native(step: StepSnapshot, kind: str) -> pd.Series | None: + src = step.d if kind == 'D' else step.n + if src is None: + return None + if step.mixed: + return src.astype(float) + return ef_kg_per_usd(src, mixed=False, c_col=step.c_col) + + +def pairwise_frame( + left: StepSnapshot, + right: StepSnapshot, + kind: str, +) -> tuple[pd.DataFrame, list[DroppedSector]]: + a = _usd_or_native(left, kind) + b = _usd_or_native(right, kind) + if a is None or b is None: + return pd.DataFrame(columns=['sector', 'sector_name', 'perc_diff']), [] + # Both mixed or both monetary: generation is comparable; no unit drop. + frame = perc_frame(a, b) + return frame, [] + + +def vs_footing_frame( + step: StepSnapshot, + footing: StepSnapshot, + kind: str, +) -> tuple[pd.DataFrame, list[DroppedSector]]: + a = _usd_or_native(step, kind) + b = _usd_or_native(footing, kind) + if a is None or b is None: + return pd.DataFrame(columns=['sector', 'sector_name', 'perc_diff']), [] + drops = footing_drops(a, b, mixed_step=step.mixed) + frame = _apply_electricity_drops(perc_frame(a, b), drops) + return frame, drops + + +def _panel_df(frame: pd.DataFrame) -> pd.DataFrame: + out = frame.copy() + out['perc_diff'] = pd.to_numeric(out['perc_diff'], errors='coerce') + return out + + +def frozen_panel_png(impl_id: ImplId, kind: str) -> Path | None: + """Return the published 1×3 panel for this implementation, if present.""" + name = FROZEN_PANEL_PNG.get((impl_id, kind)) + if name is None: + return None + path = FIGURES_DIR / name + return path if path.is_file() else None + + +def stack_panel_pngs(paths: list[Path], out: Path) -> Path: + """Stack 1×3 panel PNGs top-to-bottom without redrawing them.""" + images = [Image.open(p).convert('RGBA') for p in paths] + width = max(im.width for im in images) + aligned = [] + for im in images: + if im.width == width: + aligned.append(im) + continue + height = max(1, round(im.height * width / im.width)) + aligned.append(im.resize((width, height), Image.Resampling.LANCZOS)) + canvas = Image.new( + 'RGBA', + (width, sum(im.height for im in aligned)), + (255, 255, 255, 255), + ) + y = 0 + for im in aligned: + canvas.paste(im, (0, y)) + y += im.height + out.parent.mkdir(parents=True, exist_ok=True) + canvas.save(out) + return out + + +def _draw_hist_panel( + ax: Axes, + pair: Pair, + bundle: ImplBundle, + impl_id: ImplId, + top: ImplBundle, + bottom: ImplBundle, + step_id: StepId, + kind: str, + *, + footnote_y: float = -0.22, + text_box_fontsize: int = 7, + legend_fontsize: int = 8, +) -> None: + title = HIST_PANEL_TITLE[step_id] + frame, drops = _panel_data(pair, bundle, impl_id, top, bottom, step_id, kind) + if frame.empty or frame['perc_diff'].dropna().empty: + ax.set_title(title) + ax.text( + 0.5, + 0.5, + 'no data', + ha='center', + va='center', + transform=ax.transAxes, + ) + ax.set_xticks([]) + ax.set_yticks([]) + return + percent_histogram( + ax, + frame['perc_diff'].dropna() * 100, + xlim=DEFAULT_XLIM, + xlabel='Percentage Diff (%)', + ylabel='Count', + title=title, + text_box=_beyond_20_text(frame, 'perc_diff'), + text_box_fontsize=text_box_fontsize, + legend_fontsize=legend_fontsize, + ) + apply_axis_fonts(ax) + footnote = format_drop_footnote(drops) + if footnote: + ax.text( + 0.0, + footnote_y, + footnote, + transform=ax.transAxes, + fontsize=6, + ha='left', + va='top', + family='monospace', + wrap=True, + ) + + +def _footing_suptitle(pair: Pair, impl_id: ImplId, kind: str) -> str: + kind_label = 'direct EF (D)' if kind == 'D' else 'total EF (N)' + if pair.hist_baseline == 'peer': + vs = 'vs Cornerstone v0.3 production (non-disagg)' + elif IMPLEMENTATIONS[impl_id].footing_label == 'v0.2': + vs = 'vs Cornerstone v0.2 footing' + else: + vs = 'vs Cornerstone v0.3.1 electricity footing' + return f'{kind_label} per-sector % diff {vs} — electricity disagg steps' + + +def render_one_row_hist( + pair: Pair, + bundle: ImplBundle, + impl_id: ImplId, + kind: str, + top: ImplBundle, + bottom: ImplBundle, +) -> Figure: + """Single 1×3 vs-footing row, matching ``plot_ef.write_panel_pngs`` layout.""" + setup_mpl(font_size=13) + fig, axes = plt.subplots(1, 3, figsize=(30.0, 9.0), squeeze=False) + for c, step_id in enumerate(HIST_STEPS): + _draw_hist_panel( + axes[0][c], + pair, + bundle, + impl_id, + top, + bottom, + step_id, + kind, + footnote_y=-0.18, + text_box_fontsize=8, + legend_fontsize=10, + ) + later_ymax = max(axes[0][i].get_ylim()[1] for i in range(1, 3)) + for i in range(1, 3): + axes[0][i].set_ylim(0, later_ymax) + fig.suptitle(_footing_suptitle(pair, impl_id, kind), fontsize=16) + fig.tight_layout(rect=(0, 0.06, 1, 0.96)) + return fig + + +def render_hist_figure( + pair: Pair, + top: ImplBundle, + bottom: ImplBundle, + kind: str, +) -> Figure: + setup_mpl(font_size=11) + fig, axes = plt.subplots(2, 3, figsize=(16.0, 9.2), squeeze=False) + row_bundles = (top, bottom) + row_ids = (pair.top, pair.bottom) + for r, (bundle, impl_id) in enumerate(zip(row_bundles, row_ids)): + impl = IMPLEMENTATIONS[impl_id] + for c, step_id in enumerate(HIST_STEPS): + _draw_hist_panel( + axes[r][c], + pair, + bundle, + impl_id, + top, + bottom, + step_id, + kind, + ) + axes[r][0].set_ylabel(f'{impl.title}\nCount') + kind_label = 'direct EF (D)' if kind == 'D' else 'total EF (N)' + fig.suptitle(f'{kind_label} per-sector % diff', fontsize=16) + fig.tight_layout(rect=(0, 0.04, 1, 0.95)) + return fig + + +def _peer_baseline( + pair: Pair, + top: ImplBundle, + bottom: ImplBundle, +) -> StepSnapshot | None: + if pair.hist_baseline != 'peer': + return None + for bundle in (top, bottom): + if IMPLEMENTATIONS[bundle.impl_id].schema != 'aggregate': + continue + snap = bundle.steps.get('footing') + if snap is None and bundle.steps: + snap = next(iter(bundle.steps.values())) + if snap is not None: + return snap + return None + + +def _panel_data( + pair: Pair, + bundle: ImplBundle, + impl_id: ImplId, + top: ImplBundle, + bottom: ImplBundle, + step_id: StepId, + kind: str, +) -> tuple[pd.DataFrame, list[DroppedSector]]: + step = bundle.steps.get(step_id) + if step is None: + return pd.DataFrame(columns=['sector', 'sector_name', 'perc_diff']), [] + if pair.hist_mode == 'pairwise': + top_step = top.steps.get(step_id) + bottom_step = bottom.steps.get(step_id) + if top_step is None or bottom_step is None: + return pd.DataFrame(columns=['sector', 'sector_name', 'perc_diff']), [] + if impl_id == pair.top: + frame, drops = pairwise_frame(top_step, bottom_step, kind) + else: + frame, drops = pairwise_frame(bottom_step, top_step, kind) + return _panel_df(frame), drops + footing = _peer_baseline(pair, top, bottom) or bundle.steps.get('footing') + if footing is None: + return pd.DataFrame(columns=['sector', 'sector_name', 'perc_diff']), [] + frame, drops = vs_footing_frame(step, footing, kind) + return _panel_df(frame), drops + + +def write_hist_png( + pair: Pair, + top: ImplBundle, + bottom: ImplBundle, + kind: str, + path: Path, +) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + if pair.hist_mode == 'vs_footing': + rows: list[Path] = [] + for impl_id, bundle in ((pair.top, top), (pair.bottom, bottom)): + frozen = frozen_panel_png(impl_id, kind) + if frozen is not None: + rows.append(frozen) + continue + row_path = path.parent / f'{pair.key}_{impl_id}_{kind}_row.png' + fig = render_one_row_hist(pair, bundle, impl_id, kind, top, bottom) + save_and_close(fig, row_path) + rows.append(row_path) + return stack_panel_pngs(rows, path) + fig = render_hist_figure(pair, top, bottom, kind) + save_and_close(fig, path) + return path diff --git a/bedrock/analysis/electricity/current/diagnostics/deck/pairs.py b/bedrock/analysis/electricity/current/diagnostics/deck/pairs.py new file mode 100644 index 000000000..31264b886 --- /dev/null +++ b/bedrock/analysis/electricity/current/diagnostics/deck/pairs.py @@ -0,0 +1,311 @@ +"""Comparison pairs and electricity-step labels for the five-slide deck.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Literal + +from bedrock.utils.snapshots import releases + +ImplId = Literal['current', 'eia_gtd', 'original', 'production'] +HistMode = Literal['pairwise', 'vs_footing'] +HistBaseline = Literal['own_footing', 'peer'] +StepId = Literal['footing', 'reallocation', 'three_way', 'mixed_units'] +ClassRowStyle = Literal['eia', 'original'] +Schema = Literal['disagg', 'aggregate'] + +STEPS: tuple[StepId, ...] = ( + 'footing', + 'reallocation', + 'three_way', + 'mixed_units', +) + +HIST_STEPS: tuple[StepId, ...] = ('reallocation', 'three_way', 'mixed_units') + +STEP_COLUMN_LABEL: dict[StepId, str] = { + 'footing': 'footing', + 'reallocation': 'reallocation', + 'three_way': '3-way split', + 'mixed_units': 'unit conversion', +} + +HIST_PANEL_TITLE: dict[StepId, str] = { + 'reallocation': 'Co-production reallocation', + 'three_way': '3-way monetary split', + 'mixed_units': 'Conversion to physical units', +} + +FOOTING_CONFIG = '2025_usa_cornerstone_v0_3_electricity_footing' +PRODUCTION_CONFIG = '2025_usa_cornerstone_v0_3' +REALLOC_CONFIG = '2025_usa_cornerstone_v0_3_electricity_reallocation' +DISAGG_CONFIG = '2025_usa_cornerstone_v0_3_electricity_disaggregation' +MIXED_CONFIG = '2025_usa_cornerstone_v0_3_electricity_mixed_units' + +CONFIG_FOR_STEP: dict[StepId, str] = { + 'footing': FOOTING_CONFIG, + 'reallocation': REALLOC_CONFIG, + 'three_way': DISAGG_CONFIG, + 'mixed_units': MIXED_CONFIG, +} + +CHILD_SECTORS: tuple[str, ...] = ('221110', '221121', '221122') +AGGREGATE_SECTOR = '221100' +STAR_SECTOR = '221100*' +GENERATION_SECTOR = '221110' + +TABLE_ROW_SECTORS: tuple[str, ...] = ( + AGGREGATE_SECTOR, + STAR_SECTOR, + '221110', + '221121', + '221122', +) + +ROW_DISPLAY: dict[str, str] = { + AGGREGATE_SECTOR: '221100', + STAR_SECTOR: '221100*', + '221110': '221110 (G)', + '221121': '221121 (T)', + '221122': '221122 (D)', +} + +AGGREGATE_ONLY_NA = frozenset({STAR_SECTOR, '221110', '221121', '221122'}) + +# Sector is not in the model at this step (sample N/A), distinct from missing data. +NA_AT_STEP: dict[StepId, frozenset[str]] = { + 'footing': frozenset({STAR_SECTOR, '221110', '221121', '221122'}), + 'reallocation': frozenset({STAR_SECTOR, '221110', '221121', '221122'}), + 'three_way': frozenset({AGGREGATE_SECTOR}), + 'mixed_units': frozenset({AGGREGATE_SECTOR}), +} + +CLASS_ORDER: tuple[str, ...] = ( + 'Residential', + 'Commercial', + 'Industrial', + 'Transportation', + 'Exports', +) + +EIA_CLASS_GROUPS: tuple[tuple[str, tuple[str, ...]], ...] = ( + ('Residential', ('Residential',)), + ( + 'Com+Ind+Trans+Exports', + ('Commercial', 'Industrial', 'Transportation', 'Exports'), + ), +) + +ORIGINAL_CLASS_GROUPS: tuple[tuple[str, tuple[str, ...]], ...] = ( + ('Residential', ('Residential',)), + ('Com+Ind+Trans sales', ('Commercial', 'Industrial', 'Transportation')), +) + + +@dataclass(frozen=True) +class Implementation: + id: ImplId + title: str + footing_label: str + snapshot_key: str + class_row_style: ClassRowStyle + industrial_weights: Literal['mecs', 'dollars'] | None + schema: Schema = 'disagg' + single_config: str | None = None + + +@dataclass(frozen=True) +class Pair: + key: str + top: ImplId + bottom: ImplId + hist_mode: HistMode + filename: str + slide1_note: str + slide_ef_note: str + slide4_extra_note: str + slide5_caption: str + slide3_caption: str + hist_baseline: HistBaseline = 'own_footing' + + +IMPLEMENTATIONS: dict[ImplId, Implementation] = { + 'current': Implementation( + id='current', + title='Current (post-MECS)', + footing_label='v0.3.1', + snapshot_key=releases.v0_3_1, + class_row_style='eia', + industrial_weights='mecs', + ), + 'eia_gtd': Implementation( + id='eia_gtd', + title='EIA G/T/D (pre-MECS)', + footing_label='v0.3.1', + snapshot_key=releases.v0_3_1, + class_row_style='eia', + industrial_weights='dollars', + ), + 'original': Implementation( + id='original', + title='Original Electricity Disaggregation', + footing_label='v0.2', + snapshot_key=releases.v0_2, + class_row_style='original', + industrial_weights=None, + ), + 'production': Implementation( + id='production', + title='Cornerstone v0.3 production (non-disagg)', + footing_label='v0.3', + snapshot_key=releases.v0_3_1, + class_row_style='eia', + industrial_weights=None, + schema='aggregate', + single_config=PRODUCTION_CONFIG, + ), +} + +PAIRS: dict[str, Pair] = { + 'current_vs_eia_gtd': Pair( + key='current_vs_eia_gtd', + top='current', + bottom='eia_gtd', + hist_mode='vs_footing', + filename='current_vs_eia_gtd.pptx', + slide1_note=( + 'Class totals keep EIA Table 2.2 identities. MECS changes shares ' + 'inside manufacturing, which this slide does not show.' + ), + slide_ef_note=( + 'Baseline is the v0.3.1 electricity footing for both sides. ' + '221100 is the aggregate sector; 221100* re-aggregates G/T/D. ' + 'Steps where a sector is not in the model are N/A. Matching ' + 'values are marked same. Those same/not-same cells are the ' + 'MECS vs dollar-weight differences.' + ), + slide4_extra_note='', + slide5_caption=( + 'Top: current (post-MECS) vs v0.3.1 electricity footing. ' + 'Bottom: EIA G/T/D (pre-MECS) vs the same footing ' + '(published PPTX panel). Compare the two rows for the MECS ' + 'effect; cell-level N differences are on the table slide.' + ), + slide3_caption=( + 'Top: current (post-MECS) vs v0.3.1 electricity footing. ' + 'Bottom: EIA G/T/D (pre-MECS) vs the same footing ' + '(published PPTX panel). Compare the two rows for the MECS ' + 'effect; cell-level D differences are on the table slide.' + ), + ), + 'current_vs_original': Pair( + key='current_vs_original', + top='original', + bottom='current', + hist_mode='vs_footing', + filename='current_vs_original.pptx', + slide1_note=( + 'Goal of EIA-anchored disaggregation is to match EIA end-use class ' + 'MWh. Totals differ because EIA-anchored uses EIA trends for 2017 ' + 'eGRID (when missing) and treats exports differently.' + ), + slide_ef_note=( + 'Baseline differs between implementations. ' + '221100 is the original sector; 221100* is the re-aggregated ' + 'G/T/D block. Steps where a sector is not in the model are N/A. ' + 'Matching values are marked same.' + ), + slide4_extra_note=( + 'For EIA-anchored mixed units, 3-way and unit-conversion N may ' + 'match because generation uses a unique price conversion.' + ), + slide5_caption=( + 'Top: original vs v0.2 footing (published PPTX panel). ' + 'Bottom: current (post-MECS) vs v0.3.1 electricity footing.' + ), + slide3_caption=( + 'Top: original vs v0.2 footing (published PPTX panel). ' + 'Bottom: current (post-MECS) vs v0.3.1 electricity footing.' + ), + ), + 'eia_gtd_vs_original': Pair( + key='eia_gtd_vs_original', + top='original', + bottom='eia_gtd', + hist_mode='vs_footing', + filename='eia_gtd_vs_original.pptx', + slide1_note=( + 'Goal of EIA-anchored disaggregation is to match EIA end-use class ' + 'MWh. Totals differ because EIA-anchored uses EIA trends for 2017 ' + 'eGRID (when missing) and treats exports differently.' + ), + slide_ef_note=( + 'Baseline differs between implementations. ' + '221100 is the original sector; 221100* is the re-aggregated ' + 'G/T/D block. Steps where a sector is not in the model are N/A. ' + 'Matching values are marked same.' + ), + slide4_extra_note=( + 'For EIA-anchored mixed units, 3-way and unit-conversion N may ' + 'match because generation uses a unique price conversion.' + ), + slide5_caption=( + 'Top: original vs v0.2 footing (published PPTX panel). ' + 'Bottom: EIA G/T/D (pre-MECS) vs v0.3.1 electricity footing ' + '(published PPTX panel). Less variance on the EIA row when ' + 'generation uses a constant price.' + ), + slide3_caption=( + 'Top: original vs v0.2 footing (published PPTX panel). ' + 'Bottom: EIA G/T/D (pre-MECS) vs v0.3.1 electricity footing ' + '(published PPTX panel).' + ), + ), + 'current_vs_production': Pair( + key='current_vs_production', + top='current', + bottom='production', + hist_mode='vs_footing', + hist_baseline='peer', + filename='current_vs_production.pptx', + slide1_note=( + 'Non-disagg production has no EIA end-use class split. ' + 'Class MWh is only defined on the electricity-disaggregation path.' + ), + slide_ef_note=( + 'Production is 2025_usa_cornerstone_v0_3: aggregate 221100 at every ' + 'column, industry-average margins on, no G/T/D. Current is the ' + 'electricity-disagg chain (margins off). Matching values are ' + 'marked same. G/T/D and 221100* are N/A on production.' + ), + slide4_extra_note='', + slide5_caption=( + 'Top: current electricity disagg vs Cornerstone v0.3 production ' + '(non-disagg). Bottom: production vs itself (0% check). ' + 'Shared baseline is production D/N, not the electricity footing.' + ), + slide3_caption=( + 'Top: current electricity disagg vs Cornerstone v0.3 production ' + '(non-disagg). Bottom: production vs itself (0% check). ' + 'Shared baseline is production D/N, not the electricity footing.' + ), + ), +} + + +def class_groups_for(style: ClassRowStyle) -> tuple[tuple[str, tuple[str, ...]], ...]: + if style == 'original': + return ORIGINAL_CLASS_GROUPS + return EIA_CLASS_GROUPS + + +def config_for_step(impl: Implementation, step_id: StepId) -> str: + if impl.single_config is not None: + return impl.single_config + return CONFIG_FOR_STEP[step_id] + + +def na_sectors_at_step(impl_id: ImplId, step_id: StepId) -> frozenset[str]: + if IMPLEMENTATIONS[impl_id].schema == 'aggregate': + return AGGREGATE_ONLY_NA + return NA_AT_STEP[step_id] diff --git a/bedrock/analysis/electricity/current/diagnostics/deck/paths.py b/bedrock/analysis/electricity/current/diagnostics/deck/paths.py new file mode 100644 index 000000000..b7c47c264 --- /dev/null +++ b/bedrock/analysis/electricity/current/diagnostics/deck/paths.py @@ -0,0 +1,21 @@ +"""Output paths for the five-slide electricity comparison deck.""" + +from __future__ import annotations + +from pathlib import Path + +from bedrock.analysis.electricity.current.diagnostics.paths import OUT_DIR, ensure_dirs + +DECK_OUT_DIR = OUT_DIR / 'deck' +CACHE_DIR = DECK_OUT_DIR / 'cache' + + +def ensure_deck_dirs() -> Path: + ensure_dirs() + DECK_OUT_DIR.mkdir(parents=True, exist_ok=True) + CACHE_DIR.mkdir(parents=True, exist_ok=True) + return DECK_OUT_DIR + + +def impl_cache_dir(impl_id: str, config_stem: str) -> Path: + return CACHE_DIR / impl_id / config_stem diff --git a/bedrock/analysis/electricity/current/diagnostics/deck/pptx_write.py b/bedrock/analysis/electricity/current/diagnostics/deck/pptx_write.py new file mode 100644 index 000000000..1b5e49cb5 --- /dev/null +++ b/bedrock/analysis/electricity/current/diagnostics/deck/pptx_write.py @@ -0,0 +1,315 @@ +"""Assemble the five-slide comparison PPTX.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from pptx import Presentation +from pptx.dml.color import RGBColor +from pptx.enum.text import MSO_ANCHOR, PP_ALIGN +from pptx.oxml import parse_xml +from pptx.util import Emu, Inches, Pt + +from bedrock.analysis.electricity.current.diagnostics.deck.data import ( + ImplBundle, + fill_mixed_c_col, +) +from bedrock.analysis.electricity.current.diagnostics.deck.histograms import ( + write_hist_png, +) +from bedrock.analysis.electricity.current.diagnostics.deck.pairs import Pair +from bedrock.analysis.electricity.current.diagnostics.deck.tables import ( + TableGrid, + class_mwh_grids, + ef_grids, +) + +SLIDE_W = Inches(13.333) +SLIDE_H = Inches(7.5) +# Petrol header from the original vs EIA-anchored deck (≈ RGB 0, 91, 127). +HEADER_BG = RGBColor(0x00, 0x5B, 0x7F) +HEADER_FG = RGBColor(0xFF, 0xFF, 0xFF) +ALT_ROW = RGBColor(0xEB, 0xEB, 0xEB) +WHITE = RGBColor(0xFF, 0xFF, 0xFF) +BODY_FG = RGBColor(0x22, 0x22, 0x22) +SAME_FG = RGBColor(0x2E, 0x7D, 0x32) +NAVY = RGBColor(0x1F, 0x4E, 0x79) +TABLE_FONT_PT = 12 +TABLE_WIDTH_IN = 6.15 +TABLE_LEFTS = (0.40, 6.78) +TABLE_TOP_IN = 3.20 +HEADER_ROW_IN = 0.34 +DATA_ROW_IN = 0.30 +BORDER_WHITE = 'FFFFFF' +BORDER_EMU = '12700' + + +def _set_run_font( + run: Any, *, size_pt: float, bold: bool = False, color: RGBColor | None = None +) -> None: + run.font.size = Pt(size_pt) + run.font.bold = bold + run.font.name = 'Calibri' + if color is not None: + run.font.color.rgb = color + + +def _set_cell_borders(cell: Any) -> None: + tc = cell._tc + tc_pr = tc.get_or_add_tcPr() + for edge in ('lnL', 'lnR', 'lnT', 'lnB'): + for child in list(tc_pr): + if child.tag.endswith(edge): + tc_pr.remove(child) + tc_pr.append( + parse_xml( + f'' + f'' + f'' + f'' + ) + ) + + +def _add_title(slide: Any, text: str) -> None: + box = slide.shapes.add_textbox( + Inches(0.35), Inches(0.12), Inches(12.6), Inches(0.45) + ) + tf = box.text_frame + tf.clear() + p = tf.paragraphs[0] + run = p.add_run() + run.text = text + _set_run_font(run, size_pt=18, bold=True, color=NAVY) + + +def _add_notes( + slide: Any, text: str, top_in: float = 6.55, height_in: float = 0.8 +) -> None: + box = slide.shapes.add_textbox( + Inches(0.35), Inches(top_in), Inches(12.6), Inches(height_in) + ) + tf = box.text_frame + tf.word_wrap = True + tf.clear() + p = tf.paragraphs[0] + run = p.add_run() + run.text = text + _set_run_font(run, size_pt=10, color=RGBColor(0x33, 0x33, 0x33)) + + +def _fill_cell( + cell: Any, + text: str, + *, + header: bool, + alt: bool, + align: Any, +) -> None: + cell.text = '' + cell.vertical_anchor = MSO_ANCHOR.MIDDLE + p = cell.text_frame.paragraphs[0] + p.alignment = align + run = p.add_run() + run.text = text + if header: + _set_run_font(run, size_pt=TABLE_FONT_PT, bold=True, color=HEADER_FG) + cell.fill.solid() + cell.fill.fore_color.rgb = HEADER_BG + else: + color = SAME_FG if text == 'same' else BODY_FG + _set_run_font(run, size_pt=TABLE_FONT_PT, bold=text == 'same', color=color) + cell.fill.solid() + cell.fill.fore_color.rgb = ALT_ROW if alt else WHITE + cell.margin_left = Emu(91440) + cell.margin_right = Emu(91440) + cell.margin_top = Emu(45720) + cell.margin_bottom = Emu(45720) + cell.text_frame.word_wrap = True + _set_cell_borders(cell) + + +def _col_align(col: int, n_cols: int) -> Any: + if n_cols >= 6: + if col <= 1: + return PP_ALIGN.LEFT + return PP_ALIGN.CENTER + if col == 0: + return PP_ALIGN.LEFT + return PP_ALIGN.RIGHT + + +def _set_column_widths(table: Any, n_cols: int, width: float) -> None: + if n_cols >= 6: + shares = (1.05, 1.40, 0.95, 1.05, 0.85, 0.85) + scale = width / sum(shares) + for j, share in enumerate(shares): + table.columns[j].width = Inches(share * scale) + return + first = width * 0.34 + rest = (width - first) / max(n_cols - 1, 1) + for j in range(n_cols): + table.columns[j].width = Inches(first if j == 0 else rest) + + +def _add_table( + slide: Any, + grid: TableGrid, + *, + left: float, + top: float, + width: float, +) -> None: + n_cols = len(grid.headers) + banner = n_cols >= 6 + if not banner: + caption = slide.shapes.add_textbox( + Inches(left), Inches(top - 0.32), Inches(width), Inches(0.3) + ) + tf = caption.text_frame + tf.clear() + p = tf.paragraphs[0] + run = p.add_run() + run.text = grid.title + _set_run_font(run, size_pt=TABLE_FONT_PT, bold=True, color=NAVY) + + n_header = 2 if banner else 1 + n_rows = n_header + len(grid.rows) + height = n_header * HEADER_ROW_IN + len(grid.rows) * DATA_ROW_IN + table = slide.shapes.add_table( + n_rows, n_cols, Inches(left), Inches(top), Inches(width), Inches(height) + ).table + _set_column_widths(table, n_cols, width) + for i, row in enumerate(table.rows): + row.height = Inches(HEADER_ROW_IN if i < n_header else DATA_ROW_IN) + + header_row = 1 if banner else 0 + if banner: + for j in range(n_cols): + _fill_cell( + table.cell(0, j), '', header=True, alt=False, align=PP_ALIGN.CENTER + ) + table.cell(0, 2).merge(table.cell(0, n_cols - 1)) + _fill_cell( + table.cell(0, 2), + grid.title, + header=True, + alt=False, + align=PP_ALIGN.CENTER, + ) + for j, header in enumerate(grid.headers): + _fill_cell( + table.cell(header_row, j), + header, + header=True, + alt=False, + align=PP_ALIGN.CENTER if n_cols >= 6 else _col_align(j, n_cols), + ) + data_start = header_row + 1 + for i, row in enumerate(grid.rows): + alt = i % 2 == 0 + for j, value in enumerate(row): + _fill_cell( + table.cell(data_start + i, j), + value, + header=False, + alt=alt, + align=_col_align(j, n_cols), + ) + if banner and len(grid.rows) > 1: + origin = table.cell(data_start, 0) + origin.merge(table.cell(n_rows - 1, 0)) + origin.vertical_anchor = MSO_ANCHOR.MIDDLE + + +def _two_tables_slide( + prs: Any, + title: str, + note: str, + left_grid: TableGrid, + right_grid: TableGrid, +) -> None: + slide = prs.slides.add_slide(prs.slide_layouts[6]) + _add_title(slide, title) + _add_notes(slide, note, top_in=0.58, height_in=2.2) + _add_table( + slide, left_grid, left=TABLE_LEFTS[0], top=TABLE_TOP_IN, width=TABLE_WIDTH_IN + ) + _add_table( + slide, right_grid, left=TABLE_LEFTS[1], top=TABLE_TOP_IN, width=TABLE_WIDTH_IN + ) + + +def _image_slide(prs: Any, title: str, caption: str, png: Path) -> None: + slide = prs.slides.add_slide(prs.slide_layouts[6]) + _add_title(slide, title) + slide.shapes.add_picture( + str(png), Inches(0.25), Inches(0.6), width=Inches(12.8), height=Inches(6.0) + ) + _add_notes(slide, caption, top_in=6.65) + + +def write_pptx( + pair: Pair, + top: ImplBundle, + bottom: ImplBundle, + out_path: Path, + *, + png_dir: Path, +) -> Path: + fill_mixed_c_col(top) + fill_mixed_c_col(bottom) + out_path.parent.mkdir(parents=True, exist_ok=True) + png_dir.mkdir(parents=True, exist_ok=True) + d_png = write_hist_png(pair, top, bottom, 'D', png_dir / f'{pair.key}_D.png') + n_png = write_hist_png(pair, top, bottom, 'N', png_dir / f'{pair.key}_N.png') + + class_top, class_bottom = class_mwh_grids(pair, top, bottom) + d_top, d_bottom = ef_grids(pair, top, bottom, 'D') + n_top, n_bottom = ef_grids(pair, top, bottom, 'N') + + prs = Presentation() + prs.slide_width = SLIDE_W + prs.slide_height = SLIDE_H + + _two_tables_slide( + prs, + 'Overview of results: Electricity MWh by use class', + pair.slide1_note, + class_top, + class_bottom, + ) + _two_tables_slide( + prs, + 'Overview of results: D for electricity sectors', + pair.slide_ef_note + + ' Electricity D EFs by disaggregation step (kg CO₂e / USD).', + d_top, + d_bottom, + ) + _image_slide( + prs, + 'Overview of results: D for electricity sectors', + pair.slide3_caption, + d_png, + ) + slide4_note = pair.slide_ef_note + if pair.slide4_extra_note: + slide4_note = f'{slide4_note} {pair.slide4_extra_note}' + _two_tables_slide( + prs, + 'Overview of results: N for electricity sectors', + slide4_note + ' Electricity N EFs by disaggregation step (kg CO₂e / USD).', + n_top, + n_bottom, + ) + _image_slide( + prs, + 'Overview of results: N for electricity sectors', + pair.slide5_caption, + n_png, + ) + prs.save(str(out_path)) + return out_path diff --git a/bedrock/analysis/electricity/current/diagnostics/deck/sources.py b/bedrock/analysis/electricity/current/diagnostics/deck/sources.py new file mode 100644 index 000000000..5a6b97ba3 --- /dev/null +++ b/bedrock/analysis/electricity/current/diagnostics/deck/sources.py @@ -0,0 +1,239 @@ +"""Resolve D/N/q/class-MWh from the deck cache or a live derive. + +``current`` uses the electricity-disagg YAML chain. ``production`` uses +``2025_usa_cornerstone_v0_3`` once and repeats it at every table column. +Original and pre-MECS EIA G/T/D table/histogram values come from +``historical/original_vs_eia_anchored_deck``, not from freeze parquets. +""" + +from __future__ import annotations + +import json +import logging +from dataclasses import replace +from pathlib import Path + +import pandas as pd + +from bedrock.analysis.electricity.current.diagnostics.deck.data import ( + ImplBundle, + StepSnapshot, + class_mwh_from_parquet, + fill_mixed_c_col, + series_from_parquet, +) +from bedrock.analysis.electricity.current.diagnostics.deck.pairs import ( + STEPS, + Implementation, + StepId, + config_for_step, +) +from bedrock.analysis.electricity.current.diagnostics.deck.paths import impl_cache_dir +from bedrock.analysis.electricity.historical.original_vs_eia_anchored_deck.published import ( + PUBLISHED_IMPLS, +) + +logger = logging.getLogger(__name__) + + +def _read_c_col(folder: Path) -> float | None: + meta = folder / 'run_metadata.json' + if not meta.is_file(): + return None + payload = json.loads(meta.read_text(encoding='utf-8')) + raw = payload.get('c_col') + if raw is None: + return None + return float(raw) + + +def _load_folder(folder: Path, *, mixed: bool) -> StepSnapshot | None: + if not folder.is_dir(): + return None + d_path = folder / 'D.parquet' + n_path = folder / 'N.parquet' + q_path = folder / 'q.parquet' + x_path = folder / 'x.parquet' + class_path = folder / 'class_generation_mwh.parquet' + if not n_path.is_file() and not d_path.is_file(): + return None + class_mwh = class_mwh_target = None + if class_path.is_file(): + class_mwh, class_mwh_target = class_mwh_from_parquet(class_path) + return StepSnapshot( + d=series_from_parquet(d_path) if d_path.is_file() else None, + n=series_from_parquet(n_path) if n_path.is_file() else None, + q=series_from_parquet(q_path, 'q') if q_path.is_file() else None, + x=series_from_parquet(x_path, 'x') if x_path.is_file() else None, + mixed=mixed, + c_col=_read_c_col(folder), + class_mwh=class_mwh, + class_mwh_target=class_mwh_target, + ) + + +def _candidate_dirs(impl: Implementation, step_id: StepId) -> list[Path]: + return [impl_cache_dir(impl.id, config_for_step(impl, step_id))] + + +def load_step(impl: Implementation, step_id: StepId) -> StepSnapshot | None: + mixed = step_id == 'mixed_units' + for folder in _candidate_dirs(impl, step_id): + snap = _load_folder(folder, mixed=mixed) + if snap is not None: + return snap + return None + + +def load_footing_from_snapshot(impl: Implementation) -> StepSnapshot: + """D/N from the release snapshot used as that pipeline's EF footing.""" + from bedrock.utils.math.formulas import ( # noqa: PLC0415 + compute_d, + compute_L_matrix, + compute_M_matrix, + compute_n, + ) + from bedrock.utils.snapshots.loader import load_snapshot # noqa: PLC0415 + + cache = impl_cache_dir('footing_snapshot', impl.snapshot_key) + cached = _load_folder(cache, mixed=False) + if cached is not None and cached.d is not None and cached.n is not None: + return cached + + b = load_snapshot('B_USA_non_finetuned', impl.snapshot_key) + adom = load_snapshot('Adom_USA', impl.snapshot_key) + aimp = load_snapshot('Aimp_USA', impl.snapshot_key) + d = compute_d(B=b) + n = compute_n(M=compute_M_matrix(B=b, L=compute_L_matrix(A=adom + aimp))) + cache.mkdir(parents=True, exist_ok=True) + d.rename('D').to_frame().to_parquet(cache / 'D.parquet') + n.rename('N').to_frame().to_parquet(cache / 'N.parquet') + return StepSnapshot(d=d.astype(float), n=n.astype(float), mixed=False) + + +def _write_class_mwh(folder: Path) -> None: + from bedrock.analysis.electricity.current.eia_gtd.purchaser_tables import ( # noqa: PLC0415 + allocated_class_mwh, + class_mwh_targets, + ) + from bedrock.transform.eeio.electricity_gtd_allocation import ( # noqa: PLC0415 + get_reanchored_eia_purchaser_allocation, + ) + from bedrock.utils.config.usa_config import get_usa_config # noqa: PLC0415 + + alloc = get_reanchored_eia_purchaser_allocation() + if alloc is None: + return + eia_year = int(get_usa_config().model_base_year) + model = allocated_class_mwh(alloc) + targets = class_mwh_targets(alloc, eia_year) + rows = [] + for cls in model.index: + rows.append( + { + 'class': str(cls), + 'value': float(model.loc[cls]), + 'unit': 'MWh', + 'target': float(targets.get(str(cls), float('nan'))), + } + ) + pd.DataFrame(rows).to_parquet(folder / 'class_generation_mwh.parquet') + + +def derive_step(impl: Implementation, step_id: StepId) -> StepSnapshot: + """Live-derive one config and write the deck cache.""" + from bedrock.publish.cache_reset import clear_all_publish_caches # noqa: PLC0415 + from bedrock.transform.eeio.cornerstone_disagg_pipeline import ( # noqa: PLC0415 + electricity_conversion_factors, + electricity_mixed_units_enabled, + ) + from bedrock.transform.eeio.derived_cornerstone import ( # noqa: PLC0415 + derive_cornerstone_Aq_mixed_units, + derive_cornerstone_Aq_scaled, + ) + from bedrock.utils.config.usa_config import ( # noqa: PLC0415 + reset_usa_config, + set_global_usa_config, + ) + from bedrock.utils.validation.diagnostics_helpers import ( # noqa: PLC0415 + pull_efs_for_diagnostics, + ) + + config = config_for_step(impl, step_id) + reset_usa_config() + clear_all_publish_caches() + set_global_usa_config(config) + if electricity_mixed_units_enabled(): + aq_mon = derive_cornerstone_Aq_scaled() + c_col, _c_row = electricity_conversion_factors(aq_mon) + aq = derive_cornerstone_Aq_mixed_units() + else: + aq = derive_cornerstone_Aq_scaled() + c_col = None + efs = pull_efs_for_diagnostics() + + folder = impl_cache_dir(impl.id, config) + folder.mkdir(parents=True, exist_ok=True) + efs.D_new.to_parquet(folder / 'D.parquet') + efs.N_new.to_parquet(folder / 'N.parquet') + aq.scaled_q.astype(float).rename('q').to_frame().to_parquet(folder / 'q.parquet') + metadata = { + 'config': config, + 'impl': impl.id, + 'c_col': c_col, + 'mixed_units': bool(electricity_mixed_units_enabled()), + } + (folder / 'run_metadata.json').write_text( + json.dumps(metadata, indent=2), encoding='utf-8' + ) + if impl.schema == 'disagg' and step_id in ('three_way', 'mixed_units'): + _write_class_mwh(folder) + mixed = step_id == 'mixed_units' + loaded = _load_folder(folder, mixed=mixed) + if loaded is None: + raise RuntimeError(f'failed to reload derived snapshot at {folder}') + return loaded + + +def load_impl_bundle( + impl: Implementation, + *, + derive: bool = False, + load_snapshot_footing: bool = False, +) -> ImplBundle: + if impl.id in PUBLISHED_IMPLS: + return ImplBundle(impl_id=impl.id, steps={}) + if impl.single_config is not None: + snap = load_step(impl, 'footing') + if snap is None and derive: + snap = derive_step(impl, 'footing') + copied: dict[StepId, StepSnapshot] = {} + if snap is not None: + for step_id in STEPS: + copied[step_id] = replace(snap) + return ImplBundle(impl_id=impl.id, steps=copied) + steps: dict[StepId, StepSnapshot] = {} + for step_id in STEPS: + snap = load_step(impl, step_id) + if ( + snap is None + and step_id == 'footing' + and load_snapshot_footing + and impl.schema == 'disagg' + ): + try: + snap = load_footing_from_snapshot(impl) + except Exception as exc: + logger.warning( + 'Skipping %s footing snapshot %s: %s', + impl.id, + impl.snapshot_key, + exc, + ) + if snap is None and derive and impl.schema == 'disagg': + snap = derive_step(impl, step_id) + if snap is not None: + steps[step_id] = snap + bundle = ImplBundle(impl_id=impl.id, steps=steps) + fill_mixed_c_col(bundle) + return bundle diff --git a/bedrock/analysis/electricity/current/diagnostics/deck/tables.py b/bedrock/analysis/electricity/current/diagnostics/deck/tables.py new file mode 100644 index 000000000..c5864fe25 --- /dev/null +++ b/bedrock/analysis/electricity/current/diagnostics/deck/tables.py @@ -0,0 +1,206 @@ +"""Slide 1 class-MWh tables and slides 2/4 electricity D/N tables.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from bedrock.analysis.electricity.current.diagnostics.deck.data import ( + MISSING, + NA, + SAME, + ImplBundle, + class_total_mwh, + format_ef, + format_ratio, + format_twh, + grouped_mwh, + sector_ef_usd, + values_match, +) +from bedrock.analysis.electricity.current.diagnostics.deck.pairs import ( + IMPLEMENTATIONS, + ROW_DISPLAY, + STEP_COLUMN_LABEL, + STEPS, + TABLE_ROW_SECTORS, + Implementation, + Pair, + StepId, + class_groups_for, + na_sectors_at_step, +) +from bedrock.analysis.electricity.historical.original_vs_eia_anchored_deck.published import ( + PUBLISHED_IMPLS, + published_class_mwh_rows, + published_ef, +) + + +@dataclass(frozen=True) +class TableGrid: + title: str + headers: tuple[str, ...] + rows: tuple[tuple[str, ...], ...] + + +def _class_row_values( + impl: Implementation, + bundle: ImplBundle, +) -> list[tuple[str, str, str, str]] | None: + if impl.id in PUBLISHED_IMPLS: + published_rows: list[tuple[str, str, str, str]] = [] + for pub_mwh, pub_target, label in published_class_mwh_rows(impl.id): + published_rows.append( + ( + label, + format_twh(pub_mwh), + format_twh(pub_target), + format_ratio(pub_mwh, pub_target), + ) + ) + return published_rows + mixed = bundle.steps.get('mixed_units') + three_way = bundle.steps.get('three_way') + step = None + if mixed is not None and mixed.class_mwh is not None: + step = mixed + elif three_way is not None and three_way.class_mwh is not None: + step = three_way + if step is None or step.class_mwh is None: + return None + model = step.class_mwh + target = step.class_mwh_target + groups = class_groups_for(impl.class_row_style) + live_rows: list[tuple[str, str, str, str]] = [] + for label, members in groups: + m = grouped_mwh(model, members) + if target is None: + live_rows.append((label, format_twh(m), MISSING, MISSING)) + else: + t = grouped_mwh(target, members) + live_rows.append((label, format_twh(m), format_twh(t), format_ratio(m, t))) + m_tot = class_total_mwh(model) + if target is None: + live_rows.append(('Total', format_twh(m_tot), MISSING, MISSING)) + else: + t_tot = class_total_mwh(target) + live_rows.append( + ('Total', format_twh(m_tot), format_twh(t_tot), format_ratio(m_tot, t_tot)) + ) + return live_rows + + +def class_mwh_grids( + pair: Pair, + top: ImplBundle, + bottom: ImplBundle, +) -> tuple[TableGrid, TableGrid]: + top_impl = IMPLEMENTATIONS[pair.top] + bottom_impl = IMPLEMENTATIONS[pair.bottom] + headers = ('Comparison', 'Model', 'Target', 'Ratio') + top_rows = _class_row_values(top_impl, top) + bottom_rows = _class_row_values(bottom_impl, bottom) + if top_rows is None: + top_rows = [('(no class MWh)', MISSING, MISSING, MISSING)] + if bottom_rows is None: + bottom_rows = [('(no class MWh)', MISSING, MISSING, MISSING)] + bottom_marked: list[tuple[str, str, str, str]] = [] + top_by_label = {r[0]: r for r in top_rows} + for row in bottom_rows: + label, model, target, ratio = row + other = top_by_label.get(label) + if other is not None and other[1:] == row[1:] and model != MISSING: + bottom_marked.append((label, SAME, SAME, SAME)) + else: + bottom_marked.append(row) + return ( + TableGrid(top_impl.title, headers, tuple(top_rows)), + TableGrid(bottom_impl.title, headers, tuple(bottom_marked)), + ) + + +def _cell_for_step( + bundle: ImplBundle, + kind: str, + sector: str, + step_id: StepId, +) -> float | None | str: + if sector in na_sectors_at_step(bundle.impl_id, step_id): + return NA + if bundle.impl_id in PUBLISHED_IMPLS: + value = published_ef(bundle.impl_id, kind, sector, step_id) + return NA if value is None else value + step = bundle.steps.get(step_id) + if step is None: + return None + return sector_ef_usd(step, kind, sector) + + +def _format_cell( + value: float | None | str, + other: float | None | str, + *, + mark_same: bool, +) -> str: + if value == NA: + return NA + if isinstance(value, str): + return value + if value is None: + return MISSING + if mark_same and not isinstance(other, str) and values_match(value, other): + return SAME + return format_ef(value) + + +def ef_grids( + pair: Pair, + top: ImplBundle, + bottom: ImplBundle, + kind: str, +) -> tuple[TableGrid, TableGrid]: + top_impl = IMPLEMENTATIONS[pair.top] + bottom_impl = IMPLEMENTATIONS[pair.bottom] + return ( + _ef_grid(top_impl, top, bottom, kind, mark_same=False), + _ef_grid(bottom_impl, bottom, top, kind, mark_same=True), + ) + + +def _ef_grid( + impl: Implementation, + bundle: ImplBundle, + other: ImplBundle, + kind: str, + *, + mark_same: bool, +) -> TableGrid: + headers = ( + 'GHG', + 'Electricity sector', + impl.footing_label, + STEP_COLUMN_LABEL['reallocation'], + STEP_COLUMN_LABEL['three_way'], + STEP_COLUMN_LABEL['mixed_units'], + ) + rows: list[tuple[str, ...]] = [] + for sector in TABLE_ROW_SECTORS: + cells = [ + format_cell_pair(bundle, other, kind, sector, step, mark_same) + for step in STEPS + ] + rows.append(('Total GHG', ROW_DISPLAY[sector], *cells)) + return TableGrid(impl.title, headers, tuple(rows)) + + +def format_cell_pair( + bundle: ImplBundle, + other: ImplBundle, + kind: str, + sector: str, + step_id: StepId, + mark_same: bool, +) -> str: + value = _cell_for_step(bundle, kind, sector, step_id) + peer = _cell_for_step(other, kind, sector, step_id) + return _format_cell(value, peer, mark_same=mark_same) diff --git a/bedrock/analysis/electricity/current/diagnostics/ef_comparison/analyze_n_variance.py b/bedrock/analysis/electricity/current/diagnostics/ef_comparison/analyze_n_variance.py index d523af866..afb4a90cb 100644 --- a/bedrock/analysis/electricity/current/diagnostics/ef_comparison/analyze_n_variance.py +++ b/bedrock/analysis/electricity/current/diagnostics/ef_comparison/analyze_n_variance.py @@ -528,7 +528,7 @@ def render_mixed_units_section( "the 3-way split", "", "Scope: the **right panel** (\"Conversion to physical units\") of " - "`ef_panels_vs_v0_3_N.png` vs the **middle panel** (\"3-way monetary split\"). " + "`v0.3_eia_gtd_pre_mecs_N.png` vs the **middle panel** (\"3-way monetary split\"). " "Both panels are percent differences against the **same v0.3.1 electricity " "footing**, so the mixed panel **stacks** the 3-way undilution, the " "mixed-units `A`/`L` rewrite (flat `c_row = 1/p`), and P5 D0 re-anchor.", diff --git a/bedrock/analysis/electricity/current/diagnostics/ef_comparison/plot_ef.py b/bedrock/analysis/electricity/current/diagnostics/ef_comparison/plot_ef.py index 2c9196ada..d9ae419c3 100644 --- a/bedrock/analysis/electricity/current/diagnostics/ef_comparison/plot_ef.py +++ b/bedrock/analysis/electricity/current/diagnostics/ef_comparison/plot_ef.py @@ -169,6 +169,8 @@ def write_panel_pngs( fontsize=16, ) fig.tight_layout(rect=(0, 0.06, 1, 0.96)) + # Live write path. Published original / pre-MECS panels live in + # historical/original_vs_eia_anchored_deck/figures/. out = out_dir / f'ef_panels_vs_v0_3_{ef_kind}.png' save_and_close(fig, out) diff --git a/bedrock/analysis/electricity/current/eia_gtd/README.md b/bedrock/analysis/electricity/current/eia_gtd/README.md index 05a00196f..e9f8d77ce 100644 --- a/bedrock/analysis/electricity/current/eia_gtd/README.md +++ b/bedrock/analysis/electricity/current/eia_gtd/README.md @@ -18,9 +18,86 @@ use `get_2017_eia_purchaser_allocation`. | Table | Identity | |---|---| -| Class MWh vs D0 | `_class_mwh_targets(eia_year, alloc.egrid_mwh)` vs `alloc.mwh.groupby(alloc.end_use_class).sum()` | -| Leftover T&D | `bill − gen_dollars` (equals `t_dollars + d_dollars`) | -| Nibble vs clipped | class totals below D0 target; `clipped` is purchaser-level only | -| Optional ¢/kWh | `bill / (10 × MWh)` vs Table 2.4 — check only, not D0 | +| Class MWh vs class targets | `_class_mwh_targets(eia_year, alloc.egrid_mwh)` vs `alloc.mwh.groupby(alloc.end_use_class).sum()` | +| Leftover T&D | `electricity_purchases − gen_dollars` (equals `t_dollars + d_dollars`) | +| Nibble vs clipped | class totals below class target; `clipped` is purchaser-level only | +| Optional ¢/kWh | `electricity_purchases / (10 × MWh)` vs Table 2.4 — check only, not class targets | +| MECS vs dollar manufacturing | Dual-run `industrial_weights=mecs\|dollars` on the same electricity_purchases | Markdown lands at `current/diagnostics/output/eia_gtd_purchaser_tables.md`. + +## Why Table 7.7 stays FBA (not FBS) + +No MECS Energy FBS exists in bedrock. Tables 2.2/3.2 stay FBA and enter GHG (and CAP/HAP) FBS build as attribution sources. Table 7.7 is also left as FBA: it only supplies manufacturing purchased-kWh weights inside the electricity (Generation/Transmission/Distribution) purchaser allocation. We do not build an FBS because that path would run `estimate_suppressed_mecs_energy` and a generic NAICS→BEA crosswalk, both wrong for table 7.7 for the following reasons: + +A) `estimate_suppressed_mecs_energy` drops every row marked `D` or `Q` (i.e., treats them as “no more information than an industry that is not in the table”) and replaces `*` with 0.25, because in those tables `*` means “less than 0.5 Trillion Btu”. Table 7.7 is million kWh, not TBtu. Putting 0.25 on `*` would be a TBtu-scale guess in the wrong unit. + +Dropping `Q`/`D` is worse: EIA still publishes the manufacturing total `31-33`, so a withheld 3-digit industry is recoverable. If we drop it, those kWh disappear from manufacturing shares and that NAICS gets a zero electricity weight even though the table identity tells us the amount. + +What this path does instead: + +- `*` → 0 (too small to publish, not 0.25 TBtu) +- `Q`/`D` at 3-digit NAICS → residual fill (below) +- other `Q`/`D` → 0 + +B) 3-digit `Q`/`D` residual fill: Table 7.7 is hierarchical: `31-33` is all manufacturing purchased electricity; under that sit 3-digit industries (`311` food, `331` primary metals, `337` furniture, …), then 6-digit rows. When one 3-digit cell is `Q` or `D`, EIA still prints 31-33, which means we can calculate the leftover: + +`leftover = 31-33 − (sum of 3-digit industries that are published)` + +If exactly one 3-digit industry is Q/D, that leftover is that industry’s kWh. If leftover is not ~0 and there is not exactly one Q/D 3-digit, it hard-errors instead of guessing. This is indeed the case for 2022 MECS, which is the survey this implementation uses for model years after 2017: + +Live US Table 7.7 `Electricity total`: + +| Survey | 3-digit Q/D | `31-33` − published 3-digit | +|---|---|---| +| **2018** | none | **0** million kWh | +| **2022** | **exactly one: `337` (furniture), marked `Q`** | **9,224** million kWh | + +We need that leftover because manufacturing G/T/D MWh shares are normalized 7.7 kWh weights. Losing a whole 3-digit industry would reassign its electricity to every other manufacturer. Recovering it from `31-33` keeps the withheld industry in the manufacturing pool with the quantity EIA implied. + +C) Mapping: Using a default `FBS` map (`BEA_2017_Detail`) would not apply the 3.1 subtraction (e.g. `331313` = `3313` − `331314`/`331315`/`331318`) and could attach parent rows like `31-33` unless they are explicitly excluded. GHG combustion already uses the Cornerstone 3.1 hand map, not that generic crosswalk, which means that table 7.7 must use that same hand map, which the usual FBS path would not provide. + +## MECS NAICS/BEA vs GHG industrial combustion + +### Summary of comparison + +The treatment of table 7.7 here and MECS for GHG interpret the *same* MECS industry pool onto BEA the *same way*. On that, the methods align: same 3.1 hand map, same manufacturing vs `NON_MECS` split. We are not inventing a second NAICS story for electricity. + +What *is* different is GHG’s table-specific mechanics (static 2018 survey year use, `Q`/`D`→0, split by fuel Use). Those would be the wrong analogue for purchased kWh. + +The electricity method is: map with the Cornerstone 3.1 dictionaries (same); split multi-IO rows by Use of that row’s commodity (same concept, different commodity); use the MECS survey that matches the electricity year (different); read 7.7’s `31-33` identity instead of dropping a recoverable 3-digit cell (different). + +### Comparison of MECS across NAICS when using table 7.7 for electricity vs. GHG MECS allocation of industrial combustion + +**Same** + +| What | Method | +|---|---| +| Hand map | Which NAICS belong to which BEA, including 3.1 subtraction | +| Industry pools | Manufacturing vs `NON_MECS` membership | + +The hand map of MECS across NAICS is consistent between the GHG MECS allocation and the table 7.7 allocation. Both paths use `CORNERSTONE_INDUSTRY_TO_MECS_3_1_NAICS_*` (plus `NON_MECS_INDUSTRIES` for residual industrial). The use of the 2022 survey requires the `MECS_7_7_NAICS_OVERLAY`, which relabels four 2018 NAICS codes so 2022 Table 7.7 can be read. + +**Different** + +| What | Method (7.7 vs GHG MECS) | +|---|---| +| Survey Used | 7.7 follows EIA/eGRID year (2018 or 2022) vs 3.1 pinned 2018 | +| Suppression treatment | 3-digit Q/D leftover vs Q/D → 0 | +| Multi-IO (and residual) **split weights** | electricity purchases vs BEA fuel Use | + +### Does it make sense to align 7.7 for electricity with GHG MECS? + +**Survey year:** No, not by pinning 7.7 to 2018. G/T/D is tied to the EIA/eGRID year (2017 → MECS 2018; model year → MECS 2022). Using 2018 shares on a 2024 electricity allocation would freeze manufacturing kWh in the previous survey while class MWh and eGRID move, which would mix two survey years in one allocation. + +**Suppression treatment:** Align only if you value cross-method identity over this table. `load_mecs_3_1` zeroes `Q`/`D`. On 7.7 that would set 2022 `337` to 0 and dump 9,224 million kWh onto every other manufacturer, even though `31-33` still implies furniture’s total. The leftover fill makes sense for this table. + +**Split weights:** No. GHG does not have a generic “use BEA fuel Use” rule. It splits a MECS coal (or gas) total by BEA Use of that fuel. The parallel for 7.7 is BEA Use of electricity (purchaser electricity purchases on 221100), which is what this path already does. Splitting purchased kWh by coal/gas Use would assign furniture vs chemicals electricity with the wrong commodity. Residual `NON_MECS` is the same idea: leftover industrial electricity should follow electricity purchases, not fuel Use. + +## Census electricity cost vs Table 7.7 kWh + +Nowcasting maps annual Census “cost of purchased electricity” onto `221100`: `CSTELEC` (ASM 2018–2021, EC 2017/2022) and `EXPS_ELEC_VAL` (AIES 2023, SAS 2018–2022), NAICS-6, on the `nowcast` expense FBAs (`Census_ASM_Expenses`, `Census_EC_Expenses`, `Census_AIES_Expenses`, `Census_AIES_Service_Expenses`, `Census_SAS_Expenses`). That is the right dollar series for SUT nowcasts. It is the wrong manufacturing **MWh** weight here. + +G/T/D Industrial manufacturing is a physical split: class MWh comes from Table 2.2 / eGRID, and 7.7 is purchased kWh. Cost = kWh × rate. Intra-manufacturing rates are not flat, so cost shares would move MWh toward high-price, low-intensity sectors. Implied ¢/kWh vs Table 2.4 is already a check only, not a class target, for the same reason. + +Census cost would help later for years between MECS surveys, or for Commercial/service NAICS-6 where 7.7 does not apply. Those FBAs are not on this branch and are not wired into the allocator. diff --git a/bedrock/analysis/electricity/current/eia_gtd/__init__.py b/bedrock/analysis/electricity/current/eia_gtd/__init__.py index 668b13bc0..896c94b30 100644 --- a/bedrock/analysis/electricity/current/eia_gtd/__init__.py +++ b/bedrock/analysis/electricity/current/eia_gtd/__init__.py @@ -3,23 +3,27 @@ from bedrock.analysis.electricity.current.eia_gtd.purchaser_tables import ( MIXED_CONFIG, SPLIT_CONFIG, + class_mwh_targets_frame, class_nibble_frame, - d0_class_mwh_frame, leftover_td_class_frame, leftover_td_purchaser_frame, load_reanchored_allocation, + manufacturing_mecs_vs_dollar_frame, optional_implied_cents_kwh_frame, + p_share_from_allocation, render_purchaser_tables_md, ) __all__ = [ 'MIXED_CONFIG', 'SPLIT_CONFIG', + 'class_mwh_targets_frame', 'class_nibble_frame', - 'd0_class_mwh_frame', 'leftover_td_class_frame', 'leftover_td_purchaser_frame', 'load_reanchored_allocation', + 'manufacturing_mecs_vs_dollar_frame', 'optional_implied_cents_kwh_frame', + 'p_share_from_allocation', 'render_purchaser_tables_md', ] diff --git a/bedrock/analysis/electricity/current/eia_gtd/__main__.py b/bedrock/analysis/electricity/current/eia_gtd/__main__.py index 56075d8f2..ac26433f6 100644 --- a/bedrock/analysis/electricity/current/eia_gtd/__main__.py +++ b/bedrock/analysis/electricity/current/eia_gtd/__main__.py @@ -1,4 +1,4 @@ -"""``python -m bedrock.analysis.electricity.current.eia_gtd`` → D0 tables.""" +"""``python -m bedrock.analysis.electricity.current.eia_gtd`` → class-MWh tables.""" from __future__ import annotations @@ -17,7 +17,7 @@ def main(argv: list[str] | None = None) -> None: parser = argparse.ArgumentParser( - description='Write D0 class-MWh / leftover T&D tables from the reanchored allocation.' + description='Write class-MWh / leftover T&D tables from the reanchored allocation.' ) parser.add_argument( '--config', diff --git a/bedrock/analysis/electricity/current/eia_gtd/__tests__/test_purchaser_tables.py b/bedrock/analysis/electricity/current/eia_gtd/__tests__/test_purchaser_tables.py index 88e331ef1..4d02af658 100644 --- a/bedrock/analysis/electricity/current/eia_gtd/__tests__/test_purchaser_tables.py +++ b/bedrock/analysis/electricity/current/eia_gtd/__tests__/test_purchaser_tables.py @@ -1,4 +1,4 @@ -"""Unit tests for D0 class-MWh, leftover T&D, and nibble vs clipped tables.""" +"""Unit tests for class-MWh targets, leftover T&D, and nibble vs clipped tables.""" from __future__ import annotations @@ -6,19 +6,21 @@ import pytest from bedrock.analysis.electricity.current.eia_gtd.purchaser_tables import ( + class_mwh_targets_frame, class_nibble_frame, - d0_class_mwh_frame, leftover_td_class_frame, leftover_td_purchaser_frame, leftover_td_usd, + manufacturing_mecs_vs_dollar_frame, optional_implied_cents_kwh_frame, + p_share_from_allocation, ) from bedrock.transform.eeio.electricity_gtd_allocation import EIAPurchaserAllocation def _alloc( *, - bills: dict[str, float], + electricity_purchases: dict[str, float], classes: dict[str, str], mwh: dict[str, float], gen: dict[str, float] | None = None, @@ -27,18 +29,18 @@ def _alloc( egrid_mwh: float = 100.0, td_share: float = 0.25, ) -> EIAPurchaserAllocation: - idx = list(bills) - bill = pd.Series(bills, dtype=float) + idx = list(electricity_purchases) if gen is None: - gen = {k: 0.4 * v for k, v in bills.items()} + gen = {k: 0.4 * v for k, v in electricity_purchases.items()} + purchases = pd.Series(electricity_purchases, dtype=float) gen_s = pd.Series(gen, dtype=float).reindex(idx).astype(float) - leftover = bill - gen_s + leftover = purchases - gen_s clip = pd.Series(False, index=idx, dtype=bool) if clipped: for k, v in clipped.items(): clip[k] = v return EIAPurchaserAllocation( - bill=bill, + electricity_purchases=purchases, end_use_class=pd.Series(classes), mwh=pd.Series(mwh, dtype=float), gen_dollars=gen_s, @@ -51,9 +53,9 @@ def _alloc( ) -def test_d0_identity_matches_class_targets() -> None: +def test_class_mwh_identity_matches_class_targets() -> None: alloc = _alloc( - bills={'F01000': 100.0, '1111A0': 200.0, 'F04000': 50.0}, + electricity_purchases={'F01000': 100.0, '1111A0': 200.0, 'F04000': 50.0}, classes={ 'F01000': 'Residential', '1111A0': 'Industrial', @@ -68,19 +70,19 @@ def test_d0_identity_matches_class_targets() -> None: 'Transportation': 0.0, 'Exports': 5.0, } - frame = d0_class_mwh_frame(alloc, eia_year=2024, targets=targets) + frame = class_mwh_targets_frame(alloc, eia_year=2024, targets=targets) by_class = frame.set_index('end_use_class') assert by_class.loc['Residential', 'allocator_mwh'] == pytest.approx(10.0) assert by_class.loc['Industrial', 'allocator_mwh'] == pytest.approx(20.0) assert by_class.loc['Exports', 'allocator_mwh'] == pytest.approx(5.0) assert not bool(by_class.loc['Residential', 'nibble']) assert not bool(by_class.loc['Industrial', 'nibble']) - assert by_class.loc['Residential', 'ratio_vs_d0'] == pytest.approx(1.0) + assert by_class.loc['Residential', 'ratio_vs_class_target'] == pytest.approx(1.0) -def test_leftover_td_is_bill_minus_gen_dollars() -> None: +def test_leftover_td_is_electricity_purchases_minus_gen_dollars() -> None: alloc = _alloc( - bills={'F01000': 100.0, '1111A0': 40.0}, + electricity_purchases={'F01000': 100.0, '1111A0': 40.0}, classes={'F01000': 'Residential', '1111A0': 'Industrial'}, mwh={'F01000': 1.0, '1111A0': 0.4}, gen={'F01000': 25.0, '1111A0': 10.0}, @@ -103,7 +105,7 @@ def test_leftover_td_is_bill_minus_gen_dollars() -> None: def test_nibble_is_class_totals_clipped_is_purchaser_only() -> None: alloc = _alloc( - bills={'F01000': 10.0, '452000': 80.0, '1111A0': 50.0}, + electricity_purchases={'F01000': 10.0, '452000': 80.0, '1111A0': 50.0}, classes={ 'F01000': 'Residential', '452000': 'Commercial', @@ -131,9 +133,9 @@ def test_nibble_is_class_totals_clipped_is_purchaser_only() -> None: assert not bool(purchasers.loc['F01000', 'clipped']) -def test_optional_implied_cents_kwh_is_bill_over_mwh() -> None: +def test_optional_implied_cents_kwh_is_electricity_purchases_over_mwh() -> None: alloc = _alloc( - bills={'F01000': 160.0}, + electricity_purchases={'F01000': 160.0}, classes={'F01000': 'Residential'}, mwh={'F01000': 1.0}, ) @@ -143,3 +145,47 @@ def test_optional_implied_cents_kwh_is_bill_over_mwh() -> None: assert frame.loc['Residential', 'implied_cents_kwh'] == pytest.approx(16.0) assert frame.loc['Residential', 'table_24_cents_kwh'] == pytest.approx(16.0) assert 'Exports' not in frame.index + + +def test_p_share_from_allocation_recovers_generation_share() -> None: + alloc = _alloc( + electricity_purchases={'F01000': 200.0, '1111A0': 100.0}, + classes={'F01000': 'Residential', '1111A0': 'Industrial'}, + mwh={'F01000': 2.0, '1111A0': 1.0}, + p=40.0, + egrid_mwh=150.0, + ) + assert p_share_from_allocation(alloc) == pytest.approx(40.0 * 150.0 / 300.0) + + +def test_manufacturing_mecs_vs_dollar_flags() -> None: + mecs = _alloc( + electricity_purchases={'331110': 0.0, '1111A0': 50.0, 'F01000': 80.0}, + classes={ + '331110': 'Industrial', + '1111A0': 'Industrial', + 'F01000': 'Residential', + }, + mwh={'331110': 10.0, '1111A0': 5.0, 'F01000': 8.0}, + gen={'331110': 0.0, '1111A0': 40.0, 'F01000': 20.0}, + clipped={'331110': True, '1111A0': False, 'F01000': False}, + ) + dollars = _alloc( + electricity_purchases={'331110': 0.0, '1111A0': 50.0, 'F01000': 80.0}, + classes={ + '331110': 'Industrial', + '1111A0': 'Industrial', + 'F01000': 'Residential', + }, + mwh={'331110': 0.0, '1111A0': 20.0, 'F01000': 8.0}, + gen={'331110': 0.0, '1111A0': 20.0, 'F01000': 20.0}, + ) + frame = manufacturing_mecs_vs_dollar_frame(mecs, dollars).set_index('purchaser') + assert 'F01000' not in frame.index + assert bool(frame.loc['331110', 'manufacturing']) + assert bool(frame.loc['331110', 'zero_electricity_purchases_mecs_assignee']) + assert bool(frame.loc['331110', 'clipped_mecs']) + assert not bool(frame.loc['1111A0', 'manufacturing']) + assert bool(frame.loc['1111A0', 'cross_pool_overflow_recipient']) + assert frame.loc['331110', 'mecs_mwh'] == pytest.approx(10.0) + assert frame.loc['331110', 'dollar_mwh'] == pytest.approx(0.0) diff --git a/bedrock/analysis/electricity/current/eia_gtd/purchaser_tables.py b/bedrock/analysis/electricity/current/eia_gtd/purchaser_tables.py index b99f03c72..037ec5776 100644 --- a/bedrock/analysis/electricity/current/eia_gtd/purchaser_tables.py +++ b/bedrock/analysis/electricity/current/eia_gtd/purchaser_tables.py @@ -1,8 +1,8 @@ """Slide 7/8/27–30 tables from the reanchored ``EIAPurchaserAllocation``. -Class MWh is compared to D0 ``_class_mwh_targets``, not raw Table 2.2. -Leftover T&D is ``bill − gen_dollars``. Nibble is class totals vs those -targets; ``clipped`` is purchaser-level only. +Class MWh is compared to ``_class_mwh_targets``, not raw Table 2.2. +Leftover T&D is ``electricity_purchases − gen_dollars``. Nibble is class +totals vs those targets; ``clipped`` is purchaser-level only. """ from __future__ import annotations @@ -36,12 +36,12 @@ ) # Class nibble is logged in production, not stored as a Series. Treat a class -# as nibbled when allocated MWh sits below the D0 target by more than this. +# as nibbled when allocated MWh sits below the class target by more than this. NIBBLE_ATOL = 1e-6 def load_reanchored_allocation(config: str = MIXED_CONFIG) -> EIAPurchaserAllocation: - """Flush caches, resolve ``config``, run A/q, and return the P5 allocation. + """Flush caches, resolve ``config``, run A/q, and return the reanchored allocation. ``get_reanchored_eia_purchaser_allocation`` stays ``None`` until ``reanchor_electricity_aq_after_year_scaling`` runs. Do not use @@ -61,7 +61,7 @@ def load_reanchored_allocation(config: str = MIXED_CONFIG) -> EIAPurchaserAlloca return alloc -def d0_targets(alloc: EIAPurchaserAllocation, eia_year: int) -> dict[str, float]: +def class_mwh_targets(alloc: EIAPurchaserAllocation, eia_year: int) -> dict[str, float]: return _class_mwh_targets(eia_year, alloc.egrid_mwh) @@ -70,20 +70,22 @@ def allocated_class_mwh(alloc: EIAPurchaserAllocation) -> pd.Series: def leftover_td_usd(alloc: EIAPurchaserAllocation) -> pd.Series: - """Purchaser leftover T&D dollars: ``bill − gen_dollars``.""" - return (alloc.bill.astype(float) - alloc.gen_dollars.astype(float)).astype(float) + """Purchaser leftover T&D dollars: ``electricity_purchases − gen_dollars``.""" + return ( + alloc.electricity_purchases.astype(float) - alloc.gen_dollars.astype(float) + ).astype(float) -def d0_class_mwh_frame( +def class_mwh_targets_frame( alloc: EIAPurchaserAllocation, eia_year: int, *, targets: dict[str, float] | None = None, raw_table_22_mwh: dict[str, float] | None = None, ) -> pd.DataFrame: - """Allocator class MWh vs D0 targets (and optional raw Table 2.2).""" + """Allocator class MWh vs ``_class_mwh_targets`` (and optional raw Table 2.2).""" if targets is None: - targets = d0_targets(alloc, eia_year) + targets = class_mwh_targets(alloc, eia_year) got = allocated_class_mwh(alloc) rows: list[dict[str, Any]] = [] for cls in CLASS_ORDER: @@ -91,9 +93,9 @@ def d0_class_mwh_frame( allocated = float(got.get(cls, 0.0)) row: dict[str, Any] = { 'end_use_class': cls, - 'd0_target_mwh': target, + 'class_target_mwh': target, 'allocator_mwh': allocated, - 'ratio_vs_d0': allocated / target if target else float('nan'), + 'ratio_vs_class_target': allocated / target if target else float('nan'), 'nibble': bool(allocated + NIBBLE_ATOL < target), } if raw_table_22_mwh is not None and cls in raw_table_22_mwh: @@ -106,12 +108,14 @@ def d0_class_mwh_frame( def leftover_td_purchaser_frame(alloc: EIAPurchaserAllocation) -> pd.DataFrame: leftover = leftover_td_usd(alloc) - idx = alloc.bill.index + idx = alloc.electricity_purchases.index return pd.DataFrame( { 'purchaser': list(idx.astype(str)), 'end_use_class': alloc.end_use_class.reindex(idx).astype(str).to_numpy(), - 'bill': alloc.bill.astype(float).to_numpy(), + 'electricity_purchases': ( + alloc.electricity_purchases.astype(float).to_numpy() + ), 'gen_dollars': alloc.gen_dollars.reindex(idx).astype(float).to_numpy(), 'leftover_td': leftover.reindex(idx).to_numpy(), 't_dollars': alloc.t_dollars.reindex(idx).astype(float).to_numpy(), @@ -125,7 +129,13 @@ def leftover_td_class_frame(alloc: EIAPurchaserAllocation) -> pd.DataFrame: purchasers = leftover_td_purchaser_frame(alloc) grouped = purchasers.groupby('end_use_class', sort=False) out = grouped[ - ['bill', 'gen_dollars', 'leftover_td', 't_dollars', 'd_dollars'] + [ + 'electricity_purchases', + 'gen_dollars', + 'leftover_td', + 't_dollars', + 'd_dollars', + ] ].sum() out['n_clipped'] = grouped['clipped'].sum().astype(int) out['n_purchasers'] = grouped.size().astype(int) @@ -138,27 +148,28 @@ def class_nibble_frame( *, targets: dict[str, float] | None = None, ) -> pd.DataFrame: - """Class nibble (totals vs D0) alongside purchaser ``clipped`` counts. + """Class nibble (totals vs class targets) alongside purchaser ``clipped`` counts. Nibble has no Series flag. ``clipped`` marks purchasers that hit their - bill cap during water-fill; a clipped purchaser does not imply class nibble. + electricity-purchase cap during water-fill; a clipped purchaser does not + imply class nibble. """ - d0 = d0_class_mwh_frame(alloc, eia_year, targets=targets) + class_mwh = class_mwh_targets_frame(alloc, eia_year, targets=targets) purchasers = leftover_td_purchaser_frame(alloc) clip_counts = ( purchasers.groupby('end_use_class')['clipped'].sum().astype(int) if not purchasers.empty else pd.Series(dtype=int) ) - d0['n_clipped_purchasers'] = [ - int(clip_counts.get(cls, 0)) for cls in d0['end_use_class'] + class_mwh['n_clipped_purchasers'] = [ + int(clip_counts.get(cls, 0)) for cls in class_mwh['end_use_class'] ] - return d0[ + return class_mwh[ [ 'end_use_class', - 'd0_target_mwh', + 'class_target_mwh', 'allocator_mwh', - 'ratio_vs_d0', + 'ratio_vs_class_target', 'nibble', 'n_clipped_purchasers', ] @@ -169,16 +180,23 @@ def optional_implied_cents_kwh_frame( alloc: EIAPurchaserAllocation, table_24_cents_kwh: dict[str, float], ) -> pd.DataFrame: - """Implied retail ¢/kWh from ``bill / MWh`` vs Table 2.4 (check only).""" - bills = alloc.bill.astype(float).groupby(alloc.end_use_class).sum() + """Implied retail ¢/kWh from ``electricity_purchases / MWh`` vs Table 2.4. + + Check only; not a class-target identity. + """ + electricity_purchases = ( + alloc.electricity_purchases.astype(float).groupby(alloc.end_use_class).sum() + ) mwh = allocated_class_mwh(alloc) rows: list[dict[str, Any]] = [] for cls in CLASS_ORDER: if cls == 'Exports': continue - class_bill = float(bills.get(cls, 0.0)) + class_purchases = float(electricity_purchases.get(cls, 0.0)) class_mwh = float(mwh.get(cls, 0.0)) - implied = class_bill / (10.0 * class_mwh) if class_mwh > 0 else float('nan') + implied = ( + class_purchases / (10.0 * class_mwh) if class_mwh > 0 else float('nan') + ) listed = float(table_24_cents_kwh.get(cls, float('nan'))) rows.append( { @@ -190,6 +208,102 @@ def optional_implied_cents_kwh_frame( return pd.DataFrame(rows) +def p_share_from_allocation(alloc: EIAPurchaserAllocation) -> float: + """Recover the 2017 UGO generation share from + ``p = share × electricity_purchases / eGRID``. + """ + electricity_purchases_total = float(alloc.electricity_purchases.sum()) + if electricity_purchases_total <= 0 or float(alloc.egrid_mwh) <= 0: + raise ValueError( + 'cannot recover generation share from non-positive ' + 'electricity_purchases or eGRID' + ) + return float(alloc.p) * float(alloc.egrid_mwh) / electricity_purchases_total + + +def dual_run_industrial_allocations( + alloc: EIAPurchaserAllocation, + eia_year: int, +) -> tuple[EIAPurchaserAllocation, EIAPurchaserAllocation]: + """Re-allocate the same electricity_purchases with MECS vs dollar weights.""" + from bedrock.transform.eeio.electricity_gtd_allocation import ( # noqa: PLC0415 + ELECTRICITY_AGGREGATE, + allocate_purchaser_gtd, + ) + + p_share = p_share_from_allocation(alloc) + mecs = allocate_purchaser_gtd( + alloc.electricity_purchases, + self_use_key=ELECTRICITY_AGGREGATE, + eia_year=eia_year, + p_share_2017=p_share, + td_share_2017=float(alloc.td_share), + industrial_weights='mecs', + ) + dollars = allocate_purchaser_gtd( + alloc.electricity_purchases, + self_use_key=ELECTRICITY_AGGREGATE, + eia_year=eia_year, + p_share_2017=p_share, + td_share_2017=float(alloc.td_share), + industrial_weights='dollars', + ) + return mecs, dollars + + +def manufacturing_mecs_vs_dollar_frame( + mecs_alloc: EIAPurchaserAllocation, + dollar_alloc: EIAPurchaserAllocation, +) -> pd.DataFrame: + """Per manufacturing sector: MECS vs dollar MWh, clip flags, zero-purchase assignees. + + Cross-pool overflow marks residual Industrial purchasers whose generation + dollars rose under MECS (water-fill from clipped energy-intensive manufacturers). + """ + from bedrock.transform.eeio.electricity_gtd_allocation import ( # noqa: PLC0415 + industrial_manufacturing_pool, + ) + + idx = mecs_alloc.electricity_purchases.index.union( + dollar_alloc.electricity_purchases.index + ) + mfg_pool = industrial_manufacturing_pool() + classes = mecs_alloc.end_use_class.reindex(idx) + industrial = classes.astype(str) == 'Industrial' + is_mfg = pd.Series([str(i) in mfg_pool for i in idx], index=idx) + electricity_purchases = ( + mecs_alloc.electricity_purchases.reindex(idx).astype(float).fillna(0.0) + ) + mecs_mwh = mecs_alloc.mwh.reindex(idx).astype(float).fillna(0.0) + dollar_mwh = dollar_alloc.mwh.reindex(idx).astype(float).fillna(0.0) + mecs_gen = mecs_alloc.gen_dollars.reindex(idx).astype(float).fillna(0.0) + dollar_gen = dollar_alloc.gen_dollars.reindex(idx).astype(float).fillna(0.0) + clipped_mecs = mecs_alloc.clipped.reindex(idx).fillna(False).astype(bool) + clipped_dollars = dollar_alloc.clipped.reindex(idx).fillna(False).astype(bool) + residual = industrial & ~is_mfg + overflow = residual & (mecs_gen > dollar_gen + 1e-9) + rows = pd.DataFrame( + { + 'purchaser': idx.astype(str), + 'end_use_class': classes.astype(str).to_numpy(), + 'manufacturing': is_mfg.to_numpy(), + 'electricity_purchases': electricity_purchases.to_numpy(), + 'mecs_mwh': mecs_mwh.to_numpy(), + 'dollar_mwh': dollar_mwh.to_numpy(), + 'mwh_diff': (mecs_mwh - dollar_mwh).to_numpy(), + 'mecs_gen_dollars': mecs_gen.to_numpy(), + 'dollar_gen_dollars': dollar_gen.to_numpy(), + 'clipped_mecs': clipped_mecs.to_numpy(), + 'clipped_dollars': clipped_dollars.to_numpy(), + 'zero_electricity_purchases_mecs_assignee': ( + is_mfg & (electricity_purchases <= 0.0) & (mecs_mwh > 0.0) + ).to_numpy(), + 'cross_pool_overflow_recipient': overflow.to_numpy(), + } + ) + return rows.loc[industrial.to_numpy()].reset_index(drop=True) + + def _fmt_mwh(value: float) -> str: return f'{value / 1e6:,.1f} TWh' @@ -226,16 +340,16 @@ def render_purchaser_tables_md( table_24_cents_kwh: dict[str, float] | None = None, config: str = MIXED_CONFIG, ) -> str: - d0 = d0_class_mwh_frame( + class_mwh = class_mwh_targets_frame( alloc, eia_year, targets=targets, raw_table_22_mwh=raw_table_22_mwh ) leftover = leftover_td_class_frame(alloc) nibble = class_nibble_frame(alloc, eia_year, targets=targets) leftover_usd = leftover_td_usd(alloc) lines = [ - '# EIA-anchored purchaser allocation (D0)', + '# EIA-anchored purchaser allocation', '', - f'Config: `{config}`. EIA / D0 year: **{eia_year}**. ' + f'Config: `{config}`. EIA / class-target year: **{eia_year}**. ' f'eGRID MWh = **{alloc.egrid_mwh:,.0f}**. ' f'Generation price `p` = **{alloc.p:.6g}** USD/MWh. ' f'T&D split `td_share` = **{alloc.td_share:.4f}**.', @@ -244,34 +358,34 @@ def render_purchaser_tables_md( '`reset_usa_config` -> `clear_all_publish_caches` -> ' '`set_global_usa_config` -> `derive_cornerstone_Aq_scaled`.', '', - '## Class MWh vs D0 targets', + '## Class MWh vs class targets', '', - 'D0 identity: `(class / Total End Use) * (eGRID - Table 2.14 exports)`, ' + 'Class-target identity: `(class / Total End Use) * (eGRID - Table 2.14 exports)`, ' 'Industrial pool = Industrial + Direct Use, `F04000` = Exports. ' - 'Do **not** read Model / raw Table 2.2 ~ 1.0 as the D0 claim. ' - 'Nibble (class bills cannot cover `p *` class MWh) is the expected ' - 'exception.', + 'Do **not** read Model / raw Table 2.2 ~ 1.0 as the class-target claim. ' + 'Nibble (class electricity purchases cannot cover `p *` class MWh) ' + 'is the expected exception.', '', *_markdown_table( - d0, + class_mwh, { - 'd0_target_mwh': '{:,.0f}', + 'class_target_mwh': '{:,.0f}', 'allocator_mwh': '{:,.0f}', - 'ratio_vs_d0': '{:.4f}', + 'ratio_vs_class_target': '{:.4f}', 'raw_table_22_mwh': '{:,.0f}', 'ratio_vs_raw_22': '{:.4f}', }, ), '', - f'Totals: D0 {_fmt_mwh(float(d0["d0_target_mwh"].sum()))}; ' - f'allocator {_fmt_mwh(float(d0["allocator_mwh"].sum()))}.', + f'Totals: class targets {_fmt_mwh(float(class_mwh["class_target_mwh"].sum()))}; ' + f'allocator {_fmt_mwh(float(class_mwh["allocator_mwh"].sum()))}.', '', - '## Leftover T&D = bill - gen_dollars', + '## Leftover T&D = electricity_purchases - gen_dollars', '', *_markdown_table( leftover, { - 'bill': '${:,.2f}', + 'electricity_purchases': '${:,.2f}', 'gen_dollars': '${:,.2f}', 'leftover_td': '${:,.2f}', 't_dollars': '${:,.2f}', @@ -285,15 +399,15 @@ def render_purchaser_tables_md( '## Nibble (class) vs clipped (purchaser)', '', 'Nibble has **no Series flag**. A class is nibbled when ' - '`sum(mwh) < D0 target`. `clipped` is the only Series flag and is ' + '`sum(mwh) < class target`. `clipped` is the only Series flag and is ' 'purchaser-level water-fill.', '', *_markdown_table( nibble, { - 'd0_target_mwh': '{:,.0f}', + 'class_target_mwh': '{:,.0f}', 'allocator_mwh': '{:,.0f}', - 'ratio_vs_d0': '{:.4f}', + 'ratio_vs_class_target': '{:.4f}', }, ), '', @@ -304,8 +418,9 @@ def render_purchaser_tables_md( [ '## Optional check - implied cents/kWh vs Table 2.4', '', - 'Implied cents/kWh = `bill / (10 * MWh)`. This is a check against ' - 'retail Table 2.4, **not** the D0 identity and not production ' + 'Implied cents/kWh = `electricity_purchases / (10 * MWh)`. ' + 'This is a check against ' + 'retail Table 2.4, **not** the class-target identity and not production ' '`c_row` (production `c_row` is flat `1/p`).', '', *_markdown_table( @@ -342,10 +457,52 @@ def build_live_report(config: str = MIXED_CONFIG) -> str: } prices = electricity_end_use_retail_prices_cents_kwh(eia_year) table_24: dict[str, float] = {str(k): float(v) for k, v in prices.items()} - return render_purchaser_tables_md( + md = render_purchaser_tables_md( alloc, eia_year, raw_table_22_mwh=raw_22, table_24_cents_kwh=table_24, config=config, ) + mecs_alloc, dollar_alloc = dual_run_industrial_allocations(alloc, eia_year) + compare = manufacturing_mecs_vs_dollar_frame(mecs_alloc, dollar_alloc) + mfg = compare.loc[compare['manufacturing']] + overflow = compare.loc[compare['cross_pool_overflow_recipient']] + extra = [ + '', + '## MECS vs dollar Industrial manufacturing weights', + '', + 'Same electricity_purchases through `allocate_purchaser_gtd` with ' + '`industrial_weights=mecs` vs `dollars`. Generation share recovered as ' + '`p × eGRID / electricity_purchases_total`. Manufacturing rows use ' + 'Table 7.7 purchased kWh; ' + 'residual Industrial stays dollar-weighted. Class-wide water-fill can move ' + 'generation dollars from clipped manufacturers onto residual ag/mining/' + 'construction.', + '', + f'Manufacturing purchasers: **{int(len(mfg))}**. ' + f'Clipped under MECS: **{int(mfg["clipped_mecs"].sum()) if not mfg.empty else 0}**. ' + f'MECS assignees with zero electricity purchases: ' + f'**{int(mfg["zero_electricity_purchases_mecs_assignee"].sum()) if not mfg.empty else 0}**. ' + f'Cross-pool overflow recipients: **{int(len(overflow))}**.', + '', + ] + if not mfg.empty: + extra.extend( + _markdown_table( + mfg.assign(_abs=mfg['mwh_diff'].abs()) + .sort_values('_abs', ascending=False) + .drop(columns='_abs') + .head(25), + { + 'electricity_purchases': '${:,.2f}', + 'mecs_mwh': '{:,.0f}', + 'dollar_mwh': '{:,.0f}', + 'mwh_diff': '{:,.0f}', + 'mecs_gen_dollars': '${:,.2f}', + 'dollar_gen_dollars': '${:,.2f}', + }, + ) + ) + extra.append('') + return md + '\n'.join(extra) diff --git a/bedrock/analysis/electricity/current/vs_original_elec_disagg/compare_to_original_elec_disagg.py b/bedrock/analysis/electricity/current/vs_original_elec_disagg/compare_to_original_elec_disagg.py deleted file mode 100644 index 440446d23..000000000 --- a/bedrock/analysis/electricity/current/vs_original_elec_disagg/compare_to_original_elec_disagg.py +++ /dev/null @@ -1,68 +0,0 @@ -"""Compare live EIA-anchored G/T/D production to the original-implementation freeze. - -The historical CF report used 2018 eGRID and absolute EIA sales; that -comparison is not a production gate. -""" - -from __future__ import annotations - -from pathlib import Path - -import pandas as pd - -from bedrock.analysis.electricity.historical.original_elec_disagg_implementation.paths import ( - DISAGG_CONFIG, - MIXED_CONFIG, - OUT_DIR, - config_dir, -) - - -def original_elec_disagg_dir_for(config_stem: str) -> str: - """Return the original-implementation freeze directory for a waterfall config stem.""" - if config_stem not in (DISAGG_CONFIG, MIXED_CONFIG): - raise ValueError( - f'original-elec-disagg freeze only covers {DISAGG_CONFIG!r} and {MIXED_CONFIG!r}; ' - f'got {config_stem!r}' - ) - path = config_dir(config_stem) - if not path.is_dir(): - raise FileNotFoundError( - f'original-elec-disagg freeze missing at {path}. ' - 'The freeze is committed under ' - 'bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/; ' - 'if files are absent, they are missing or not tracked.' - ) - return str(path) - - -def original_elec_disagg_output_dir() -> str: - return str(OUT_DIR) - - -def load_freeze_q(config_stem: str) -> pd.Series: - path = Path(original_elec_disagg_dir_for(config_stem)) / 'q.parquet' - frame = pd.read_parquet(path) - if 'q' in frame.columns: - return frame['q'].astype(float) - if frame.shape[1] == 1: - return frame.iloc[:, 0].astype(float) - squeezed = frame.squeeze() - if not isinstance(squeezed, pd.Series): - raise TypeError(f'expected a Series from {path}, got {type(squeezed)}') - return squeezed.astype(float) - - -def compare_q_to_freeze( - live_q: pd.Series, - config_stem: str, -) -> pd.DataFrame: - """Percent-diff live vs original-elec-disagg freeze ``q`` (not a CI gate).""" - frozen = load_freeze_q(config_stem) - idx = live_q.index.union(frozen.index) - live = live_q.reindex(idx).astype(float) - base = frozen.reindex(idx).astype(float) - out = pd.DataFrame({'live': live, 'freeze': base}) - out['abs_diff'] = out['live'] - out['freeze'] - out['pct_diff'] = out['abs_diff'] / out['freeze'].replace(0.0, pd.NA) - return out diff --git a/bedrock/analysis/electricity/historical/README.md b/bedrock/analysis/electricity/historical/README.md index 4a7596078..9ea68cad8 100644 --- a/bedrock/analysis/electricity/historical/README.md +++ b/bedrock/analysis/electricity/historical/README.md @@ -1,20 +1,15 @@ # Historical electricity analyses -Last commit of the original UGO / Table 8.3 / Table 2.4 production path: +Published original vs EIA-anchored (pre-MECS) comparison tables and figures +live in `original_vs_eia_anchored_deck/`. The five-slide deck reads that pack +for original and pre-MECS EIA G/T/D; current (post-MECS) stays live. -`af4a82994d9302bb6efef33633f3b2eb251b31b7` -(`feat(eeio): freeze current electricity 3-way and mixed-units production`). +Full-sector freeze parquets are not in git. They were moved to Dropbox +`Documentation/Archive/`: -EIA-anchored G/T/D replaced that path in a later commit. Method discussion: +- `original_elec_disagg_implementation_P0_af4a829_2026-08-22/` +- `eia_gtd_pre_mecs_dollar_weights/` + +Method discussion: [Discussion #85](https://github.com/cornerstone-data/methods/discussions/85) and [Discussion #88](https://github.com/cornerstone-data/methods/discussions/88). - -The freeze under `original_elec_disagg_implementation/output//` is -**read-only**. It records the original implementation for - -- `2025_usa_cornerstone_v0_3_electricity_disaggregation` -- `2025_usa_cornerstone_v0_3_electricity_mixed_units` (also has `c_row.parquet`) - -Do not run historical Python against current production. Compare live EIA -output to the freeze with -`bedrock.analysis.electricity.current.vs_original_elec_disagg.compare_to_original_elec_disagg`. diff --git a/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/__tests__/test_original_elec_disagg_freeze_exists.py b/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/__tests__/test_original_elec_disagg_freeze_exists.py deleted file mode 100644 index bb9112c16..000000000 --- a/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/__tests__/test_original_elec_disagg_freeze_exists.py +++ /dev/null @@ -1,27 +0,0 @@ -"""The original electricity-disagg freeze must stay on disk and tracked. - -Production now uses EIA-anchored G/T/D allocation. The freeze under -``bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/`` -records the previous UGO / Table 8.3 / Table 2.4 path. -""" - -from __future__ import annotations - -from bedrock.analysis.electricity.historical.original_elec_disagg_implementation.paths import ( - DISAGG_CONFIG, - MIXED_CONFIG, - SNAPSHOT_FILES, - config_dir, -) - - -def test_original_elec_disagg_freeze_exists() -> None: - for stem in (DISAGG_CONFIG, MIXED_CONFIG): - folder = config_dir(stem) - assert folder.is_dir(), f'original-elec-disagg freeze missing at {folder}' - for name in SNAPSHOT_FILES: - path = folder / name - assert path.is_file(), f'original-elec-disagg freeze file missing: {path}' - if stem == MIXED_CONFIG: - c_row = folder / 'c_row.parquet' - assert c_row.is_file(), f'mixed-units freeze missing c_row.parquet: {c_row}' diff --git a/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_disaggregation/BLy.parquet b/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_disaggregation/BLy.parquet deleted file mode 100644 index e7eb85c3e..000000000 Binary files a/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_disaggregation/BLy.parquet and /dev/null differ diff --git a/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_disaggregation/D.parquet b/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_disaggregation/D.parquet deleted file mode 100644 index d45228550..000000000 Binary files a/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_disaggregation/D.parquet and /dev/null differ diff --git a/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_disaggregation/E.parquet b/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_disaggregation/E.parquet deleted file mode 100644 index f4279f860..000000000 Binary files a/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_disaggregation/E.parquet and /dev/null differ diff --git a/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_disaggregation/N.parquet b/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_disaggregation/N.parquet deleted file mode 100644 index 6d85c9dd8..000000000 Binary files a/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_disaggregation/N.parquet and /dev/null differ diff --git a/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_disaggregation/class_generation_mwh.parquet b/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_disaggregation/class_generation_mwh.parquet deleted file mode 100644 index 9c733a7d0..000000000 Binary files a/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_disaggregation/class_generation_mwh.parquet and /dev/null differ diff --git a/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_disaggregation/electricity_rows_y.parquet b/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_disaggregation/electricity_rows_y.parquet deleted file mode 100644 index 1c589e5b0..000000000 Binary files a/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_disaggregation/electricity_rows_y.parquet and /dev/null differ diff --git a/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_disaggregation/intersection_3x3.parquet b/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_disaggregation/intersection_3x3.parquet deleted file mode 100644 index 067c0347e..000000000 Binary files a/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_disaggregation/intersection_3x3.parquet and /dev/null differ diff --git a/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_disaggregation/q.parquet b/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_disaggregation/q.parquet deleted file mode 100644 index 43c9a4185..000000000 Binary files a/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_disaggregation/q.parquet and /dev/null differ diff --git a/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_disaggregation/run_metadata.json b/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_disaggregation/run_metadata.json deleted file mode 100644 index 1ac5563c5..000000000 --- a/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_disaggregation/run_metadata.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "config": "2025_usa_cornerstone_v0_3_electricity_disaggregation", - "git_sha": "537cb0aa8d5ac22d2aefeb208dc83056765364e7", - "datetime_utc": "2026-08-22T22:31:03.275266+00:00", - "flags": { - "implement_electricity_reallocation": true, - "implement_electricity_disaggregation": true, - "implement_electricity_mixed_units": false, - "apply_io_year_adjustments": true, - "model_base_year": 2024, - "usa_ghg_data_year": 2024 - }, - "mixed_units": false, - "p": "N/A", - "p_note": "no single production p on 3-way-only path", - "c_col": null, - "generation_use_y_total": 155218960135.1052, - "U[G,G]_udom": 8486136959.918574, - "F01000_generation": 58825292862.53748 -} diff --git a/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_disaggregation/use_y_generation.parquet b/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_disaggregation/use_y_generation.parquet deleted file mode 100644 index b2486920b..000000000 Binary files a/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_disaggregation/use_y_generation.parquet and /dev/null differ diff --git a/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_disaggregation/x.parquet b/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_disaggregation/x.parquet deleted file mode 100644 index 002ad70f9..000000000 Binary files a/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_disaggregation/x.parquet and /dev/null differ diff --git a/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_mixed_units/BLy.parquet b/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_mixed_units/BLy.parquet deleted file mode 100644 index baa92b58c..000000000 Binary files a/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_mixed_units/BLy.parquet and /dev/null differ diff --git a/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_mixed_units/D.parquet b/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_mixed_units/D.parquet deleted file mode 100644 index 70d8364d9..000000000 Binary files a/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_mixed_units/D.parquet and /dev/null differ diff --git a/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_mixed_units/E.parquet b/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_mixed_units/E.parquet deleted file mode 100644 index f4279f860..000000000 Binary files a/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_mixed_units/E.parquet and /dev/null differ diff --git a/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_mixed_units/N.parquet b/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_mixed_units/N.parquet deleted file mode 100644 index a4d61c6ce..000000000 Binary files a/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_mixed_units/N.parquet and /dev/null differ diff --git a/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_mixed_units/c_row.parquet b/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_mixed_units/c_row.parquet deleted file mode 100644 index 3908b432e..000000000 Binary files a/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_mixed_units/c_row.parquet and /dev/null differ diff --git a/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_mixed_units/class_generation_mwh.parquet b/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_mixed_units/class_generation_mwh.parquet deleted file mode 100644 index 01f076c11..000000000 Binary files a/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_mixed_units/class_generation_mwh.parquet and /dev/null differ diff --git a/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_mixed_units/electricity_rows_y.parquet b/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_mixed_units/electricity_rows_y.parquet deleted file mode 100644 index 1c589e5b0..000000000 Binary files a/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_mixed_units/electricity_rows_y.parquet and /dev/null differ diff --git a/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_mixed_units/intersection_3x3.parquet b/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_mixed_units/intersection_3x3.parquet deleted file mode 100644 index 067c0347e..000000000 Binary files a/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_mixed_units/intersection_3x3.parquet and /dev/null differ diff --git a/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_mixed_units/q.parquet b/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_mixed_units/q.parquet deleted file mode 100644 index 7610795e9..000000000 Binary files a/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_mixed_units/q.parquet and /dev/null differ diff --git a/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_mixed_units/run_metadata.json b/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_mixed_units/run_metadata.json deleted file mode 100644 index 0e9a51590..000000000 --- a/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_mixed_units/run_metadata.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "config": "2025_usa_cornerstone_v0_3_electricity_mixed_units", - "git_sha": "537cb0aa8d5ac22d2aefeb208dc83056765364e7", - "datetime_utc": "2026-08-22T22:31:26.707640+00:00", - "flags": { - "implement_electricity_reallocation": true, - "implement_electricity_disaggregation": true, - "implement_electricity_mixed_units": true, - "apply_io_year_adjustments": true, - "model_base_year": 2024, - "usa_ghg_data_year": 2024 - }, - "mixed_units": true, - "p": 71.55492202425425, - "p_note": "implied q_$[221110]/eGRID (today 1/c_col); not D0 p", - "c_col": 0.013975279012407282, - "generation_use_y_total": 4312557440.341, - "U[G,G]_udom": 8486136959.918574, - "F01000_generation": 561492440.0008721 -} diff --git a/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_mixed_units/use_y_generation.parquet b/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_mixed_units/use_y_generation.parquet deleted file mode 100644 index 717a9f2ab..000000000 Binary files a/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_mixed_units/use_y_generation.parquet and /dev/null differ diff --git a/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_mixed_units/x.parquet b/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_mixed_units/x.parquet deleted file mode 100644 index 002ad70f9..000000000 Binary files a/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/output/2025_usa_cornerstone_v0_3_electricity_mixed_units/x.parquet and /dev/null differ diff --git a/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/paths.py b/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/paths.py deleted file mode 100644 index e6ee6d7f0..000000000 --- a/bedrock/analysis/electricity/historical/original_elec_disagg_implementation/paths.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Paths for the original electricity-disaggregation implementation freeze.""" - -from __future__ import annotations - -from pathlib import Path - -PACKAGE_DIR = Path(__file__).resolve().parent -OUT_DIR = PACKAGE_DIR / 'output' - -DISAGG_CONFIG = '2025_usa_cornerstone_v0_3_electricity_disaggregation' -MIXED_CONFIG = '2025_usa_cornerstone_v0_3_electricity_mixed_units' - -SNAPSHOT_FILES = ( - 'q.parquet', - 'x.parquet', - 'use_y_generation.parquet', - 'intersection_3x3.parquet', - 'electricity_rows_y.parquet', - 'E.parquet', - 'D.parquet', - 'N.parquet', - 'BLy.parquet', - 'class_generation_mwh.parquet', - 'run_metadata.json', -) - - -def config_dir(config_stem: str) -> Path: - return OUT_DIR / config_stem diff --git a/bedrock/analysis/electricity/historical/original_vs_eia_anchored_deck/README.md b/bedrock/analysis/electricity/historical/original_vs_eia_anchored_deck/README.md new file mode 100644 index 000000000..cbfe9002d --- /dev/null +++ b/bedrock/analysis/electricity/historical/original_vs_eia_anchored_deck/README.md @@ -0,0 +1,21 @@ +# Original vs EIA-anchored (pre-MECS) published comparison + +Checked-in extract of + +`electricity disagg results comparison original vs eia anchored.pptx` + +under Dropbox `Documentation/`. The comparison deck reads **only** this pack +for original and pre-MECS EIA G/T/D tables and vs-footing histogram panels. + +| File | Contents | +|---|---| +| `tables.yaml` | Slide 1 class MWh; slides 2/4 electricity-sector D/N | +| `figures/v0.2_original_electricity_disagg_{D,N}.png` | Original vs v0.2 footing | +| `figures/v0.3_eia_gtd_pre_mecs_{D,N}.png` | EIA G/T/D pre-MECS vs v0.3.1 footing | + +Full-sector freeze parquets (later vintage than these tables) are **not** in +the repo. They were moved to Dropbox + +`Documentation/Archive/original_elec_disagg_implementation_P0_af4a829_2026-08-22/` +and +`Documentation/Archive/eia_gtd_pre_mecs_dollar_weights/`. diff --git a/bedrock/analysis/electricity/historical/original_vs_eia_anchored_deck/__init__.py b/bedrock/analysis/electricity/historical/original_vs_eia_anchored_deck/__init__.py new file mode 100644 index 000000000..3e3b573a6 --- /dev/null +++ b/bedrock/analysis/electricity/historical/original_vs_eia_anchored_deck/__init__.py @@ -0,0 +1 @@ +"""Published original vs EIA-anchored (pre-MECS) comparison tables and figures.""" diff --git a/bedrock/analysis/electricity/historical/original_vs_eia_anchored_deck/figures/v0.2_original_electricity_disagg_D.png b/bedrock/analysis/electricity/historical/original_vs_eia_anchored_deck/figures/v0.2_original_electricity_disagg_D.png new file mode 100644 index 000000000..5fb001bbb Binary files /dev/null and b/bedrock/analysis/electricity/historical/original_vs_eia_anchored_deck/figures/v0.2_original_electricity_disagg_D.png differ diff --git a/bedrock/analysis/electricity/historical/original_vs_eia_anchored_deck/figures/v0.2_original_electricity_disagg_N.png b/bedrock/analysis/electricity/historical/original_vs_eia_anchored_deck/figures/v0.2_original_electricity_disagg_N.png new file mode 100644 index 000000000..92f0b6ff4 Binary files /dev/null and b/bedrock/analysis/electricity/historical/original_vs_eia_anchored_deck/figures/v0.2_original_electricity_disagg_N.png differ diff --git a/bedrock/analysis/electricity/historical/original_vs_eia_anchored_deck/figures/v0.3_eia_gtd_pre_mecs_D.png b/bedrock/analysis/electricity/historical/original_vs_eia_anchored_deck/figures/v0.3_eia_gtd_pre_mecs_D.png new file mode 100644 index 000000000..c98a85f0b Binary files /dev/null and b/bedrock/analysis/electricity/historical/original_vs_eia_anchored_deck/figures/v0.3_eia_gtd_pre_mecs_D.png differ diff --git a/bedrock/analysis/electricity/historical/original_vs_eia_anchored_deck/figures/v0.3_eia_gtd_pre_mecs_N.png b/bedrock/analysis/electricity/historical/original_vs_eia_anchored_deck/figures/v0.3_eia_gtd_pre_mecs_N.png new file mode 100644 index 000000000..1b9df52c4 Binary files /dev/null and b/bedrock/analysis/electricity/historical/original_vs_eia_anchored_deck/figures/v0.3_eia_gtd_pre_mecs_N.png differ diff --git a/bedrock/analysis/electricity/historical/original_vs_eia_anchored_deck/paths.py b/bedrock/analysis/electricity/historical/original_vs_eia_anchored_deck/paths.py new file mode 100644 index 000000000..e74fd25c4 --- /dev/null +++ b/bedrock/analysis/electricity/historical/original_vs_eia_anchored_deck/paths.py @@ -0,0 +1,24 @@ +"""Paths for the published original vs EIA-anchored (pre-MECS) comparison pack.""" + +from __future__ import annotations + +from pathlib import Path + +PACKAGE_DIR = Path(__file__).resolve().parent +FIGURES_DIR = PACKAGE_DIR / 'figures' +TABLES_YAML = PACKAGE_DIR / 'tables.yaml' + +PANEL_PNG: dict[tuple[str, str], str] = { + ('original', 'D'): 'v0.2_original_electricity_disagg_D.png', + ('original', 'N'): 'v0.2_original_electricity_disagg_N.png', + ('eia_gtd', 'D'): 'v0.3_eia_gtd_pre_mecs_D.png', + ('eia_gtd', 'N'): 'v0.3_eia_gtd_pre_mecs_N.png', +} + + +def panel_png(impl_id: str, kind: str) -> Path | None: + name = PANEL_PNG.get((impl_id, kind)) + if name is None: + return None + path = FIGURES_DIR / name + return path if path.is_file() else None diff --git a/bedrock/analysis/electricity/historical/original_vs_eia_anchored_deck/published.py b/bedrock/analysis/electricity/historical/original_vs_eia_anchored_deck/published.py new file mode 100644 index 000000000..c1efcf575 --- /dev/null +++ b/bedrock/analysis/electricity/historical/original_vs_eia_anchored_deck/published.py @@ -0,0 +1,55 @@ +"""Load published class-MWh / D / N grids from the original-vs-EIA PPTX extract.""" + +from __future__ import annotations + +from functools import lru_cache +from typing import Any + +import yaml + +from bedrock.analysis.electricity.historical.original_vs_eia_anchored_deck.paths import ( + TABLES_YAML, +) + +PUBLISHED_IMPLS = frozenset({'original', 'eia_gtd'}) + + +@lru_cache(maxsize=1) +def load_tables() -> dict[str, Any]: + payload = yaml.safe_load(TABLES_YAML.read_text(encoding='utf-8')) + if not isinstance(payload, dict) or 'implementations' not in payload: + raise ValueError(f'{TABLES_YAML} missing implementations') + return payload + + +def impl_tables(impl_id: str) -> dict[str, Any]: + implementations = load_tables()['implementations'] + if impl_id not in implementations: + raise KeyError(f'no published tables for {impl_id!r}') + return implementations[impl_id] + + +def published_ef( + impl_id: str, + kind: str, + sector: str, + step_id: str, +) -> float | None: + """Return the published kg/USD value, or ``None`` when the PPTX cell is N/A.""" + block = impl_tables(impl_id)[kind] + if sector not in block: + return None + raw = block[sector].get(step_id) + if raw is None: + return None + return float(raw) + + +def published_class_mwh_rows(impl_id: str) -> list[tuple[float, float, str]]: + """``(model_mwh, target_mwh, label)`` rows including Total.""" + rows = [] + for row in impl_tables(impl_id)['class_mwh']: + rows.append( + (float(row['model_mwh']), float(row['target_mwh']), str(row['label'])) + ) + return rows diff --git a/bedrock/analysis/electricity/historical/original_vs_eia_anchored_deck/tables.yaml b/bedrock/analysis/electricity/historical/original_vs_eia_anchored_deck/tables.yaml new file mode 100644 index 000000000..6a8318f4f --- /dev/null +++ b/bedrock/analysis/electricity/historical/original_vs_eia_anchored_deck/tables.yaml @@ -0,0 +1,124 @@ +# Published original vs EIA-anchored G/T/D (pre-MECS) comparison tables. +# Source: electricity disagg results comparison original vs eia anchored.pptx +# (Dropbox Documentation/). Do not regenerate from freeze parquets. +source: + pptx: electricity disagg results comparison original vs eia anchored.pptx + dropbox_documentation: D:\Dropbox\ERG Work\2025_07_15_Cornerstone\Tasks\Electricity Disagg\Documentation + +implementations: + original: + footing_label: v0.2 + class_mwh: + - {label: Residential, model_mwh: 771500000.0, target_mwh: 1450000000.0} + - {label: Com+Ind+Trans sales, model_mwh: 3417700000.0, target_mwh: 2424200000.0} + - {label: Total, model_mwh: 4191000000.0, target_mwh: 4011200000.0} + D: + '221100': + footing: 2.386 + reallocation: 2.393 + three_way: null + mixed_units: null + '221100*': + footing: null + reallocation: null + three_way: 2.448 + mixed_units: 2.448 + '221110': + footing: null + reallocation: null + three_way: 7.138 + mixed_units: 7.138 + '221121': + footing: null + reallocation: null + three_way: 0.225 + mixed_units: 0.225 + '221122': + footing: null + reallocation: null + three_way: 0.0 + mixed_units: 0.0 + N: + '221100': + footing: 2.776 + reallocation: 2.784 + three_way: null + mixed_units: null + '221100*': + footing: null + reallocation: null + three_way: 3.216 + mixed_units: 3.510 + '221110': + footing: null + reallocation: null + three_way: 9.213 + mixed_units: 10.070 + '221121': + footing: null + reallocation: null + three_way: 0.424 + mixed_units: 0.426 + '221122': + footing: null + reallocation: null + three_way: 0.083 + mixed_units: 0.085 + eia_gtd: + footing_label: v0.3.1 + class_mwh: + - {label: Residential, model_mwh: 1548800000.0, target_mwh: 1548800000.0} + - {label: Com+Ind+Trans+Exports, model_mwh: 2763800000.0, target_mwh: 2763800000.0} + - {label: Total, model_mwh: 4312600000.0, target_mwh: 4312600000.0} + D: + '221100': + footing: 2.409 + reallocation: 2.416 + three_way: null + mixed_units: null + '221100*': + footing: null + reallocation: null + three_way: 2.468 + mixed_units: 2.468 + '221110': + footing: null + reallocation: null + three_way: 7.197 + mixed_units: 7.197 + '221121': + footing: null + reallocation: null + three_way: 0.226 + mixed_units: 0.226 + '221122': + footing: null + reallocation: null + three_way: 0.0 + mixed_units: 0.0 + N: + '221100': + footing: 2.812 + reallocation: 2.820 + three_way: null + mixed_units: null + '221100*': + footing: null + reallocation: null + three_way: 3.034 + mixed_units: 3.034 + '221110': + footing: null + reallocation: null + three_way: 8.622 + mixed_units: 8.622 + '221121': + footing: null + reallocation: null + three_way: 0.360 + mixed_units: 0.360 + '221122': + footing: null + reallocation: null + three_way: 0.119 + mixed_units: 0.119 diff --git a/bedrock/extract/eia/EIA_MECS.py b/bedrock/extract/eia/EIA_MECS.py index a9cef2576..fac3d73cf 100644 --- a/bedrock/extract/eia/EIA_MECS.py +++ b/bedrock/extract/eia/EIA_MECS.py @@ -382,7 +382,7 @@ def _eia_clean_mecs_energy( df_data_region.columns = table_dict[year][table]['col_names'] df_rse_region.columns = table_dict[year][table]['col_names'] # if table name ends in 2, the units must be stripped from the column names listed in the table dict - if table[-1] in ['2', '0', '5', '6']: + if table[-1] in ['2', '0', '5', '6', '7']: df_data_region.columns = [ name.split(' | ', 2)[0] for name in table_dict[year][table]['col_names'] ] @@ -462,8 +462,14 @@ def _eia_clean_mecs_energy( # if table name ends in 2, units are 'trillion Btu' elif table[-1] in ['2', '5', '6']: df_data_region['Unit'] = 'Trillion Btu' - if table[-3] == '7': # 7_2 + if table[-3] == '7': # Table 7.2 df_data_region['Unit'] = 'USD / million btu' + elif str(table).rstrip().endswith('7.7'): + # YAML col_names are the three electricity columns only (million kWh). + # Full EIA 7.7 also has NG (bcf) and steam (billion Btu); those are + # not extracted. Other Table 7.x do not hit this branch (7.1 / 7.2 / + # 7.10 dispatch on a different last character). + df_data_region['Unit'] = 'million kWh' elif table[-1] == '0': df_data_region['Unit'] = 'million USD' diff --git a/bedrock/extract/eia/EIA_MECS_Energy.yaml b/bedrock/extract/eia/EIA_MECS_Energy.yaml index ea813ff5d..7a5d5e1c4 100644 --- a/bedrock/extract/eia/EIA_MECS_Energy.yaml +++ b/bedrock/extract/eia/EIA_MECS_Energy.yaml @@ -26,6 +26,7 @@ tables: - Table2_2 - Table3_1 - Table3_2 +- table7_7 # - table3_6 # - table7_2 # - table7_10 @@ -529,6 +530,26 @@ table_dict: regions: *regions_2018_72 rse_regions: *rse_regions_2018_72 data_type: 'money' + Table 7.7: + col_names: &col_names_table_7_7 + - NAICS Code + - Subsector and Industry + - Electricity total + - Electricity from Local Utility + - Electricity from Sources Other than Local Utility + regions: ®ions_2018_77 + Total United States: [18, 100] + Northeast Region: [103, 185] + Midwest Region: [188, 270] + South Region: [273, 355] + West Region: [358, 440] + rse_regions: &rse_regions_2018_77 + Total United States: [15, 97] + Northeast Region: [100, 182] + Midwest Region: [185, 267] + South Region: [270, 352] + West Region: [355, 437] + data_type: 'fuel consumption' '2022': Table 2.1: @@ -598,3 +619,18 @@ table_dict: rse_regions: *rse_regions_2022_31 data_type: 'fuel consumption' + Table 7.7: + col_names: *col_names_table_7_7 + regions: + Total United States: [18, 98] + Northeast Region: [101, 181] + Midwest Region: [184, 264] + South Region: [267, 347] + West Region: [350, 430] + rse_regions: + Total United States: [15, 95] + Northeast Region: [98, 178] + Midwest Region: [181, 261] + South Region: [264, 344] + West Region: [347, 427] + data_type: 'fuel consumption' diff --git a/bedrock/publish/cache_reset.py b/bedrock/publish/cache_reset.py index 02016c976..bca717fd2 100644 --- a/bedrock/publish/cache_reset.py +++ b/bedrock/publish/cache_reset.py @@ -128,7 +128,14 @@ def _clear_electricity_caches_if_loaded() -> None: _clear_cached_attrs(ed, _ELECTRICITY_DISAGG_CACHED_ATTRS) gtd = sys.modules.get('bedrock.transform.eeio.electricity_gtd_allocation') if gtd is not None: - _clear_cached_attrs(gtd, ('get_2017_eia_purchaser_allocation',)) + _clear_cached_attrs( + gtd, + ( + 'get_2017_eia_purchaser_allocation', + 'mecs_purchased_kwh', + '_mecs_purchased_kwh_cached', + ), + ) clear_reanchored = getattr(gtd, 'clear_reanchored_electricity_q', None) if callable(clear_reanchored): clear_reanchored() diff --git a/bedrock/transform/eeio/__tests__/test_electricity_disaggregation.py b/bedrock/transform/eeio/__tests__/test_electricity_disaggregation.py index b155d100f..85899b12e 100644 --- a/bedrock/transform/eeio/__tests__/test_electricity_disaggregation.py +++ b/bedrock/transform/eeio/__tests__/test_electricity_disaggregation.py @@ -2,7 +2,9 @@ from __future__ import annotations -from typing import Callable +from contextlib import contextmanager +from typing import Any, Callable, Iterator +from unittest.mock import patch import numpy as np import pandas as pd @@ -44,6 +46,7 @@ disaggregate_use_industry_columns, get_2017_eia_purchaser_allocation, ) +from bedrock.transform.eeio.electricity_gtd_allocation import mecs_purchased_kwh from bedrock.utils.config.usa_config import reset_usa_config, set_global_usa_config from bedrock.utils.economic.inflation_helpers_cornerstone import ( clear_cornerstone_inflation_caches, @@ -67,6 +70,7 @@ build_electricity_detail_GO_growth_ratios, applied_utilities_summary_q_growth_ratio, get_2017_eia_purchaser_allocation, + mecs_purchased_kwh, _derive_post_reallocation_checkpoint_for_disagg, derive_cornerstone_V, derive_cornerstone_Vnorm_scrap_corrected, @@ -97,6 +101,22 @@ def _clear_all_caches() -> None: clear_reanchored_electricity_q() +@contextmanager +def dollar_industrial_weights() -> Iterator[None]: + """Force dollar Industrial weights so unmarked tests do not load Table 7.7.""" + import bedrock.transform.eeio.electricity_gtd_allocation as gtd # noqa: PLC0415 + + orig = gtd.allocate_purchaser_gtd + + def _wrapped(*args: Any, **kwargs: Any) -> Any: + kwargs = dict(kwargs) + kwargs['industrial_weights'] = 'dollars' + return orig(*args, **kwargs) + + with patch.object(gtd, 'allocate_purchaser_gtd', _wrapped): + yield + + def _setup_config(config_name: str) -> None: _clear_all_caches() reset_usa_config(should_reset_env_var=True) @@ -283,8 +303,9 @@ def test_getter_does_not_call_io_bundle( ) as y_mock: get_2017_eia_purchaser_allocation.cache_clear() _derive_post_reallocation_checkpoint_for_disagg.cache_clear() - alloc = get_2017_eia_purchaser_allocation() - assert ELECTRICITY_AGGREGATE in alloc.bill.index + with dollar_industrial_weights(): + alloc = get_2017_eia_purchaser_allocation() + assert ELECTRICITY_AGGREGATE in alloc.electricity_purchases.index bundle_mock.assert_not_called() y_mock.assert_not_called() finally: @@ -399,7 +420,7 @@ def test_adom_times_q_matches_allocated_udom_and_written_y(self) -> None: col_s = str(col) if col_s in ELECTRICITY_DISAGG_SECTORS: continue - if col_s not in alloc.bill.index: + if col_s not in alloc.electricity_purchases.index: continue assert _frame_cell_float(udom, g, col_s) == pytest.approx( float(alloc.gen_dollars[col_s]), rel=1e-6, abs=1.0 @@ -411,7 +432,9 @@ def test_adom_times_q_matches_allocated_udom_and_written_y(self) -> None: float(alloc.d_dollars[col_s]), rel=1e-6, abs=1.0 ) gen_self = float(alloc.gen_dollars[ELECTRICITY_AGGREGATE]) - leftover = float(alloc.bill[ELECTRICITY_AGGREGATE]) - gen_self + leftover = ( + float(alloc.electricity_purchases[ELECTRICITY_AGGREGATE]) - gen_self + ) assert _frame_cell_float(udom, g, g) == pytest.approx( gen_self, rel=1e-6, abs=1.0 ) @@ -428,7 +451,7 @@ def test_adom_times_q_matches_allocated_udom_and_written_y(self) -> None: assert _frame_cell_float(udom, i, j) == pytest.approx(0.0, abs=1e-6) fd_keys = [ k - for k in alloc.bill.index + for k in alloc.electricity_purchases.index if k in set(FINAL_DEMANDS) and k != IMPORT_FD_CODE ] assert float(y.loc[g]) == pytest.approx( @@ -488,7 +511,6 @@ def test_generation_mwh_class_mix_exports_and_clip( export = eia_table_2_14_export_mwh(2017) t22 = eia_table_2_2_end_use_mwh(2017) assert float(alloc.egrid_mwh) == pytest.approx(egrid, rel=1e-9) - assert not bool(alloc.clipped.any()) assert float(alloc.mwh.sum()) == pytest.approx(egrid, rel=1e-4, abs=1.0) assert float(alloc.mwh[EXPORT_FD_CODE]) == pytest.approx( export, rel=1e-4, abs=1.0 @@ -549,7 +571,7 @@ def test_import_fd_column_uses_child_sum_when_aggregate_missing( empty = pd.Series(dtype=float) alloc = EIAPurchaserAllocation( - bill=empty, + electricity_purchases=empty, end_use_class=empty, mwh=empty, gen_dollars=empty, diff --git a/bedrock/transform/eeio/__tests__/test_electricity_gtd_mecs.py b/bedrock/transform/eeio/__tests__/test_electricity_gtd_mecs.py new file mode 100644 index 000000000..36254468e --- /dev/null +++ b/bedrock/transform/eeio/__tests__/test_electricity_gtd_mecs.py @@ -0,0 +1,543 @@ +"""Unit tests for MECS Table 7.7 Industrial manufacturing MWh shares.""" + +from __future__ import annotations + +from unittest.mock import patch + +import pandas as pd +import pytest + +from bedrock.publish.cache_reset import _clear_electricity_caches_if_loaded +from bedrock.transform.allocation.mappings.v7.ceda_mecs import NON_MECS_INDUSTRIES +from bedrock.transform.eeio.electricity_end_use_mapping import build_end_use_map +from bedrock.transform.eeio.electricity_gtd_allocation import ( + ELECTRICITY_AGGREGATE, + EXPORT_FD_CODE, + MECS_7_7_NAICS_OVERLAY, + TABLE_7_7_DESCRIPTION, + TABLE_7_7_ELECTRICITY_TOTAL, + EIAPurchaserAllocation, + _dedupe_overlay, + _fill_three_digit_suppressed, + _mecs_purchased_kwh_cached, + _overlaid_3_1_mapping, + _overlaid_3_1_subtraction, + _overlay_naics_tuple, + _required_7_7_naics, + allocate_purchaser_gtd, + industrial_manufacturing_pool, + io_manufacturing_purchased_kwh, + mecs_purchased_kwh, + mecs_year_for_eia_year, +) +from bedrock.utils.mapping.location import US_FIPS +from bedrock.utils.schemas.cornerstone_schemas import ELECTRICITY_DISAGG_SECTORS + +_MFG_A = '331110' +_MFG_B = '327310' +_RES_AG = '1111A0' +_COMMERCIAL = '452000' +_RESIDENTIAL_FD = 'F01000' + + +def _fake_7_7_frame( + *, + amounts_mkwh: dict[str, float] | None = None, + suppressed: dict[str, str | float | None] | None = None, + description: str = TABLE_7_7_DESCRIPTION, + flow_name: str = TABLE_7_7_ELECTRICITY_TOTAL, +) -> pd.DataFrame: + needed = sorted(_required_7_7_naics() | {'31-33'}) + three = [c for c in needed if len(c) == 3 and c.isdigit()] + amt = {c: 1.0 for c in needed} + amt['31-33'] = float(sum(amt[c] for c in three)) + if amounts_mkwh: + amt.update(amounts_mkwh) + supp: dict[str, str | float | None] = {c: None for c in needed} + if suppressed: + supp.update(suppressed) + return pd.DataFrame( + { + 'Description': description, + 'Location': US_FIPS, + 'FlowName': flow_name, + 'ActivityConsumedBy': needed, + 'FlowAmount': [amt[c] for c in needed], + 'Suppressed': [supp[c] for c in needed], + } + ) + + +def _allocate( + electricity_purchases: pd.Series, + *, + industrial_weights: str, + targets: dict[str, float] | None = None, + egrid: float = 1_000.0, + p_share: float = 0.1, + td_share: float = 0.3, + eia_year: int = 2017, +) -> EIAPurchaserAllocation: + if targets is None: + targets = { + 'Residential': 200.0, + 'Commercial': 200.0, + 'Industrial': 400.0, + 'Transportation': 0.0, + 'Exports': 200.0, + } + with ( + patch( + 'bedrock.transform.eeio.electricity_gtd_allocation.egrid_mwh_for_io_year', + return_value=egrid, + ), + patch( + 'bedrock.transform.eeio.electricity_gtd_allocation._class_mwh_targets', + return_value=targets, + ), + ): + return allocate_purchaser_gtd( + electricity_purchases, + self_use_key=ELECTRICITY_AGGREGATE, + eia_year=eia_year, + p_share_2017=p_share, + td_share_2017=td_share, + industrial_weights=industrial_weights, # type: ignore[arg-type] + ) + + +def test_mecs_year_for_eia_year() -> None: + assert mecs_year_for_eia_year(2017) == 2018 + assert mecs_year_for_eia_year(2016) == 2018 + assert mecs_year_for_eia_year(2018) == 2022 + assert mecs_year_for_eia_year(2024) == 2022 + + +def test_overlay_dedupes_paper_and_auto_naics() -> None: + assert _dedupe_overlay(('322121', '322122')) == ('322120',) + assert _dedupe_overlay(('336111', '336112')) == ('336110',) + for src, dst in MECS_7_7_NAICS_OVERLAY.items(): + assert _dedupe_overlay((src,)) == (dst,) + assert _overlay_naics_tuple(('322121', '322122'), {'322120', '322'}) == ('322120',) + assert _overlay_naics_tuple(('322121', '322122'), {'322121', '322122', '322'}) == ( + '322121', + '322122', + ) + + +def test_mapping_and_subtraction_io_keys_are_disjoint() -> None: + available = set(_required_7_7_naics()) + mapped = {str(c) for k in _overlaid_3_1_mapping(available) for c in k} + subtracted = {str(c) for k in _overlaid_3_1_subtraction(available) for c in k} + assert not (mapped & subtracted) + + +def test_manufacturing_pool_is_io_keys_not_naics_values() -> None: + pool = industrial_manufacturing_pool() + assert _MFG_A in pool + assert _MFG_B in pool + assert '31-33' not in pool + assert '322121' not in pool + assert '322120' in pool + + +def test_non_mecs_industrial_codes_land_in_residual() -> None: + mfg = industrial_manufacturing_pool() + end_use = build_end_use_map() + industrial = {c for c, cls in end_use.items() if cls == 'Industrial'} + industrial.add(ELECTRICITY_AGGREGATE) + residual = industrial - mfg + for code in NON_MECS_INDUSTRIES: + if code in industrial: + assert code in residual + assert code not in mfg + + +def test_manufacturing_pool_matches_ghg_combustion_mecs_io_keys() -> None: + """Same Cornerstone 3.1 IO keys as GHG industrial coal/gas combustion.""" + from bedrock.transform.allocation.co2.industrial_coal import ( # noqa: PLC0415 + _get_mecs_3_1_naics_mappings as coal_maps, + ) + from bedrock.transform.allocation.co2.industrial_natural_gas import ( # noqa: PLC0415 + _get_mecs_3_1_naics_mappings as gas_maps, + ) + from bedrock.transform.allocation.utils import flatten_items # noqa: PLC0415 + + coal_map, coal_sub = coal_maps() + gas_map, gas_sub = gas_maps() + assert coal_map is gas_map + assert coal_sub is gas_sub + ghg_mfg = {str(c) for c in flatten_items(coal_map.keys())} + ghg_mfg.update(str(c) for c in flatten_items(coal_sub.keys())) + assert industrial_manufacturing_pool() == frozenset(ghg_mfg) + assert not (industrial_manufacturing_pool() & set(NON_MECS_INDUSTRIES)) + + +def test_industrial_class_is_mecs_union_non_mecs_plus_utilities_and_fd() -> None: + """EIA Industrial ≠ 3.1 keys ∪ NON_MECS; extras stay dollar residual, not dropped. + + A is commodity-by-commodity, so electricity_purchases columns are commodities + FD. + ``331314`` is a 3.1 industry key with no matching commodity column. + """ + from bedrock.utils.schemas.cornerstone_schemas import ( # noqa: PLC0415 + CORNERSTONE_COMMODITIES_ELEC, + ) + + industrial = {c for c, cls in build_end_use_map().items() if cls == 'Industrial'} + mfg = set(industrial_manufacturing_pool()) + non_mecs = set(NON_MECS_INDUSTRIES) + commodities = set(map(str, CORNERSTONE_COMMODITIES_ELEC)) + industrial_extras = frozenset( + { + '221110', + '221121', + '221122', + '221200', + '221300', + 'F02E00', + } + ) + assert non_mecs <= industrial + assert (mfg & commodities) <= industrial + assert industrial - mfg - non_mecs == industrial_extras + assert '331314' in mfg + assert '331314' not in commodities + assert '331314' not in industrial + + +def test_self_use_industrial_fd_and_gtd_children_are_residual() -> None: + mfg = industrial_manufacturing_pool() + for code in (ELECTRICITY_AGGREGATE, 'F02E00', *ELECTRICITY_DISAGG_SECTORS): + assert code not in mfg + + +def test_fill_three_digit_assigns_single_qd_residual() -> None: + kwh = pd.Series( + { + '31-33': 100.0e6, + '311': 40.0e6, + '331': 50.0e6, + '337': float('nan'), + } + ) + suppressed = pd.Series({'31-33': None, '311': None, '331': None, '337': 'Q'}) + out = _fill_three_digit_suppressed(kwh, suppressed) + assert float(out['337']) == pytest.approx(10.0e6) + + +def test_fill_three_digit_treats_zero_flowamount_qd_as_suppressed() -> None: + kwh = pd.Series( + { + '31-33': 100.0e6, + '311': 40.0e6, + '331': 50.0e6, + '337': 0.0, + } + ) + suppressed = pd.Series({'31-33': None, '311': None, '331': None, '337': 'Q'}) + out = _fill_three_digit_suppressed(kwh, suppressed) + assert float(out['337']) == pytest.approx(10.0e6) + + +def test_fill_three_digit_ok_when_residual_near_zero() -> None: + kwh = pd.Series({'31-33': 100.0e6, '311': 60.0e6, '331': 40.0e6}) + suppressed = pd.Series({'31-33': None, '311': None, '331': None}) + out = _fill_three_digit_suppressed(kwh, suppressed) + assert float(out['311']) == pytest.approx(60.0e6) + assert float(out.sum()) == pytest.approx(200.0e6) + + +def test_fill_three_digit_hard_error_if_leftover_and_not_one_qd() -> None: + kwh = pd.Series( + { + '31-33': 100.0e6, + '311': 10.0e6, + '331': float('nan'), + '337': float('nan'), + } + ) + suppressed = pd.Series({'31-33': None, '311': None, '331': 'Q', '337': 'D'}) + with pytest.raises(ValueError, match='3-digit residual'): + _fill_three_digit_suppressed(kwh, suppressed) + kwh_none = pd.Series({'31-33': 100.0e6, '311': 10.0e6}) + suppressed_none = pd.Series({'31-33': None, '311': None}) + with pytest.raises(ValueError, match='3-digit residual'): + _fill_three_digit_suppressed(kwh_none, suppressed_none) + + +def test_fill_three_digit_zeros_remaining_non_three_digit_qd() -> None: + kwh = pd.Series( + { + '31-33': 100.0e6, + '311': 100.0e6, + '331110': float('nan'), + } + ) + suppressed = pd.Series({'31-33': None, '311': None, '331110': 'Q'}) + out = _fill_three_digit_suppressed(kwh, suppressed) + assert float(out['331110']) == pytest.approx(0.0) + + +def test_empty_us_electricity_total_hard_errors() -> None: + _mecs_purchased_kwh_cached.cache_clear() + empty = pd.DataFrame( + { + 'Description': ['Table 3.1'], + 'Location': [US_FIPS], + 'FlowName': ['Net Electricity'], + 'ActivityConsumedBy': ['311'], + 'FlowAmount': [1.0], + 'Suppressed': [None], + } + ) + with patch( + 'bedrock.extract.flowbyactivity.getFlowByActivity', + return_value=empty, + ): + with pytest.raises(ValueError, match='no US Table 7.7'): + mecs_purchased_kwh(2018) + _mecs_purchased_kwh_cached.cache_clear() + + +def test_star_is_zero_and_missing_mapped_naics_hard_errors() -> None: + _mecs_purchased_kwh_cached.cache_clear() + frame = _fake_7_7_frame() + frame = frame.loc[frame['ActivityConsumedBy'] != '331110'].reset_index(drop=True) + with patch( + 'bedrock.extract.flowbyactivity.getFlowByActivity', + return_value=frame, + ): + with pytest.raises(ValueError, match='missing NAICS'): + mecs_purchased_kwh(2018) + _mecs_purchased_kwh_cached.cache_clear() + + +def test_io_kwh_sum_excludes_manufacturing_total_row() -> None: + _mecs_purchased_kwh_cached.cache_clear() + frame = _fake_7_7_frame() + electricity_purchases = pd.Series( + 1.0, index=list(industrial_manufacturing_pool()), dtype=float + ) + with patch( + 'bedrock.extract.flowbyactivity.getFlowByActivity', + return_value=frame, + ): + naics = mecs_purchased_kwh(2018) + io_kwh = io_manufacturing_purchased_kwh(electricity_purchases, 2018) + assert '31-33' not in io_kwh.index + assert float(io_kwh.sum()) != pytest.approx(float(naics.sum())) + _mecs_purchased_kwh_cached.cache_clear() + + +def test_one_to_one_zero_electricity_purchases_still_gets_kwh() -> None: + _mecs_purchased_kwh_cached.cache_clear() + frame = _fake_7_7_frame() + pool = list(industrial_manufacturing_pool()) + electricity_purchases = pd.Series(1.0, index=pool, dtype=float) + electricity_purchases[_MFG_A] = 0.0 + with patch( + 'bedrock.extract.flowbyactivity.getFlowByActivity', + return_value=frame, + ): + io_kwh = io_manufacturing_purchased_kwh(electricity_purchases, 2018) + assert _MFG_A in io_kwh.index + assert float(io_kwh[_MFG_A]) > 0.0 + _mecs_purchased_kwh_cached.cache_clear() + + +def test_many_to_one_zero_electricity_purchases_member_gets_zero_kwh() -> None: + _mecs_purchased_kwh_cached.cache_clear() + naics_to_io: dict[str, list[str]] = {} + for io_key, naics_vals in _overlaid_3_1_mapping(set(_required_7_7_naics())).items(): + for n in naics_vals: + bucket = naics_to_io.setdefault(n, []) + for io in io_key: + if str(io) not in bucket: + bucket.append(str(io)) + shared = next((ios for ios in naics_to_io.values() if len(ios) >= 2), None) + if shared is None: + pytest.skip('no many:1 MECS NAICS in the overlay mapping') + frame = _fake_7_7_frame() + pool = list(industrial_manufacturing_pool()) + electricity_purchases = pd.Series(1.0, index=pool, dtype=float) + electricity_purchases[shared[0]] = 0.0 + with patch( + 'bedrock.extract.flowbyactivity.getFlowByActivity', + return_value=frame, + ): + io_kwh = io_manufacturing_purchased_kwh(electricity_purchases, 2018) + assert float(io_kwh.get(shared[0], 0.0)) == pytest.approx(0.0) + assert float(sum(float(io_kwh.get(c, 0.0)) for c in shared[1:])) > 0.0 + _mecs_purchased_kwh_cached.cache_clear() + + +def test_two_pool_identities_and_class_total() -> None: + electricity_purchases = pd.Series( + { + _MFG_A: 600.0, + _MFG_B: 200.0, + _RES_AG: 200.0, + ELECTRICITY_AGGREGATE: 100.0, + _RESIDENTIAL_FD: 2_000.0, + _COMMERCIAL: 2_000.0, + EXPORT_FD_CODE: 2_000.0, + }, + dtype=float, + ) + io_kwh = pd.Series({_MFG_A: 3.0, _MFG_B: 1.0}, dtype=float) + industrial_t = 400.0 + with patch( + 'bedrock.transform.eeio.electricity_gtd_allocation.io_manufacturing_purchased_kwh', + return_value=io_kwh, + ): + mecs = _allocate(electricity_purchases, industrial_weights='mecs') + dollars = _allocate(electricity_purchases, industrial_weights='dollars') + industrial = mecs.end_use_class == 'Industrial' + assert float(mecs.mwh[industrial].sum()) == pytest.approx( + float(dollars.mwh[industrial].sum()) + ) + assert float(mecs.mwh[industrial].sum()) == pytest.approx(industrial_t) + mfg_electricity_purchases = 600.0 + 200.0 + industrial_electricity_purchases = mfg_electricity_purchases + 200.0 + 100.0 + pool_mfg = ( + industrial_t * mfg_electricity_purchases / industrial_electricity_purchases + ) + pool_res = industrial_t - pool_mfg + assert float(mecs.mwh[_MFG_A]) == pytest.approx(pool_mfg * 3.0 / 4.0) + assert float(mecs.mwh[_MFG_B]) == pytest.approx(pool_mfg * 1.0 / 4.0) + assert float(mecs.mwh[_RES_AG]) == pytest.approx(pool_res * 200.0 / 300.0) + assert float(mecs.mwh[ELECTRICITY_AGGREGATE]) == pytest.approx( + pool_res * 100.0 / 300.0 + ) + assert float(mecs.mwh[_RESIDENTIAL_FD]) == pytest.approx( + float(dollars.mwh[_RESIDENTIAL_FD]) + ) + assert float(mecs.mwh[_COMMERCIAL]) == pytest.approx( + float(dollars.mwh[_COMMERCIAL]) + ) + assert float(mecs.mwh[EXPORT_FD_CODE]) == pytest.approx( + float(dollars.mwh[EXPORT_FD_CODE]) + ) + + +def test_dollars_path_does_not_load_mecs() -> None: + electricity_purchases = pd.Series( + { + _MFG_A: 100.0, + _RES_AG: 50.0, + _RESIDENTIAL_FD: 200.0, + _COMMERCIAL: 200.0, + EXPORT_FD_CODE: 200.0, + }, + dtype=float, + ) + with patch( + 'bedrock.transform.eeio.electricity_gtd_allocation.mecs_purchased_kwh' + ) as mecs_mock: + _allocate(electricity_purchases, industrial_weights='dollars') + mecs_mock.assert_not_called() + + +def test_manufacturing_kwh_sum_zero_falls_back_to_dollars_inside_mfg() -> None: + electricity_purchases = pd.Series( + { + _MFG_A: 80.0, + _MFG_B: 20.0, + _RES_AG: 100.0, + _RESIDENTIAL_FD: 1_000.0, + _COMMERCIAL: 1_000.0, + EXPORT_FD_CODE: 1_000.0, + }, + dtype=float, + ) + with patch( + 'bedrock.transform.eeio.electricity_gtd_allocation.io_manufacturing_purchased_kwh', + return_value=pd.Series({_MFG_A: 0.0, _MFG_B: 0.0}, dtype=float), + ): + mecs = _allocate(electricity_purchases, industrial_weights='mecs') + dollars = _allocate(electricity_purchases, industrial_weights='dollars') + assert float(mecs.mwh[_MFG_A]) == pytest.approx(float(dollars.mwh[_MFG_A])) + assert float(mecs.mwh[_MFG_B]) == pytest.approx(float(dollars.mwh[_MFG_B])) + assert float(mecs.mwh[_RES_AG]) == pytest.approx(float(dollars.mwh[_RES_AG])) + + +def test_zero_residual_electricity_purchases_assign_industrial_mwh_to_manufacturing() -> ( + None +): + electricity_purchases = pd.Series( + { + _MFG_A: 200.0, + _RES_AG: -10.0, + _RESIDENTIAL_FD: 1_000.0, + _COMMERCIAL: 1_000.0, + EXPORT_FD_CODE: 1_000.0, + }, + dtype=float, + ) + with patch( + 'bedrock.transform.eeio.electricity_gtd_allocation.io_manufacturing_purchased_kwh', + return_value=pd.Series({_MFG_A: 5.0}, dtype=float), + ): + mecs = _allocate(electricity_purchases, industrial_weights='mecs') + industrial = mecs.end_use_class == 'Industrial' + assert float(mecs.mwh[industrial].sum()) == pytest.approx(400.0) + assert float(mecs.mwh[_MFG_A]) == pytest.approx(400.0) + assert float(mecs.mwh[_RES_AG]) == pytest.approx(0.0) + + +def test_industrial_nonnegative_use_sum_zero_hard_errors() -> None: + electricity_purchases = pd.Series( + { + _MFG_A: -5.0, + _RES_AG: 0.0, + _RESIDENTIAL_FD: 1_000.0, + _COMMERCIAL: 1_000.0, + EXPORT_FD_CODE: 1_000.0, + }, + dtype=float, + ) + with pytest.raises(ValueError, match='cannot split'): + _allocate(electricity_purchases, industrial_weights='mecs') + + +def test_mecs_purchased_kwh_returns_copy() -> None: + _mecs_purchased_kwh_cached.cache_clear() + frame = _fake_7_7_frame() + with patch( + 'bedrock.extract.flowbyactivity.getFlowByActivity', + return_value=frame, + ): + first = mecs_purchased_kwh(2018) + first.iloc[0] = -1.0 + second = mecs_purchased_kwh(2018) + assert float(second.iloc[0]) != pytest.approx(-1.0) + _mecs_purchased_kwh_cached.cache_clear() + + +def test_cache_reset_clears_mecs_purchased_kwh() -> None: + _mecs_purchased_kwh_cached.cache_clear() + frame = _fake_7_7_frame() + with patch( + 'bedrock.extract.flowbyactivity.getFlowByActivity', + return_value=frame, + ): + mecs_purchased_kwh(2018) + assert _mecs_purchased_kwh_cached.cache_info().currsize == 1 + _clear_electricity_caches_if_loaded() + assert _mecs_purchased_kwh_cached.cache_info().currsize == 0 + + +@pytest.mark.eeio_integration +@pytest.mark.parametrize('year', [2018, 2022]) +def test_post_overlay_naics_exist_in_table_7_7(year: int) -> None: + _mecs_purchased_kwh_cached.cache_clear() + kwh = mecs_purchased_kwh(year) + _required_7_7_naics(set(kwh.index.astype(str))) + leftover = float(kwh.loc['31-33']) - float( + kwh.loc[[c for c in kwh.index if len(str(c)) == 3 and str(c).isdigit()]].sum() + ) + if year == 2018: + assert abs(leftover) <= 1.0e6 + if year == 2022: + assert float(kwh.loc['337']) > 0.0 + _mecs_purchased_kwh_cached.cache_clear() diff --git a/bedrock/transform/eeio/__tests__/test_electricity_mixed_units.py b/bedrock/transform/eeio/__tests__/test_electricity_mixed_units.py index afbfdd819..82f86a093 100644 --- a/bedrock/transform/eeio/__tests__/test_electricity_mixed_units.py +++ b/bedrock/transform/eeio/__tests__/test_electricity_mixed_units.py @@ -2,7 +2,8 @@ from __future__ import annotations -from typing import Callable, cast +from contextlib import contextmanager +from typing import Any, Callable, Iterator, cast from unittest.mock import Mock, patch import numpy as np @@ -52,6 +53,7 @@ electricity_output_factor, get_2017_eia_purchaser_allocation, ) +from bedrock.transform.eeio.electricity_gtd_allocation import mecs_purchased_kwh from bedrock.utils.config.usa_config import ( get_usa_config, reset_usa_config, @@ -82,6 +84,7 @@ build_electricity_disagg_go_weights, build_electricity_disagg_use_intersection_weights, get_2017_eia_purchaser_allocation, + mecs_purchased_kwh, _derive_post_reallocation_checkpoint_for_disagg, derive_cornerstone_V, derive_cornerstone_Vnorm_scrap_corrected, @@ -119,6 +122,21 @@ def _clear_caches() -> None: eia_table_3_1_total_mwh.cache_clear() +@contextmanager +def _dollar_industrial_weights() -> Iterator[None]: + import bedrock.transform.eeio.electricity_gtd_allocation as gtd # noqa: PLC0415 + + orig = gtd.allocate_purchaser_gtd + + def _wrapped(*args: Any, **kwargs: Any) -> Any: + kwargs = dict(kwargs) + kwargs['industrial_weights'] = 'dollars' + return orig(*args, **kwargs) + + with patch.object(gtd, 'allocate_purchaser_gtd', _wrapped): + yield + + def _setup(config_name: str) -> None: _clear_caches() reset_usa_config(should_reset_env_var=True) @@ -203,8 +221,9 @@ def test_build_end_use_map_includes_electricity_children() -> None: def test_mixed_units_flag_off_is_noop() -> None: _setup('test_usa_config_waste_disagg_electricity_disaggregation.yaml') try: - aq_mon = derive_cornerstone_Aq_scaled() - aq_mixed = derive_cornerstone_Aq_mixed_units() + with _dollar_industrial_weights(): + aq_mon = derive_cornerstone_Aq_scaled() + aq_mixed = derive_cornerstone_Aq_mixed_units() pd.testing.assert_frame_equal(aq_mon.Adom, aq_mixed.Adom) pd.testing.assert_series_equal(aq_mon.scaled_q, aq_mixed.scaled_q) finally: @@ -226,7 +245,8 @@ def test_output_mwh_anchor( ) -> None: _setup(mixed_units_config) try: - aq = derive_cornerstone_Aq_mixed_units() + with _dollar_industrial_weights(): + aq = derive_cornerstone_Aq_mixed_units() assert aq.scaled_q[GENERATION_SECTOR] == pytest.approx(4_000_000_000.0) finally: _teardown() @@ -280,12 +300,14 @@ def test_apply_mixed_units_bly_diff_exemptions() -> None: def test_y_nab_stays_monetary_under_mixed_gate(mixed_units_config: str) -> None: _setup('test_usa_config_waste_disagg_electricity_disaggregation.yaml') try: - y_off = derive_cornerstone_y_nab().copy() + with _dollar_industrial_weights(): + y_off = derive_cornerstone_y_nab().copy() finally: _teardown() _setup(mixed_units_config) try: - y_on = derive_cornerstone_y_nab() + with _dollar_industrial_weights(): + y_on = derive_cornerstone_y_nab() pd.testing.assert_series_equal(y_off, y_on) finally: _teardown() @@ -306,9 +328,10 @@ def test_y_nab_mixed_differs_from_monetary_under_gate( ) -> None: _setup(mixed_units_config) try: - y_mon = derive_cornerstone_y_nab() - y_mix = derive_cornerstone_y_nab_mixed_units() - aq = derive_cornerstone_Aq_mixed_units() + with _dollar_industrial_weights(): + y_mon = derive_cornerstone_y_nab() + y_mix = derive_cornerstone_y_nab_mixed_units() + aq = derive_cornerstone_Aq_mixed_units() assert y_mix[GENERATION_SECTOR] != pytest.approx(y_mon[GENERATION_SECTOR]) y_back = backcompute_y_from_A_and_q(A=aq.Adom, q=aq.scaled_q) pd.testing.assert_series_equal(y_mix, y_back) diff --git a/bedrock/transform/eeio/electricity_gtd_allocation.py b/bedrock/transform/eeio/electricity_gtd_allocation.py index 55c6adc79..deda0d16d 100644 --- a/bedrock/transform/eeio/electricity_gtd_allocation.py +++ b/bedrock/transform/eeio/electricity_gtd_allocation.py @@ -2,6 +2,20 @@ Pure allocator plus the 2017 cached getter and Use/Y/A/q writers. Table 2.2 / 2.14 / 3.1 loaders live in ``egrid_generation``. + +Table 7.7 purchased kWh is read from the ``EIA_MECS_Energy`` **FBA** parquet. +This repo does not publish a standalone MECS Energy FBS: Tables 2.2 / 3.2 +also stay FBA and enter GHG only as attribution sources during FBS build. +7.7 is the same pattern with a different consumer (G/T/D purchaser weights, +not sector-attributed flows). An FBS pass would apply +``estimate_suppressed_mecs_energy`` and the generic NAICS crosswalk; this +path needs 3-digit Q/D residual fill and the Cornerstone MECS 3.1 hand map. + +Manufacturing NAICS→BEA IO uses ``CORNERSTONE_INDUSTRY_TO_MECS_3_1_NAICS_*``, +the same maps as GHG industrial coal/gas combustion. Residual Industrial +purchasers are ``NON_MECS_INDUSTRIES`` (electricity purchases here; BEA +fuel Use in GHG). Multi-IO mapping keys split 7.7 kWh by electricity +purchases; GHG splits fuel by BEA Use of that commodity. """ from __future__ import annotations @@ -9,8 +23,9 @@ import functools import logging import warnings +from collections.abc import Mapping from dataclasses import dataclass -from typing import cast +from typing import Literal, cast import numpy as np import pandas as pd @@ -52,7 +67,9 @@ def _as_float_series(obj: object) -> pd.Series: def set_reanchored_electricity_q(q: pd.Series) -> None: - """Record published G/T/D ``q`` after A/q reanchor, for the GHG-year ``x`` split.""" + """Record published G/T/D ``q`` after A/q reanchor, for the GHG-year + ``x`` split. + """ global _REANCHORED_ELECTRICITY_Q _REANCHORED_ELECTRICITY_Q = ( q.reindex(ELECTRICITY_DISAGG_SECTORS).astype(float).copy() @@ -100,12 +117,35 @@ def reanchored_electricity_q_shares() -> pd.Series | None: _WATER_FILL_ATOL = 1e-9 +IndustrialWeighting = Literal['mecs', 'dollars'] + +# 2017 NAICS (MECS 2018 Table 7.7 and Cornerstone 3.1 map values) → 2022 NAICS +# (MECS 2022 Table 7.7). Paper mills 322121/322122 collapsed to 322120; automobile +# and light-truck 336111/336112 collapsed to 336110. IO keys stay 2017/Cornerstone. +# ``_resolve_7_7_naics`` keeps the 2017 code when that vintage still publishes it +# and remaps only when the loaded 7.7 frame has the 2022 aggregate. +MECS_7_7_NAICS_OVERLAY: dict[str, str] = { + '322121': '322120', + '322122': '322120', + '336111': '336110', + '336112': '336110', +} + +TABLE_7_7_DESCRIPTION = 'Table 7.7' +TABLE_7_7_ELECTRICITY_TOTAL = 'Electricity total' +MANUFACTURING_TOTAL_NAICS = '31-33' +_MKWH_TO_KWH = 1.0e6 +_THREE_DIGIT_RESIDUAL_ATOL_KWH = 1.0e6 +_Q_D_CODES = frozenset({'Q', 'D'}) + @dataclass(frozen=True) class EIAPurchaserAllocation: - """Per-purchaser generation / T&D split aligned to ``bills.index``.""" + """Per-purchaser generation / T&D split aligned to + ``electricity_purchases.index``. + """ - bill: pd.Series + electricity_purchases: pd.Series end_use_class: pd.Series mwh: pd.Series gen_dollars: pd.Series @@ -160,10 +200,10 @@ def _class_mwh_targets( return targets -def _end_use_classes(bills: pd.Series, self_use_key: str) -> pd.Series: +def _end_use_classes(electricity_purchases: pd.Series, self_use_key: str) -> pd.Series: end_use_map = build_end_use_map() - classes = pd.Series(index=bills.index, dtype=object) - for key in bills.index: + classes = pd.Series(index=electricity_purchases.index, dtype=object) + for key in electricity_purchases.index: if str(key) == self_use_key: classes[key] = 'Industrial' else: @@ -172,18 +212,18 @@ def _end_use_classes(bills: pd.Series, self_use_key: str) -> pd.Series: def _water_fill_gen( - bills: pd.Series, + electricity_purchases: pd.Series, proportional: pd.Series, members: list[str], ) -> tuple[pd.Series, pd.Series]: - """Clip gen to positive bills; put overflow on remaining slack in class.""" + """Clip gen to non-negative electricity purchases; overflow stays in class.""" gen = pd.Series(0.0, index=members, dtype=float) clipped = pd.Series(False, index=members, dtype=bool) - cap = {j: max(float(bills[j]), 0.0) for j in members} + cap = {j: max(float(electricity_purchases[j]), 0.0) for j in members} overflow = 0.0 for j in members: prop = float(proportional[j]) - if float(bills[j]) <= 0: + if float(electricity_purchases[j]) <= 0: gen[j] = 0.0 overflow += prop continue @@ -211,67 +251,466 @@ def _water_fill_gen( return gen, clipped +def mecs_year_for_eia_year(eia_year: int) -> Literal[2018, 2022]: + """Map an EIA/eGRID year to the MECS survey used for purchased-kWh shares. + + 2017 and earlier use the 2018 survey; later years (including + ``model_base_year``) use 2022. Production callers are ``eia_year=2017`` + and ``eia_year=model_year``. No interpolation between survey years. + """ + return 2018 if eia_year <= 2017 else 2022 + + +def industrial_manufacturing_pool() -> frozenset[str]: + """Cornerstone IO codes in flattened 3.1 mapping keys ∪ subtraction keys. + + Manufacturing purchasers inside the Industrial end-use class are + this set intersected with that class's purchaser columns. + """ + from bedrock.transform.allocation.mappings.cornerstone.mecs import ( # noqa: PLC0415 + CORNERSTONE_INDUSTRY_TO_MECS_3_1_NAICS_MAPPING, + CORNERSTONE_INDUSTRY_TO_MECS_3_1_NAICS_SUBTRACTION_MAPPING, + ) + from bedrock.transform.allocation.utils import flatten_items # noqa: PLC0415 + + codes = set(flatten_items(CORNERSTONE_INDUSTRY_TO_MECS_3_1_NAICS_MAPPING.keys())) + codes.update( + flatten_items(CORNERSTONE_INDUSTRY_TO_MECS_3_1_NAICS_SUBTRACTION_MAPPING.keys()) + ) + return frozenset(str(c) for c in codes) + + +def _dedupe_overlay(naics: tuple[str, ...]) -> tuple[str, ...]: + seen: set[str] = set() + out: list[str] = [] + for code in naics: + mapped = MECS_7_7_NAICS_OVERLAY.get(str(code), str(code)) + if mapped not in seen: + seen.add(mapped) + out.append(mapped) + return tuple(out) + + +def _resolve_7_7_naics(code: str, available: set[str]) -> str: + """Prefer the 7.7 overlay code when that vintage publishes it. + + 2022 collapses 322121/322122 → 322120 and 336111/336112 → 336110; 2018 + still lists the older 6-digit rows. + """ + mapped = MECS_7_7_NAICS_OVERLAY.get(str(code), str(code)) + if mapped in available: + return mapped + if str(code) in available: + return str(code) + raise ValueError( + f'Table 7.7 US Electricity total missing NAICS {code!r} (overlay {mapped!r})' + ) + + +def _overlay_naics_tuple( + naics: tuple[str, ...], + available: set[str], +) -> tuple[str, ...]: + seen: set[str] = set() + out: list[str] = [] + for code in naics: + resolved = _resolve_7_7_naics(str(code), available) + if resolved not in seen: + seen.add(resolved) + out.append(resolved) + return tuple(out) + + +def _overlaid_3_1_mapping( + available: set[str], +) -> dict[tuple[str, ...], tuple[str, ...]]: + from bedrock.transform.allocation.mappings.cornerstone.mecs import ( # noqa: PLC0415 + CORNERSTONE_INDUSTRY_TO_MECS_3_1_NAICS_MAPPING, + ) + + return { + k: _overlay_naics_tuple(v, available) + for k, v in CORNERSTONE_INDUSTRY_TO_MECS_3_1_NAICS_MAPPING.items() + } + + +def _overlaid_3_1_subtraction( + available: set[str], +) -> dict[tuple[str, ...], tuple[tuple[str, ...], tuple[str, ...]]]: + from bedrock.transform.allocation.mappings.cornerstone.mecs import ( # noqa: PLC0415 + CORNERSTONE_INDUSTRY_TO_MECS_3_1_NAICS_SUBTRACTION_MAPPING, + ) + + return { + k: ( + _overlay_naics_tuple(parent, available), + _overlay_naics_tuple(children, available), + ) + for k, (parent, children) in ( + CORNERSTONE_INDUSTRY_TO_MECS_3_1_NAICS_SUBTRACTION_MAPPING.items() + ) + } + + +def _required_7_7_naics(available: set[str] | None = None) -> frozenset[str]: + from bedrock.transform.allocation.mappings.cornerstone.mecs import ( # noqa: PLC0415 + CORNERSTONE_INDUSTRY_TO_MECS_3_1_NAICS_MAPPING, + CORNERSTONE_INDUSTRY_TO_MECS_3_1_NAICS_SUBTRACTION_MAPPING, + ) + + raw: set[str] = set() + for vals in CORNERSTONE_INDUSTRY_TO_MECS_3_1_NAICS_MAPPING.values(): + raw.update(str(c) for c in vals) + for ( + parent, + children, + ) in CORNERSTONE_INDUSTRY_TO_MECS_3_1_NAICS_SUBTRACTION_MAPPING.values(): + raw.update(str(c) for c in parent) + raw.update(str(c) for c in children) + if available is None: + needed = set(raw) + needed.update(MECS_7_7_NAICS_OVERLAY.get(c, c) for c in raw) + needed.update(MECS_7_7_NAICS_OVERLAY.values()) + return frozenset(needed) + needed = set() + for vals in CORNERSTONE_INDUSTRY_TO_MECS_3_1_NAICS_MAPPING.values(): + needed.update(_overlay_naics_tuple(vals, available)) + for ( + parent, + children, + ) in CORNERSTONE_INDUSTRY_TO_MECS_3_1_NAICS_SUBTRACTION_MAPPING.values(): + needed.update(_overlay_naics_tuple(parent, available)) + needed.update(_overlay_naics_tuple(children, available)) + return frozenset(needed) + + +def _is_three_digit_naics(code: str) -> bool: + return len(code) == 3 and code.isdigit() + + +def _fill_three_digit_suppressed(kwh: pd.Series, suppressed: pd.Series) -> pd.Series: + """Fill at most one 3-digit Q/D from ``31-33`` minus published 3-digit rows. + + The leftover is identified only when exactly one 3-digit cell is Q/D. + Two or more Q/D with a nonzero leftover is unidentified; that raises + rather than splitting. Live 2018 has none; 2022 has one (337). + """ + out = kwh.astype(float).copy() + if MANUFACTURING_TOTAL_NAICS not in out.index: + raise ValueError( + 'Table 7.7 US Electricity total is missing manufacturing total ' + f'{MANUFACTURING_TOTAL_NAICS!r}' + ) + total = float(out.loc[MANUFACTURING_TOTAL_NAICS]) + three = [str(c) for c in out.index if _is_three_digit_naics(str(c))] + published: list[str] = [] + qd: list[str] = [] + for code in three: + letter = '' + if code in suppressed.index and pd.notna(suppressed.loc[code]): + letter = str(suppressed.loc[code]).strip() + val = float(out.loc[code]) + if letter in _Q_D_CODES: + qd.append(code) + elif np.isfinite(val): + published.append(code) + pub_sum = float(out.loc[published].sum()) if published else 0.0 + leftover = total - pub_sum + if abs(leftover) > _THREE_DIGIT_RESIDUAL_ATOL_KWH and len(qd) != 1: + raise ValueError( + 'Table 7.7 US Electricity total 3-digit residual ' + f'{leftover / _MKWH_TO_KWH:.6g} million kWh is not ~0 and ' + f'there are {len(qd)} Q/D 3-digit industries {qd!r} (need 0 or 1)' + ) + if abs(leftover) > _THREE_DIGIT_RESIDUAL_ATOL_KWH and len(qd) == 1: + out.loc[qd[0]] = leftover + return out.fillna(0.0) + + +@functools.cache +def _mecs_purchased_kwh_cached(mecs_year: int) -> pd.Series: + from bedrock.extract.flowbyactivity import getFlowByActivity # noqa: PLC0415 + from bedrock.utils.mapping.location import US_FIPS # noqa: PLC0415 + + df = getFlowByActivity('EIA_MECS_Energy', int(mecs_year)) + desc = df['Description'].astype(str) + loc = df['Location'].astype(str) + flow = df['FlowName'].astype(str) + sub = df.loc[ + (desc == TABLE_7_7_DESCRIPTION) + & (loc == US_FIPS) + & (flow == TABLE_7_7_ELECTRICITY_TOTAL) + ].copy() + if sub.empty: + raise ValueError( + f'EIA_MECS_Energy {mecs_year} has no US Table 7.7 ' + f'{TABLE_7_7_ELECTRICITY_TOTAL!r} rows' + ) + naics = sub['ActivityConsumedBy'].astype(str).str.strip() + amount_mkwh = pd.to_numeric(sub['FlowAmount'], errors='coerce') + suppressed = sub['Suppressed'] + is_star = suppressed.astype(str).str.strip() == '*' + amount_mkwh = amount_mkwh.mask(is_star, 0.0) + kwh = pd.Series( + amount_mkwh.to_numpy(dtype=float) * _MKWH_TO_KWH, + index=pd.Index(naics.to_numpy(), name='naics'), + dtype=float, + ) + supp = pd.Series( + suppressed.to_numpy(), + index=kwh.index, + ) + if kwh.index.has_duplicates: + kwh = kwh.groupby(level=0).sum() + supp = supp.groupby(level=0).first() + kwh = _fill_three_digit_suppressed(kwh, supp) + available = set(kwh.index.astype(str)) + _required_7_7_naics(available) + return kwh.astype(float) + + +def mecs_purchased_kwh(mecs_year: int) -> pd.Series: + """US Table 7.7 Electricity-total purchased kWh by MECS NAICS. + + Rows are Table 7.7, US, ``Electricity total``. ``*`` is 0; a single + 3-digit ``Q``/``D`` is filled from the ``31-33`` residual; other + suppressed cells are 0. The index includes ``31-33`` for that residual + only. Cached by year; returns a copy. + """ + return _mecs_purchased_kwh_cached(int(mecs_year)).copy() + + +def _split_kwh_across_io( + io_codes: tuple[str, ...], + kwh: float, + electricity_purchases: pd.Series, +) -> dict[str, float]: + if not io_codes: + return {} + if len(io_codes) == 1: + return {io_codes[0]: float(kwh)} + weights = ( + electricity_purchases.reindex(list(io_codes)) + .astype(float) + .clip(lower=0.0) + .fillna(0.0) + ) + wsum = float(weights.sum()) + if wsum <= 0: + return {c: 0.0 for c in io_codes} + return {c: float(kwh) * float(weights[c]) / wsum for c in io_codes} + + +def io_manufacturing_purchased_kwh( + electricity_purchases: pd.Series, mecs_year: int +) -> pd.Series: + """Map Table 7.7 purchased kWh onto manufacturing IO columns. + + Overlay 7.7 NAICS, split shared rows by non-negative purchaser use, + clamp negative parent−child leftover to 0. Returns kWh on IO index + (not class MWh). + """ + naics_kwh = mecs_purchased_kwh(mecs_year) + available = set(naics_kwh.index.astype(str)) + mapping = _overlaid_3_1_mapping(available) + subtraction = _overlaid_3_1_subtraction(available) + mapped_io = {str(c) for c in _flatten_io_keys(mapping)} + subtracted_io = {str(c) for c in _flatten_io_keys(subtraction)} + both = mapped_io & subtracted_io + if both: + raise ValueError( + f'IO codes assigned by both 3.1 mapping and subtraction: {sorted(both)}' + ) + naics_to_io: dict[str, list[str]] = {} + for io_key, naics_vals in mapping.items(): + for n in naics_vals: + bucket = naics_to_io.setdefault(n, []) + for io in io_key: + code = str(io) + if code not in bucket: + bucket.append(code) + assigned: dict[str, float] = {} + for n, io_list in naics_to_io.items(): + if n not in naics_kwh.index: + raise ValueError(f'Table 7.7 {mecs_year} missing NAICS {n!r} after overlay') + chunk = _split_kwh_across_io( + tuple(io_list), float(naics_kwh.loc[n]), electricity_purchases + ) + for io, val in chunk.items(): + assigned[io] = assigned.get(io, 0.0) + val + for io_key, (parent, children) in subtraction.items(): + parent_kwh = sum(float(naics_kwh.loc[p]) for p in parent) + child_kwh = sum(float(naics_kwh.loc[c]) for c in children) + leftover = max(0.0, parent_kwh - child_kwh) + chunk = _split_kwh_across_io( + tuple(str(c) for c in io_key), leftover, electricity_purchases + ) + for io, val in chunk.items(): + assigned[io] = assigned.get(io, 0.0) + val + return pd.Series(assigned, dtype=float) + + +def _flatten_io_keys(mapping: Mapping[tuple[str, ...], object]) -> list[str]: + out: list[str] = [] + for key in mapping: + out.extend(str(c) for c in key) + return out + + +def _industrial_mecs_class_mwh( + electricity_purchases: pd.Series, + members: list[str], + target_mwh: float, + eia_year: int, +) -> pd.Series: + """Two-pool Industrial MWh: MECS shares inside manufacturing, dollars in residual. + + Direct Use in the Industrial class target rides on Table 7.7 purchase + shares (manufacturing) and dollar electricity-purchase shares + (residual). After this assignment, class-wide water-fill can move + generation dollars from clipped energy-intensive manufacturers onto + residual ag/mining/construction. + """ + mfg_pool = industrial_manufacturing_pool() + mfg = [m for m in members if m in mfg_pool] + res = [m for m in members if m not in mfg_pool] + clip_all = ( + electricity_purchases.reindex(members).astype(float).clip(lower=0.0).fillna(0.0) + ) + den = float(clip_all.sum()) + if den <= 0: + raise ValueError( + 'Industrial non-negative electricity use sums to 0; cannot split ' + 'manufacturing vs residual pools' + ) + clip_mfg = ( + electricity_purchases.reindex(mfg).astype(float).clip(lower=0.0).fillna(0.0) + ) + pool_mfg = float(target_mwh) * float(clip_mfg.sum()) / den + pool_res = float(target_mwh) - pool_mfg + clip_res = ( + electricity_purchases.reindex(res).astype(float).clip(lower=0.0).fillna(0.0) + ) + res_sum = float(clip_res.sum()) + if res_sum <= 0 and pool_res > _WATER_FILL_ATOL: + logger.info( + 'Industrial residual non-negative electricity use sums to 0 with ' + 'residual pool %.6g MWh; adding that pool to manufacturing', + pool_res, + ) + pool_mfg += pool_res + pool_res = 0.0 + out = pd.Series(0.0, index=members, dtype=float) + if mfg: + mecs_year = mecs_year_for_eia_year(eia_year) + io_kwh = ( + io_manufacturing_purchased_kwh(electricity_purchases, mecs_year) + .reindex(mfg) + .fillna(0.0) + ) + ksum = float(io_kwh.sum()) + if ksum <= 0: + wsum = float(clip_mfg.sum()) + if wsum > 0: + out.loc[mfg] = pool_mfg * (clip_mfg / wsum) + else: + out.loc[mfg] = 0.0 + else: + out.loc[mfg] = pool_mfg * (io_kwh / ksum) + if res and pool_res > 0 and res_sum > 0: + out.loc[res] = pool_res * (clip_res / res_sum) + return out + + def allocate_purchaser_gtd( - bills: pd.Series, + electricity_purchases: pd.Series, *, self_use_key: str, eia_year: int, p_share_2017: float, td_share_2017: float, + industrial_weights: IndustrialWeighting = 'mecs', ) -> EIAPurchaserAllocation: - """Allocate domestic electricity bills to generation, transmission, and distribution. - - ``bills`` is domestic Use columns union Y columns. Do not pass Uimp — - imported Use is written onto the generation row separately. - ``self_use_key`` is always ``'221100'``. Class dollar weights use - ``clip(lower=0)`` for shares only; if ``bill <= 0``, ``gen = 0``. + """Allocate domestic electricity purchaser use to G/T/D. + + ``electricity_purchases`` is domestic Use of electricity (intermediate) + plus Y (FD). + Do not pass Uimp — imported Use is written onto the generation row + separately. ``self_use_key`` is always ``'221100'``. Class dollar + weights use non-negative values for shares only; if a purchaser's use + is ``<= 0``, ``gen = 0``. ``industrial_weights='mecs'`` (default) uses + Table 7.7 purchased kWh inside manufacturing and dollar weights for + residual Industrial. ``industrial_weights='dollars'`` keeps non-negative + use weights for the whole Industrial class (diagnostics and tests that + must not load Table 7.7). """ if self_use_key != ELECTRICITY_AGGREGATE: raise ValueError( f'self_use_key must be {ELECTRICITY_AGGREGATE!r}, got {self_use_key!r}' ) + if industrial_weights not in ('mecs', 'dollars'): + raise ValueError( + f'industrial_weights must be mecs or dollars, got {industrial_weights!r}' + ) if not np.isfinite(p_share_2017) or p_share_2017 <= 0: raise ValueError( 'UGO generation share is missing or non-positive; ' 'Table 8.3 is not a p backup' ) - if IMPORT_FD_CODE in bills.index: - bills = bills.drop(index=IMPORT_FD_CODE) - bills = bills.astype(float) - classes = _end_use_classes(bills, self_use_key) + if IMPORT_FD_CODE in electricity_purchases.index: + electricity_purchases = electricity_purchases.drop(index=IMPORT_FD_CODE) + electricity_purchases = electricity_purchases.astype(float) + classes = _end_use_classes(electricity_purchases, self_use_key) egrid_mwh = egrid_mwh_for_io_year(eia_year) class_mwh = _class_mwh_targets(eia_year, egrid_mwh) - bill_total = float(bills.sum()) - p = (p_share_2017 * bill_total) / egrid_mwh if egrid_mwh else float('nan') + electricity_purchases_total = float(electricity_purchases.sum()) + p = ( + (p_share_2017 * electricity_purchases_total) / egrid_mwh + if egrid_mwh + else float('nan') + ) if not np.isfinite(p) or p <= 0: raise ValueError(f'non-positive generation price p={p!r}') - mwh = pd.Series(0.0, index=bills.index, dtype=float) - gen = pd.Series(0.0, index=bills.index, dtype=float) - clipped = pd.Series(False, index=bills.index, dtype=bool) + mwh = pd.Series(0.0, index=electricity_purchases.index, dtype=float) + gen = pd.Series(0.0, index=electricity_purchases.index, dtype=float) + clipped = pd.Series(False, index=electricity_purchases.index, dtype=bool) for cls, target_mwh in class_mwh.items(): members = [str(k) for k in classes.index if classes[k] == cls] if not members: continue - weights = bills.loc[members].clip(lower=0.0) - wsum = float(weights.sum()) - if wsum > 0: - mwh.loc[members] = target_mwh * (weights / wsum) - class_bill_sum = float(bills.loc[members].sum()) + if cls == 'Industrial' and industrial_weights == 'mecs': + mwh.loc[members] = _industrial_mecs_class_mwh( + electricity_purchases, members, float(target_mwh), eia_year + ) + else: + weights = electricity_purchases.loc[members].clip(lower=0.0) + wsum = float(weights.sum()) + if wsum > 0: + mwh.loc[members] = target_mwh * (weights / wsum) + class_electricity_purchases_sum = float( + electricity_purchases.loc[members].sum() + ) class_needed = p * float(target_mwh) - if class_needed > _WATER_FILL_ATOL and class_bill_sum < class_needed: - scale = class_bill_sum / class_needed + if ( + class_needed > _WATER_FILL_ATOL + and class_electricity_purchases_sum < class_needed + ): + scale = class_electricity_purchases_sum / class_needed mwh.loc[members] = mwh.loc[members] * scale logger.info( - 'nibble class %s: bills %.6g < p × MWh %.6g; scale=%.6g', + 'nibble class %s: electricity purchases %.6g ' + '< p × MWh %.6g; scale=%.6g', cls, - class_bill_sum, + class_electricity_purchases_sum, class_needed, scale, ) proportional = mwh.loc[members] * p - gen_cls, clip_cls = _water_fill_gen(bills, proportional, members) + gen_cls, clip_cls = _water_fill_gen( + electricity_purchases, proportional, members + ) gen.loc[members] = gen_cls clipped.loc[members] = clip_cls @@ -282,11 +721,11 @@ def allocate_purchaser_gtd( int(len(clipped)), ) - leftover = bills - gen + leftover = electricity_purchases - gen t_dollars = leftover * float(td_share_2017) d_dollars = leftover * (1.0 - float(td_share_2017)) return EIAPurchaserAllocation( - bill=bills, + electricity_purchases=electricity_purchases, end_use_class=classes, mwh=mwh.astype(float), gen_dollars=gen.astype(float), @@ -300,9 +739,9 @@ def allocate_purchaser_gtd( def _electricity_column_dollars(frame: pd.DataFrame, col: str) -> float: - """Import bill on ``221100`` if still present, else the G/T/D child sum. + """Import dollars on ``221100`` if still present, else the G/T/D child sum. - After correspondence, the aggregate row is often gone and the bill sits + After correspondence, the aggregate row is often gone and the import sits on the children. Prefer the aggregate when it is non-zero so the two are not added together. """ @@ -378,7 +817,7 @@ def write_purchaser_gtd_use_and_y( orig_imp = 0.0 if agg in Uimp.index and col in Uimp.columns: orig_imp = _cell_float(Uimp, agg, str(col)) - if col_s in allocation.bill.index: + if col_s in allocation.electricity_purchases.index: Udom.at[GENERATION_SECTOR, col] = float(allocation.gen_dollars[col_s]) Udom.at[TRANSMISSION_SECTOR, col] = float(allocation.t_dollars[col_s]) Udom.at[DISTRIBUTION_SECTOR, col] = float(allocation.d_dollars[col_s]) @@ -403,7 +842,7 @@ def write_purchaser_gtd_use_and_y( Y.at[GENERATION_SECTOR, col] = _electricity_column_dollars(Y, str(col)) Y.at[TRANSMISSION_SECTOR, col] = 0.0 Y.at[DISTRIBUTION_SECTOR, col] = 0.0 - elif col_s in allocation.bill.index: + elif col_s in allocation.electricity_purchases.index: Y.at[GENERATION_SECTOR, col] = float(allocation.gen_dollars[col_s]) Y.at[TRANSMISSION_SECTOR, col] = float(allocation.t_dollars[col_s]) Y.at[DISTRIBUTION_SECTOR, col] = float(allocation.d_dollars[col_s]) @@ -429,7 +868,7 @@ def write_gtd_use_intersection( Udom = _ensure_column_codes(Udom, children) Uimp = _ensure_column_codes(Uimp, children) - t_dom = float(allocation.bill[agg]) + t_dom = float(allocation.electricity_purchases[agg]) gen_self = float(allocation.gen_dollars[agg]) leftover = t_dom - gen_self t_self = leftover * float(allocation.td_share) @@ -478,7 +917,10 @@ def make_last_weights_from_domestic_use_y( @functools.cache def get_2017_eia_purchaser_allocation() -> EIAPurchaserAllocation: - """Cached 2017 domestic bills → G/T/D. Does not call the IO bundle or Ytot.""" + """Cached 2017 domestic electricity purchases → G/T/D. + + Does not call the IO bundle or Ytot. + """ from bedrock.transform.eeio.electricity_disaggregation import ( # noqa: PLC0415 _derive_post_reallocation_checkpoint_for_disagg, _derive_y_before_electricity_disagg_lazy, @@ -491,12 +933,12 @@ def get_2017_eia_purchaser_allocation() -> EIAPurchaserAllocation: _as_float_series(udom.loc[agg]) if agg in udom.index else pd.Series(dtype=float) ) y_row = _as_float_series(y.loc[agg]) if agg in y.index else pd.Series(dtype=float) - bills = use_row.add(y_row, fill_value=0.0) - if IMPORT_FD_CODE in bills.index: - bills = bills.drop(index=IMPORT_FD_CODE) + electricity_purchases = use_row.add(y_row, fill_value=0.0) + if IMPORT_FD_CODE in electricity_purchases.index: + electricity_purchases = electricity_purchases.drop(index=IMPORT_FD_CODE) p_share, td_share = _go_p_and_td_shares() return allocate_purchaser_gtd( - bills, + electricity_purchases, self_use_key=ELECTRICITY_AGGREGATE, eia_year=2017, p_share_2017=p_share, @@ -521,7 +963,7 @@ def apply_purchaser_allocation_to_y(Y: pd.DataFrame) -> pd.DataFrame: Y.at[GENERATION_SECTOR, col] = _electricity_column_dollars(Y, str(col)) Y.at[TRANSMISSION_SECTOR, col] = 0.0 Y.at[DISTRIBUTION_SECTOR, col] = 0.0 - elif col_s in allocation.bill.index: + elif col_s in allocation.electricity_purchases.index: Y.at[GENERATION_SECTOR, col] = float(allocation.gen_dollars[col_s]) Y.at[TRANSMISSION_SECTOR, col] = float(allocation.t_dollars[col_s]) Y.at[DISTRIBUTION_SECTOR, col] = float(allocation.d_dollars[col_s]) @@ -612,7 +1054,7 @@ def _spill_generation_nonfuel( return Udom, Uimp -def _scaled_export_fd_bill( +def _scaled_export_fd_electricity_purchases( *, original_year: int, target_year: int, @@ -686,9 +1128,9 @@ def _inflate_summary_year_scaled_aq( return adom, q -def _purchaser_bills_from_aq( - adom_bills: pd.DataFrame, - q_bills: pd.Series, +def _purchaser_electricity_purchases_from_aq( + adom: pd.DataFrame, + q: pd.Series, ) -> pd.Series: from bedrock.transform.eeio.cornerstone_disagg_pipeline import ( # noqa: PLC0415 derive_disagg_Ytot_with_trade, @@ -696,26 +1138,28 @@ def _purchaser_bills_from_aq( from bedrock.utils.math.formulas import backcompute_y_from_A_and_q # noqa: PLC0415 elec = list(ELECTRICITY_DISAGG_SECTORS) - a_elec = adom_bills.loc[elec].sum(axis=0).astype(float) - industry_bills = a_elec * q_bills.reindex(a_elec.index).astype(float) - self_bill = float(industry_bills.reindex(elec).fillna(0.0).sum()) - industry_bills = industry_bills.drop(labels=elec, errors='ignore') - industry_bills[ELECTRICITY_AGGREGATE] = self_bill + a_elec = adom.loc[elec].sum(axis=0).astype(float) + industry_purchases = a_elec * q.reindex(a_elec.index).astype(float) + self_electricity_purchases = float( + industry_purchases.reindex(elec).fillna(0.0).sum() + ) + industry_purchases = industry_purchases.drop(labels=elec, errors='ignore') + industry_purchases[ELECTRICITY_AGGREGATE] = self_electricity_purchases - y_snap = backcompute_y_from_A_and_q(A=adom_bills, q=q_bills) + y_snap = backcompute_y_from_A_and_q(A=adom, q=q) y_elec_total = float(y_snap.reindex(elec).fillna(0.0).sum()) y2017 = derive_disagg_Ytot_with_trade() y2017_elec = y2017.loc[elec].sum(axis=0).astype(float) y2017_sum = float(y2017_elec.sum()) - fd_bills = pd.Series(dtype=float) + fd_purchases = pd.Series(dtype=float) if y2017_sum > 0: for col, val in y2017_elec.items(): col_s = str(col) if col_s in (EXPORT_FD_CODE, IMPORT_FD_CODE): continue - fd_bills[col_s] = y_elec_total * (float(val) / y2017_sum) - bills = industry_bills.add(fd_bills, fill_value=0.0) - return bills.astype(float) + fd_purchases[col_s] = y_elec_total * (float(val) / y2017_sum) + electricity_purchases = industry_purchases.add(fd_purchases, fill_value=0.0) + return electricity_purchases.astype(float) def reanchor_electricity_aq_after_year_scaling( @@ -737,7 +1181,7 @@ def reanchor_electricity_aq_after_year_scaling( q = aq.scaled_q.astype(float).copy() elec = list(ELECTRICITY_DISAGG_SECTORS) - adom_bills, q_bills = _inflate_summary_year_scaled_aq( + adom_purchases, q_purchases = _inflate_summary_year_scaled_aq( original_year=original_year, target_year=target_year, model_year=model_year, @@ -748,8 +1192,10 @@ def reanchor_electricity_aq_after_year_scaling( ) pre = get_summary_year_scaled_aq(original_year, target_year) - bills = _purchaser_bills_from_aq(adom_bills, q_bills) - bills[EXPORT_FD_CODE] = _scaled_export_fd_bill( + electricity_purchases = _purchaser_electricity_purchases_from_aq( + adom_purchases, q_purchases + ) + electricity_purchases[EXPORT_FD_CODE] = _scaled_export_fd_electricity_purchases( original_year=original_year, target_year=target_year, model_year=model_year, @@ -758,7 +1204,7 @@ def reanchor_electricity_aq_after_year_scaling( ) p_share, td_share = _go_p_and_td_shares() allocation = allocate_purchaser_gtd( - bills, + electricity_purchases, self_use_key=ELECTRICITY_AGGREGATE, eia_year=model_year, p_share_2017=p_share, @@ -774,7 +1220,7 @@ def reanchor_electricity_aq_after_year_scaling( col_s = str(col) if col_s in elec: continue - if col_s in allocation.bill.index: + if col_s in allocation.electricity_purchases.index: udom.at[GENERATION_SECTOR, col] = float(allocation.gen_dollars[col_s]) udom.at[TRANSMISSION_SECTOR, col] = float(allocation.t_dollars[col_s]) udom.at[DISTRIBUTION_SECTOR, col] = float(allocation.d_dollars[col_s]) @@ -783,7 +1229,7 @@ def reanchor_electricity_aq_after_year_scaling( uimp.at[TRANSMISSION_SECTOR, col] = 0.0 uimp.at[DISTRIBUTION_SECTOR, col] = 0.0 - t_dom = float(allocation.bill[ELECTRICITY_AGGREGATE]) + t_dom = float(allocation.electricity_purchases[ELECTRICITY_AGGREGATE]) gen_self = float(allocation.gen_dollars[ELECTRICITY_AGGREGATE]) leftover = t_dom - gen_self uimp_gg = float(uimp.loc[elec, elec].sum().sum()) @@ -799,7 +1245,9 @@ def reanchor_electricity_aq_after_year_scaling( y = y.astype(float) for code in elec: y.loc[code] = 0.0 - fd_keys = [k for k in allocation.bill.index if k in set(FINAL_DEMANDS)] + fd_keys = [ + k for k in allocation.electricity_purchases.index if k in set(FINAL_DEMANDS) + ] for col in fd_keys: if col == IMPORT_FD_CODE: continue diff --git a/bedrock/utils/economic/__tests__/test_inflation_helpers_cornerstone.py b/bedrock/utils/economic/__tests__/test_inflation_helpers_cornerstone.py index 08725aba6..ea48ac06a 100644 --- a/bedrock/utils/economic/__tests__/test_inflation_helpers_cornerstone.py +++ b/bedrock/utils/economic/__tests__/test_inflation_helpers_cornerstone.py @@ -1,6 +1,8 @@ from __future__ import annotations -from typing import Callable +from contextlib import contextmanager +from typing import Any, Callable, Iterator +from unittest.mock import patch import pytest @@ -31,6 +33,7 @@ build_electricity_disagg_use_intersection_weights, get_2017_eia_purchaser_allocation, ) +from bedrock.transform.eeio.electricity_gtd_allocation import mecs_purchased_kwh from bedrock.utils.config.usa_config import ( get_usa_config, reset_usa_config, @@ -66,6 +69,7 @@ build_electricity_disagg_use_intersection_weights, build_electricity_detail_GO_growth_ratios, get_2017_eia_purchaser_allocation, + mecs_purchased_kwh, _derive_post_reallocation_checkpoint_for_disagg, derive_cornerstone_V, derive_cornerstone_Vnorm_scrap_corrected, @@ -96,6 +100,21 @@ def _clear_all_caches() -> None: clear_reanchored_electricity_q() +@contextmanager +def _dollar_industrial_weights() -> Iterator[None]: + import bedrock.transform.eeio.electricity_gtd_allocation as gtd # noqa: PLC0415 + + orig = gtd.allocate_purchaser_gtd + + def _wrapped(*args: Any, **kwargs: Any) -> Any: + kwargs = dict(kwargs) + kwargs['industrial_weights'] = 'dollars' + return orig(*args, **kwargs) + + with patch.object(gtd, 'allocate_purchaser_gtd', _wrapped): + yield + + def _setup_config(config_name: str) -> None: _clear_all_caches() reset_usa_config(should_reset_env_var=True) @@ -193,8 +212,9 @@ def test_industry_price_ratio_apply_io_plus_elec_is_industry_elec_indexed() -> N _setup_config('2025_usa_cornerstone_v0_3_electricity_disaggregation.yaml') try: - industry = get_cornerstone_industry_price_ratio(2017, 2024) - commodity = get_vnorm_adjusted_commodity_price_ratio(2017, 2024) + with _dollar_industrial_weights(): + industry = get_cornerstone_industry_price_ratio(2017, 2024) + commodity = get_vnorm_adjusted_commodity_price_ratio(2017, 2024) assert list(industry.index) == CORNERSTONE_INDUSTRIES_ELEC assert list(commodity.index) == CORNERSTONE_COMMODITIES_ELEC assert '331314' in industry.index @@ -220,28 +240,29 @@ def test_industry_pi_under_elec_is_industries_elec_indexed() -> None: _setup_config('2025_usa_cornerstone_v0_3_electricity_disaggregation.yaml') try: - pi = _cornerstone_indexed_industry_pi(2022) - assert list(pi.index) == active_cornerstone_industries() - assert ELECTRICITY_AGGREGATE_SECTOR not in pi.index - for code in ELECTRICITY_DISAGG_SECTORS: - assert float(pi.loc[code]) == pytest.approx(parent_pi) - - from bedrock.utils.taxonomy.mappings.bea_v2017_industry__bea_v2017_summary import ( # noqa: PLC0415 - load_bea_v2017_industry_to_bea_v2017_summary, - ) - - x_y = derive_cornerstone_x() - bea_fixed: dict[str, list[str]] = { - str(k): [str(s) for s in v] - for k, v in load_bea_v2017_industry_to_bea_v2017_summary().items() - } - parent_summaries = list(bea_fixed.get(ELECTRICITY_AGGREGATE_SECTOR, ['22'])) - bea_fixed.pop(ELECTRICITY_AGGREGATE_SECTOR, None) - for child in ELECTRICITY_DISAGG_SECTORS: - bea_fixed[child] = list(parent_summaries) - expected_22 = _aggregate_industry_pi(pi, x_y, bea_fixed)['22'] - assert float( - _get_summary_industry_price_index(2022).loc['22'] - ) == pytest.approx(expected_22) + with _dollar_industrial_weights(): + pi = _cornerstone_indexed_industry_pi(2022) + assert list(pi.index) == active_cornerstone_industries() + assert ELECTRICITY_AGGREGATE_SECTOR not in pi.index + for code in ELECTRICITY_DISAGG_SECTORS: + assert float(pi.loc[code]) == pytest.approx(parent_pi) + + from bedrock.utils.taxonomy.mappings.bea_v2017_industry__bea_v2017_summary import ( # noqa: PLC0415 + load_bea_v2017_industry_to_bea_v2017_summary, + ) + + x_y = derive_cornerstone_x() + bea_fixed: dict[str, list[str]] = { + str(k): [str(s) for s in v] + for k, v in load_bea_v2017_industry_to_bea_v2017_summary().items() + } + parent_summaries = list(bea_fixed.get(ELECTRICITY_AGGREGATE_SECTOR, ['22'])) + bea_fixed.pop(ELECTRICITY_AGGREGATE_SECTOR, None) + for child in ELECTRICITY_DISAGG_SECTORS: + bea_fixed[child] = list(parent_summaries) + expected_22 = _aggregate_industry_pi(pi, x_y, bea_fixed)['22'] + assert float( + _get_summary_industry_price_index(2022).loc['22'] + ) == pytest.approx(expected_22) finally: _teardown() diff --git a/pyproject.toml b/pyproject.toml index c3d177c52..cbf8e76e3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,7 @@ dependencies = [ "python-dotenv >= 1.1.0", "click>=8.1.7,<9.0.0", "matplotlib>=3.10.6", + "python-pptx>=1.0.0,<2.0.0", ] [dependency-groups] diff --git a/scripts/extract_raw_data.py b/scripts/extract_raw_data.py index fb91735f5..682b46561 100644 --- a/scripts/extract_raw_data.py +++ b/scripts/extract_raw_data.py @@ -9,6 +9,7 @@ FBA_RAW_DATA = ( ('EIA_MECS_Energy', '2018'), + ('EIA_MECS_Energy', '2022'), ('USDA_CoA_Cropland', '2022'), ('USDA_CoA_Cropland_NAICS', '2022'), ) diff --git a/uv.lock b/uv.lock index 1bd25a670..86421eb9c 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = "==3.11.*" [[package]] @@ -158,6 +158,7 @@ dependencies = [ { name = "pycountry" }, { name = "pydantic" }, { name = "python-dotenv" }, + { name = "python-pptx" }, { name = "pyyaml" }, { name = "stewi" }, { name = "tabula-py" }, @@ -200,6 +201,7 @@ requires-dist = [ { name = "pycountry", specifier = ">=24.6.1" }, { name = "pydantic", specifier = ">=2.7.1,<3.0.0" }, { name = "python-dotenv", specifier = ">=1.1.0" }, + { name = "python-pptx", specifier = ">=1.0.0,<2.0.0" }, { name = "pyyaml", specifier = ">=6.0,<7.0.0" }, { name = "stewi", git = "https://github.com/cornerstone-data/standardizedinventories.git" }, { name = "tabula-py", specifier = "==2.10.0" }, @@ -1262,6 +1264,36 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/82/3d/14ce75ef66813643812f3093ab17e46d3a206942ce7376d31ec2d36229e7/lark-1.3.1-py3-none-any.whl", hash = "sha256:c629b661023a014c37da873b4ff58a817398d12635d3bbb2c5a03be7fe5d1e12", size = 113151, upload-time = "2025-10-27T18:25:54.882Z" }, ] +[[package]] +name = "lxml" +version = "6.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ad/a9/970b8fa0ecc4fbf1dfaed0d89bbc1fc1421b25ec26a2038c91e872dc6c8e/lxml-6.1.2.tar.gz", hash = "sha256:1055241852f2b02068af4a625a5d32c087db193c12251928af2562ecd2239f18", size = 4210626, upload-time = "2026-08-19T04:58:15.341Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8d/2d/c292b75049d8b919a515a439646307b971a5f72cd99aaf77d59c9a99e7c4/lxml-6.1.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:da6a4f55f0e3308c07354b1ee239c5550afc212f81629a6067db505ace3b667a", size = 8563059, upload-time = "2026-08-19T04:58:26.559Z" }, + { url = "https://files.pythonhosted.org/packages/69/55/16395f232cb28182c72a1fb4d9d187163fd05a581a98c37f33e945b77a6d/lxml-6.1.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4f4d2c36fd5997d30ff19c29fb93293401d0daaf87512297d47610e6883964b5", size = 4613599, upload-time = "2026-08-19T04:58:30.589Z" }, + { url = "https://files.pythonhosted.org/packages/08/20/a65a084596ccd7fd1ed0668b4cf3b68e700da4eac830a0f22ac569f19a73/lxml-6.1.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1d55a614d2f0457b1f7511c1b7bec0db0dcdd4af4d09d226829eb054c647527c", size = 4935619, upload-time = "2026-08-19T04:58:45.181Z" }, + { url = "https://files.pythonhosted.org/packages/e1/35/008bf5a5f8809a90a3e62909d8d4458f09b7c034c365b508990bdc38b5b7/lxml-6.1.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:575fef7f30048b744dffb3e4ff64a18cac7dba3fd26efdea5730ade9d1bdeb33", size = 5078913, upload-time = "2026-08-19T04:58:53.376Z" }, + { url = "https://files.pythonhosted.org/packages/4f/cf/041b4c15ba3b0421ed828af60993f23cf6e5ea8801efb773b19e248fc6a5/lxml-6.1.2-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:79b428c3242e63bdacf3b526a34e0b8b26583846fc597da84b8f0c3d5ea446b2", size = 5012236, upload-time = "2026-08-19T04:59:06.663Z" }, + { url = "https://files.pythonhosted.org/packages/06/42/89a2760cd2f2cda28ef5b9591ec775a6a5183d193e7b62ddb936b1565167/lxml-6.1.2-cp311-cp311-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:12ecfea07d767f6accbf30b014e1c477b5eabb13eb4e8c748215efb52c0e314a", size = 5211283, upload-time = "2026-08-19T04:59:31.561Z" }, + { url = "https://files.pythonhosted.org/packages/5d/0d/f5607ff466d0d8874d7b778c3ccb64f65ccc0ac430e1961969fd450b899c/lxml-6.1.2-cp311-cp311-manylinux_2_28_i686.whl", hash = "sha256:bfcbee8ffff4188f4c6d97eceeff36d8eb983cf838933cbc12ce5f5dd51476c6", size = 5343352, upload-time = "2026-08-19T04:59:41.056Z" }, + { url = "https://files.pythonhosted.org/packages/63/6a/77713b73265d043a513d9e7df2458f07b2a14709f95e3a35a34834785fde/lxml-6.1.2-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:822d9397033edbe530a13bb1e0091c0e817536b6aba87a9b4ad626ed779ca0bd", size = 4673191, upload-time = "2026-08-19T05:00:01.85Z" }, + { url = "https://files.pythonhosted.org/packages/a7/c7/e4179e0b9f71859bf9a56b3da91db4c7e85c47072018e7b63e019ff65c9f/lxml-6.1.2-cp311-cp311-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4303f904fb6c41b58dc70743b1d8a470aba6c9897427c48324cff1a95673ddb4", size = 5281079, upload-time = "2026-08-19T05:00:20.59Z" }, + { url = "https://files.pythonhosted.org/packages/22/f4/358200b95081db4fd02c4d81938a07080ae7636f9149befda1c0e5189c40/lxml-6.1.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:cdd35422de747237f451e821766e2b6be3dd2c31955c1ecd7f17984c5b9bb62d", size = 5055515, upload-time = "2026-08-19T05:00:29.28Z" }, + { url = "https://files.pythonhosted.org/packages/fe/06/8fe708d90022bd13122c359d38f3f751e4fa71b871eace7fa81212dadfa5/lxml-6.1.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:b3ca02ef3b5920b88119c82eb6badfb2d082b1f681d528a856dcce17c8706da8", size = 4722745, upload-time = "2026-08-19T05:00:49.132Z" }, + { url = "https://files.pythonhosted.org/packages/ad/1d/9d374182c2ee79a5097d4950bfca9e28011eeacdf614db022b9905266b5c/lxml-6.1.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:4bf14db2f0214003ec7f46c4300e2065668fc93e20448c1c95bac2e952072168", size = 5268962, upload-time = "2026-08-19T05:01:15.762Z" }, + { url = "https://files.pythonhosted.org/packages/72/89/d0835e464b84d92c43d838bbeaef02f9ac374ab2bb6972411e4c3e80975d/lxml-6.1.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:2afd1688e372d8eafaa6f56c589399e0a87d086a0c110f6346b0b50f42e67e25", size = 5235564, upload-time = "2026-08-19T05:03:11.298Z" }, + { url = "https://files.pythonhosted.org/packages/bf/ea/0b8acc86d702b9fa1a0194fc7e653087912d340cb10507f4a5bc369d04b3/lxml-6.1.2-cp311-cp311-win32.whl", hash = "sha256:aea814342f6afd20d832937ff8b333cd6506428a39c0c4c70c2380aab1887bfb", size = 3600342, upload-time = "2026-08-19T05:03:14.238Z" }, + { url = "https://files.pythonhosted.org/packages/65/5c/04480497142794bfb2d98c01ea9972e9b3d0f6b1f017073cabb74ab0b8c1/lxml-6.1.2-cp311-cp311-win_amd64.whl", hash = "sha256:b3db5497af55f7a557c95265dd3b91c75dc56364a7b59f258c45fa5576dce058", size = 4032771, upload-time = "2026-08-19T05:03:16.934Z" }, + { url = "https://files.pythonhosted.org/packages/f9/88/4c5ca0f808a80b7eaad073269f1fc53992c5c7c905df13d3953d886834b1/lxml-6.1.2-cp311-cp311-win_arm64.whl", hash = "sha256:e8dc3d29f2ed2bbf24c205a86326d6681230ace55abfb3f9d5230f42078ad63d", size = 3674380, upload-time = "2026-08-19T05:03:19.158Z" }, + { url = "https://files.pythonhosted.org/packages/b9/c9/11bfea1b3afc7a27ce74222b2e12b97005f3b81aa0011313769a14afd60a/lxml-6.1.2-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:4622c5616683faf63791b349e6c8dad7717412dc5f29f4febe7575f110609a86", size = 3942892, upload-time = "2026-08-19T05:03:21.567Z" }, + { url = "https://files.pythonhosted.org/packages/c4/98/9885a4505758885c113af2bc2335a9fced99cb01e07e42895a62f1eb97fb/lxml-6.1.2-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:733dfb492ec3dfef8350a5cc896e90d202c5171e791e1609e77563751d69a15d", size = 4213061, upload-time = "2026-08-19T05:03:24.259Z" }, + { url = "https://files.pythonhosted.org/packages/2c/5e/e80d9e7d6e54b0693df60c7eeeed4aa19e2e3936dadf0676e6a3e8ac1ee1/lxml-6.1.2-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4618b20f43dc98b49569b1dc822176140ea0f2598d672a6989187ba49bcbfec1", size = 4322013, upload-time = "2026-08-19T05:03:26.764Z" }, + { url = "https://files.pythonhosted.org/packages/52/22/2e896cfba4e86b805eb8a3259cbdc1601971dc8fda5b1db2044ec2a3e6f0/lxml-6.1.2-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f93bc5e25992f5545709000d840c6cafdbd022781a7a0ed79d58a5633733a4e8", size = 4257333, upload-time = "2026-08-19T05:03:29.355Z" }, + { url = "https://files.pythonhosted.org/packages/b0/1d/9dbdbfa284ea96aee7c368e0ac73994f7e1375281070c355bcd85d4f7a77/lxml-6.1.2-pp311-pypy311_pp73-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:662432a6103e671d971e06e75ed146d9ff67f39d2c98c2f26613b6057f54eafc", size = 4410828, upload-time = "2026-08-19T05:03:31.948Z" }, + { url = "https://files.pythonhosted.org/packages/f9/f2/fea24b044219458c252e0a0a08074a27dc9e28edb85f83533e36e3ddb57d/lxml-6.1.2-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:ba0dfead73be5be9ad0b7fbf9f31ff29c1b1eae858816dfc8d85099d6e4af0d6", size = 3511278, upload-time = "2026-08-19T05:03:34.597Z" }, +] + [[package]] name = "markdown-it-py" version = "4.0.0" @@ -2009,6 +2041,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/51/e5/fecf13f06e5e5f67e8837d777d1bc43fac0ed2b77a676804df5c34744727/python_json_logger-4.0.0-py3-none-any.whl", hash = "sha256:af09c9daf6a813aa4cc7180395f50f2a9e5fa056034c9953aec92e381c5ba1e2", size = 15548, upload-time = "2025-10-06T04:15:17.553Z" }, ] +[[package]] +name = "python-pptx" +version = "1.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "lxml" }, + { name = "pillow" }, + { name = "typing-extensions" }, + { name = "xlsxwriter" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/52/a9/0c0db8d37b2b8a645666f7fd8accea4c6224e013c42b1d5c17c93590cd06/python_pptx-1.0.2.tar.gz", hash = "sha256:479a8af0eaf0f0d76b6f00b0887732874ad2e3188230315290cd1f9dd9cc7095", size = 10109297, upload-time = "2024-08-07T17:33:37.772Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d9/4f/00be2196329ebbff56ce564aa94efb0fbc828d00de250b1980de1a34ab49/python_pptx-1.0.2-py3-none-any.whl", hash = "sha256:160838e0b8565a8b1f67947675886e9fea18aa5e795db7ae531606d68e785cba", size = 472788, upload-time = "2024-08-07T17:33:28.192Z" }, +] + [[package]] name = "python-slugify" version = "8.0.4"