Add fair default rate design modules

context/README.md

@@ -60,11 +60,12 @@ BAT framework, residual allocation, and how they connect to the literature.
 
 Cost-reflective TOU design: theory and window optimization.
 
-| File                                  | Purpose                                                                                                                                                                                                         |
-| ------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| cost_reflective_tou_rate_design.md    | Theory and practice of cost-reflective TOU rate design: demand-weighted MC averages, cost-causation ratios, period selection, assumptions, demand flexibility implications, and partial vs. general equilibrium |
-| tou_window_optimization.md            | TOU window width ($N$) sweep: welfare-loss metric derivation, HP-demand weighting proof, sweep algorithm, CLI/Justfile, NY results                                                                              |
-| demand_flex_elasticity_calibration.md | Per-utility elasticity calibration: Arcturus 2.0 meta-analysis anchor, diagnostic methodology, results table (ε = -0.10 or -0.12 per utility), two savings mechanisms, known limitations                        |
+| File                                  | Purpose                                                                                                                                                                                                                                                                                                                                    |
+| ------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| cost_reflective_tou_rate_design.md    | Theory and practice of cost-reflective TOU rate design: demand-weighted MC averages, cost-causation ratios, period selection, assumptions, demand flexibility implications, and partial vs. general equilibrium                                                                                                                            |
+| tou_window_optimization.md            | TOU window width ($N$) sweep: welfare-loss metric derivation, HP-demand weighting proof, sweep algorithm, CLI/Justfile, NY results                                                                                                                                                                                                         |
+| demand_flex_elasticity_calibration.md | Per-utility elasticity calibration: Arcturus 2.0 meta-analysis anchor, diagnostic methodology, results table (ε = -0.10 or -0.12 per utility), two savings mechanisms, known limitations                                                                                                                                                   |
+| fair_default_rate_design.md           | Math for designing a single residential default tariff that eliminates the HP cross-subsidy $X_S$ from BAT outputs while collecting class RR. Three closed-form strategies (fixed-only, seasonal-only, combined cost-reflective ratio), uniqueness analysis (exactly one $(r_w, r_s)$ per chosen $F$), feasibility region, worked example. |
 
 ### methods/marginal_costs/
 

rate_design/hp_rates/Justfile

@@ -13,7 +13,7 @@
 # Tier 1: Identity (set by state wrapper, dispatch, or manual source)
 # =============================================================================
 
-set allow-duplicate-variables
+set allow-duplicate-variables := true
 
 state := env_var_or_default('STATE', '')
 utility := env_var_or_default('UTILITY', '')
@@ -57,6 +57,7 @@ use_resstock_loads := env_var_or_default('USE_RESSTOCK_LOADS', 'false')
 # "default" = preserve the actual utility rate structure.
 
 base_tariff_pattern := env_var_or_default('BASE_TARIFF_PATTERN', 'flat')
+mc_seasonal_ratio := env_var_or_default('MC_SEASONAL_RATIO', '')
 
 # =============================================================================
 # Tier 3: Derived paths (all computed from Tier 1 + 2)
@@ -77,7 +78,7 @@ path_default_rev_requirement := path_rev_requirement / utility + ".yaml"
 path_differentiated_rev_requirement := path_rev_requirement / utility + "_hp_vs_nonhp.yaml"
 path_tariff_maps := path_config / "tariff_maps"
 path_dist_and_sub_tx_mc := "s3://data.sb/switchbox/marginal_costs/" + state + "/dist_and_sub_tx/utility=" + utility + "/year=" + mc_year + "/data.parquet"
-path_bulk_tx_mc := env_var_or_default('BULK_TX_MC', "")
+path_bulk_tx_mc := env_var_or_default('BULK_TX_MC', "s3://data.sb/switchbox/marginal_costs/" + state + "/bulk_tx/utility=" + utility + "/year=" + mc_year + "/data.parquet")
 
 # Supply MC: default to per-utility S3 parquets (NY-style).
 # Cambium-based states (RI) override via SUPPLY_ENERGY_MC / SUPPLY_CAPACITY_MC / SUPPLY_ANCILLARY_MC in state.env.
@@ -430,6 +431,86 @@ create-flat-discount-tariff base_tariff_json flat_inputs_csv label output_path:
       "{{ label }}" \
       "{{ output_path }}"
 
+compute-fair-default-inputs path_run_dir subclass_value cross_subsidy_col="BAT_percustomer" path_output_dir="" path_base_tariff_json="" group_col="has_hp" group_value_to_subclass="" mc_seasonal_ratio_value=mc_seasonal_ratio path_periods_yaml=path_periods_yaml fixed_charge_floor="0.0":
+    #!/usr/bin/env bash
+    set -euo pipefail
+    path_effective_base_tariff="{{ path_base_tariff_json }}"
+    if [ -z "${path_effective_base_tariff}" ]; then
+        path_effective_base_tariff="{{ path_tariffs_electric }}/{{ utility }}_{{ base_tariff_pattern }}_calibrated.json"
+    fi
+    just -f {{ path_repo }}/utils/Justfile compute-fair-default-inputs \
+      "{{ path_run_dir }}" \
+      "{{ path_resstock_release }}" \
+      "{{ state_upper }}" \
+      "{{ upgrade }}" \
+      "{{ subclass_value }}" \
+      "{{ cross_subsidy_col }}" \
+      "{{ path_output_dir }}" \
+      "${path_effective_base_tariff}" \
+      "{{ group_col }}" \
+      "{{ group_value_to_subclass }}" \
+      "{{ mc_seasonal_ratio_value }}" \
+      "{{ path_periods_yaml }}" \
+      "{{ fixed_charge_floor }}"
+
+create-fair-default-tariff strategy label path_output_path path_inputs_csv path_base_tariff_json="" path_periods_yaml=path_periods_yaml allow_infeasible="false":
+    #!/usr/bin/env bash
+    set -euo pipefail
+    path_effective_base_tariff="{{ path_base_tariff_json }}"
+    if [ -z "${path_effective_base_tariff}" ]; then
+        path_effective_base_tariff="{{ path_tariffs_electric }}/{{ utility }}_{{ base_tariff_pattern }}_calibrated.json"
+    fi
+    just -f {{ path_repo }}/utils/Justfile create-fair-default-tariff \
+      "${path_effective_base_tariff}" \
+      "{{ path_inputs_csv }}" \
+      "{{ strategy }}" \
+      "{{ label }}" \
+      "{{ path_output_path }}" \
+      "{{ path_periods_yaml }}" \
+      "{{ allow_infeasible }}"
+
+create-all-fair-default-tariffs path_run_dir subclass_value="true" path_tariff_output_dir=path_tariffs_electric path_base_tariff_json="" cross_subsidy_col="BAT_percustomer" group_col="has_hp" group_value_to_subclass="" path_output_dir="" mc_seasonal_ratio_value=mc_seasonal_ratio path_periods_yaml=path_periods_yaml fixed_charge_floor="0.0":
+    #!/usr/bin/env bash
+    set -euo pipefail
+    path_effective_output_dir="{{ path_output_dir }}"
+    if [ -z "${path_effective_output_dir}" ]; then
+        path_effective_output_dir="{{ path_run_dir }}"
+    fi
+    path_effective_base_tariff="{{ path_base_tariff_json }}"
+    if [ -z "${path_effective_base_tariff}" ]; then
+        path_effective_base_tariff="{{ path_tariffs_electric }}/{{ utility }}_{{ base_tariff_pattern }}_calibrated.json"
+    fi
+    if [ -z "{{ mc_seasonal_ratio_value }}" ]; then
+        echo "ERROR: create-all-fair-default-tariffs requires mc_seasonal_ratio (set MC_SEASONAL_RATIO or pass mc_seasonal_ratio_value)." >&2
+        exit 1
+    fi
+    just compute-fair-default-inputs "{{ path_run_dir }}" "{{ subclass_value }}" \
+        "{{ cross_subsidy_col }}" "${path_effective_output_dir}" \
+        "${path_effective_base_tariff}" "{{ group_col }}" "{{ group_value_to_subclass }}" \
+        "{{ mc_seasonal_ratio_value }}" "{{ path_periods_yaml }}" "{{ fixed_charge_floor }}"
+    path_inputs_csv="${path_effective_output_dir}/fair_default_inputs.csv"
+    just create-fair-default-tariff \
+        fixed_charge_only \
+        {{ utility }}_default_fair_fixed_charge_only \
+        "{{ path_tariff_output_dir }}/{{ utility }}_default_fair_fixed_charge_only.json" \
+        "${path_inputs_csv}" \
+        "${path_effective_base_tariff}" \
+        "{{ path_periods_yaml }}"
+    just create-fair-default-tariff \
+        seasonal_rates_only \
+        {{ utility }}_default_fair_seasonal_rates_only \
+        "{{ path_tariff_output_dir }}/{{ utility }}_default_fair_seasonal_rates_only.json" \
+        "${path_inputs_csv}" \
+        "${path_effective_base_tariff}" \
+        "{{ path_periods_yaml }}"
+    just create-fair-default-tariff \
+        fixed_plus_seasonal_mc \
+        {{ utility }}_default_fair_fixed_plus_seasonal_mc \
+        "{{ path_tariff_output_dir }}/{{ utility }}_default_fair_fixed_plus_seasonal_mc.json" \
+        "${path_inputs_csv}" \
+        "${path_effective_base_tariff}" \
+        "{{ path_periods_yaml }}"
+
 copy-calibrated-tariff-from-run run_dir output_dir="":
     just -f {{ path_repo }}/utils/Justfile copy-calibrated-tariff-from-run \
       "{{ run_dir }}" \
@@ -458,8 +539,232 @@ compute-subclass-rev-requirements:
       "${run2_dir}"
 
 # =============================================================================
-# RUNS: scenario execution (run-1 through run-16)
+# FAIR-DEFAULT: paths, tariff preparation, scenario generation, and runs 101-124
+# =============================================================================
+
+path_scenarios_fair_default := path_config / "scenarios" / "fair_default"
+path_scenario_fair_default_config := path_scenarios_fair_default / ("scenarios_" + utility + ".yaml")
+
+# Generate fair-default scenario YAMLs (one per utility, in fair_default/ subdir)
+create-fair-default-scenario-yamls:
+    uv run python {{ path_repo }}/utils/pre/create_fair_default_scenario_yamls.py \
+      --state {{ state }} --utility {{ utility }}
+
+# Build the 12 uncalibrated fair-default tariff JSONs for one utility.
+
+# Requires run-1 and run-2 outputs to already exist (resolved via latest_run_output.sh).
+prepare-fair-default-tariffs:
+    #!/usr/bin/env bash
+    set -euo pipefail
+    run1_dir=$(bash "{{ latest_output }}" "{{ path_scenario_config }}" 1)
+    run2_dir=$(bash "{{ latest_output }}" "{{ path_scenario_config }}" 2)
+    echo ">> prepare-fair-default-tariffs: run-1 delivery dir → ${run1_dir}" >&2
+    echo ">> prepare-fair-default-tariffs: run-2 supply dir   → ${run2_dir}" >&2
+    uv run python {{ path_repo }}/utils/mid/prepare_fair_default_tariffs.py \
+      --state {{ state }} --utility {{ utility }} \
+      --run-dir-delivery "${run1_dir}" \
+      --run-dir-supply "${run2_dir}" \
+      --output-dir "{{ path_tariffs_electric }}" \
+      --resstock-base "{{ path_resstock_release }}" \
+      --path-dist-and-sub-tx-mc "{{ path_dist_and_sub_tx_mc }}" \
+      --path-bulk-tx-mc "{{ path_bulk_tx_mc }}" \
+      --path-supply-energy-mc "{{ path_supply_energy_mc }}" \
+      --path-supply-capacity-mc "{{ path_supply_capacity_mc }}" \
+      --base-tariff-delivery "{{ path_tariffs_electric }}/{{ utility }}_{{ base_tariff_pattern }}_calibrated.json" \
+      --base-tariff-supply "{{ path_tariffs_electric }}/{{ utility }}_{{ base_tariff_pattern }}_supply_calibrated.json" \
+      --periods-yaml "{{ path_periods_yaml }}" \
+      --allow-infeasible
+
+# Generic dispatcher for fair-default runs (101-124): dispatches against
+
+# the fair_default/ scenario YAML instead of the main scenarios_<util>.yaml.
+run-fd N:
+    just run-scenario-from "{{ path_scenario_fair_default_config }}" {{ N }}
+
+# Precalc fair-default runs (precalc delivery/supply): run then promote calibrated tariff.
+run-101:
+    just run-fd 101
+    run_dir=$(bash "{{ latest_output }}" "{{ path_scenario_fair_default_config }}" 101); \
+      just copy-calibrated-tariff-from-run "${run_dir}"
+
+run-102:
+    just run-fd 102
+    run_dir=$(bash "{{ latest_output }}" "{{ path_scenario_fair_default_config }}" 102); \
+      just copy-calibrated-tariff-from-run "${run_dir}"
+
+run-103:
+    just run-fd 103
+
+run-104:
+    just run-fd 104
+
+run-105:
+    just run-fd 105
+    run_dir=$(bash "{{ latest_output }}" "{{ path_scenario_fair_default_config }}" 105); \
+      just copy-calibrated-tariff-from-run "${run_dir}"
+
+run-106:
+    just run-fd 106
+    run_dir=$(bash "{{ latest_output }}" "{{ path_scenario_fair_default_config }}" 106); \
+      just copy-calibrated-tariff-from-run "${run_dir}"
+
+run-107:
+    just run-fd 107
+
+run-108:
+    just run-fd 108
+
+run-109:
+    just run-fd 109
+    run_dir=$(bash "{{ latest_output }}" "{{ path_scenario_fair_default_config }}" 109); \
+      just copy-calibrated-tariff-from-run "${run_dir}"
+
+run-110:
+    just run-fd 110
+    run_dir=$(bash "{{ latest_output }}" "{{ path_scenario_fair_default_config }}" 110); \
+      just copy-calibrated-tariff-from-run "${run_dir}"
+
+run-111:
+    just run-fd 111
+
+run-112:
+    just run-fd 112
+
+run-113:
+    just run-fd 113
+    run_dir=$(bash "{{ latest_output }}" "{{ path_scenario_fair_default_config }}" 113); \
+      just copy-calibrated-tariff-from-run "${run_dir}"
+
+run-114:
+    just run-fd 114
+    run_dir=$(bash "{{ latest_output }}" "{{ path_scenario_fair_default_config }}" 114); \
+      just copy-calibrated-tariff-from-run "${run_dir}"
+
+run-115:
+    just run-fd 115
+
+run-116:
+    just run-fd 116
+
+run-117:
+    just run-fd 117
+    run_dir=$(bash "{{ latest_output }}" "{{ path_scenario_fair_default_config }}" 117); \
+      just copy-calibrated-tariff-from-run "${run_dir}"
+
+run-118:
+    just run-fd 118
+    run_dir=$(bash "{{ latest_output }}" "{{ path_scenario_fair_default_config }}" 118); \
+      just copy-calibrated-tariff-from-run "${run_dir}"
+
+run-119:
+    just run-fd 119
+
+run-120:
+    just run-fd 120
+
+run-121:
+    just run-fd 121
+    run_dir=$(bash "{{ latest_output }}" "{{ path_scenario_fair_default_config }}" 121); \
+      just copy-calibrated-tariff-from-run "${run_dir}"
+
+run-122:
+    just run-fd 122
+    run_dir=$(bash "{{ latest_output }}" "{{ path_scenario_fair_default_config }}" 122); \
+      just copy-calibrated-tariff-from-run "${run_dir}"
+
+run-123:
+    just run-fd 123
+
+run-124:
+    just run-fd 124
+
+# Pre-rate setup that includes fair-default scaffolding (generates scenario YAMLs
+
+# and tariff maps for the fair_default/ subdir in addition to the baseline pre steps).
+all-pre-fair-rate:
+    just all-pre
+    just create-fair-default-scenario-yamls
+    just -f {{ path_repo }}/utils/Justfile write-tariff-maps-all "{{ path_scenarios_fair_default }}"
+    uv run python {{ path_repo }}/utils/pre/validate_config.py \
+      --scenario-config "{{ path_scenario_fair_default_config }}" \
+      --state "{{ state_upper }}" \
+      --utility "{{ utility }}" \
+      --upgrade "{{ upgrade }}" \
+      --year "{{ year }}" \
+      --path-dist-and-sub-tx-mc "{{ path_dist_and_sub_tx_mc }}" \
+      {{ if path_bulk_tx_mc != "" { "--path-bulk-tx-mc \"" + path_bulk_tx_mc + "\"" } else { "" } }} \
+      --path-supply-energy-mc "{{ path_supply_energy_mc }}" \
+      --path-supply-capacity-mc "{{ path_supply_capacity_mc }}" \
+      --path-electric-utility-stats "{{ path_electric_utility_stats }}" \
+      --path-resstock-loads "{{ path_resstock_loads_00 }}" \
+      --fair-default-dir "{{ path_scenarios_fair_default }}"
+
+# Full end-to-end: pre-rate + run-1/2 + tariff prep + 24 fair-default runs (101-124).
+# Dependencies: run-1 → copy-calibrated, run-2 → copy-calibrated → prepare-fair-default-tariffs
+
+# → runs 101-124 in groups of 4 (precalc-del, precalc-sup, u2-del, u2-sup).
+fair-default-rate-runs:
+    #!/usr/bin/env bash
+    set -euo pipefail
+    just all-pre-fair-rate
+    just run-1
+    just run-2
+    run1_dir=$(bash "{{ latest_output }}" "{{ path_scenario_config }}" 1)
+    just copy-calibrated-tariff-from-run "${run1_dir}"
+    run2_dir=$(bash "{{ latest_output }}" "{{ path_scenario_config }}" 2)
+    just copy-calibrated-tariff-from-run "${run2_dir}"
+    just prepare-fair-default-tariffs
+    just run-101
+    just run-102
+    just run-103
+    just run-104
+    just run-105
+    just run-106
+    just run-107
+    just run-108
+    just run-109
+    just run-110
+    just run-111
+    just run-112
+    just run-113
+    just run-114
+    just run-115
+    just run-116
+    just run-117
+    just run-118
+    just run-119
+    just run-120
+    just run-121
+    just run-122
+    just run-123
+    just run-124
+
+# =============================================================================
+# RUNS: scenario execution (run-1 through run-36)
 # =============================================================================
+# Dispatch a single run from an arbitrary scenario YAML file (used by run-fd).
+
+# Passes --scenario-config directly to run_scenario.py.
+run-scenario-from path_scenario_config run_num *extra_args:
+    #!/usr/bin/env bash
+    set -euo pipefail
+    : "${RDP_BATCH:?Set RDP_BATCH before running (e.g. ny_20260305c_r1-8)}"
+    log_dir="${HOME}/rdp_run_logs"
+    mkdir -p "${log_dir}"
+    log_file="${log_dir}/{{ utility }}_run{{ run_num }}_${RDP_BATCH}.log"
+    echo ">> run-scenario-from {{ run_num }}: logging to ${log_file}" >&2
+    echo "git_commit: $(git -C "{{ path_repo }}" rev-parse HEAD 2>/dev/null || echo unknown)" > "${log_file}"
+    echo "git_dirty:  $(git -C "{{ path_repo }}" status --porcelain 2>/dev/null | wc -l | tr -d ' ') files" >> "${log_file}"
+    echo "timestamp:  $(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "${log_file}"
+    uv run python {{ path_repo }}/rate_design/hp_rates/run_scenario.py \
+        --scenario-config "{{ path_scenario_config }}" \
+        --state "{{ state }}" \
+        --run-num "{{ run_num }}" \
+        --output-dir "{{ path_outputs_base }}/${RDP_BATCH}" \
+        {{ if env_var_or_default('RDP_NUM_WORKERS', '') != '' { "--num-workers " + env_var_or_default('RDP_NUM_WORKERS', '') } else { "" } }} \
+        {{ if path_supply_ancillary_mc != "" { "--path-supply-ancillary-mc \"" + path_supply_ancillary_mc + "\"" } else { "" } }} \
+        {{ extra_args }} \
+        2>&1 | tee -a "${log_file}"
 
 run-scenario run_num *extra_args:
     #!/usr/bin/env bash