From b1a043a9f10fb34b53501c8c984469c076653197 Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Fri, 21 Aug 2026 16:24:52 -0600 Subject: [PATCH 01/18] add eia mecs 2022 to source catalog --- bedrock/utils/config/source_catalog.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/bedrock/utils/config/source_catalog.yaml b/bedrock/utils/config/source_catalog.yaml index 5acd5ef13..374f9fc0b 100644 --- a/bedrock/utils/config/source_catalog.yaml +++ b/bedrock/utils/config/source_catalog.yaml @@ -214,6 +214,7 @@ EIA_MECS_Energy: activity_schema: {2010: NAICS_2007_Code, 2014: NAICS_2012_Code, 2018: NAICS_2017_Code, + 2022: NAICS_2017_Code, } sector_hierarchy: "parent-incompleteChild" EIA_MECS_Energy_Allocation_CEDA: From 10683b579537154e5b53c2e30dcb39c7bace71b5 Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Fri, 21 Aug 2026 17:19:56 -0600 Subject: [PATCH 02/18] add Energy FBS for use in ghg energy allocation --- bedrock/extract/eia/EIA_MECS.py | 236 +++++++++++++++++- .../energy/Energy_Cornerstone_2018.yaml | 203 +++++++++++++++ .../energy/Energy_Cornerstone_2022.yaml | 203 +++++++++++++++ 3 files changed, 639 insertions(+), 3 deletions(-) create mode 100644 bedrock/transform/energy/Energy_Cornerstone_2018.yaml create mode 100644 bedrock/transform/energy/Energy_Cornerstone_2022.yaml diff --git a/bedrock/extract/eia/EIA_MECS.py b/bedrock/extract/eia/EIA_MECS.py index a9cef2576..2d11593a2 100644 --- a/bedrock/extract/eia/EIA_MECS.py +++ b/bedrock/extract/eia/EIA_MECS.py @@ -18,10 +18,21 @@ from bedrock.extract.eia.EIA_CBECS_Land import calculate_total_facility_land_area from bedrock.extract.flowbyactivity import FlowByActivity, getFlowByActivity from bedrock.extract.generateflowbyactivity import generateFlowByActivity -from bedrock.transform.flowbyclean import load_prepare_clean_source +from bedrock.transform.flowbyclean import ( + define_parentincompletechild_descendants, + load_prepare_clean_source, +) from bedrock.transform.flowbyfunctions import assign_fips_location_system -from bedrock.utils.config.common import WITHDRAWN_KEYWORD -from bedrock.utils.io.gcp import download_extract_input_from_gcs_if_not_exists +from bedrock.utils.config.common import WITHDRAWN_KEYWORD, get_catalog_info +from bedrock.utils.economic.units import ( + HEATING_OIL_MMBTU_PER_GALLON, + PROPANE_MMBTU_PER_GALLON, +) +from bedrock.utils.io.gcp import ( + download_extract_input_from_gcs_if_not_exists, + load_from_gcs, +) +from bedrock.utils.io.gcp_paths import gcs_extract_input_path from bedrock.utils.io.local_extract_input_data import local_extract_input_dir from bedrock.utils.logging.flowsa_log import log from bedrock.utils.mapping.location import ( @@ -656,6 +667,225 @@ def estimate_suppressed_mecs_energy( return unsuppressed.drop(columns='Suppressed') +def keep_chemical_manufacturing(fba: FlowByActivity, **_kwargs: Any) -> FlowByActivity: + '''Keep MECS NAICS 325 (chemicals) and descendants. CEDA NEU NG is chemicals-only.''' + return fba.query("ActivityConsumedBy.str.startswith('325')") + + +def multiply_bea_by_mecs_petroleum_energy_fraction( + fba: FlowByActivity, **_kwargs: Any +) -> FlowByActivity: + ''' + clean_fba_after_attribution. CEDA industrial petrol weights are BEA 324110 + purchases times MECS Other energy fraction Table 3.1 / (Table 2.1 + Table 3.1). + Sectors with no MECS ratio keep 1.0. Restricts to ag/mining/construction/ + manufacturing plus natural gas distribution. + + Loads MECS from config clean_source (FBA only — selection / estimate_suppressed + from YAML; does not run prepare_fbs). + ''' + clean_source = fba.config.get('clean_source') + if not clean_source: + raise ValueError( + 'clean_source is required for multiply_bea_by_mecs_petroleum_energy_fraction' + ) + if isinstance(clean_source, str): + name, src_config = clean_source, {} + else: + ((name, src_config),) = clean_source.items() + year = int( + src_config.get( + 'year', + (fba.config.get('clean_parameter') or {}).get( + 'year', fba.config.get('year') + ), + ) + ) + mecs = FlowByActivity( + getFlowByActivity(name, year), + full_name=name, + config={**get_catalog_info(name), **src_config, 'year': year}, + ) + mecs = ( + mecs.function_socket('estimate_suppressed') + .select_by_fields() + .assign(Flowable=lambda x: x.FlowName) + ) + # Residualize each MECS table separately (define groups by Flowable only). + # define_parentincompletechild_descendants expects group_id / group_total. + residualized = [] + for desc in ['Table 2.1', 'Table 3.1']: + table = ( + mecs.query(f'Description == "{desc}"') + .drop(columns=['group_id', 'group_total'], errors='ignore') + .reset_index(drop=True) + ) + table = table.assign(group_id=table.index, group_total=table.FlowAmount) + residualized.append(define_parentincompletechild_descendants(table)) + mecs = FlowByActivity( + pd.concat(residualized), + full_name=mecs.full_name, + config=mecs.config, + ) + t21 = ( + mecs.query("Description == 'Table 2.1'") + .groupby('ActivityConsumedBy')['FlowAmount'] + .sum() + ) + t31 = ( + mecs.query("Description == 'Table 3.1'") + .groupby('ActivityConsumedBy')['FlowAmount'] + .sum() + ) + idx = t21.index.union(t31.index) + t21 = t21.reindex(idx).fillna(0) + t31 = t31.reindex(idx).fillna(0) + den = t21 + t31 + ratios = (t31 / den).where(den != 0).fillna(1.0) + + activity = fba['ActivityConsumedBy'].astype(str) + industrial = activity.str.startswith( + ('11', '21', '23', '31', '32', '33') + ) | activity.eq('221200') + out = fba.loc[industrial].copy() + if out.empty: + log.warning('No industrial BEA 324110 rows left to apply MECS petrol fraction') + return out + + sector_col = ( + 'SectorConsumedBy' + if 'SectorConsumedBy' in out.columns + else 'ActivityConsumedBy' + ) + ratio_index = set(ratios.index.astype(str)) + + def ratio_for_sector(sector: str) -> float: + for n in range(len(sector), 1, -1): + key = sector[:n] + if key in ratio_index: + return float(ratios[key]) + return 1.0 + + matched = out[sector_col].astype(str).map(ratio_for_sector) + return out.assign(FlowAmount=out['FlowAmount'] * matched) + + +def household_petroleum_transport_fraction(config: dict[str, Any]) -> float: + ''' + Share of PCE gasoline-and-other-energy that is transport rather than + residential heat oil/propane. Same formula CEDA uses on F01000. + + All inputs come from config clean_source (GHGI annex FBA) and + clean_parameter (PCE + residential fuel prices + fuel price groups). + ''' + clean_parameter = config.get('clean_parameter') or {} + year = int(clean_parameter.get('year', config.get('year'))) + + pce_cfg = clean_parameter.get('pce') + propane_cfg = clean_parameter.get('propane_price') + heat_oil_cfg = clean_parameter.get('heating_oil_price') + heat_fuels = clean_parameter.get('residential_heat_fuels') + if not all((pce_cfg, propane_cfg, heat_oil_cfg, heat_fuels)): + raise ValueError( + 'clean_parameter must define pce, propane_price, ' + 'heating_oil_price, and residential_heat_fuels' + ) + assert pce_cfg is not None + assert propane_cfg is not None + assert heat_oil_cfg is not None + assert heat_fuels is not None + + pce_tbl = load_from_gcs( + name=pce_cfg['file'], + sub_bucket=gcs_extract_input_path(pce_cfg['extract_input']), + local_dir=local_extract_input_dir(pce_cfg['extract_input']), + loader=lambda pth: pd.read_csv( + pth, + skiprows=int(pce_cfg.get('skiprows', 3)), + index_col=int(pce_cfg.get('index_col', 1)), + ) + .dropna() + .drop(columns=list(pce_cfg.get('drop_columns', ['Line']))), + ) + pce_tbl.index = pce_tbl.index.str.strip() + pce_tbl.columns = pce_tbl.columns.astype(int) + pce_years = [c for c in pce_tbl.columns if c <= year] or list(pce_tbl.columns) + pce_year = int(max(pce_years)) + if pce_year != year: + log.warning( + f'BEA PCE has no {year} gasoline-and-other-energy; using {pce_year}' + ) + pce = float(pce_tbl.loc[pce_cfg['line'], pce_year]) + + prices: dict[str, float] = {} + for key, price_cfg in ( + ('propane_price', propane_cfg), + ('heating_oil_price', heat_oil_cfg), + ): + price_tbl = load_from_gcs( + name=price_cfg['file'], + sub_bucket=gcs_extract_input_path(price_cfg['extract_input']), + local_dir=local_extract_input_dir(price_cfg['extract_input']), + loader=lambda pth, rows=int(price_cfg.get('skiprows', 4)): pd.read_csv( + pth, skiprows=rows + ), + ) + month_col = price_cfg.get('month_column', 'Month') + value_col = price_cfg['value_column'] + price_tbl['Year'] = price_tbl[month_col].str.extract(r'(\d{4})').astype(int) + by_year = price_tbl.groupby('Year')[value_col].mean() + years = [y for y in by_year.index if y <= year] or list(by_year.index) + prices[key] = float(by_year[int(max(years))]) + + clean_source = config.get('clean_source') + if not clean_source: + raise ValueError( + 'clean_source is required for household_petroleum_transport_fraction' + ) + if isinstance(clean_source, str): + name, src_config = clean_source, {} + else: + ((name, src_config),) = clean_source.items() + annex_year = int(src_config.get('year', year)) + annex = FlowByActivity( + getFlowByActivity(name, annex_year), + full_name=name, + config={**get_catalog_info(name), **src_config, 'year': annex_year}, + ).select_by_fields() + + propane_priced = list(heat_fuels['propane_priced']) + heating_oil_priced = list(heat_fuels['heating_oil_priced']) + kerosene_lpg = float( + annex.loc[annex['FlowName'].isin(propane_priced), 'FlowAmount'].sum() + ) + distillate = float( + annex.loc[annex['FlowName'].isin(heating_oil_priced), 'FlowAmount'].sum() + ) + res_heat = (kerosene_lpg * (prices['propane_price'] / PROPANE_MMBTU_PER_GALLON)) + ( + distillate * (prices['heating_oil_price'] / HEATING_OIL_MMBTU_PER_GALLON) + ) + if pce == 0: + log.warning('PCE gasoline-and-other-energy is 0; F01000 transport share is 1.0') + return 1.0 + return (pce - res_heat) / pce + + +def scale_household_petroleum_to_transport_share( + fba: FlowByActivity, **_kwargs: Any +) -> FlowByActivity: + ''' + clean_fba_after_attribution. Scale F01000 petroleum by the transport share + of PCE gasoline-and-other-energy (residential heat oil/propane removed). + ''' + scale = household_petroleum_transport_fraction(fba.config) + mask = fba['ActivityConsumedBy'] == 'F01000' + if 'SectorConsumedBy' in fba.columns: + mask = mask | (fba['SectorConsumedBy'] == 'F01000') + out = fba.copy() + out.loc[mask, 'FlowAmount'] = out.loc[mask, 'FlowAmount'] * scale + return out + + def clean_mapped_mecs_energy_fba_to_state( fba: FlowByActivity, **_: Any ) -> FlowByActivity: diff --git a/bedrock/transform/energy/Energy_Cornerstone_2018.yaml b/bedrock/transform/energy/Energy_Cornerstone_2018.yaml new file mode 100644 index 000000000..dcd655d12 --- /dev/null +++ b/bedrock/transform/energy/Energy_Cornerstone_2018.yaml @@ -0,0 +1,203 @@ +# Manufacturing energy FBS for Cornerstone. MECS survey year 2018. +# Replicates CEDA allocation steps as quantity weights: MECS mapped with +# parent-incomplete-child + Cornerstone_2025, then nested BEA splits; +# petrol energy is BEA 324110 times MECS Other energy fraction. + +!include:Cornerstone_2025_target.yaml +year: &mecs_year 2018 +geoscale: national + +_attribution_sources: + BEA: &bea + year: 2017 + activity_to_sector_mapping: Cornerstone_2025 + exclusion_fields: + ActivityConsumedBy: ['F03000', 'F04000', 'F05000', 'F02E00', + 'F06E00', 'F07E00', 'F10E00', 'F02R00', + 'T001', 'T004', 'T007', 'T019'] + ActivityProducedBy: ['T007', 'T013', 'T015', 'T016', 'TOP', + 'MCIF'] + attribution_method: equal + transport_consumers: &transport_consumers + - '481000' + - '482000' + - '483000' + - '484000' + - '485000' + - '486000' + - '48A000' + - '492000' + - 'S00500' + - 'S00600' + - '491000' + - 'GSLGO' + - 'S00203' + - 'F01000' + +source_names: + EIA_MECS_Energy: + year: *mecs_year + activity_to_sector_mapping: Cornerstone_2025 + selection_fields: + Location: '00000' + Description: [Table 2.1, Table 3.1] + exclusion_fields: + ActivityConsumedBy: '31-33' + clean_fba_before_activity_sets: + - !script_function:EIA_MECS estimate_suppressed_mecs_energy + activity_sets: + coal: + selection_fields: + Description: + Table 3.1: energy + FlowName: Coal + attribution_method: proportional + attribution_source: + BEA_Detail_Use_AfterRedef: + <<: *bea + selection_fields: + ActivityProducedBy: {'212100': ''} + + ng: + selection_fields: + Description: + Table 3.1: energy + FlowName: 'Natural Gas' + attribution_method: proportional + attribution_source: + BEA_Detail_Use_AfterRedef: + <<: *bea + selection_fields: + ActivityProducedBy: {'221200': ''} + + neu_ng: + selection_fields: + Description: + Table 2.1: non energy + FlowName: 'Natural Gas' + clean_fba: !script_function:EIA_MECS keep_chemical_manufacturing + attribution_method: proportional + attribution_source: + BEA_Detail_Use_AfterRedef: + <<: *bea + selection_fields: + ActivityProducedBy: {'221200': ''} + + neu_petrol_asphalt: + selection_fields: + Description: + Table 2.1: non energy + FlowName: + Other: Petroleum + ActivityConsumedBy: ['324121', '324122'] + attribution_method: proportional + attribution_source: + BEA_Detail_Use_AfterRedef: + <<: *bea + selection_fields: + ActivityProducedBy: {'324110': ''} + + neu_petrol_hgl: + selection_fields: + Description: + Table 2.1: non energy + FlowName: + 'Hydrocarbon Gas Liquids, excluding natural gasoline': Petroleum + attribution_method: proportional + attribution_source: + BEA_Detail_Use_AfterRedef: + <<: *bea + selection_fields: + ActivityProducedBy: {'324110': ''} + + neu_petrol_other: + selection_fields: + Description: + Table 2.1: non energy + FlowName: + Other: Petroleum + exclusion_fields: + ActivityConsumedBy: ['324121', '324122'] + attribution_method: proportional + attribution_source: + BEA_Detail_Use_AfterRedef: + <<: *bea + selection_fields: + ActivityProducedBy: {'324110': ''} + + BEA_Detail_Use_AfterRedef: + <<: *bea + activity_sets: + petrol: + selection_fields: + ActivityProducedBy: {'324110': ''} + exclusion_fields: + ActivityConsumedBy: *transport_consumers + assign_fields: + FlowName: Petroleum + Description: energy + clean_source: + EIA_MECS_Energy: + year: *mecs_year + selection_fields: + Location: '00000' + FlowName: Other + Description: [Table 2.1, Table 3.1] + exclusion_fields: + ActivityConsumedBy: '31-33' + estimate_suppressed: !script_function:EIA_MECS estimate_suppressed_mecs_energy + clean_fba_after_attribution: !script_function:EIA_MECS multiply_bea_by_mecs_petroleum_energy_fraction + attribution_method: direct + neu_coal_coke: + selection_fields: + ActivityConsumedBy: '2122A0' + exclusion_fields: + ActivityProducedBy: '324110' + assign_fields: + FlowName: Coal and Coke + Description: non energy + attribution_method: direct + neu_transport: + selection_fields: + ActivityProducedBy: {'324110': ''} + ActivityConsumedBy: *transport_consumers + assign_fields: + FlowName: Transport + Description: non energy + # Annex A-10 = 2018 energy consumption (TBtu) from EPA GHGI. + clean_source: + EPA_GHGI_T_A_10: + year: *mecs_year + selection_fields: + Unit: TBtu + ActivityConsumedBy: Residential + FlowName: + - Kerosene + - LPG (Propane) + - Distillate Fuel Oil + clean_parameter: + year: *mecs_year + pce: + extract_input: BEA_PCE + file: BEA Personal Consumption Expenditures by Major Type of Product_June27_2024.csv + skiprows: 3 + index_col: 1 + drop_columns: [Line] + line: Gasoline and other energy goods + propane_price: + extract_input: EIA_EnergyPrice + file: U.S._Propane_Residential_Price.csv + skiprows: 4 + month_column: Month + value_column: U.S. Propane Residential Price Dollars per Gallon + heating_oil_price: + extract_input: EIA_EnergyPrice + file: U.S._No._2_Heating_Oil_Residential_Price.csv + skiprows: 4 + month_column: Month + value_column: U.S. No. 2 Heating Oil Residential Price Dollars per Gallon + residential_heat_fuels: + propane_priced: [Kerosene, LPG (Propane)] + heating_oil_priced: [Distillate Fuel Oil] + clean_fba_after_attribution: !script_function:EIA_MECS scale_household_petroleum_to_transport_share + attribution_method: direct diff --git a/bedrock/transform/energy/Energy_Cornerstone_2022.yaml b/bedrock/transform/energy/Energy_Cornerstone_2022.yaml new file mode 100644 index 000000000..327022fa4 --- /dev/null +++ b/bedrock/transform/energy/Energy_Cornerstone_2022.yaml @@ -0,0 +1,203 @@ +# Manufacturing energy FBS for Cornerstone. MECS survey year 2022. +# Replicates CEDA allocation steps as quantity weights: MECS mapped with +# parent-incomplete-child + Cornerstone_2025, then nested BEA splits; +# petrol energy is BEA 324110 times MECS Other energy fraction. + +!include:Cornerstone_2025_target.yaml +year: &mecs_year 2022 +geoscale: national + +_attribution_sources: + BEA: &bea + year: 2017 + activity_to_sector_mapping: Cornerstone_2025 + exclusion_fields: + ActivityConsumedBy: ['F03000', 'F04000', 'F05000', 'F02E00', + 'F06E00', 'F07E00', 'F10E00', 'F02R00', + 'T001', 'T004', 'T007', 'T019'] + ActivityProducedBy: ['T007', 'T013', 'T015', 'T016', 'TOP', + 'MCIF'] + attribution_method: equal + transport_consumers: &transport_consumers + - '481000' + - '482000' + - '483000' + - '484000' + - '485000' + - '486000' + - '48A000' + - '492000' + - 'S00500' + - 'S00600' + - '491000' + - 'GSLGO' + - 'S00203' + - 'F01000' + +source_names: + EIA_MECS_Energy: + year: *mecs_year + activity_to_sector_mapping: Cornerstone_2025 + selection_fields: + Location: '00000' + Description: [Table 2.1, Table 3.1] + exclusion_fields: + ActivityConsumedBy: '31-33' + clean_fba_before_activity_sets: + - !script_function:EIA_MECS estimate_suppressed_mecs_energy + activity_sets: + coal: + selection_fields: + Description: + Table 3.1: energy + FlowName: Coal + attribution_method: proportional + attribution_source: + BEA_Detail_Use_AfterRedef: + <<: *bea + selection_fields: + ActivityProducedBy: {'212100': ''} + + ng: + selection_fields: + Description: + Table 3.1: energy + FlowName: 'Natural Gas' + attribution_method: proportional + attribution_source: + BEA_Detail_Use_AfterRedef: + <<: *bea + selection_fields: + ActivityProducedBy: {'221200': ''} + + neu_ng: + selection_fields: + Description: + Table 2.1: non energy + FlowName: 'Natural Gas' + clean_fba: !script_function:EIA_MECS keep_chemical_manufacturing + attribution_method: proportional + attribution_source: + BEA_Detail_Use_AfterRedef: + <<: *bea + selection_fields: + ActivityProducedBy: {'221200': ''} + + neu_petrol_asphalt: + selection_fields: + Description: + Table 2.1: non energy + FlowName: + Other: Petroleum + ActivityConsumedBy: ['324121', '324122'] + attribution_method: proportional + attribution_source: + BEA_Detail_Use_AfterRedef: + <<: *bea + selection_fields: + ActivityProducedBy: {'324110': ''} + + neu_petrol_hgl: + selection_fields: + Description: + Table 2.1: non energy + FlowName: + 'Hydrocarbon Gas Liquids, excluding natural gasoline': Petroleum + attribution_method: proportional + attribution_source: + BEA_Detail_Use_AfterRedef: + <<: *bea + selection_fields: + ActivityProducedBy: {'324110': ''} + + neu_petrol_other: + selection_fields: + Description: + Table 2.1: non energy + FlowName: + Other: Petroleum + exclusion_fields: + ActivityConsumedBy: ['324121', '324122'] + attribution_method: proportional + attribution_source: + BEA_Detail_Use_AfterRedef: + <<: *bea + selection_fields: + ActivityProducedBy: {'324110': ''} + + BEA_Detail_Use_AfterRedef: + <<: *bea + activity_sets: + petrol: + selection_fields: + ActivityProducedBy: {'324110': ''} + exclusion_fields: + ActivityConsumedBy: *transport_consumers + assign_fields: + FlowName: Petroleum + Description: energy + clean_source: + EIA_MECS_Energy: + year: *mecs_year + selection_fields: + Location: '00000' + FlowName: Other + Description: [Table 2.1, Table 3.1] + exclusion_fields: + ActivityConsumedBy: '31-33' + estimate_suppressed: !script_function:EIA_MECS estimate_suppressed_mecs_energy + clean_fba_after_attribution: !script_function:EIA_MECS multiply_bea_by_mecs_petroleum_energy_fraction + attribution_method: direct + neu_coal_coke: + selection_fields: + ActivityConsumedBy: '2122A0' + exclusion_fields: + ActivityProducedBy: '324110' + assign_fields: + FlowName: Coal and Coke + Description: non energy + attribution_method: direct + neu_transport: + selection_fields: + ActivityProducedBy: {'324110': ''} + ActivityConsumedBy: *transport_consumers + assign_fields: + FlowName: Transport + Description: non energy + # Annex A-6 = 2022 energy consumption (TBtu) from EPA GHGI. + clean_source: + EPA_GHGI_T_A_6: + year: *mecs_year + selection_fields: + Unit: TBtu + ActivityConsumedBy: Residential + FlowName: + - Kerosene + - LPG (Propane) + - Distillate Fuel Oil + clean_parameter: + year: *mecs_year + pce: + extract_input: BEA_PCE + file: BEA Personal Consumption Expenditures by Major Type of Product_June27_2024.csv + skiprows: 3 + index_col: 1 + drop_columns: [Line] + line: Gasoline and other energy goods + propane_price: + extract_input: EIA_EnergyPrice + file: U.S._Propane_Residential_Price.csv + skiprows: 4 + month_column: Month + value_column: U.S. Propane Residential Price Dollars per Gallon + heating_oil_price: + extract_input: EIA_EnergyPrice + file: U.S._No._2_Heating_Oil_Residential_Price.csv + skiprows: 4 + month_column: Month + value_column: U.S. No. 2 Heating Oil Residential Price Dollars per Gallon + residential_heat_fuels: + propane_priced: [Kerosene, LPG (Propane)] + heating_oil_priced: [Distillate Fuel Oil] + clean_fba_after_attribution: !script_function:EIA_MECS scale_household_petroleum_to_transport_share + attribution_method: direct From 00d7c053e4181b217e4fda94ac3bc00ea68cafb9 Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Fri, 21 Aug 2026 17:20:50 -0600 Subject: [PATCH 03/18] replace EIA_MECS_Energy_Allocation_CEDA with Energy FBS call --- .../ghg/GHG_national_Cornerstone_2017.yaml | 42 +++++++++--------- .../ghg/GHG_national_Cornerstone_2018.yaml | 42 +++++++++--------- .../ghg/GHG_national_Cornerstone_2019.yaml | 43 ++++++++++--------- .../ghg/GHG_national_Cornerstone_2020.yaml | 42 +++++++++--------- .../ghg/GHG_national_Cornerstone_2021.yaml | 43 ++++++++++--------- .../ghg/GHG_national_Cornerstone_2022.yaml | 43 ++++++++++--------- .../ghg/GHG_national_Cornerstone_2023.yaml | 43 ++++++++++--------- .../ghg/GHG_national_Cornerstone_2024.yaml | 43 ++++++++++--------- 8 files changed, 173 insertions(+), 168 deletions(-) diff --git a/bedrock/transform/ghg/GHG_national_Cornerstone_2017.yaml b/bedrock/transform/ghg/GHG_national_Cornerstone_2017.yaml index abab0007b..abf9accdd 100644 --- a/bedrock/transform/ghg/GHG_national_Cornerstone_2017.yaml +++ b/bedrock/transform/ghg/GHG_national_Cornerstone_2017.yaml @@ -25,9 +25,9 @@ sources_to_cache: estimate_suppressed: !clean_function:flowbyclean estimate_suppressed_sectors_equal_attribution _attribution_sources: - EIA_MECS_Energy_Allocation_CEDA: &mecs_energy_alloc - activity_to_sector_mapping: CEDA_2025 - year: *mecs_year + # Nested EPA_GHGI_T_3_25 below is inventory, not UMD T_3_25 (petroleum-systems CH4). + Energy_Cornerstone_2018: &mecs_energy_alloc + data_format: FBS geoscale: national BEA: &bea # 2017 Make and Use tables @@ -586,11 +586,11 @@ source_names: - Petroleum Industrial attribution_method: proportional attribution_source: - EIA_MECS_Energy_Allocation_CEDA: + Energy_Cornerstone_2018: <<: *mecs_energy_alloc selection_fields: - FlowName: Petroleum - Description: 'energy' + Flowable: Petroleum + Class: Money natural_gas_nonmanufacturing: selection_fields: @@ -609,11 +609,11 @@ source_names: - Natural Gas Industrial - Manufacturing attribution_method: proportional attribution_source: - EIA_MECS_Energy_Allocation_CEDA: + Energy_Cornerstone_2018: <<: *mecs_energy_alloc selection_fields: - FlowName: Natural Gas - Description: "energy" + Flowable: Natural Gas + Class: Energy coal_nonmanufacturing: # empty in some years (i.e., all coal consumption for manufacturing) selection_fields: @@ -632,11 +632,11 @@ source_names: - Coal Industrial - Manufacturing attribution_method: proportional attribution_source: - EIA_MECS_Energy_Allocation_CEDA: + Energy_Cornerstone_2018: <<: *mecs_energy_alloc selection_fields: - FlowName: Coal - Description: 'energy' + Flowable: Coal + Class: Energy # Intentionally left out 'Wood Commercial' and 'Wood Industrial' @@ -720,33 +720,33 @@ source_names: Industry HGL: Industry Petroleum Products Non-energy attribution_method: proportional attribution_source: - EIA_MECS_Energy_Allocation_CEDA: + Energy_Cornerstone_2018: <<: *mecs_energy_alloc selection_fields: - FlowName: Petroleum - Description: "non energy" + Flowable: Petroleum + Class: Other natural_gas_neu: selection_fields: PrimaryActivity: Industry Natural Gas to Chemical Plants attribution_method: proportional attribution_source: - EIA_MECS_Energy_Allocation_CEDA: + Energy_Cornerstone_2018: <<: *mecs_energy_alloc selection_fields: - FlowName: Natural Gas - Description: "non energy" + Flowable: Natural Gas + Class: Other transportation_lubricants: selection_fields: PrimaryActivity: Transportation Lubricants attribution_method: proportional attribution_source: - EIA_MECS_Energy_Allocation_CEDA: + Energy_Cornerstone_2018: <<: *mecs_energy_alloc selection_fields: - FlowName: Transport - Description: "non energy" + Flowable: Transport + Class: Money ## Other Emissions UMD_GHGIA_T_4_33: # HFCs from HCFC-22 production diff --git a/bedrock/transform/ghg/GHG_national_Cornerstone_2018.yaml b/bedrock/transform/ghg/GHG_national_Cornerstone_2018.yaml index 47d51d3a2..da7434721 100644 --- a/bedrock/transform/ghg/GHG_national_Cornerstone_2018.yaml +++ b/bedrock/transform/ghg/GHG_national_Cornerstone_2018.yaml @@ -25,9 +25,9 @@ sources_to_cache: estimate_suppressed: !clean_function:flowbyclean estimate_suppressed_sectors_equal_attribution _attribution_sources: - EIA_MECS_Energy_Allocation_CEDA: &mecs_energy_alloc - activity_to_sector_mapping: CEDA_2025 - year: *mecs_year + # Nested EPA_GHGI_T_3_25 below is inventory, not UMD T_3_25 (petroleum-systems CH4). + Energy_Cornerstone_2018: &mecs_energy_alloc + data_format: FBS geoscale: national BEA: &bea # 2017 Make and Use tables @@ -586,11 +586,11 @@ source_names: - Petroleum Industrial attribution_method: proportional attribution_source: - EIA_MECS_Energy_Allocation_CEDA: + Energy_Cornerstone_2018: <<: *mecs_energy_alloc selection_fields: - FlowName: Petroleum - Description: 'energy' + Flowable: Petroleum + Class: Money natural_gas_nonmanufacturing: selection_fields: @@ -609,11 +609,11 @@ source_names: - Natural Gas Industrial - Manufacturing attribution_method: proportional attribution_source: - EIA_MECS_Energy_Allocation_CEDA: + Energy_Cornerstone_2018: <<: *mecs_energy_alloc selection_fields: - FlowName: Natural Gas - Description: "energy" + Flowable: Natural Gas + Class: Energy coal_nonmanufacturing: # empty in some years (i.e., all coal consumption for manufacturing) selection_fields: @@ -632,11 +632,11 @@ source_names: - Coal Industrial - Manufacturing attribution_method: proportional attribution_source: - EIA_MECS_Energy_Allocation_CEDA: + Energy_Cornerstone_2018: <<: *mecs_energy_alloc selection_fields: - FlowName: Coal - Description: 'energy' + Flowable: Coal + Class: Energy # Intentionally left out 'Wood Commercial' and 'Wood Industrial' @@ -720,33 +720,33 @@ source_names: Industry HGL: Industry Petroleum Products Non-energy attribution_method: proportional attribution_source: - EIA_MECS_Energy_Allocation_CEDA: + Energy_Cornerstone_2018: <<: *mecs_energy_alloc selection_fields: - FlowName: Petroleum - Description: "non energy" + Flowable: Petroleum + Class: Other natural_gas_neu: selection_fields: PrimaryActivity: Industry Natural Gas to Chemical Plants attribution_method: proportional attribution_source: - EIA_MECS_Energy_Allocation_CEDA: + Energy_Cornerstone_2018: <<: *mecs_energy_alloc selection_fields: - FlowName: Natural Gas - Description: "non energy" + Flowable: Natural Gas + Class: Other transportation_lubricants: selection_fields: PrimaryActivity: Transportation Lubricants attribution_method: proportional attribution_source: - EIA_MECS_Energy_Allocation_CEDA: + Energy_Cornerstone_2018: <<: *mecs_energy_alloc selection_fields: - FlowName: Transport - Description: "non energy" + Flowable: Transport + Class: Money ## Other Emissions UMD_GHGIA_T_4_33: # HFCs from HCFC-22 production diff --git a/bedrock/transform/ghg/GHG_national_Cornerstone_2019.yaml b/bedrock/transform/ghg/GHG_national_Cornerstone_2019.yaml index 7b5bef271..ca78b4883 100644 --- a/bedrock/transform/ghg/GHG_national_Cornerstone_2019.yaml +++ b/bedrock/transform/ghg/GHG_national_Cornerstone_2019.yaml @@ -22,9 +22,9 @@ sources_to_cache: estimate_suppressed: !clean_function:flowbyclean estimate_suppressed_sectors_equal_attribution _attribution_sources: - EIA_MECS_Energy_Allocation_CEDA: &mecs_energy_alloc - activity_to_sector_mapping: CEDA_2025 - year: *mecs_year + # Nested EPA_GHGI_T_3_25 below is inventory, not UMD T_3_25 (petroleum-systems CH4). + Energy_Cornerstone_2018: &mecs_energy_alloc + data_format: FBS geoscale: national BEA: &bea # 2017 Make and Use tables @@ -583,11 +583,11 @@ source_names: - Petroleum Industrial attribution_method: proportional attribution_source: - EIA_MECS_Energy_Allocation_CEDA: + Energy_Cornerstone_2018: <<: *mecs_energy_alloc selection_fields: - FlowName: Petroleum - Description: 'energy' + Flowable: Petroleum + Class: Money natural_gas_nonmanufacturing: selection_fields: @@ -606,11 +606,11 @@ source_names: - Natural Gas Industrial - Manufacturing attribution_method: proportional attribution_source: - EIA_MECS_Energy_Allocation_CEDA: + Energy_Cornerstone_2018: <<: *mecs_energy_alloc selection_fields: - FlowName: Natural Gas - Description: "energy" + Flowable: Natural Gas + Class: Energy coal_nonmanufacturing: # empty in some years (i.e., all coal consumption for manufacturing) selection_fields: @@ -629,11 +629,11 @@ source_names: - Coal Industrial - Manufacturing attribution_method: proportional attribution_source: - EIA_MECS_Energy_Allocation_CEDA: + Energy_Cornerstone_2018: <<: *mecs_energy_alloc selection_fields: - FlowName: Coal - Description: 'energy' + Flowable: Coal + Class: Energy # Intentionally left out 'Wood Commercial' and 'Wood Industrial' @@ -717,33 +717,34 @@ source_names: Industry HGL: Industry Petroleum Products Non-energy attribution_method: proportional attribution_source: - EIA_MECS_Energy_Allocation_CEDA: + Energy_Cornerstone_2018: <<: *mecs_energy_alloc selection_fields: - FlowName: Petroleum - Description: "non energy" + Flowable: Petroleum + Class: Other + natural_gas_neu: selection_fields: PrimaryActivity: Industry Natural Gas to Chemical Plants attribution_method: proportional attribution_source: - EIA_MECS_Energy_Allocation_CEDA: + Energy_Cornerstone_2018: <<: *mecs_energy_alloc selection_fields: - FlowName: Natural Gas - Description: "non energy" + Flowable: Natural Gas + Class: Other transportation_lubricants: selection_fields: PrimaryActivity: Transportation Lubricants attribution_method: proportional attribution_source: - EIA_MECS_Energy_Allocation_CEDA: + Energy_Cornerstone_2018: <<: *mecs_energy_alloc selection_fields: - FlowName: Transport - Description: "non energy" + Flowable: Transport + Class: Money ## Other Emissions UMD_GHGIA_T_4_33: # HFCs from HCFC-22 production diff --git a/bedrock/transform/ghg/GHG_national_Cornerstone_2020.yaml b/bedrock/transform/ghg/GHG_national_Cornerstone_2020.yaml index 4fcae51be..2def398c6 100644 --- a/bedrock/transform/ghg/GHG_national_Cornerstone_2020.yaml +++ b/bedrock/transform/ghg/GHG_national_Cornerstone_2020.yaml @@ -22,9 +22,9 @@ sources_to_cache: estimate_suppressed: !clean_function:flowbyclean estimate_suppressed_sectors_equal_attribution _attribution_sources: - EIA_MECS_Energy_Allocation_CEDA: &mecs_energy_alloc - activity_to_sector_mapping: CEDA_2025 - year: *mecs_year + # Nested EPA_GHGI_T_3_25 below is inventory, not UMD T_3_25 (petroleum-systems CH4). + Energy_Cornerstone_2018: &mecs_energy_alloc + data_format: FBS geoscale: national BEA: &bea # 2017 Make and Use tables @@ -583,11 +583,11 @@ source_names: - Petroleum Industrial attribution_method: proportional attribution_source: - EIA_MECS_Energy_Allocation_CEDA: + Energy_Cornerstone_2018: <<: *mecs_energy_alloc selection_fields: - FlowName: Petroleum - Description: 'energy' + Flowable: Petroleum + Class: Money natural_gas_nonmanufacturing: selection_fields: @@ -606,11 +606,11 @@ source_names: - Natural Gas Industrial - Manufacturing attribution_method: proportional attribution_source: - EIA_MECS_Energy_Allocation_CEDA: + Energy_Cornerstone_2018: <<: *mecs_energy_alloc selection_fields: - FlowName: Natural Gas - Description: "energy" + Flowable: Natural Gas + Class: Energy coal_nonmanufacturing: # empty in some years (i.e., all coal consumption for manufacturing) selection_fields: @@ -629,11 +629,11 @@ source_names: - Coal Industrial - Manufacturing attribution_method: proportional attribution_source: - EIA_MECS_Energy_Allocation_CEDA: + Energy_Cornerstone_2018: <<: *mecs_energy_alloc selection_fields: - FlowName: Coal - Description: 'energy' + Flowable: Coal + Class: Energy # Intentionally left out 'Wood Commercial' and 'Wood Industrial' @@ -717,33 +717,33 @@ source_names: Industry HGL: Industry Petroleum Products Non-energy attribution_method: proportional attribution_source: - EIA_MECS_Energy_Allocation_CEDA: + Energy_Cornerstone_2018: <<: *mecs_energy_alloc selection_fields: - FlowName: Petroleum - Description: "non energy" + Flowable: Petroleum + Class: Other natural_gas_neu: selection_fields: PrimaryActivity: Industry Natural Gas to Chemical Plants attribution_method: proportional attribution_source: - EIA_MECS_Energy_Allocation_CEDA: + Energy_Cornerstone_2018: <<: *mecs_energy_alloc selection_fields: - FlowName: Natural Gas - Description: "non energy" + Flowable: Natural Gas + Class: Other transportation_lubricants: selection_fields: PrimaryActivity: Transportation Lubricants attribution_method: proportional attribution_source: - EIA_MECS_Energy_Allocation_CEDA: + Energy_Cornerstone_2018: <<: *mecs_energy_alloc selection_fields: - FlowName: Transport - Description: "non energy" + Flowable: Transport + Class: Money ## Other Emissions UMD_GHGIA_T_4_33: # HFCs from HCFC-22 production diff --git a/bedrock/transform/ghg/GHG_national_Cornerstone_2021.yaml b/bedrock/transform/ghg/GHG_national_Cornerstone_2021.yaml index 0d62079c7..219b5dfb0 100644 --- a/bedrock/transform/ghg/GHG_national_Cornerstone_2021.yaml +++ b/bedrock/transform/ghg/GHG_national_Cornerstone_2021.yaml @@ -22,9 +22,9 @@ sources_to_cache: estimate_suppressed: !clean_function:flowbyclean estimate_suppressed_sectors_equal_attribution _attribution_sources: - EIA_MECS_Energy_Allocation_CEDA: &mecs_energy_alloc - activity_to_sector_mapping: CEDA_2025 - year: *mecs_year + # Nested EPA_GHGI_T_3_25 below is inventory, not UMD T_3_25 (petroleum-systems CH4). + Energy_Cornerstone_2022: &mecs_energy_alloc + data_format: FBS geoscale: national BEA: &bea # 2017 Make and Use tables @@ -583,11 +583,11 @@ source_names: - Petroleum Industrial attribution_method: proportional attribution_source: - EIA_MECS_Energy_Allocation_CEDA: + Energy_Cornerstone_2022: <<: *mecs_energy_alloc selection_fields: - FlowName: Petroleum - Description: 'energy' + Flowable: Petroleum + Class: Money natural_gas_nonmanufacturing: selection_fields: @@ -606,11 +606,11 @@ source_names: - Natural Gas Industrial - Manufacturing attribution_method: proportional attribution_source: - EIA_MECS_Energy_Allocation_CEDA: + Energy_Cornerstone_2022: <<: *mecs_energy_alloc selection_fields: - FlowName: Natural Gas - Description: "energy" + Flowable: Natural Gas + Class: Energy coal_nonmanufacturing: # empty in some years (i.e., all coal consumption for manufacturing) selection_fields: @@ -629,11 +629,11 @@ source_names: - Coal Industrial - Manufacturing attribution_method: proportional attribution_source: - EIA_MECS_Energy_Allocation_CEDA: + Energy_Cornerstone_2022: <<: *mecs_energy_alloc selection_fields: - FlowName: Coal - Description: 'energy' + Flowable: Coal + Class: Energy # Intentionally left out 'Wood Commercial' and 'Wood Industrial' @@ -717,33 +717,34 @@ source_names: Industry HGL: Industry Petroleum Products Non-energy attribution_method: proportional attribution_source: - EIA_MECS_Energy_Allocation_CEDA: + Energy_Cornerstone_2022: <<: *mecs_energy_alloc selection_fields: - FlowName: Petroleum - Description: "non energy" + Flowable: Petroleum + Class: Other + natural_gas_neu: selection_fields: PrimaryActivity: Industry Natural Gas to Chemical Plants attribution_method: proportional attribution_source: - EIA_MECS_Energy_Allocation_CEDA: + Energy_Cornerstone_2022: <<: *mecs_energy_alloc selection_fields: - FlowName: Natural Gas - Description: "non energy" + Flowable: Natural Gas + Class: Other transportation_lubricants: selection_fields: PrimaryActivity: Transportation Lubricants attribution_method: proportional attribution_source: - EIA_MECS_Energy_Allocation_CEDA: + Energy_Cornerstone_2022: <<: *mecs_energy_alloc selection_fields: - FlowName: Transport - Description: "non energy" + Flowable: Transport + Class: Money ## Other Emissions UMD_GHGIA_T_4_33: # HFCs from HCFC-22 production diff --git a/bedrock/transform/ghg/GHG_national_Cornerstone_2022.yaml b/bedrock/transform/ghg/GHG_national_Cornerstone_2022.yaml index cecc98153..755bde6c9 100644 --- a/bedrock/transform/ghg/GHG_national_Cornerstone_2022.yaml +++ b/bedrock/transform/ghg/GHG_national_Cornerstone_2022.yaml @@ -22,9 +22,9 @@ sources_to_cache: estimate_suppressed: !clean_function:flowbyclean estimate_suppressed_sectors_equal_attribution _attribution_sources: - EIA_MECS_Energy_Allocation_CEDA: &mecs_energy_alloc - activity_to_sector_mapping: CEDA_2025 - year: *mecs_year + # Nested EPA_GHGI_T_3_25 below is inventory, not UMD T_3_25 (petroleum-systems CH4). + Energy_Cornerstone_2022: &mecs_energy_alloc + data_format: FBS geoscale: national BEA: &bea # 2017 Make and Use tables @@ -583,11 +583,11 @@ source_names: - Petroleum Industrial attribution_method: proportional attribution_source: - EIA_MECS_Energy_Allocation_CEDA: + Energy_Cornerstone_2022: <<: *mecs_energy_alloc selection_fields: - FlowName: Petroleum - Description: 'energy' + Flowable: Petroleum + Class: Money natural_gas_nonmanufacturing: selection_fields: @@ -606,11 +606,11 @@ source_names: - Natural Gas Industrial - Manufacturing attribution_method: proportional attribution_source: - EIA_MECS_Energy_Allocation_CEDA: + Energy_Cornerstone_2022: <<: *mecs_energy_alloc selection_fields: - FlowName: Natural Gas - Description: "energy" + Flowable: Natural Gas + Class: Energy coal_nonmanufacturing: # empty in some years (i.e., all coal consumption for manufacturing) selection_fields: @@ -629,11 +629,11 @@ source_names: - Coal Industrial - Manufacturing attribution_method: proportional attribution_source: - EIA_MECS_Energy_Allocation_CEDA: + Energy_Cornerstone_2022: <<: *mecs_energy_alloc selection_fields: - FlowName: Coal - Description: 'energy' + Flowable: Coal + Class: Energy # Intentionally left out 'Wood Commercial' and 'Wood Industrial' @@ -717,33 +717,34 @@ source_names: Industry HGL: Industry Petroleum Products Non-energy attribution_method: proportional attribution_source: - EIA_MECS_Energy_Allocation_CEDA: + Energy_Cornerstone_2022: <<: *mecs_energy_alloc selection_fields: - FlowName: Petroleum - Description: "non energy" + Flowable: Petroleum + Class: Other + natural_gas_neu: selection_fields: PrimaryActivity: Industry Natural Gas to Chemical Plants attribution_method: proportional attribution_source: - EIA_MECS_Energy_Allocation_CEDA: + Energy_Cornerstone_2022: <<: *mecs_energy_alloc selection_fields: - FlowName: Natural Gas - Description: "non energy" + Flowable: Natural Gas + Class: Other transportation_lubricants: selection_fields: PrimaryActivity: Transportation Lubricants attribution_method: proportional attribution_source: - EIA_MECS_Energy_Allocation_CEDA: + Energy_Cornerstone_2022: <<: *mecs_energy_alloc selection_fields: - FlowName: Transport - Description: "non energy" + Flowable: Transport + Class: Money ## Other Emissions UMD_GHGIA_T_4_33: # HFCs from HCFC-22 production diff --git a/bedrock/transform/ghg/GHG_national_Cornerstone_2023.yaml b/bedrock/transform/ghg/GHG_national_Cornerstone_2023.yaml index 774986303..aa44d281e 100644 --- a/bedrock/transform/ghg/GHG_national_Cornerstone_2023.yaml +++ b/bedrock/transform/ghg/GHG_national_Cornerstone_2023.yaml @@ -22,9 +22,9 @@ sources_to_cache: estimate_suppressed: !clean_function:flowbyclean estimate_suppressed_sectors_equal_attribution _attribution_sources: - EIA_MECS_Energy_Allocation_CEDA: &mecs_energy_alloc - activity_to_sector_mapping: CEDA_2025 - year: *mecs_year + # Nested EPA_GHGI_T_3_25 below is inventory, not UMD T_3_25 (petroleum-systems CH4). + Energy_Cornerstone_2022: &mecs_energy_alloc + data_format: FBS geoscale: national BEA: &bea # 2017 Make and Use tables @@ -583,11 +583,11 @@ source_names: - Petroleum Industrial attribution_method: proportional attribution_source: - EIA_MECS_Energy_Allocation_CEDA: + Energy_Cornerstone_2022: <<: *mecs_energy_alloc selection_fields: - FlowName: Petroleum - Description: 'energy' + Flowable: Petroleum + Class: Money natural_gas_nonmanufacturing: selection_fields: @@ -606,11 +606,11 @@ source_names: - Natural Gas Industrial - Manufacturing attribution_method: proportional attribution_source: - EIA_MECS_Energy_Allocation_CEDA: + Energy_Cornerstone_2022: <<: *mecs_energy_alloc selection_fields: - FlowName: Natural Gas - Description: "energy" + Flowable: Natural Gas + Class: Energy coal_nonmanufacturing: # empty in some years (i.e., all coal consumption for manufacturing) selection_fields: @@ -629,11 +629,11 @@ source_names: - Coal Industrial - Manufacturing attribution_method: proportional attribution_source: - EIA_MECS_Energy_Allocation_CEDA: + Energy_Cornerstone_2022: <<: *mecs_energy_alloc selection_fields: - FlowName: Coal - Description: 'energy' + Flowable: Coal + Class: Energy # Intentionally left out 'Wood Commercial' and 'Wood Industrial' @@ -717,33 +717,34 @@ source_names: Industry HGL: Industry Petroleum Products Non-energy attribution_method: proportional attribution_source: - EIA_MECS_Energy_Allocation_CEDA: + Energy_Cornerstone_2022: <<: *mecs_energy_alloc selection_fields: - FlowName: Petroleum - Description: "non energy" + Flowable: Petroleum + Class: Other + natural_gas_neu: selection_fields: PrimaryActivity: Industry Natural Gas to Chemical Plants attribution_method: proportional attribution_source: - EIA_MECS_Energy_Allocation_CEDA: + Energy_Cornerstone_2022: <<: *mecs_energy_alloc selection_fields: - FlowName: Natural Gas - Description: "non energy" + Flowable: Natural Gas + Class: Other transportation_lubricants: selection_fields: PrimaryActivity: Transportation Lubricants attribution_method: proportional attribution_source: - EIA_MECS_Energy_Allocation_CEDA: + Energy_Cornerstone_2022: <<: *mecs_energy_alloc selection_fields: - FlowName: Transport - Description: "non energy" + Flowable: Transport + Class: Money ## Other Emissions UMD_GHGIA_T_4_33: # HFCs from HCFC-22 production diff --git a/bedrock/transform/ghg/GHG_national_Cornerstone_2024.yaml b/bedrock/transform/ghg/GHG_national_Cornerstone_2024.yaml index a06afde12..34d0d29b5 100644 --- a/bedrock/transform/ghg/GHG_national_Cornerstone_2024.yaml +++ b/bedrock/transform/ghg/GHG_national_Cornerstone_2024.yaml @@ -22,9 +22,9 @@ sources_to_cache: estimate_suppressed: !clean_function:flowbyclean estimate_suppressed_sectors_equal_attribution _attribution_sources: - EIA_MECS_Energy_Allocation_CEDA: &mecs_energy_alloc - activity_to_sector_mapping: CEDA_2025 - year: *mecs_year + # Nested EPA_GHGI_T_3_25 below is inventory, not UMD T_3_25 (petroleum-systems CH4). + Energy_Cornerstone_2022: &mecs_energy_alloc + data_format: FBS geoscale: national BEA: &bea # 2017 Make and Use tables @@ -583,11 +583,11 @@ source_names: - Petroleum Industrial attribution_method: proportional attribution_source: - EIA_MECS_Energy_Allocation_CEDA: + Energy_Cornerstone_2022: <<: *mecs_energy_alloc selection_fields: - FlowName: Petroleum - Description: 'energy' + Flowable: Petroleum + Class: Money natural_gas_nonmanufacturing: selection_fields: @@ -606,11 +606,11 @@ source_names: - Natural Gas Industrial - Manufacturing attribution_method: proportional attribution_source: - EIA_MECS_Energy_Allocation_CEDA: + Energy_Cornerstone_2022: <<: *mecs_energy_alloc selection_fields: - FlowName: Natural Gas - Description: "energy" + Flowable: Natural Gas + Class: Energy coal_nonmanufacturing: # empty in some years (i.e., all coal consumption for manufacturing) selection_fields: @@ -629,11 +629,11 @@ source_names: - Coal Industrial - Manufacturing attribution_method: proportional attribution_source: - EIA_MECS_Energy_Allocation_CEDA: + Energy_Cornerstone_2022: <<: *mecs_energy_alloc selection_fields: - FlowName: Coal - Description: 'energy' + Flowable: Coal + Class: Energy # Intentionally left out 'Wood Commercial' and 'Wood Industrial' @@ -717,33 +717,34 @@ source_names: Industry HGL: Industry Petroleum Products Non-energy attribution_method: proportional attribution_source: - EIA_MECS_Energy_Allocation_CEDA: + Energy_Cornerstone_2022: <<: *mecs_energy_alloc selection_fields: - FlowName: Petroleum - Description: "non energy" + Flowable: Petroleum + Class: Other + natural_gas_neu: selection_fields: PrimaryActivity: Industry Natural Gas to Chemical Plants attribution_method: proportional attribution_source: - EIA_MECS_Energy_Allocation_CEDA: + Energy_Cornerstone_2022: <<: *mecs_energy_alloc selection_fields: - FlowName: Natural Gas - Description: "non energy" + Flowable: Natural Gas + Class: Other transportation_lubricants: selection_fields: PrimaryActivity: Transportation Lubricants attribution_method: proportional attribution_source: - EIA_MECS_Energy_Allocation_CEDA: + Energy_Cornerstone_2022: <<: *mecs_energy_alloc selection_fields: - FlowName: Transport - Description: "non energy" + Flowable: Transport + Class: Money ## Other Emissions UMD_GHGIA_T_4_33: # HFCs from HCFC-22 production From 85bcbde847d20db4d157d889a7af7075f91652e7 Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Thu, 27 Aug 2026 13:48:37 -0600 Subject: [PATCH 04/18] updates to manufacturing energy fbs approach to replicate ceda approach --- bedrock/extract/eia/EIA_MECS.py | 190 ++++++------------ ...> Energy_manufacturing_national_2018.yaml} | 141 +++++-------- ...> Energy_manufacturing_national_2022.yaml} | 138 +++++-------- 3 files changed, 150 insertions(+), 319 deletions(-) rename bedrock/transform/energy/{Energy_Cornerstone_2018.yaml => Energy_manufacturing_national_2018.yaml} (60%) rename bedrock/transform/energy/{Energy_Cornerstone_2022.yaml => Energy_manufacturing_national_2022.yaml} (61%) diff --git a/bedrock/extract/eia/EIA_MECS.py b/bedrock/extract/eia/EIA_MECS.py index 2d11593a2..a9d5d9fe3 100644 --- a/bedrock/extract/eia/EIA_MECS.py +++ b/bedrock/extract/eia/EIA_MECS.py @@ -18,10 +18,7 @@ from bedrock.extract.eia.EIA_CBECS_Land import calculate_total_facility_land_area from bedrock.extract.flowbyactivity import FlowByActivity, getFlowByActivity from bedrock.extract.generateflowbyactivity import generateFlowByActivity -from bedrock.transform.flowbyclean import ( - define_parentincompletechild_descendants, - load_prepare_clean_source, -) +from bedrock.transform.flowbyclean import load_prepare_clean_source from bedrock.transform.flowbyfunctions import assign_fips_location_system from bedrock.utils.config.common import WITHDRAWN_KEYWORD, get_catalog_info from bedrock.utils.economic.units import ( @@ -672,61 +669,50 @@ def keep_chemical_manufacturing(fba: FlowByActivity, **_kwargs: Any) -> FlowByAc return fba.query("ActivityConsumedBy.str.startswith('325')") +# 2013 MECS Other energy / (energy + nonfuel) when the mapped NAICS has no +# Table 2.1 / 3.1 Other ratio. Keys are MECS NAICS prefixes. +PETROL_FUEL_RATIO_FALLBACKS: dict[str, float] = { + '311221': 0.93, + '324110': 0.43, + '324199': 0.45, + '324121': 0.45, + '324122': 0.45, + '325110': 0.95, + '325180': 0.91, + '325194': 0.76, + '325211': 0.99, + '325212': 0.84, + '325311': 0.50, + '327211': 0.00, + '327310': 0.98, + '331110': 0.96, + '3312': 0.70, + '336': 0.83, +} + + def multiply_bea_by_mecs_petroleum_energy_fraction( fba: FlowByActivity, **_kwargs: Any ) -> FlowByActivity: ''' - clean_fba_after_attribution. CEDA industrial petrol weights are BEA 324110 + clean_fba_after_attribution. Industrial petrol weights are BEA 324110 purchases times MECS Other energy fraction Table 3.1 / (Table 2.1 + Table 3.1). - Sectors with no MECS ratio keep 1.0. Restricts to ag/mining/construction/ - manufacturing plus natural gas distribution. - - Loads MECS from config clean_source (FBA only — selection / estimate_suppressed - from YAML; does not run prepare_fbs). + Join on mapped NAICS (SectorConsumedBy after Cornerstone_2025), prefix-matching + published MECS lines. Sectors with no MECS ratio use 2013 fallback values, + else 1.0. Restricts to ag/mining/construction/manufacturing plus 221200. ''' - clean_source = fba.config.get('clean_source') - if not clean_source: - raise ValueError( - 'clean_source is required for multiply_bea_by_mecs_petroleum_energy_fraction' - ) + clean_source = fba.config['clean_source'] if isinstance(clean_source, str): name, src_config = clean_source, {} else: ((name, src_config),) = clean_source.items() - year = int( - src_config.get( - 'year', - (fba.config.get('clean_parameter') or {}).get( - 'year', fba.config.get('year') - ), - ) - ) + year = int(src_config['year']) mecs = FlowByActivity( getFlowByActivity(name, year), full_name=name, config={**get_catalog_info(name), **src_config, 'year': year}, ) - mecs = ( - mecs.function_socket('estimate_suppressed') - .select_by_fields() - .assign(Flowable=lambda x: x.FlowName) - ) - # Residualize each MECS table separately (define groups by Flowable only). - # define_parentincompletechild_descendants expects group_id / group_total. - residualized = [] - for desc in ['Table 2.1', 'Table 3.1']: - table = ( - mecs.query(f'Description == "{desc}"') - .drop(columns=['group_id', 'group_total'], errors='ignore') - .reset_index(drop=True) - ) - table = table.assign(group_id=table.index, group_total=table.FlowAmount) - residualized.append(define_parentincompletechild_descendants(table)) - mecs = FlowByActivity( - pd.concat(residualized), - full_name=mecs.full_name, - config=mecs.config, - ) + mecs = mecs.function_socket('estimate_suppressed').select_by_fields() t21 = ( mecs.query("Description == 'Table 2.1'") .groupby('ActivityConsumedBy')['FlowAmount'] @@ -743,27 +729,25 @@ def multiply_bea_by_mecs_petroleum_energy_fraction( den = t21 + t31 ratios = (t31 / den).where(den != 0).fillna(1.0) - activity = fba['ActivityConsumedBy'].astype(str) - industrial = activity.str.startswith( - ('11', '21', '23', '31', '32', '33') - ) | activity.eq('221200') - out = fba.loc[industrial].copy() - if out.empty: - log.warning('No industrial BEA 324110 rows left to apply MECS petrol fraction') - return out - sector_col = ( 'SectorConsumedBy' - if 'SectorConsumedBy' in out.columns + if 'SectorConsumedBy' in fba.columns else 'ActivityConsumedBy' ) + sector = fba[sector_col].astype(str) + industrial = sector.str.startswith( + ('11', '21', '23', '31', '32', '33') + ) | sector.eq('221200') + out = fba.loc[industrial].copy() ratio_index = set(ratios.index.astype(str)) - def ratio_for_sector(sector: str) -> float: - for n in range(len(sector), 1, -1): - key = sector[:n] + def ratio_for_sector(naics: str) -> float: + for n in range(len(naics), 0, -1): + key = naics[:n] if key in ratio_index: return float(ratios[key]) + if key in PETROL_FUEL_RATIO_FALLBACKS: + return PETROL_FUEL_RATIO_FALLBACKS[key] return 1.0 matched = out[sector_col].astype(str).map(ratio_for_sector) @@ -774,26 +758,13 @@ def household_petroleum_transport_fraction(config: dict[str, Any]) -> float: ''' Share of PCE gasoline-and-other-energy that is transport rather than residential heat oil/propane. Same formula CEDA uses on F01000. - - All inputs come from config clean_source (GHGI annex FBA) and - clean_parameter (PCE + residential fuel prices + fuel price groups). ''' - clean_parameter = config.get('clean_parameter') or {} - year = int(clean_parameter.get('year', config.get('year'))) - - pce_cfg = clean_parameter.get('pce') - propane_cfg = clean_parameter.get('propane_price') - heat_oil_cfg = clean_parameter.get('heating_oil_price') - heat_fuels = clean_parameter.get('residential_heat_fuels') - if not all((pce_cfg, propane_cfg, heat_oil_cfg, heat_fuels)): - raise ValueError( - 'clean_parameter must define pce, propane_price, ' - 'heating_oil_price, and residential_heat_fuels' - ) - assert pce_cfg is not None - assert propane_cfg is not None - assert heat_oil_cfg is not None - assert heat_fuels is not None + clean_parameter = config['clean_parameter'] + year = int(clean_parameter.get('year', config['year'])) + pce_cfg = clean_parameter['pce'] + propane_cfg = clean_parameter['propane_price'] + heat_oil_cfg = clean_parameter['heating_oil_price'] + heat_fuels = clean_parameter['residential_heat_fuels'] pce_tbl = load_from_gcs( name=pce_cfg['file'], @@ -809,12 +780,7 @@ def household_petroleum_transport_fraction(config: dict[str, Any]) -> float: ) pce_tbl.index = pce_tbl.index.str.strip() pce_tbl.columns = pce_tbl.columns.astype(int) - pce_years = [c for c in pce_tbl.columns if c <= year] or list(pce_tbl.columns) - pce_year = int(max(pce_years)) - if pce_year != year: - log.warning( - f'BEA PCE has no {year} gasoline-and-other-energy; using {pce_year}' - ) + pce_year = int(max(int(c) for c in pce_tbl.columns if int(c) <= year)) pce = float(pce_tbl.loc[pce_cfg['line'], pce_year]) prices: dict[str, float] = {} @@ -822,13 +788,12 @@ def household_petroleum_transport_fraction(config: dict[str, Any]) -> float: ('propane_price', propane_cfg), ('heating_oil_price', heat_oil_cfg), ): + skiprows = int(price_cfg.get('skiprows', 4)) price_tbl = load_from_gcs( name=price_cfg['file'], sub_bucket=gcs_extract_input_path(price_cfg['extract_input']), local_dir=local_extract_input_dir(price_cfg['extract_input']), - loader=lambda pth, rows=int(price_cfg.get('skiprows', 4)): pd.read_csv( - pth, skiprows=rows - ), + loader=lambda pth: pd.read_csv(pth, skiprows=skiprows), ) month_col = price_cfg.get('month_column', 'Month') value_col = price_cfg['value_column'] @@ -837,11 +802,7 @@ def household_petroleum_transport_fraction(config: dict[str, Any]) -> float: years = [y for y in by_year.index if y <= year] or list(by_year.index) prices[key] = float(by_year[int(max(years))]) - clean_source = config.get('clean_source') - if not clean_source: - raise ValueError( - 'clean_source is required for household_petroleum_transport_fraction' - ) + clean_source = config['clean_source'] if isinstance(clean_source, str): name, src_config = clean_source, {} else: @@ -853,20 +814,20 @@ def household_petroleum_transport_fraction(config: dict[str, Any]) -> float: config={**get_catalog_info(name), **src_config, 'year': annex_year}, ).select_by_fields() - propane_priced = list(heat_fuels['propane_priced']) - heating_oil_priced = list(heat_fuels['heating_oil_priced']) kerosene_lpg = float( - annex.loc[annex['FlowName'].isin(propane_priced), 'FlowAmount'].sum() + annex.loc[ + annex['FlowName'].isin(list(heat_fuels['propane_priced'])), 'FlowAmount' + ].sum() ) distillate = float( - annex.loc[annex['FlowName'].isin(heating_oil_priced), 'FlowAmount'].sum() + annex.loc[ + annex['FlowName'].isin(list(heat_fuels['heating_oil_priced'])), + 'FlowAmount', + ].sum() ) res_heat = (kerosene_lpg * (prices['propane_price'] / PROPANE_MMBTU_PER_GALLON)) + ( distillate * (prices['heating_oil_price'] / HEATING_OIL_MMBTU_PER_GALLON) ) - if pce == 0: - log.warning('PCE gasoline-and-other-energy is 0; F01000 transport share is 1.0') - return 1.0 return (pce - res_heat) / pce @@ -968,41 +929,6 @@ def mecs_land_fba_cleanup(fba: FlowByActivity, **_: Any) -> FlowByActivity: return fba -def clean_mecs_energy_fba_for_bea_summary( - fba: FlowByActivity, **_kwargs: Any -) -> FlowByActivity: - naics_3 = fba.query('ActivityConsumedBy.str.len() == 3') - naics_4 = fba.query( - 'ActivityConsumedBy.str.len() == 4 ' - '& ActivityConsumedBy.str.startswith("336")' - ) - naics_4_sum = ( - naics_4.assign(ActivityConsumedBy='336') - .aggregate_flowby()[['Flowable', 'FlowAmount', 'Unit', 'ActivityConsumedBy']] - .rename(columns={'FlowAmount': 'naics_4_sum'}) - ) - - merged = naics_3.merge(naics_4_sum, how='left').fillna({'naics_4_sum': 0}) - subtracted = merged.assign(FlowAmount=merged.FlowAmount - merged.naics_4_sum).drop( - columns='naics_4_sum' - ) - - subtracted.config['naics_4_list'] = list(naics_4.ActivityConsumedBy.unique()) - - return subtracted - - -def clean_mapped_mecs_energy_fba_for_bea_summary( - fba: FlowByActivity, **_kwargs: Any -) -> FlowByActivity: - _naics_4_list = fba.config['naics_4_list'] - - return fba.query( - '~(SectorConsumedBy in @_naics_4_list ' - '& ActivityConsumedBy != SectorConsumedBy)' - ) - - if __name__ == "__main__": generateFlowByActivity(source='EIA_MECS_Energy', year=2018) fba = getFlowByActivity('EIA_MECS_Energy', 2018) diff --git a/bedrock/transform/energy/Energy_Cornerstone_2018.yaml b/bedrock/transform/energy/Energy_manufacturing_national_2018.yaml similarity index 60% rename from bedrock/transform/energy/Energy_Cornerstone_2018.yaml rename to bedrock/transform/energy/Energy_manufacturing_national_2018.yaml index dcd655d12..a55370b23 100644 --- a/bedrock/transform/energy/Energy_Cornerstone_2018.yaml +++ b/bedrock/transform/energy/Energy_manufacturing_national_2018.yaml @@ -1,24 +1,29 @@ -# Manufacturing energy FBS for Cornerstone. MECS survey year 2018. +# Manufacturing energy FBS used in GHG Cornerstone FBS methods. +# MECS survey year 2018. # Replicates CEDA allocation steps as quantity weights: MECS mapped with -# parent-incomplete-child + Cornerstone_2025, then nested BEA splits; -# petrol energy is BEA 324110 times MECS Other energy fraction. +# parent-incomplete-child + Cornerstone_2025 !include:Cornerstone_2025_target.yaml year: &mecs_year 2018 geoscale: national _attribution_sources: - BEA: &bea - year: 2017 - activity_to_sector_mapping: Cornerstone_2025 - exclusion_fields: - ActivityConsumedBy: ['F03000', 'F04000', 'F05000', 'F02E00', - 'F06E00', 'F07E00', 'F10E00', 'F02R00', - 'T001', 'T004', 'T007', 'T019'] - ActivityProducedBy: ['T007', 'T013', 'T015', 'T016', 'TOP', - 'MCIF'] - attribution_method: equal - transport_consumers: &transport_consumers + transport_industries: &transport_industries + - '481000' + - '482000' + - '483000' + - '484000' + - '485000' + - '486000' + - '48A000' + - '492000' + - 'S00500' + - 'S00600' + - '491000' + - 'GSLGO' + - 'S00203' + - 'F01000' + petrol_exclude: &petrol_exclude - '481000' - '482000' - '483000' @@ -46,96 +51,45 @@ source_names: clean_fba_before_activity_sets: - !script_function:EIA_MECS estimate_suppressed_mecs_energy activity_sets: - coal: - selection_fields: - Description: - Table 3.1: energy - FlowName: Coal - attribution_method: proportional - attribution_source: - BEA_Detail_Use_AfterRedef: - <<: *bea - selection_fields: - ActivityProducedBy: {'212100': ''} - - ng: - selection_fields: - Description: - Table 3.1: energy - FlowName: 'Natural Gas' - attribution_method: proportional - attribution_source: - BEA_Detail_Use_AfterRedef: - <<: *bea - selection_fields: - ActivityProducedBy: {'221200': ''} - - neu_ng: - selection_fields: - Description: - Table 2.1: non energy - FlowName: 'Natural Gas' - clean_fba: !script_function:EIA_MECS keep_chemical_manufacturing - attribution_method: proportional - attribution_source: - BEA_Detail_Use_AfterRedef: - <<: *bea - selection_fields: - ActivityProducedBy: {'221200': ''} - - neu_petrol_asphalt: - selection_fields: - Description: - Table 2.1: non energy - FlowName: - Other: Petroleum - ActivityConsumedBy: ['324121', '324122'] - attribution_method: proportional - attribution_source: - BEA_Detail_Use_AfterRedef: - <<: *bea - selection_fields: - ActivityProducedBy: {'324110': ''} - - neu_petrol_hgl: + direct_table_3_1: selection_fields: - Description: - Table 2.1: non energy + Description: Table 3.1 FlowName: - 'Hydrocarbon Gas Liquids, excluding natural gasoline': Petroleum - attribution_method: proportional - attribution_source: - BEA_Detail_Use_AfterRedef: - <<: *bea - selection_fields: - ActivityProducedBy: {'324110': ''} - - neu_petrol_other: + - Coal + - Natural Gas + attribution_method: direct + direct_table_2_1: selection_fields: - Description: - Table 2.1: non energy + Description: Table 2.1 FlowName: + 'Hydrocarbon Gas Liquids, excluding natural gasoline': HGL Other: Petroleum - exclusion_fields: - ActivityConsumedBy: ['324121', '324122'] - attribution_method: proportional - attribution_source: - BEA_Detail_Use_AfterRedef: - <<: *bea - selection_fields: - ActivityProducedBy: {'324110': ''} + attribution_method: direct + direct_table_2_1_ng: + selection_fields: + Description: Table 2.1 + FlowName: Natural Gas + clean_fba: !script_function:EIA_MECS keep_chemical_manufacturing + attribution_method: direct BEA_Detail_Use_AfterRedef: - <<: *bea + year: 2017 + activity_to_sector_mapping: Cornerstone_2025 + exclusion_fields: + ActivityConsumedBy: ['F03000', 'F04000', 'F05000', 'F02E00', + 'F06E00', 'F07E00', 'F10E00', 'F02R00', + 'T001', 'T004', 'T007', 'T019'] + ActivityProducedBy: ['T007', 'T013', 'T015', 'T016', 'TOP', + 'MCIF'] activity_sets: petrol: selection_fields: ActivityProducedBy: {'324110': ''} exclusion_fields: - ActivityConsumedBy: *transport_consumers + ActivityConsumedBy: *petrol_exclude assign_fields: FlowName: Petroleum - Description: energy + Class: Money clean_source: EIA_MECS_Energy: year: *mecs_year @@ -155,16 +109,15 @@ source_names: ActivityProducedBy: '324110' assign_fields: FlowName: Coal and Coke - Description: non energy + Class: Other attribution_method: direct neu_transport: selection_fields: ActivityProducedBy: {'324110': ''} - ActivityConsumedBy: *transport_consumers + ActivityConsumedBy: *transport_industries assign_fields: FlowName: Transport - Description: non energy - # Annex A-10 = 2018 energy consumption (TBtu) from EPA GHGI. + Class: Money clean_source: EPA_GHGI_T_A_10: year: *mecs_year diff --git a/bedrock/transform/energy/Energy_Cornerstone_2022.yaml b/bedrock/transform/energy/Energy_manufacturing_national_2022.yaml similarity index 61% rename from bedrock/transform/energy/Energy_Cornerstone_2022.yaml rename to bedrock/transform/energy/Energy_manufacturing_national_2022.yaml index 327022fa4..d339fa432 100644 --- a/bedrock/transform/energy/Energy_Cornerstone_2022.yaml +++ b/bedrock/transform/energy/Energy_manufacturing_national_2022.yaml @@ -1,24 +1,28 @@ # Manufacturing energy FBS for Cornerstone. MECS survey year 2022. # Replicates CEDA allocation steps as quantity weights: MECS mapped with -# parent-incomplete-child + Cornerstone_2025, then nested BEA splits; -# petrol energy is BEA 324110 times MECS Other energy fraction. +# parent-incomplete-child + Cornerstone_2025 !include:Cornerstone_2025_target.yaml year: &mecs_year 2022 geoscale: national _attribution_sources: - BEA: &bea - year: 2017 - activity_to_sector_mapping: Cornerstone_2025 - exclusion_fields: - ActivityConsumedBy: ['F03000', 'F04000', 'F05000', 'F02E00', - 'F06E00', 'F07E00', 'F10E00', 'F02R00', - 'T001', 'T004', 'T007', 'T019'] - ActivityProducedBy: ['T007', 'T013', 'T015', 'T016', 'TOP', - 'MCIF'] - attribution_method: equal - transport_consumers: &transport_consumers + transport_industries: &transport_industries + - '481000' + - '482000' + - '483000' + - '484000' + - '485000' + - '486000' + - '48A000' + - '492000' + - 'S00500' + - 'S00600' + - '491000' + - 'GSLGO' + - 'S00203' + - 'F01000' + petrol_exclude: &petrol_exclude - '481000' - '482000' - '483000' @@ -46,96 +50,45 @@ source_names: clean_fba_before_activity_sets: - !script_function:EIA_MECS estimate_suppressed_mecs_energy activity_sets: - coal: - selection_fields: - Description: - Table 3.1: energy - FlowName: Coal - attribution_method: proportional - attribution_source: - BEA_Detail_Use_AfterRedef: - <<: *bea - selection_fields: - ActivityProducedBy: {'212100': ''} - - ng: - selection_fields: - Description: - Table 3.1: energy - FlowName: 'Natural Gas' - attribution_method: proportional - attribution_source: - BEA_Detail_Use_AfterRedef: - <<: *bea - selection_fields: - ActivityProducedBy: {'221200': ''} - - neu_ng: - selection_fields: - Description: - Table 2.1: non energy - FlowName: 'Natural Gas' - clean_fba: !script_function:EIA_MECS keep_chemical_manufacturing - attribution_method: proportional - attribution_source: - BEA_Detail_Use_AfterRedef: - <<: *bea - selection_fields: - ActivityProducedBy: {'221200': ''} - - neu_petrol_asphalt: - selection_fields: - Description: - Table 2.1: non energy - FlowName: - Other: Petroleum - ActivityConsumedBy: ['324121', '324122'] - attribution_method: proportional - attribution_source: - BEA_Detail_Use_AfterRedef: - <<: *bea - selection_fields: - ActivityProducedBy: {'324110': ''} - - neu_petrol_hgl: + direct_table_3_1: selection_fields: - Description: - Table 2.1: non energy + Description: Table 3.1 FlowName: - 'Hydrocarbon Gas Liquids, excluding natural gasoline': Petroleum - attribution_method: proportional - attribution_source: - BEA_Detail_Use_AfterRedef: - <<: *bea - selection_fields: - ActivityProducedBy: {'324110': ''} - - neu_petrol_other: + - Coal + - Natural Gas + attribution_method: direct + direct_table_2_1: selection_fields: - Description: - Table 2.1: non energy + Description: Table 2.1 FlowName: + 'Hydrocarbon Gas Liquids, excluding natural gasoline': HGL Other: Petroleum - exclusion_fields: - ActivityConsumedBy: ['324121', '324122'] - attribution_method: proportional - attribution_source: - BEA_Detail_Use_AfterRedef: - <<: *bea - selection_fields: - ActivityProducedBy: {'324110': ''} + attribution_method: direct + direct_table_2_1_ng: + selection_fields: + Description: Table 2.1 + FlowName: Natural Gas + clean_fba: !script_function:EIA_MECS keep_chemical_manufacturing + attribution_method: direct BEA_Detail_Use_AfterRedef: - <<: *bea + year: 2017 + activity_to_sector_mapping: Cornerstone_2025 + exclusion_fields: + ActivityConsumedBy: ['F03000', 'F04000', 'F05000', 'F02E00', + 'F06E00', 'F07E00', 'F10E00', 'F02R00', + 'T001', 'T004', 'T007', 'T019'] + ActivityProducedBy: ['T007', 'T013', 'T015', 'T016', 'TOP', + 'MCIF'] activity_sets: petrol: selection_fields: ActivityProducedBy: {'324110': ''} exclusion_fields: - ActivityConsumedBy: *transport_consumers + ActivityConsumedBy: *petrol_exclude assign_fields: FlowName: Petroleum - Description: energy + Class: Money clean_source: EIA_MECS_Energy: year: *mecs_year @@ -155,16 +108,15 @@ source_names: ActivityProducedBy: '324110' assign_fields: FlowName: Coal and Coke - Description: non energy + Class: Other attribution_method: direct neu_transport: selection_fields: ActivityProducedBy: {'324110': ''} - ActivityConsumedBy: *transport_consumers + ActivityConsumedBy: *transport_industries assign_fields: FlowName: Transport - Description: non energy - # Annex A-6 = 2022 energy consumption (TBtu) from EPA GHGI. + Class: Money clean_source: EPA_GHGI_T_A_6: year: *mecs_year From dd252dafbb0c0a3c2e3c773dc159d395a1576e60 Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Thu, 27 Aug 2026 13:56:30 -0600 Subject: [PATCH 05/18] update comment --- .../transform/energy/Energy_manufacturing_national_2022.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bedrock/transform/energy/Energy_manufacturing_national_2022.yaml b/bedrock/transform/energy/Energy_manufacturing_national_2022.yaml index d339fa432..9628878e3 100644 --- a/bedrock/transform/energy/Energy_manufacturing_national_2022.yaml +++ b/bedrock/transform/energy/Energy_manufacturing_national_2022.yaml @@ -1,4 +1,5 @@ -# Manufacturing energy FBS for Cornerstone. MECS survey year 2022. +# Manufacturing energy FBS used in GHG Cornerstone FBS methods. +# MECS survey year 2022. # Replicates CEDA allocation steps as quantity weights: MECS mapped with # parent-incomplete-child + Cornerstone_2025 From 91854fbec4cd42c7c2fbe164eeff4411064d0662 Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Thu, 27 Aug 2026 13:59:32 -0600 Subject: [PATCH 06/18] use additional activity sets for UMD_GHGIA_T_3_14 to align with energy mecs fbs data/approach --- .../ghg/GHG_national_Cornerstone_2017.yaml | 47 +++++++++++++++---- .../ghg/GHG_national_Cornerstone_2018.yaml | 47 +++++++++++++++---- .../ghg/GHG_national_Cornerstone_2019.yaml | 47 +++++++++++++++---- .../ghg/GHG_national_Cornerstone_2020.yaml | 47 +++++++++++++++---- .../ghg/GHG_national_Cornerstone_2021.yaml | 47 +++++++++++++++---- .../ghg/GHG_national_Cornerstone_2022.yaml | 47 +++++++++++++++---- .../ghg/GHG_national_Cornerstone_2023.yaml | 47 +++++++++++++++---- .../ghg/GHG_national_Cornerstone_2024.yaml | 47 +++++++++++++++---- 8 files changed, 304 insertions(+), 72 deletions(-) diff --git a/bedrock/transform/ghg/GHG_national_Cornerstone_2017.yaml b/bedrock/transform/ghg/GHG_national_Cornerstone_2017.yaml index abf9accdd..c4d37022f 100644 --- a/bedrock/transform/ghg/GHG_national_Cornerstone_2017.yaml +++ b/bedrock/transform/ghg/GHG_national_Cornerstone_2017.yaml @@ -26,7 +26,7 @@ sources_to_cache: _attribution_sources: # Nested EPA_GHGI_T_3_25 below is inventory, not UMD T_3_25 (petroleum-systems CH4). - Energy_Cornerstone_2018: &mecs_energy_alloc + Energy_manufacturing_national_2018: &mecs_energy_alloc data_format: FBS geoscale: national @@ -586,7 +586,7 @@ source_names: - Petroleum Industrial attribution_method: proportional attribution_source: - Energy_Cornerstone_2018: + Energy_manufacturing_national_2018: <<: *mecs_energy_alloc selection_fields: Flowable: Petroleum @@ -609,7 +609,7 @@ source_names: - Natural Gas Industrial - Manufacturing attribution_method: proportional attribution_source: - Energy_Cornerstone_2018: + Energy_manufacturing_national_2018: <<: *mecs_energy_alloc selection_fields: Flowable: Natural Gas @@ -632,7 +632,7 @@ source_names: - Coal Industrial - Manufacturing attribution_method: proportional attribution_source: - Energy_Cornerstone_2018: + Energy_manufacturing_national_2018: <<: *mecs_energy_alloc selection_fields: Flowable: Coal @@ -703,10 +703,36 @@ source_names: - Industry Industrial Other Coal attribution_method: direct - petroleum_neu: + petroleum_neu_asphalt: selection_fields: PrimaryActivity: 'Industry Asphalt & Road Oil': Industry Petroleum Products Non-energy + attribution_method: proportional + attribution_source: + Energy_manufacturing_national_2018: + <<: *mecs_energy_alloc + selection_fields: + Flowable: Petroleum + Class: Other + SectorConsumedBy: + - '324121' + - '324122' + + petroleum_neu_hgl: + selection_fields: + PrimaryActivity: + Industry HGL: Industry Petroleum Products Non-energy + attribution_method: proportional + attribution_source: + Energy_manufacturing_national_2018: + <<: *mecs_energy_alloc + selection_fields: + Flowable: HGL + Class: Other + + petroleum_neu: + selection_fields: + PrimaryActivity: Industry Distillate Fuel Oil: Industry Petroleum Products Non-energy Industry Lubricants: Industry Petroleum Products Non-energy Industry Miscellaneous Products: Industry Petroleum Products Non-energy @@ -717,21 +743,24 @@ source_names: Industry Still Gas: Industry Petroleum Products Non-energy Industry Waxes: Industry Petroleum Products Non-energy Industry Natural Gasoline: Industry Petroleum Products Non-energy - Industry HGL: Industry Petroleum Products Non-energy attribution_method: proportional attribution_source: - Energy_Cornerstone_2018: + Energy_manufacturing_national_2018: <<: *mecs_energy_alloc selection_fields: Flowable: Petroleum Class: Other + exclusion_fields: + SectorConsumedBy: + - '324121' + - '324122' natural_gas_neu: selection_fields: PrimaryActivity: Industry Natural Gas to Chemical Plants attribution_method: proportional attribution_source: - Energy_Cornerstone_2018: + Energy_manufacturing_national_2018: <<: *mecs_energy_alloc selection_fields: Flowable: Natural Gas @@ -742,7 +771,7 @@ source_names: PrimaryActivity: Transportation Lubricants attribution_method: proportional attribution_source: - Energy_Cornerstone_2018: + Energy_manufacturing_national_2018: <<: *mecs_energy_alloc selection_fields: Flowable: Transport diff --git a/bedrock/transform/ghg/GHG_national_Cornerstone_2018.yaml b/bedrock/transform/ghg/GHG_national_Cornerstone_2018.yaml index da7434721..fa16e739c 100644 --- a/bedrock/transform/ghg/GHG_national_Cornerstone_2018.yaml +++ b/bedrock/transform/ghg/GHG_national_Cornerstone_2018.yaml @@ -26,7 +26,7 @@ sources_to_cache: _attribution_sources: # Nested EPA_GHGI_T_3_25 below is inventory, not UMD T_3_25 (petroleum-systems CH4). - Energy_Cornerstone_2018: &mecs_energy_alloc + Energy_manufacturing_national_2018: &mecs_energy_alloc data_format: FBS geoscale: national @@ -586,7 +586,7 @@ source_names: - Petroleum Industrial attribution_method: proportional attribution_source: - Energy_Cornerstone_2018: + Energy_manufacturing_national_2018: <<: *mecs_energy_alloc selection_fields: Flowable: Petroleum @@ -609,7 +609,7 @@ source_names: - Natural Gas Industrial - Manufacturing attribution_method: proportional attribution_source: - Energy_Cornerstone_2018: + Energy_manufacturing_national_2018: <<: *mecs_energy_alloc selection_fields: Flowable: Natural Gas @@ -632,7 +632,7 @@ source_names: - Coal Industrial - Manufacturing attribution_method: proportional attribution_source: - Energy_Cornerstone_2018: + Energy_manufacturing_national_2018: <<: *mecs_energy_alloc selection_fields: Flowable: Coal @@ -703,10 +703,36 @@ source_names: - Industry Industrial Other Coal attribution_method: direct - petroleum_neu: + petroleum_neu_asphalt: selection_fields: PrimaryActivity: 'Industry Asphalt & Road Oil': Industry Petroleum Products Non-energy + attribution_method: proportional + attribution_source: + Energy_manufacturing_national_2018: + <<: *mecs_energy_alloc + selection_fields: + Flowable: Petroleum + Class: Other + SectorConsumedBy: + - '324121' + - '324122' + + petroleum_neu_hgl: + selection_fields: + PrimaryActivity: + Industry HGL: Industry Petroleum Products Non-energy + attribution_method: proportional + attribution_source: + Energy_manufacturing_national_2018: + <<: *mecs_energy_alloc + selection_fields: + Flowable: HGL + Class: Other + + petroleum_neu: + selection_fields: + PrimaryActivity: Industry Distillate Fuel Oil: Industry Petroleum Products Non-energy Industry Lubricants: Industry Petroleum Products Non-energy Industry Miscellaneous Products: Industry Petroleum Products Non-energy @@ -717,21 +743,24 @@ source_names: Industry Still Gas: Industry Petroleum Products Non-energy Industry Waxes: Industry Petroleum Products Non-energy Industry Natural Gasoline: Industry Petroleum Products Non-energy - Industry HGL: Industry Petroleum Products Non-energy attribution_method: proportional attribution_source: - Energy_Cornerstone_2018: + Energy_manufacturing_national_2018: <<: *mecs_energy_alloc selection_fields: Flowable: Petroleum Class: Other + exclusion_fields: + SectorConsumedBy: + - '324121' + - '324122' natural_gas_neu: selection_fields: PrimaryActivity: Industry Natural Gas to Chemical Plants attribution_method: proportional attribution_source: - Energy_Cornerstone_2018: + Energy_manufacturing_national_2018: <<: *mecs_energy_alloc selection_fields: Flowable: Natural Gas @@ -742,7 +771,7 @@ source_names: PrimaryActivity: Transportation Lubricants attribution_method: proportional attribution_source: - Energy_Cornerstone_2018: + Energy_manufacturing_national_2018: <<: *mecs_energy_alloc selection_fields: Flowable: Transport diff --git a/bedrock/transform/ghg/GHG_national_Cornerstone_2019.yaml b/bedrock/transform/ghg/GHG_national_Cornerstone_2019.yaml index ca78b4883..95d6aa957 100644 --- a/bedrock/transform/ghg/GHG_national_Cornerstone_2019.yaml +++ b/bedrock/transform/ghg/GHG_national_Cornerstone_2019.yaml @@ -23,7 +23,7 @@ sources_to_cache: _attribution_sources: # Nested EPA_GHGI_T_3_25 below is inventory, not UMD T_3_25 (petroleum-systems CH4). - Energy_Cornerstone_2018: &mecs_energy_alloc + Energy_manufacturing_national_2018: &mecs_energy_alloc data_format: FBS geoscale: national @@ -583,7 +583,7 @@ source_names: - Petroleum Industrial attribution_method: proportional attribution_source: - Energy_Cornerstone_2018: + Energy_manufacturing_national_2018: <<: *mecs_energy_alloc selection_fields: Flowable: Petroleum @@ -606,7 +606,7 @@ source_names: - Natural Gas Industrial - Manufacturing attribution_method: proportional attribution_source: - Energy_Cornerstone_2018: + Energy_manufacturing_national_2018: <<: *mecs_energy_alloc selection_fields: Flowable: Natural Gas @@ -629,7 +629,7 @@ source_names: - Coal Industrial - Manufacturing attribution_method: proportional attribution_source: - Energy_Cornerstone_2018: + Energy_manufacturing_national_2018: <<: *mecs_energy_alloc selection_fields: Flowable: Coal @@ -700,10 +700,36 @@ source_names: - Industry Industrial Other Coal attribution_method: direct - petroleum_neu: + petroleum_neu_asphalt: selection_fields: PrimaryActivity: 'Industry Asphalt & Road Oil': Industry Petroleum Products Non-energy + attribution_method: proportional + attribution_source: + Energy_manufacturing_national_2018: + <<: *mecs_energy_alloc + selection_fields: + Flowable: Petroleum + Class: Other + SectorConsumedBy: + - '324121' + - '324122' + + petroleum_neu_hgl: + selection_fields: + PrimaryActivity: + Industry HGL: Industry Petroleum Products Non-energy + attribution_method: proportional + attribution_source: + Energy_manufacturing_national_2018: + <<: *mecs_energy_alloc + selection_fields: + Flowable: HGL + Class: Other + + petroleum_neu: + selection_fields: + PrimaryActivity: Industry Distillate Fuel Oil: Industry Petroleum Products Non-energy Industry Lubricants: Industry Petroleum Products Non-energy Industry Miscellaneous Products: Industry Petroleum Products Non-energy @@ -714,14 +740,17 @@ source_names: Industry Still Gas: Industry Petroleum Products Non-energy Industry Waxes: Industry Petroleum Products Non-energy Industry Natural Gasoline: Industry Petroleum Products Non-energy - Industry HGL: Industry Petroleum Products Non-energy attribution_method: proportional attribution_source: - Energy_Cornerstone_2018: + Energy_manufacturing_national_2018: <<: *mecs_energy_alloc selection_fields: Flowable: Petroleum Class: Other + exclusion_fields: + SectorConsumedBy: + - '324121' + - '324122' natural_gas_neu: @@ -729,7 +758,7 @@ source_names: PrimaryActivity: Industry Natural Gas to Chemical Plants attribution_method: proportional attribution_source: - Energy_Cornerstone_2018: + Energy_manufacturing_national_2018: <<: *mecs_energy_alloc selection_fields: Flowable: Natural Gas @@ -740,7 +769,7 @@ source_names: PrimaryActivity: Transportation Lubricants attribution_method: proportional attribution_source: - Energy_Cornerstone_2018: + Energy_manufacturing_national_2018: <<: *mecs_energy_alloc selection_fields: Flowable: Transport diff --git a/bedrock/transform/ghg/GHG_national_Cornerstone_2020.yaml b/bedrock/transform/ghg/GHG_national_Cornerstone_2020.yaml index 2def398c6..15acc2d5f 100644 --- a/bedrock/transform/ghg/GHG_national_Cornerstone_2020.yaml +++ b/bedrock/transform/ghg/GHG_national_Cornerstone_2020.yaml @@ -23,7 +23,7 @@ sources_to_cache: _attribution_sources: # Nested EPA_GHGI_T_3_25 below is inventory, not UMD T_3_25 (petroleum-systems CH4). - Energy_Cornerstone_2018: &mecs_energy_alloc + Energy_manufacturing_national_2018: &mecs_energy_alloc data_format: FBS geoscale: national @@ -583,7 +583,7 @@ source_names: - Petroleum Industrial attribution_method: proportional attribution_source: - Energy_Cornerstone_2018: + Energy_manufacturing_national_2018: <<: *mecs_energy_alloc selection_fields: Flowable: Petroleum @@ -606,7 +606,7 @@ source_names: - Natural Gas Industrial - Manufacturing attribution_method: proportional attribution_source: - Energy_Cornerstone_2018: + Energy_manufacturing_national_2018: <<: *mecs_energy_alloc selection_fields: Flowable: Natural Gas @@ -629,7 +629,7 @@ source_names: - Coal Industrial - Manufacturing attribution_method: proportional attribution_source: - Energy_Cornerstone_2018: + Energy_manufacturing_national_2018: <<: *mecs_energy_alloc selection_fields: Flowable: Coal @@ -700,10 +700,36 @@ source_names: - Industry Industrial Other Coal attribution_method: direct - petroleum_neu: + petroleum_neu_asphalt: selection_fields: PrimaryActivity: 'Industry Asphalt & Road Oil': Industry Petroleum Products Non-energy + attribution_method: proportional + attribution_source: + Energy_manufacturing_national_2018: + <<: *mecs_energy_alloc + selection_fields: + Flowable: Petroleum + Class: Other + SectorConsumedBy: + - '324121' + - '324122' + + petroleum_neu_hgl: + selection_fields: + PrimaryActivity: + Industry HGL: Industry Petroleum Products Non-energy + attribution_method: proportional + attribution_source: + Energy_manufacturing_national_2018: + <<: *mecs_energy_alloc + selection_fields: + Flowable: HGL + Class: Other + + petroleum_neu: + selection_fields: + PrimaryActivity: Industry Distillate Fuel Oil: Industry Petroleum Products Non-energy Industry Lubricants: Industry Petroleum Products Non-energy Industry Miscellaneous Products: Industry Petroleum Products Non-energy @@ -714,21 +740,24 @@ source_names: Industry Still Gas: Industry Petroleum Products Non-energy Industry Waxes: Industry Petroleum Products Non-energy Industry Natural Gasoline: Industry Petroleum Products Non-energy - Industry HGL: Industry Petroleum Products Non-energy attribution_method: proportional attribution_source: - Energy_Cornerstone_2018: + Energy_manufacturing_national_2018: <<: *mecs_energy_alloc selection_fields: Flowable: Petroleum Class: Other + exclusion_fields: + SectorConsumedBy: + - '324121' + - '324122' natural_gas_neu: selection_fields: PrimaryActivity: Industry Natural Gas to Chemical Plants attribution_method: proportional attribution_source: - Energy_Cornerstone_2018: + Energy_manufacturing_national_2018: <<: *mecs_energy_alloc selection_fields: Flowable: Natural Gas @@ -739,7 +768,7 @@ source_names: PrimaryActivity: Transportation Lubricants attribution_method: proportional attribution_source: - Energy_Cornerstone_2018: + Energy_manufacturing_national_2018: <<: *mecs_energy_alloc selection_fields: Flowable: Transport diff --git a/bedrock/transform/ghg/GHG_national_Cornerstone_2021.yaml b/bedrock/transform/ghg/GHG_national_Cornerstone_2021.yaml index 219b5dfb0..b645fbbd2 100644 --- a/bedrock/transform/ghg/GHG_national_Cornerstone_2021.yaml +++ b/bedrock/transform/ghg/GHG_national_Cornerstone_2021.yaml @@ -23,7 +23,7 @@ sources_to_cache: _attribution_sources: # Nested EPA_GHGI_T_3_25 below is inventory, not UMD T_3_25 (petroleum-systems CH4). - Energy_Cornerstone_2022: &mecs_energy_alloc + Energy_manufacturing_national_2022: &mecs_energy_alloc data_format: FBS geoscale: national @@ -583,7 +583,7 @@ source_names: - Petroleum Industrial attribution_method: proportional attribution_source: - Energy_Cornerstone_2022: + Energy_manufacturing_national_2022: <<: *mecs_energy_alloc selection_fields: Flowable: Petroleum @@ -606,7 +606,7 @@ source_names: - Natural Gas Industrial - Manufacturing attribution_method: proportional attribution_source: - Energy_Cornerstone_2022: + Energy_manufacturing_national_2022: <<: *mecs_energy_alloc selection_fields: Flowable: Natural Gas @@ -629,7 +629,7 @@ source_names: - Coal Industrial - Manufacturing attribution_method: proportional attribution_source: - Energy_Cornerstone_2022: + Energy_manufacturing_national_2022: <<: *mecs_energy_alloc selection_fields: Flowable: Coal @@ -700,10 +700,36 @@ source_names: - Industry Industrial Other Coal attribution_method: direct - petroleum_neu: + petroleum_neu_asphalt: selection_fields: PrimaryActivity: 'Industry Asphalt & Road Oil': Industry Petroleum Products Non-energy + attribution_method: proportional + attribution_source: + Energy_manufacturing_national_2022: + <<: *mecs_energy_alloc + selection_fields: + Flowable: Petroleum + Class: Other + SectorConsumedBy: + - '324121' + - '324122' + + petroleum_neu_hgl: + selection_fields: + PrimaryActivity: + Industry HGL: Industry Petroleum Products Non-energy + attribution_method: proportional + attribution_source: + Energy_manufacturing_national_2022: + <<: *mecs_energy_alloc + selection_fields: + Flowable: HGL + Class: Other + + petroleum_neu: + selection_fields: + PrimaryActivity: Industry Distillate Fuel Oil: Industry Petroleum Products Non-energy Industry Lubricants: Industry Petroleum Products Non-energy Industry Miscellaneous Products: Industry Petroleum Products Non-energy @@ -714,14 +740,17 @@ source_names: Industry Still Gas: Industry Petroleum Products Non-energy Industry Waxes: Industry Petroleum Products Non-energy Industry Natural Gasoline: Industry Petroleum Products Non-energy - Industry HGL: Industry Petroleum Products Non-energy attribution_method: proportional attribution_source: - Energy_Cornerstone_2022: + Energy_manufacturing_national_2022: <<: *mecs_energy_alloc selection_fields: Flowable: Petroleum Class: Other + exclusion_fields: + SectorConsumedBy: + - '324121' + - '324122' natural_gas_neu: @@ -729,7 +758,7 @@ source_names: PrimaryActivity: Industry Natural Gas to Chemical Plants attribution_method: proportional attribution_source: - Energy_Cornerstone_2022: + Energy_manufacturing_national_2022: <<: *mecs_energy_alloc selection_fields: Flowable: Natural Gas @@ -740,7 +769,7 @@ source_names: PrimaryActivity: Transportation Lubricants attribution_method: proportional attribution_source: - Energy_Cornerstone_2022: + Energy_manufacturing_national_2022: <<: *mecs_energy_alloc selection_fields: Flowable: Transport diff --git a/bedrock/transform/ghg/GHG_national_Cornerstone_2022.yaml b/bedrock/transform/ghg/GHG_national_Cornerstone_2022.yaml index 755bde6c9..42f59ff5d 100644 --- a/bedrock/transform/ghg/GHG_national_Cornerstone_2022.yaml +++ b/bedrock/transform/ghg/GHG_national_Cornerstone_2022.yaml @@ -23,7 +23,7 @@ sources_to_cache: _attribution_sources: # Nested EPA_GHGI_T_3_25 below is inventory, not UMD T_3_25 (petroleum-systems CH4). - Energy_Cornerstone_2022: &mecs_energy_alloc + Energy_manufacturing_national_2022: &mecs_energy_alloc data_format: FBS geoscale: national @@ -583,7 +583,7 @@ source_names: - Petroleum Industrial attribution_method: proportional attribution_source: - Energy_Cornerstone_2022: + Energy_manufacturing_national_2022: <<: *mecs_energy_alloc selection_fields: Flowable: Petroleum @@ -606,7 +606,7 @@ source_names: - Natural Gas Industrial - Manufacturing attribution_method: proportional attribution_source: - Energy_Cornerstone_2022: + Energy_manufacturing_national_2022: <<: *mecs_energy_alloc selection_fields: Flowable: Natural Gas @@ -629,7 +629,7 @@ source_names: - Coal Industrial - Manufacturing attribution_method: proportional attribution_source: - Energy_Cornerstone_2022: + Energy_manufacturing_national_2022: <<: *mecs_energy_alloc selection_fields: Flowable: Coal @@ -700,10 +700,36 @@ source_names: - Industry Industrial Other Coal attribution_method: direct - petroleum_neu: + petroleum_neu_asphalt: selection_fields: PrimaryActivity: 'Industry Asphalt & Road Oil': Industry Petroleum Products Non-energy + attribution_method: proportional + attribution_source: + Energy_manufacturing_national_2022: + <<: *mecs_energy_alloc + selection_fields: + Flowable: Petroleum + Class: Other + SectorConsumedBy: + - '324121' + - '324122' + + petroleum_neu_hgl: + selection_fields: + PrimaryActivity: + Industry HGL: Industry Petroleum Products Non-energy + attribution_method: proportional + attribution_source: + Energy_manufacturing_national_2022: + <<: *mecs_energy_alloc + selection_fields: + Flowable: HGL + Class: Other + + petroleum_neu: + selection_fields: + PrimaryActivity: Industry Distillate Fuel Oil: Industry Petroleum Products Non-energy Industry Lubricants: Industry Petroleum Products Non-energy Industry Miscellaneous Products: Industry Petroleum Products Non-energy @@ -714,14 +740,17 @@ source_names: Industry Still Gas: Industry Petroleum Products Non-energy Industry Waxes: Industry Petroleum Products Non-energy Industry Natural Gasoline: Industry Petroleum Products Non-energy - Industry HGL: Industry Petroleum Products Non-energy attribution_method: proportional attribution_source: - Energy_Cornerstone_2022: + Energy_manufacturing_national_2022: <<: *mecs_energy_alloc selection_fields: Flowable: Petroleum Class: Other + exclusion_fields: + SectorConsumedBy: + - '324121' + - '324122' natural_gas_neu: @@ -729,7 +758,7 @@ source_names: PrimaryActivity: Industry Natural Gas to Chemical Plants attribution_method: proportional attribution_source: - Energy_Cornerstone_2022: + Energy_manufacturing_national_2022: <<: *mecs_energy_alloc selection_fields: Flowable: Natural Gas @@ -740,7 +769,7 @@ source_names: PrimaryActivity: Transportation Lubricants attribution_method: proportional attribution_source: - Energy_Cornerstone_2022: + Energy_manufacturing_national_2022: <<: *mecs_energy_alloc selection_fields: Flowable: Transport diff --git a/bedrock/transform/ghg/GHG_national_Cornerstone_2023.yaml b/bedrock/transform/ghg/GHG_national_Cornerstone_2023.yaml index aa44d281e..75db5de8a 100644 --- a/bedrock/transform/ghg/GHG_national_Cornerstone_2023.yaml +++ b/bedrock/transform/ghg/GHG_national_Cornerstone_2023.yaml @@ -23,7 +23,7 @@ sources_to_cache: _attribution_sources: # Nested EPA_GHGI_T_3_25 below is inventory, not UMD T_3_25 (petroleum-systems CH4). - Energy_Cornerstone_2022: &mecs_energy_alloc + Energy_manufacturing_national_2022: &mecs_energy_alloc data_format: FBS geoscale: national @@ -583,7 +583,7 @@ source_names: - Petroleum Industrial attribution_method: proportional attribution_source: - Energy_Cornerstone_2022: + Energy_manufacturing_national_2022: <<: *mecs_energy_alloc selection_fields: Flowable: Petroleum @@ -606,7 +606,7 @@ source_names: - Natural Gas Industrial - Manufacturing attribution_method: proportional attribution_source: - Energy_Cornerstone_2022: + Energy_manufacturing_national_2022: <<: *mecs_energy_alloc selection_fields: Flowable: Natural Gas @@ -629,7 +629,7 @@ source_names: - Coal Industrial - Manufacturing attribution_method: proportional attribution_source: - Energy_Cornerstone_2022: + Energy_manufacturing_national_2022: <<: *mecs_energy_alloc selection_fields: Flowable: Coal @@ -700,10 +700,36 @@ source_names: - Industry Industrial Other Coal attribution_method: direct - petroleum_neu: + petroleum_neu_asphalt: selection_fields: PrimaryActivity: 'Industry Asphalt & Road Oil': Industry Petroleum Products Non-energy + attribution_method: proportional + attribution_source: + Energy_manufacturing_national_2022: + <<: *mecs_energy_alloc + selection_fields: + Flowable: Petroleum + Class: Other + SectorConsumedBy: + - '324121' + - '324122' + + petroleum_neu_hgl: + selection_fields: + PrimaryActivity: + Industry HGL: Industry Petroleum Products Non-energy + attribution_method: proportional + attribution_source: + Energy_manufacturing_national_2022: + <<: *mecs_energy_alloc + selection_fields: + Flowable: HGL + Class: Other + + petroleum_neu: + selection_fields: + PrimaryActivity: Industry Distillate Fuel Oil: Industry Petroleum Products Non-energy Industry Lubricants: Industry Petroleum Products Non-energy Industry Miscellaneous Products: Industry Petroleum Products Non-energy @@ -714,14 +740,17 @@ source_names: Industry Still Gas: Industry Petroleum Products Non-energy Industry Waxes: Industry Petroleum Products Non-energy Industry Natural Gasoline: Industry Petroleum Products Non-energy - Industry HGL: Industry Petroleum Products Non-energy attribution_method: proportional attribution_source: - Energy_Cornerstone_2022: + Energy_manufacturing_national_2022: <<: *mecs_energy_alloc selection_fields: Flowable: Petroleum Class: Other + exclusion_fields: + SectorConsumedBy: + - '324121' + - '324122' natural_gas_neu: @@ -729,7 +758,7 @@ source_names: PrimaryActivity: Industry Natural Gas to Chemical Plants attribution_method: proportional attribution_source: - Energy_Cornerstone_2022: + Energy_manufacturing_national_2022: <<: *mecs_energy_alloc selection_fields: Flowable: Natural Gas @@ -740,7 +769,7 @@ source_names: PrimaryActivity: Transportation Lubricants attribution_method: proportional attribution_source: - Energy_Cornerstone_2022: + Energy_manufacturing_national_2022: <<: *mecs_energy_alloc selection_fields: Flowable: Transport diff --git a/bedrock/transform/ghg/GHG_national_Cornerstone_2024.yaml b/bedrock/transform/ghg/GHG_national_Cornerstone_2024.yaml index 34d0d29b5..a90e6c43d 100644 --- a/bedrock/transform/ghg/GHG_national_Cornerstone_2024.yaml +++ b/bedrock/transform/ghg/GHG_national_Cornerstone_2024.yaml @@ -23,7 +23,7 @@ sources_to_cache: _attribution_sources: # Nested EPA_GHGI_T_3_25 below is inventory, not UMD T_3_25 (petroleum-systems CH4). - Energy_Cornerstone_2022: &mecs_energy_alloc + Energy_manufacturing_national_2022: &mecs_energy_alloc data_format: FBS geoscale: national @@ -583,7 +583,7 @@ source_names: - Petroleum Industrial attribution_method: proportional attribution_source: - Energy_Cornerstone_2022: + Energy_manufacturing_national_2022: <<: *mecs_energy_alloc selection_fields: Flowable: Petroleum @@ -606,7 +606,7 @@ source_names: - Natural Gas Industrial - Manufacturing attribution_method: proportional attribution_source: - Energy_Cornerstone_2022: + Energy_manufacturing_national_2022: <<: *mecs_energy_alloc selection_fields: Flowable: Natural Gas @@ -629,7 +629,7 @@ source_names: - Coal Industrial - Manufacturing attribution_method: proportional attribution_source: - Energy_Cornerstone_2022: + Energy_manufacturing_national_2022: <<: *mecs_energy_alloc selection_fields: Flowable: Coal @@ -700,10 +700,36 @@ source_names: - Industry Industrial Other Coal attribution_method: direct - petroleum_neu: + petroleum_neu_asphalt: selection_fields: PrimaryActivity: 'Industry Asphalt & Road Oil': Industry Petroleum Products Non-energy + attribution_method: proportional + attribution_source: + Energy_manufacturing_national_2022: + <<: *mecs_energy_alloc + selection_fields: + Flowable: Petroleum + Class: Other + SectorConsumedBy: + - '324121' + - '324122' + + petroleum_neu_hgl: + selection_fields: + PrimaryActivity: + Industry HGL: Industry Petroleum Products Non-energy + attribution_method: proportional + attribution_source: + Energy_manufacturing_national_2022: + <<: *mecs_energy_alloc + selection_fields: + Flowable: HGL + Class: Other + + petroleum_neu: + selection_fields: + PrimaryActivity: Industry Distillate Fuel Oil: Industry Petroleum Products Non-energy Industry Lubricants: Industry Petroleum Products Non-energy Industry Miscellaneous Products: Industry Petroleum Products Non-energy @@ -714,14 +740,17 @@ source_names: Industry Still Gas: Industry Petroleum Products Non-energy Industry Waxes: Industry Petroleum Products Non-energy Industry Natural Gasoline: Industry Petroleum Products Non-energy - Industry HGL: Industry Petroleum Products Non-energy attribution_method: proportional attribution_source: - Energy_Cornerstone_2022: + Energy_manufacturing_national_2022: <<: *mecs_energy_alloc selection_fields: Flowable: Petroleum Class: Other + exclusion_fields: + SectorConsumedBy: + - '324121' + - '324122' natural_gas_neu: @@ -729,7 +758,7 @@ source_names: PrimaryActivity: Industry Natural Gas to Chemical Plants attribution_method: proportional attribution_source: - Energy_Cornerstone_2022: + Energy_manufacturing_national_2022: <<: *mecs_energy_alloc selection_fields: Flowable: Natural Gas @@ -740,7 +769,7 @@ source_names: PrimaryActivity: Transportation Lubricants attribution_method: proportional attribution_source: - Energy_Cornerstone_2022: + Energy_manufacturing_national_2022: <<: *mecs_energy_alloc selection_fields: Flowable: Transport From d5b4bacb9dd1fa3978524ac0180b0adac1c0fb25 Mon Sep 17 00:00:00 2001 From: Catherine Birney <60186515+catherinebirney@users.noreply.github.com> Date: Fri, 21 Aug 2026 08:16:25 -0600 Subject: [PATCH 07/18] Prevent dropping all FBS data due to ValueError (#663) (cherry picked from commit d84a592011895651ee1a9933b79a5d9122add655) --- bedrock/extract/epa/EPA_GHGI.py | 6 +++- bedrock/extract/flowbyactivity.py | 56 ++++++++++++++----------------- bedrock/transform/flowby.py | 29 ++++++++-------- bedrock/transform/flowbysector.py | 19 ++++++----- 4 files changed, 55 insertions(+), 55 deletions(-) diff --git a/bedrock/extract/epa/EPA_GHGI.py b/bedrock/extract/epa/EPA_GHGI.py index d468bb7dd..4cb336e2a 100644 --- a/bedrock/extract/epa/EPA_GHGI.py +++ b/bedrock/extract/epa/EPA_GHGI.py @@ -168,7 +168,11 @@ def ghg_load_gcs(**kwargs: dict[str, Any]) -> List[pd.DataFrame]: if year == '2023' and table == '3-25b': # Skip 3-25b for current year (use 3-25 instead) continue - df = _load_ghg_table(table) + try: + df = _load_ghg_table(table) + except Exception as exc: + log.warning(f'Skipping EPA GHGI table {table} for year {year}: {exc}') + continue if df is not None and len(df.columns) > 1: years = YEARS.copy() years.remove(year) diff --git a/bedrock/extract/flowbyactivity.py b/bedrock/extract/flowbyactivity.py index d6c7206e2..e52379c99 100644 --- a/bedrock/extract/flowbyactivity.py +++ b/bedrock/extract/flowbyactivity.py @@ -715,37 +715,31 @@ def prepare_fbs( drop_cols = [] if 'activity_sets' in self.config: - try: - return FlowBySector( - pd.concat( - [ - fba.prepare_fbs( - external_config_path=external_config_path, - download_sources_ok=download_sources_ok, - skip_select_by=True, - retain_activity_columns=retain_activity_columns, - fbs_method_name=fbs_method_name, - ) - for fba in ( - self.select_by_fields() - .function_socket('clean_fba_before_activity_sets') - .activity_sets() - ) - ] - ).reset_index(drop=True), - convert_df_to_flowby=True, - ) - except ValueError: - # This discards every activity_set, not just the one that - # failed, so a fault in one silently zeroes the whole method. - # Log it - the silence is what makes this class of bug expensive - # to find. - log.exception( - 'Discarding ALL activity_sets for %s: one of them raised ' - 'while being prepared. The method will return no rows.', - self.full_name, - ) - return FlowBySector(pd.DataFrame(), convert_df_to_flowby=True) + prepared = [] + for fba in ( + self.select_by_fields() + .function_socket('clean_fba_before_activity_sets') + .activity_sets() + ): + try: + prepared.append( + fba.prepare_fbs( + external_config_path=external_config_path, + download_sources_ok=download_sources_ok, + skip_select_by=True, + retain_activity_columns=retain_activity_columns, + fbs_method_name=fbs_method_name, + ) + ) + except ValueError as exc: + log.exception(f'{fba.full_name} failed while preparing FBS: {exc}') + raise ValueError( + f'{fba.full_name} failed while preparing FBS: {exc}' + ) from exc + return FlowBySector( + pd.concat(prepared).reset_index(drop=True), + convert_df_to_flowby=True, + ) log.info(f'Processing FlowBySector for {self.full_name}') # Primary FlowBySector generation approach: return FlowBySector( diff --git a/bedrock/transform/flowby.py b/bedrock/transform/flowby.py index 8871b9549..0971ffc66 100644 --- a/bedrock/transform/flowby.py +++ b/bedrock/transform/flowby.py @@ -107,7 +107,8 @@ def get_flowby_from_config( f'{config.get("data_format")}` for source {name}' ) raise ValueError( - 'Unrecognized data format, check assignment in ' + f'Unrecognized data format {config.get("data_format")!r} for ' + f'source {name}, check assignment in ' '.../flowsa/methods/method_status.yaml or assign ' 'within the method yaml. Data formats allowed: ' '"FBA", "FBS", "FBS_outside_flowsa".' @@ -890,7 +891,10 @@ def attribute_flows_to_sectors( f'Attribution method for {fb.full_name} not ' f'recognized: {attribution_method}' ) - raise ValueError('Attribution method not recognized') + raise ValueError( + f'Attribution method for {fb.full_name} not ' + f'recognized: {attribution_method}' + ) else: if all(fb.groupby('group_id')['group_id'].agg('count') == 1): @@ -1620,6 +1624,7 @@ def equally_attribute(self: 'FB') -> 'FB': fba = self.add_primary_secondary_columns('Sector') + # Joint pool per flow group — do not partition by SectorSourceName. groupby_cols = ['group_id', 'Location'] for rank in ['Primary', 'Secondary']: # continue if values are all np.nan @@ -1843,20 +1848,16 @@ def add_primary_secondary_columns( } ) - def _identify_secondary(row: _FlowBySeries) -> str: - sectors = [ - row[f'{col_type}ProducedBy'], - row[f'{col_type}ConsumedBy'], - ] - sectors.remove(row[f'Primary{col_type}']) - return sectors[0] - + # Secondary is the other of ProducedBy/ConsumedBy. Vectorized + # so an empty frame (matching miss) does not raise: pandas + # apply(axis='columns') on 0 rows returns a DataFrame, and + # assign then errors with "Cannot set a DataFrame with + # multiple columns to the single column SecondarySector". fb = fb.assign( **{ - f'Secondary{col_type}': ( - fb.apply(_identify_secondary, axis='columns') - # ^^^ Applying with axis='columns' applies TO each row. - .astype('object') + f'Secondary{col_type}': fb[f'{col_type}ConsumedBy'].where( + fb[f'Primary{col_type}'].eq(fb[f'{col_type}ProducedBy']), + fb[f'{col_type}ProducedBy'], ) } ) diff --git a/bedrock/transform/flowbysector.py b/bedrock/transform/flowbysector.py index 9e16e2316..a82dd2367 100644 --- a/bedrock/transform/flowbysector.py +++ b/bedrock/transform/flowbysector.py @@ -358,15 +358,16 @@ def prepare_fbs( ) -> FlowBySector: if 'activity_sets' in self.config: - try: - return pd.concat( # type: ignore[return-value] - [ - fbs.prepare_fbs() # type: ignore[operator] - for fbs in (self.select_by_fields().activity_sets()) - ] - ).reset_index(drop=True) - except ValueError: - return FlowBySector(pd.DataFrame(), convert_df_to_flowby=True) + prepared = [] + for fbs in self.select_by_fields().activity_sets(): + try: + prepared.append(fbs.prepare_fbs()) # type: ignore[operator] + except ValueError as exc: + log.exception(f'{fbs.full_name} failed while preparing FBS: {exc}') + raise ValueError( + f'{fbs.full_name} failed while preparing FBS: {exc}' + ) from exc + return pd.concat(prepared).reset_index(drop=True) return ( self.function_socket('clean_fbs') .select_by_fields() From ccfe417e3d754014c8199c19a65fe30dcd31b3d6 Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Thu, 27 Aug 2026 15:03:14 -0600 Subject: [PATCH 08/18] prevent key error --- bedrock/extract/eia/EIA_MECS.py | 2 +- bedrock/transform/flowby.py | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/bedrock/extract/eia/EIA_MECS.py b/bedrock/extract/eia/EIA_MECS.py index a9d5d9fe3..fc8683dcc 100644 --- a/bedrock/extract/eia/EIA_MECS.py +++ b/bedrock/extract/eia/EIA_MECS.py @@ -666,7 +666,7 @@ def estimate_suppressed_mecs_energy( def keep_chemical_manufacturing(fba: FlowByActivity, **_kwargs: Any) -> FlowByActivity: '''Keep MECS NAICS 325 (chemicals) and descendants. CEDA NEU NG is chemicals-only.''' - return fba.query("ActivityConsumedBy.str.startswith('325')") + return fba.query("ActivityConsumedBy.str.startswith('325')").reset_index(drop=True) # 2013 MECS Other energy / (energy + nonfuel) when the mapped NAICS has no diff --git a/bedrock/transform/flowby.py b/bedrock/transform/flowby.py index 0971ffc66..26c0a504c 100644 --- a/bedrock/transform/flowby.py +++ b/bedrock/transform/flowby.py @@ -1738,14 +1738,17 @@ def extract_target_geoscale(text: str) -> str | None: # if Geo Corr column is missing from the df or if all Geo Corr column is all 0s, add score based on geo if 'GeographicalCorrelation' not in fbs: - if fbs['LocationSystem'][0] == 'Census_Region': + if fbs.empty: + return fbs + location_system = fbs['LocationSystem'].iloc[0] + if location_system == 'Census_Region': fbs = fbs.assign(GeographicalCorrelation=4) - elif fbs['LocationSystem'][0] == 'Census_Division': + elif location_system == 'Census_Division': fbs = fbs.assign(GeographicalCorrelation=3) else: # assign geo corr score by FIPS year try: - loc_match = re.search(r"\d{4}", fbs['LocationSystem'][0]) + loc_match = re.search(r"\d{4}", location_system) assert loc_match is not None fips = geo_get_all_fips(int(loc_match.group())).rename( # type: ignore[arg-type] columns={ From 751592ebdd06551cb407d843296abd2cc076ca81 Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Thu, 27 Aug 2026 15:03:30 -0600 Subject: [PATCH 09/18] use umd data over epa --- .../transform/energy/Energy_manufacturing_national_2018.yaml | 2 +- .../transform/energy/Energy_manufacturing_national_2022.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bedrock/transform/energy/Energy_manufacturing_national_2018.yaml b/bedrock/transform/energy/Energy_manufacturing_national_2018.yaml index a55370b23..90dfd8985 100644 --- a/bedrock/transform/energy/Energy_manufacturing_national_2018.yaml +++ b/bedrock/transform/energy/Energy_manufacturing_national_2018.yaml @@ -119,7 +119,7 @@ source_names: FlowName: Transport Class: Money clean_source: - EPA_GHGI_T_A_10: + UMD_GHGIA_T_A5_1_S6: year: *mecs_year selection_fields: Unit: TBtu diff --git a/bedrock/transform/energy/Energy_manufacturing_national_2022.yaml b/bedrock/transform/energy/Energy_manufacturing_national_2022.yaml index 9628878e3..017098c77 100644 --- a/bedrock/transform/energy/Energy_manufacturing_national_2022.yaml +++ b/bedrock/transform/energy/Energy_manufacturing_national_2022.yaml @@ -119,7 +119,7 @@ source_names: FlowName: Transport Class: Money clean_source: - EPA_GHGI_T_A_6: + UMD_GHGIA_T_A5_1_S2: year: *mecs_year selection_fields: Unit: TBtu From 519d2bd00297870a787f396b2fda8eaf4f4a0fa0 Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Thu, 27 Aug 2026 16:10:02 -0600 Subject: [PATCH 10/18] bea attribution over equal --- .../Energy_manufacturing_national_2018.yaml | 51 +++++++++++++++---- .../Energy_manufacturing_national_2022.yaml | 51 +++++++++++++++---- 2 files changed, 84 insertions(+), 18 deletions(-) diff --git a/bedrock/transform/energy/Energy_manufacturing_national_2018.yaml b/bedrock/transform/energy/Energy_manufacturing_national_2018.yaml index 90dfd8985..7b20a8dd0 100644 --- a/bedrock/transform/energy/Energy_manufacturing_national_2018.yaml +++ b/bedrock/transform/energy/Energy_manufacturing_national_2018.yaml @@ -8,6 +8,16 @@ year: &mecs_year 2018 geoscale: national _attribution_sources: + BEA_use: &bea_use + year: 2017 + activity_to_sector_mapping: Cornerstone_2025 + exclusion_fields: + ActivityConsumedBy: ['F03000', 'F04000', 'F05000', 'F02E00', + 'F06E00', 'F07E00', 'F10E00', 'F02R00', + 'T001', 'T004', 'T007', 'T019'] + ActivityProducedBy: ['T007', 'T013', 'T015', 'T016', 'TOP', + 'MCIF'] + attribution_method: equal transport_industries: &transport_industries - '481000' - '482000' @@ -51,26 +61,49 @@ source_names: clean_fba_before_activity_sets: - !script_function:EIA_MECS estimate_suppressed_mecs_energy activity_sets: - direct_table_3_1: + table_3_1_coal: selection_fields: Description: Table 3.1 - FlowName: - - Coal - - Natural Gas - attribution_method: direct - direct_table_2_1: + FlowName: Coal + attribution_method: proportional + attribution_source: + BEA_Detail_Use_AfterRedef: + <<: *bea_use + selection_fields: + ActivityProducedBy: {'212100': ''} + table_3_1_ng: + selection_fields: + Description: Table 3.1 + FlowName: Natural Gas + attribution_method: proportional + attribution_source: + BEA_Detail_Use_AfterRedef: + <<: *bea_use + selection_fields: + ActivityProducedBy: {'221200': ''} + table_2_1: selection_fields: Description: Table 2.1 FlowName: 'Hydrocarbon Gas Liquids, excluding natural gasoline': HGL Other: Petroleum - attribution_method: direct - direct_table_2_1_ng: + attribution_method: proportional + attribution_source: + BEA_Detail_Use_AfterRedef: + <<: *bea_use + selection_fields: + ActivityProducedBy: {'324110': ''} + table_2_1_ng: selection_fields: Description: Table 2.1 FlowName: Natural Gas clean_fba: !script_function:EIA_MECS keep_chemical_manufacturing - attribution_method: direct + attribution_method: proportional + attribution_source: + BEA_Detail_Use_AfterRedef: + <<: *bea_use + selection_fields: + ActivityProducedBy: {'221200': ''} BEA_Detail_Use_AfterRedef: year: 2017 diff --git a/bedrock/transform/energy/Energy_manufacturing_national_2022.yaml b/bedrock/transform/energy/Energy_manufacturing_national_2022.yaml index 017098c77..227dfbc15 100644 --- a/bedrock/transform/energy/Energy_manufacturing_national_2022.yaml +++ b/bedrock/transform/energy/Energy_manufacturing_national_2022.yaml @@ -8,6 +8,16 @@ year: &mecs_year 2022 geoscale: national _attribution_sources: + BEA_use: &bea_use + year: 2017 + activity_to_sector_mapping: Cornerstone_2025 + exclusion_fields: + ActivityConsumedBy: ['F03000', 'F04000', 'F05000', 'F02E00', + 'F06E00', 'F07E00', 'F10E00', 'F02R00', + 'T001', 'T004', 'T007', 'T019'] + ActivityProducedBy: ['T007', 'T013', 'T015', 'T016', 'TOP', + 'MCIF'] + attribution_method: equal transport_industries: &transport_industries - '481000' - '482000' @@ -51,26 +61,49 @@ source_names: clean_fba_before_activity_sets: - !script_function:EIA_MECS estimate_suppressed_mecs_energy activity_sets: - direct_table_3_1: + table_3_1_coal: selection_fields: Description: Table 3.1 - FlowName: - - Coal - - Natural Gas - attribution_method: direct - direct_table_2_1: + FlowName: Coal + attribution_method: proportional + attribution_source: + BEA_Detail_Use_AfterRedef: + <<: *bea_use + selection_fields: + ActivityProducedBy: {'212100': ''} + table_3_1_ng: + selection_fields: + Description: Table 3.1 + FlowName: Natural Gas + attribution_method: proportional + attribution_source: + BEA_Detail_Use_AfterRedef: + <<: *bea_use + selection_fields: + ActivityProducedBy: {'221200': ''} + table_2_1: selection_fields: Description: Table 2.1 FlowName: 'Hydrocarbon Gas Liquids, excluding natural gasoline': HGL Other: Petroleum - attribution_method: direct - direct_table_2_1_ng: + attribution_method: proportional + attribution_source: + BEA_Detail_Use_AfterRedef: + <<: *bea_use + selection_fields: + ActivityProducedBy: {'324110': ''} + table_2_1_ng: selection_fields: Description: Table 2.1 FlowName: Natural Gas clean_fba: !script_function:EIA_MECS keep_chemical_manufacturing - attribution_method: direct + attribution_method: proportional + attribution_source: + BEA_Detail_Use_AfterRedef: + <<: *bea_use + selection_fields: + ActivityProducedBy: {'221200': ''} BEA_Detail_Use_AfterRedef: year: 2017 From 35ab5c0357f08b5d7cba3e561b0e5613dd062a34 Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Fri, 28 Aug 2026 08:41:56 -0600 Subject: [PATCH 11/18] update naics year --- bedrock/utils/config/source_catalog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bedrock/utils/config/source_catalog.yaml b/bedrock/utils/config/source_catalog.yaml index 374f9fc0b..0ea650893 100644 --- a/bedrock/utils/config/source_catalog.yaml +++ b/bedrock/utils/config/source_catalog.yaml @@ -214,7 +214,7 @@ EIA_MECS_Energy: activity_schema: {2010: NAICS_2007_Code, 2014: NAICS_2012_Code, 2018: NAICS_2017_Code, - 2022: NAICS_2017_Code, + 2022: NAICS_2022_Code, } sector_hierarchy: "parent-incompleteChild" EIA_MECS_Energy_Allocation_CEDA: From a022f3049d92064ff2bcfd4acf4df622cde1cf38 Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Mon, 31 Aug 2026 13:58:18 -0600 Subject: [PATCH 12/18] update petrol attribution --- bedrock/extract/eia/EIA_MECS.py | 91 +++++++++---------- .../Energy_manufacturing_national_2018.yaml | 1 + .../Energy_manufacturing_national_2022.yaml | 1 + 3 files changed, 47 insertions(+), 46 deletions(-) diff --git a/bedrock/extract/eia/EIA_MECS.py b/bedrock/extract/eia/EIA_MECS.py index fc8683dcc..bcc9aaae5 100644 --- a/bedrock/extract/eia/EIA_MECS.py +++ b/bedrock/extract/eia/EIA_MECS.py @@ -669,26 +669,24 @@ def keep_chemical_manufacturing(fba: FlowByActivity, **_kwargs: Any) -> FlowByAc return fba.query("ActivityConsumedBy.str.startswith('325')").reset_index(drop=True) -# 2013 MECS Other energy / (energy + nonfuel) when the mapped NAICS has no -# Table 2.1 / 3.1 Other ratio. Keys are MECS NAICS prefixes. -PETROL_FUEL_RATIO_FALLBACKS: dict[str, float] = { - '311221': 0.93, - '324110': 0.43, - '324199': 0.45, - '324121': 0.45, - '324122': 0.45, - '325110': 0.95, - '325180': 0.91, - '325194': 0.76, - '325211': 0.99, - '325212': 0.84, - '325311': 0.50, - '327211': 0.00, - '327310': 0.98, - '331110': 0.96, - '3312': 0.70, - '336': 0.83, -} +def keep_industrial_petroleum_consumers( + fba: FlowByActivity, **_kwargs: Any +) -> FlowByActivity: + ''' + clean_fba on petrol activity set. Keep ag/mining/construction/manufacturing + and 221200 consumers. + ''' + sector_col = ( + 'SectorConsumedBy' + if 'SectorConsumedBy' in fba.columns + else 'ActivityConsumedBy' + ) + sector = fba[sector_col].astype(str) + keep = sector.str.startswith(('11', '21', '23', '31', '32', '33')) | sector.isin( + frozenset({'221200'}) + ) + + return fba.loc[keep].reset_index(drop=True) def multiply_bea_by_mecs_petroleum_energy_fraction( @@ -697,9 +695,8 @@ def multiply_bea_by_mecs_petroleum_energy_fraction( ''' clean_fba_after_attribution. Industrial petrol weights are BEA 324110 purchases times MECS Other energy fraction Table 3.1 / (Table 2.1 + Table 3.1). - Join on mapped NAICS (SectorConsumedBy after Cornerstone_2025), prefix-matching - published MECS lines. Sectors with no MECS ratio use 2013 fallback values, - else 1.0. Restricts to ag/mining/construction/manufacturing plus 221200. + MECS ratios are mapped via Cornerstone_2025 and joined on SectorConsumedBy; + sectors with no MECS line use 1.0. ''' clean_source = fba.config['clean_source'] if isinstance(clean_source, str): @@ -710,17 +707,35 @@ def multiply_bea_by_mecs_petroleum_energy_fraction( mecs = FlowByActivity( getFlowByActivity(name, year), full_name=name, - config={**get_catalog_info(name), **src_config, 'year': year}, + config={ + **get_catalog_info(name), + **src_config, + 'year': year, + 'activity_to_sector_mapping': fba.config['activity_to_sector_mapping'], + 'target_naics_year': fba.config['target_naics_year'], + 'industry_spec': fba.config['industry_spec'], + }, + ) + mecs = ( + mecs.function_socket('estimate_suppressed') + .select_by_fields() + .convert_units_and_flows() + .reset_index(drop=True) + .reset_index(names='group_id') + .assign(group_total=lambda x: x.FlowAmount) ) - mecs = mecs.function_socket('estimate_suppressed').select_by_fields() + target_year = fba.config['target_naics_year'] + sector_col = 'SectorConsumedBy' t21 = ( mecs.query("Description == 'Table 2.1'") - .groupby('ActivityConsumedBy')['FlowAmount'] + .map_to_sectors(target_year=target_year) + .groupby(sector_col)['FlowAmount'] .sum() ) t31 = ( mecs.query("Description == 'Table 3.1'") - .groupby('ActivityConsumedBy')['FlowAmount'] + .map_to_sectors(target_year=target_year) + .groupby(sector_col)['FlowAmount'] .sum() ) idx = t21.index.union(t31.index) @@ -729,29 +744,13 @@ def multiply_bea_by_mecs_petroleum_energy_fraction( den = t21 + t31 ratios = (t31 / den).where(den != 0).fillna(1.0) - sector_col = ( + bea_sector_col = ( 'SectorConsumedBy' if 'SectorConsumedBy' in fba.columns else 'ActivityConsumedBy' ) - sector = fba[sector_col].astype(str) - industrial = sector.str.startswith( - ('11', '21', '23', '31', '32', '33') - ) | sector.eq('221200') - out = fba.loc[industrial].copy() - ratio_index = set(ratios.index.astype(str)) - - def ratio_for_sector(naics: str) -> float: - for n in range(len(naics), 0, -1): - key = naics[:n] - if key in ratio_index: - return float(ratios[key]) - if key in PETROL_FUEL_RATIO_FALLBACKS: - return PETROL_FUEL_RATIO_FALLBACKS[key] - return 1.0 - - matched = out[sector_col].astype(str).map(ratio_for_sector) - return out.assign(FlowAmount=out['FlowAmount'] * matched) + matched = fba[bea_sector_col].astype(str).map(ratios).fillna(1.0) + return fba.assign(FlowAmount=fba['FlowAmount'] * matched) def household_petroleum_transport_fraction(config: dict[str, Any]) -> float: diff --git a/bedrock/transform/energy/Energy_manufacturing_national_2018.yaml b/bedrock/transform/energy/Energy_manufacturing_national_2018.yaml index 7b20a8dd0..2156e75e2 100644 --- a/bedrock/transform/energy/Energy_manufacturing_national_2018.yaml +++ b/bedrock/transform/energy/Energy_manufacturing_national_2018.yaml @@ -123,6 +123,7 @@ source_names: assign_fields: FlowName: Petroleum Class: Money + clean_fba: !script_function:EIA_MECS keep_industrial_petroleum_consumers clean_source: EIA_MECS_Energy: year: *mecs_year diff --git a/bedrock/transform/energy/Energy_manufacturing_national_2022.yaml b/bedrock/transform/energy/Energy_manufacturing_national_2022.yaml index 227dfbc15..343dfd150 100644 --- a/bedrock/transform/energy/Energy_manufacturing_national_2022.yaml +++ b/bedrock/transform/energy/Energy_manufacturing_national_2022.yaml @@ -123,6 +123,7 @@ source_names: assign_fields: FlowName: Petroleum Class: Money + clean_fba: !script_function:EIA_MECS keep_industrial_petroleum_consumers clean_source: EIA_MECS_Energy: year: *mecs_year From ad3056d526b68fbaa14ce53385434ff7756814cb Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Tue, 1 Sep 2026 15:09:37 -0600 Subject: [PATCH 13/18] drop use of tags --- .../Energy_manufacturing_national_2018.yaml | 38 +++---------------- .../Energy_manufacturing_national_2022.yaml | 38 +++---------------- 2 files changed, 12 insertions(+), 64 deletions(-) diff --git a/bedrock/transform/energy/Energy_manufacturing_national_2018.yaml b/bedrock/transform/energy/Energy_manufacturing_national_2018.yaml index 2156e75e2..3de014015 100644 --- a/bedrock/transform/energy/Energy_manufacturing_national_2018.yaml +++ b/bedrock/transform/energy/Energy_manufacturing_national_2018.yaml @@ -18,36 +18,6 @@ _attribution_sources: ActivityProducedBy: ['T007', 'T013', 'T015', 'T016', 'TOP', 'MCIF'] attribution_method: equal - transport_industries: &transport_industries - - '481000' - - '482000' - - '483000' - - '484000' - - '485000' - - '486000' - - '48A000' - - '492000' - - 'S00500' - - 'S00600' - - '491000' - - 'GSLGO' - - 'S00203' - - 'F01000' - petrol_exclude: &petrol_exclude - - '481000' - - '482000' - - '483000' - - '484000' - - '485000' - - '486000' - - '48A000' - - '492000' - - 'S00500' - - 'S00600' - - '491000' - - 'GSLGO' - - 'S00203' - - 'F01000' source_names: EIA_MECS_Energy: @@ -119,7 +89,9 @@ source_names: selection_fields: ActivityProducedBy: {'324110': ''} exclusion_fields: - ActivityConsumedBy: *petrol_exclude + ActivityConsumedBy: ['481000', '482000', '483000', '484000', + '485000', '486000', '48A000', '492000', 'S00500', 'S00600', + '491000', 'GSLGO', 'S00203', 'F01000'] assign_fields: FlowName: Petroleum Class: Money @@ -148,7 +120,9 @@ source_names: neu_transport: selection_fields: ActivityProducedBy: {'324110': ''} - ActivityConsumedBy: *transport_industries + ActivityConsumedBy: ['481000', '482000', '483000', + '484000', '485000', '486000', '48A000', '492000', + 'S00500', 'S00600', '491000', 'GSLGO', 'S00203', 'F01000'] assign_fields: FlowName: Transport Class: Money diff --git a/bedrock/transform/energy/Energy_manufacturing_national_2022.yaml b/bedrock/transform/energy/Energy_manufacturing_national_2022.yaml index 343dfd150..fac104a5c 100644 --- a/bedrock/transform/energy/Energy_manufacturing_national_2022.yaml +++ b/bedrock/transform/energy/Energy_manufacturing_national_2022.yaml @@ -18,36 +18,6 @@ _attribution_sources: ActivityProducedBy: ['T007', 'T013', 'T015', 'T016', 'TOP', 'MCIF'] attribution_method: equal - transport_industries: &transport_industries - - '481000' - - '482000' - - '483000' - - '484000' - - '485000' - - '486000' - - '48A000' - - '492000' - - 'S00500' - - 'S00600' - - '491000' - - 'GSLGO' - - 'S00203' - - 'F01000' - petrol_exclude: &petrol_exclude - - '481000' - - '482000' - - '483000' - - '484000' - - '485000' - - '486000' - - '48A000' - - '492000' - - 'S00500' - - 'S00600' - - '491000' - - 'GSLGO' - - 'S00203' - - 'F01000' source_names: EIA_MECS_Energy: @@ -119,7 +89,9 @@ source_names: selection_fields: ActivityProducedBy: {'324110': ''} exclusion_fields: - ActivityConsumedBy: *petrol_exclude + ActivityConsumedBy: ['481000', '482000', '483000', '484000', + '485000', '486000', '48A000', '492000', 'S00500', 'S00600', + '491000', 'GSLGO', 'S00203', 'F01000'] assign_fields: FlowName: Petroleum Class: Money @@ -148,7 +120,9 @@ source_names: neu_transport: selection_fields: ActivityProducedBy: {'324110': ''} - ActivityConsumedBy: *transport_industries + ActivityConsumedBy: ['481000', '482000', '483000', + '484000', '485000', '486000', '48A000', '492000', + 'S00500', 'S00600', '491000', 'GSLGO', 'S00203', 'F01000'] assign_fields: FlowName: Transport Class: Money From 87d7f45960b894b7a5f4ff3f2eba079cad3f3e35 Mon Sep 17 00:00:00 2001 From: Catherine Birney <60186515+catherinebirney@users.noreply.github.com> Date: Thu, 3 Sep 2026 16:06:42 -0600 Subject: [PATCH 14/18] By mecs fix (#847) --------- Co-authored-by: Ben Young --- bedrock/extract/eia/EIA_MECS.py | 7 + bedrock/extract/eia/EIA_MECS_Energy.yaml | 3 + bedrock/extract/flowbyactivity.py | 113 ++++++++-------- bedrock/transform/flowby.py | 34 ++++- bedrock/transform/flowbyclean.py | 86 +++++++++++- .../ghg/GHG_national_Cornerstone_2017.yaml | 2 + .../ghg/GHG_national_Cornerstone_2018.yaml | 2 + .../ghg/GHG_national_Cornerstone_2019.yaml | 2 + .../ghg/GHG_national_Cornerstone_2020.yaml | 2 + .../ghg/GHG_national_Cornerstone_2021.yaml | 2 + .../ghg/GHG_national_Cornerstone_2022.yaml | 2 + .../ghg/GHG_national_Cornerstone_2023.yaml | 2 + .../ghg/GHG_national_Cornerstone_2024.yaml | 2 + bedrock/utils/mapping/naics.py | 124 +++++++++++++++++- 14 files changed, 311 insertions(+), 72 deletions(-) diff --git a/bedrock/extract/eia/EIA_MECS.py b/bedrock/extract/eia/EIA_MECS.py index bcc9aaae5..792981757 100644 --- a/bedrock/extract/eia/EIA_MECS.py +++ b/bedrock/extract/eia/EIA_MECS.py @@ -398,6 +398,13 @@ def _eia_clean_mecs_energy( name.split(' | ', 2)[0] for name in table_dict[year][table]['col_names'] ] + skip_cols = [ + c for c in df_data_region.columns if str(c).startswith('SKIP') + ] + if skip_cols: + df_data_region = df_data_region.drop(columns=skip_cols) + df_rse_region = df_rse_region.drop(columns=skip_cols) + if table[-1] == '5': major_name = "" df_data_region = df_data_region.dropna() diff --git a/bedrock/extract/eia/EIA_MECS_Energy.yaml b/bedrock/extract/eia/EIA_MECS_Energy.yaml index ea813ff5d..fdaf46a56 100644 --- a/bedrock/extract/eia/EIA_MECS_Energy.yaml +++ b/bedrock/extract/eia/EIA_MECS_Energy.yaml @@ -536,13 +536,16 @@ table_dict: - NAICS Code - Subsector and Industry - Total | trillion Btu + - SKIP - Residual Fuel Oil | million bbl - Distillate Fuel Oil | million bbl - Natural Gas | billion cu ft + - SKIP - Hydrocarbon Gas Liquids, excluding natural gasoline | million bbl - Coal | million short tons - Coke and Breeze | million short tons - Hydrogen | trillion Btu + - SKIP - Other | trillion Btu regions: ®ions_2022_21 Total United States : [15,97] diff --git a/bedrock/extract/flowbyactivity.py b/bedrock/extract/flowbyactivity.py index e52379c99..6f789d13f 100644 --- a/bedrock/extract/flowbyactivity.py +++ b/bedrock/extract/flowbyactivity.py @@ -466,8 +466,7 @@ def map_to_sectors( crosswalk. """ from bedrock.transform.flowbyclean import ( # noqa: PLC0415 - define_parentincompletechild_descendants, - drop_parentincompletechild_descendants, + map_parentincompletechild_sectors, ) # determine activity schema and use for mapping @@ -559,11 +558,6 @@ def map_to_sectors( **{f"Sector{direction}": np.nan} ).assign(**{f"{direction}SectorType": np.nan}) else: - if self.config.get('sector_hierarchy') == 'parent-incompleteChild': - # add descendants column - fba_w_naics = define_parentincompletechild_descendants( - fba_w_naics, activity_col=f'Activity{direction}' - ) if "NAICS" in activity_schema: primary_sector_key = naics_key secondary_sector_key = None @@ -571,59 +565,64 @@ def map_to_sectors( primary_sector_key = activity_to_source_naics_crosswalk secondary_sector_key = naics_key - activity_to_target_naics_crosswalk = subset_sector_key( - fba_w_naics, - f'Activity{direction}', - str(source_year), - primary_sector_key=primary_sector_key, - secondary_sector_key=secondary_sector_key, - ) - - fba_w_naics = ( - fba_w_naics.merge( - activity_to_target_naics_crosswalk, - how='left', - on=[ - 'Class', - 'Flowable', - 'Context', - 'ActivityProducedBy', - 'ActivityConsumedBy', - ], - ) - .rename( - columns={ - 'target_naics': f'Sector{direction}', # when activities are sector-like - 'Sector': f'Sector{direction}', # when activities are text based - 'SectorType': f'{direction}SectorType', - } - ) - .drop( - columns=[ - 'ActivitySourceName', - 'SectorSourceName', - 'source_naics', # when activities are sector-like - 'Activity', # when activities are text based - ], - errors='ignore', - ) - ) - # drop original DQ scores in favor of modified scores after mapping - dq_cols = ['DataReliability', 'DataCollection'] - for c in dq_cols: - fba_w_naics.loc[fba_w_naics[f'{c}_y'].notnull(), f'{c}_x'] = ( - fba_w_naics[f'{c}_y'] + if self.config.get('sector_hierarchy') == 'parent-incompleteChild': + fba_w_naics = map_parentincompletechild_sectors( + fba_w_naics, + activity_col=f'Activity{direction}', + sector_col=f'Sector{direction}', + sector_type_col=f'{direction}SectorType', + source_year=source_year, + primary_sector_key=primary_sector_key, + secondary_sector_key=secondary_sector_key, + naics_key=naics_key, ) - fba_w_naics = fba_w_naics.drop(columns=[f'{c}_y']).rename( - columns={f'{c}_x': c} + else: + activity_to_target_naics_crosswalk = subset_sector_key( + fba_w_naics, + f'Activity{direction}', + str(source_year), + primary_sector_key=primary_sector_key, + secondary_sector_key=secondary_sector_key, ) - if ( - fba_w_naics.config.get('sector_hierarchy') - == 'parent-incompleteChild' - ): - fba_w_naics = drop_parentincompletechild_descendants( - fba_w_naics, sector_col=f'Sector{direction}' + + fba_w_naics = ( + fba_w_naics.merge( + activity_to_target_naics_crosswalk, + how='left', + on=[ + 'Class', + 'Flowable', + 'Context', + 'ActivityProducedBy', + 'ActivityConsumedBy', + ], + ) + .rename( + columns={ + 'target_naics': f'Sector{direction}', # when activities are sector-like + 'Sector': f'Sector{direction}', # when activities are text based + 'SectorType': f'{direction}SectorType', + } + ) + .drop( + columns=[ + 'ActivitySourceName', + 'SectorSourceName', + 'source_naics', # when activities are sector-like + 'Activity', # when activities are text based + ], + errors='ignore', + ) ) + # drop original DQ scores in favor of modified scores after mapping + dq_cols = ['DataReliability', 'DataCollection'] + for c in dq_cols: + fba_w_naics.loc[fba_w_naics[f'{c}_y'].notnull(), f'{c}_x'] = ( + fba_w_naics[f'{c}_y'] + ) + fba_w_naics = fba_w_naics.drop(columns=[f'{c}_y']).rename( + columns={f'{c}_x': c} + ) # assign data quality scores based on highest value, if there are data for both SCB and SPB for dq in ['DataReliability', 'DataCollection', 'TechnologicalCorrelation']: if f'{dq}_x' in fba_w_naics.columns: diff --git a/bedrock/transform/flowby.py b/bedrock/transform/flowby.py index 26c0a504c..5e6220605 100644 --- a/bedrock/transform/flowby.py +++ b/bedrock/transform/flowby.py @@ -747,12 +747,15 @@ def aggregate_flowby( # check flowamounts equal after aggregating self_flow = self['FlowAmount'].sum() agg_flow = aggregated['FlowAmount'].sum() - percent_inc = int(((agg_flow - self_flow) * 100) / self_flow) - if percent_inc > 0: + if not np.isclose(agg_flow, self_flow): + percent_diff = ( + ((agg_flow - self_flow) / self_flow) * 100 if self_flow else np.nan + ) log.warning( 'There is an error in aggregating dataframe, as new ' - 'flow totals do not match original dataframe ' - 'flowtotals, there is a {percent_inc}% difference.' + f'flow totals do not match original dataframe ' + f'flowtotals, there is a {percent_diff:.6f}% difference ' + f'(pre={self_flow}, post={agg_flow}).' ) return aggregated # type: ignore[return-value] @@ -1237,7 +1240,9 @@ def proportionally_attribute( denominator=( merged.assign( FlowAmount_other=( - merged.FlowAmount_other * denominator_flag + merged.FlowAmount + * merged.FlowAmount_other + * denominator_flag ) ) .groupby(groupby_cols)['FlowAmount_other'] @@ -1288,10 +1293,14 @@ def proportionally_attribute( ).to_string() ) ) - + # must include group_total in this formula as a weight for the specific case of + # parent-incompleteChild data combined with requiring a sector year conversion proportionally_attributed = non_zero_denominator.assign( FlowAmount=lambda x: ( - x.FlowAmount * x.FlowAmount_other / x.denominator + x.group_total + * x.FlowAmount + * x.FlowAmount_other + / x.denominator ) ) @@ -1665,6 +1674,17 @@ def equally_attribute(self: 'FB') -> 'FB': ) groupby_cols.append(f'{rank}Sector') + # must include group_total in this formula as a weight for the specific case of + # parent-incompleteChild data combined with requiring a sector year conversion + split_sum = fba.groupby('group_id')['FlowAmount'].transform('sum') + fba = fba.assign( + FlowAmount=lambda x: np.where( + split_sum != 0, + x.group_total * x.FlowAmount / split_sum, + x.FlowAmount, + ) + ) + return fba.drop( columns=[ 'PrimarySector', diff --git a/bedrock/transform/flowbyclean.py b/bedrock/transform/flowbyclean.py index b78a53571..d67c78502 100644 --- a/bedrock/transform/flowbyclean.py +++ b/bedrock/transform/flowbyclean.py @@ -20,6 +20,7 @@ from bedrock.utils.mapping.location import US_FIPS from bedrock.utils.mapping.naics import ( map_source_sectors_to_more_aggregated_sectors, + subset_sector_key, ) from bedrock.utils.validation.validation import ( compare_summation_at_sector_lengths_between_two_dfs, @@ -656,13 +657,26 @@ def drop_parentincompletechild_descendants( the dataset, a row mapping 3112 to 311221 will not be dropped, since no more detailed information on 311221 is given. Further attribution/ disaggregation should be done using another datatset such as the QCEW. + + Also drops when the target is an ancestor of a published descendant (e.g. + parent residual mapped to NAICS-5 32511 while 325110 is published). ''' + def _overlaps_published(target: str, descendants: str) -> bool: + t = str(target) + for d in descendants.split(): + if not d: + continue + d = str(d) + if t.startswith(d) or d.startswith(t): + return True + return False + fba2 = ( fba.assign( to_keep=fba.apply( - lambda x: not any( - [str(x[sector_col]).startswith(d) for d in x.descendants.split()] + lambda x: not _overlaps_published( + str(x[sector_col]), str(x.descendants) ), axis='columns', ) @@ -674,6 +688,74 @@ def drop_parentincompletechild_descendants( return fba2 +def map_parentincompletechild_sectors( + fba: FlowByActivity, + *, + activity_col: str, + sector_col: str, + sector_type_col: str, + source_year: int, + primary_sector_key: pd.DataFrame, + secondary_sector_key: pd.DataFrame | None, + naics_key: pd.DataFrame, # noqa: ARG001 — kept for caller API stability +) -> FlowByActivity: + ''' + Map parent-incompleteChild activities (MECS) at industry_spec resolution, + then drop rows whose sector target overlaps a published descendant + (bidirectional prefix check fixes ancestor rollup cases such as 32511 vs + published 325110). + ''' + merge_on = [ + 'Class', + 'Flowable', + 'Context', + 'ActivityProducedBy', + 'ActivityConsumedBy', + ] + fba = define_parentincompletechild_descendants(fba, activity_col=activity_col) + crosswalk = subset_sector_key( + fba, + activity_col, + str(source_year), + primary_sector_key=primary_sector_key, + secondary_sector_key=secondary_sector_key, + ) + mapped = ( + fba.merge(crosswalk, how='left', on=merge_on) + .rename( + columns={ + 'target_naics': sector_col, + 'Sector': sector_col, + 'SectorType': sector_type_col, + } + ) + .drop( + columns=[ + 'ActivitySourceName', + 'SectorSourceName', + 'source_naics', + 'Activity', + ], + errors='ignore', + ) + ) + for c in ['DataReliability', 'DataCollection']: + if f'{c}_y' in mapped.columns: + mapped.loc[mapped[f'{c}_y'].notnull(), f'{c}_x'] = mapped[f'{c}_y'] + mapped = mapped.drop(columns=[f'{c}_y']).rename(columns={f'{c}_x': c}) + mapped = drop_parentincompletechild_descendants(mapped, sector_col=sector_col) + if 'group_id' in mapped.columns and 'group_total' in mapped.columns: + mapped['group_total'] = mapped.groupby('group_id')['group_total'].transform( + 'first' + ) + return FlowByActivity( + mapped, + full_name=fba.full_name, + config=fba.config, + w_sector=True, + ) + + @deprecated("No known use") def proxy_sector_data(fba: FlowByActivity, **_kwargs: Any) -> FlowByActivity: """ diff --git a/bedrock/transform/ghg/GHG_national_Cornerstone_2017.yaml b/bedrock/transform/ghg/GHG_national_Cornerstone_2017.yaml index c4d37022f..523a9ed11 100644 --- a/bedrock/transform/ghg/GHG_national_Cornerstone_2017.yaml +++ b/bedrock/transform/ghg/GHG_national_Cornerstone_2017.yaml @@ -714,6 +714,7 @@ source_names: selection_fields: Flowable: Petroleum Class: Other + # Allocated to asphalt sectors only SectorConsumedBy: - '324121' - '324122' @@ -751,6 +752,7 @@ source_names: Flowable: Petroleum Class: Other exclusion_fields: + # Skips petroleum products used for asphalt sectors SectorConsumedBy: - '324121' - '324122' diff --git a/bedrock/transform/ghg/GHG_national_Cornerstone_2018.yaml b/bedrock/transform/ghg/GHG_national_Cornerstone_2018.yaml index fa16e739c..d9f7eac01 100644 --- a/bedrock/transform/ghg/GHG_national_Cornerstone_2018.yaml +++ b/bedrock/transform/ghg/GHG_national_Cornerstone_2018.yaml @@ -714,6 +714,7 @@ source_names: selection_fields: Flowable: Petroleum Class: Other + # Allocated to asphalt sectors only SectorConsumedBy: - '324121' - '324122' @@ -751,6 +752,7 @@ source_names: Flowable: Petroleum Class: Other exclusion_fields: + # Skips petroleum products used for asphalt sectors SectorConsumedBy: - '324121' - '324122' diff --git a/bedrock/transform/ghg/GHG_national_Cornerstone_2019.yaml b/bedrock/transform/ghg/GHG_national_Cornerstone_2019.yaml index 95d6aa957..8238e2ba0 100644 --- a/bedrock/transform/ghg/GHG_national_Cornerstone_2019.yaml +++ b/bedrock/transform/ghg/GHG_national_Cornerstone_2019.yaml @@ -711,6 +711,7 @@ source_names: selection_fields: Flowable: Petroleum Class: Other + # Allocated to asphalt sectors only SectorConsumedBy: - '324121' - '324122' @@ -748,6 +749,7 @@ source_names: Flowable: Petroleum Class: Other exclusion_fields: + # Skips petroleum products used for asphalt sectors SectorConsumedBy: - '324121' - '324122' diff --git a/bedrock/transform/ghg/GHG_national_Cornerstone_2020.yaml b/bedrock/transform/ghg/GHG_national_Cornerstone_2020.yaml index 15acc2d5f..62fdc6d69 100644 --- a/bedrock/transform/ghg/GHG_national_Cornerstone_2020.yaml +++ b/bedrock/transform/ghg/GHG_national_Cornerstone_2020.yaml @@ -711,6 +711,7 @@ source_names: selection_fields: Flowable: Petroleum Class: Other + # Allocated to asphalt sectors only SectorConsumedBy: - '324121' - '324122' @@ -748,6 +749,7 @@ source_names: Flowable: Petroleum Class: Other exclusion_fields: + # Skips petroleum products used for asphalt sectors SectorConsumedBy: - '324121' - '324122' diff --git a/bedrock/transform/ghg/GHG_national_Cornerstone_2021.yaml b/bedrock/transform/ghg/GHG_national_Cornerstone_2021.yaml index b645fbbd2..b84b9873b 100644 --- a/bedrock/transform/ghg/GHG_national_Cornerstone_2021.yaml +++ b/bedrock/transform/ghg/GHG_national_Cornerstone_2021.yaml @@ -711,6 +711,7 @@ source_names: selection_fields: Flowable: Petroleum Class: Other + # Allocated to asphalt sectors only SectorConsumedBy: - '324121' - '324122' @@ -748,6 +749,7 @@ source_names: Flowable: Petroleum Class: Other exclusion_fields: + # Skips petroleum products used for asphalt sectors SectorConsumedBy: - '324121' - '324122' diff --git a/bedrock/transform/ghg/GHG_national_Cornerstone_2022.yaml b/bedrock/transform/ghg/GHG_national_Cornerstone_2022.yaml index 42f59ff5d..606a811d6 100644 --- a/bedrock/transform/ghg/GHG_national_Cornerstone_2022.yaml +++ b/bedrock/transform/ghg/GHG_national_Cornerstone_2022.yaml @@ -711,6 +711,7 @@ source_names: selection_fields: Flowable: Petroleum Class: Other + # Allocated to asphalt sectors only SectorConsumedBy: - '324121' - '324122' @@ -748,6 +749,7 @@ source_names: Flowable: Petroleum Class: Other exclusion_fields: + # Skips petroleum products used for asphalt sectors SectorConsumedBy: - '324121' - '324122' diff --git a/bedrock/transform/ghg/GHG_national_Cornerstone_2023.yaml b/bedrock/transform/ghg/GHG_national_Cornerstone_2023.yaml index 75db5de8a..8df45a337 100644 --- a/bedrock/transform/ghg/GHG_national_Cornerstone_2023.yaml +++ b/bedrock/transform/ghg/GHG_national_Cornerstone_2023.yaml @@ -711,6 +711,7 @@ source_names: selection_fields: Flowable: Petroleum Class: Other + # Allocated to asphalt sectors only SectorConsumedBy: - '324121' - '324122' @@ -748,6 +749,7 @@ source_names: Flowable: Petroleum Class: Other exclusion_fields: + # Skips petroleum products used for asphalt sectors SectorConsumedBy: - '324121' - '324122' diff --git a/bedrock/transform/ghg/GHG_national_Cornerstone_2024.yaml b/bedrock/transform/ghg/GHG_national_Cornerstone_2024.yaml index a90e6c43d..5515586dc 100644 --- a/bedrock/transform/ghg/GHG_national_Cornerstone_2024.yaml +++ b/bedrock/transform/ghg/GHG_national_Cornerstone_2024.yaml @@ -711,6 +711,7 @@ source_names: selection_fields: Flowable: Petroleum Class: Other + # Allocated to asphalt sectors only SectorConsumedBy: - '324121' - '324122' @@ -748,6 +749,7 @@ source_names: Flowable: Petroleum Class: Other exclusion_fields: + # Skips petroleum products used for asphalt sectors SectorConsumedBy: - '324121' - '324122' diff --git a/bedrock/utils/mapping/naics.py b/bedrock/utils/mapping/naics.py index eccefeca1..cf7e16ca4 100644 --- a/bedrock/utils/mapping/naics.py +++ b/bedrock/utils/mapping/naics.py @@ -555,6 +555,25 @@ def generate_naics_crosswalk_conversion_ratios( # Combine all ratios into a single DataFrame ratios_df = pd.concat(all_ratios, ignore_index=True) + # Sources that are already valid target-year codes do not need conversion + # fan-out. Truncation invents keys like "311" that collide with real + # target-year aggregates and inflate FlowAmount on merge. + valid_targets = set(ratios_df['target'].dropna().unique()) + ratios_df = ratios_df[~ratios_df['source'].isin(valid_targets)] + # Keep 1:1 identity rows so cw_list still recognizes those codes as + # already-valid target-year NAICS (merge stays one row, ratio 1). + target_list = list(valid_targets) + identity = pd.DataFrame( + { + 'source': target_list, + 'target': target_list, + 'naics_count': 1, + 'allocation_ratio': 1.0, + 'length': [len(str(s)) for s in target_list], + } + ) + ratios_df = pd.concat([ratios_df, identity], ignore_index=True) + # TODO: modify how unofficial sectors are added - ensure correct mapping between years # append the unofficial sector codes year_match = re.search(r'\d+', sectorsourcename) @@ -638,6 +657,25 @@ def replace_sectors_with_targetsectors( continue # merge df with the melted sector crosswalk df = df.merge(cw_melt, left_on=c, right_on='NAICS', how='left') + matched = df['allocation_ratio'].notna() + if matched.any(): + ratio_sums = ( + df.loc[matched, ['NAICS', targetsectorsourcename, 'allocation_ratio']] + .drop_duplicates() + .groupby('NAICS')['allocation_ratio'] + .sum() + ) + not_one = ratio_sums[~np.isclose(ratio_sums.to_numpy(dtype=float), 1.0)] + if len(not_one): + details = ', '.join(f'{src}={val}' for src, val in not_one.items()) + raise ValueError( + f'NAICS year conversion allocation_ratio does not sum to 1 ' + f'for source sector(s): {details}' + ) + unmatched = df[c].isin(non_naics) & df['allocation_ratio'].isna() + if unmatched.any(): + missing = sorted(df.loc[unmatched, c].astype(str).unique()) + log.warning(f'No allocation_ratio for NAICS year conversion of: {missing}') # if there is a value in the sectorsourcename column, # use that value to replace sector in column c if value in # column c is in the non_naics list @@ -710,9 +748,33 @@ def convert_naics_year( lambda x: x.split(".")[0] if isinstance(x, str) else x ) - if "NAICS" in activity_schema and "ActivityProducedBy" in df_load.columns: + # parent-incompleteChild: sectors already mapped; converting activities would duplicate values under one group_id. + if ( + "NAICS" in activity_schema + and "ActivityProducedBy" in df_load.columns + and getattr(df_load, 'config', {}).get('sector_hierarchy') + != 'parent-incompleteChild' + ): column_headers += ['ActivityProducedBy', 'ActivityConsumedBy'] + # used to ensure that the group_totals for each group_id do not change after naics year conversion + pre_group_totals = None + pre_flow_sums = None + if ( + "NAICS" in activity_schema + and getattr(df_load, 'config', {}).get('sector_hierarchy') + == 'parent-incompleteChild' + and 'group_id' in df_load.columns + and 'group_total' in df_load.columns + ): + pre_group_totals = df_load.drop_duplicates(subset=['group_id']).set_index( + 'group_id' + )['group_total'] + # Only when converting years: catch merge inflation (per-group FlowAmount + # must match pre-convert; not compared to group_total). + if targetsectorsourcename != sectorsourcename: + pre_flow_sums = df_load.groupby('group_id')['FlowAmount'].sum() + # load the mastercrosswalk and subset by sectorsourcename, # save values to list if targetsectorsourcename == sectorsourcename: @@ -788,6 +850,38 @@ def convert_naics_year( ) nonsectors = check_if_sectors_are_naics(df, cw_list, column_headers) + # Before dropping unconverted sectors: year convert must conserve FlowAmount + # per group_id + if pre_flow_sums is not None and 'group_id' in df.columns: + post_flow_sums = df.groupby('group_id')['FlowAmount'].sum() + shared = pre_flow_sums.index.intersection(post_flow_sums.index) + mismatched = shared[ + ~np.isclose( + pre_flow_sums.loc[shared].to_numpy(dtype=float), + post_flow_sums.loc[shared].to_numpy(dtype=float), + ) + ] + missing = pre_flow_sums.index.difference(post_flow_sums.index) + missing_nonzero = missing[ + ~np.isclose(pre_flow_sums.loc[missing].to_numpy(dtype=float), 0.0) + ] + if len(mismatched) or len(missing_nonzero): + details = [] + for gid in mismatched: + details.append( + f'group_id={gid}: pre_flow={pre_flow_sums.loc[gid]}, ' + f'post_flow={post_flow_sums.loc[gid]}' + ) + for gid in missing_nonzero: + details.append( + f'group_id={gid}: pre_flow={pre_flow_sums.loc[gid]}, ' + f'post=missing' + ) + raise ValueError( + f'NAICS year conversion did not conserve FlowAmount for ' + f'{dfname}: ' + '; '.join(details) + ) + if len(nonsectors) != 0: log.info(f'Dropping non {targetsectorsourcename}s from dataframe: {nonsectors}') for c in column_headers: @@ -805,12 +899,30 @@ def convert_naics_year( # aggregate data if hasattr(df, 'aggregate_flowby'): if "NAICS" in activity_schema: - df2 = ( - df.drop(columns=['group_id', 'group_total']).aggregate_flowby( - columns_to_group_by=df.groupby_cols # type: ignore[operator] + preserve_group_id = ( + getattr(df, 'config', {}).get('sector_hierarchy') + == 'parent-incompleteChild' + and 'group_id' in df.columns + ) + if preserve_group_id: + # MECS parent-incompleteChild: keep original group_id through NAICS + # year conversion so proportional BEA attribution conserves mass + # across crosswalk fan-out rows. QCEW and other NAICS-like sources + # use the default branch below (many:1 merge, new group_id per row). + df2 = df.aggregate_flowby( + columns_to_group_by=df.groupby_cols + ['group_id'] # type: ignore[operator] ) - ).reset_index(drop=True) - df2 = df2.assign(group_id=df2.index, group_total=df2['FlowAmount']) + if pre_group_totals is not None: + # Convert already scaled FlowAmount by allocation_ratio. + # Restore group_total only; it is the residual checksum. + df2['group_total'] = df2['group_id'].map(pre_group_totals) + else: + df2 = ( + df.drop(columns=['group_id', 'group_total']).aggregate_flowby( + columns_to_group_by=df.groupby_cols # type: ignore[operator] + ) + ).reset_index(drop=True) + df2 = df2.assign(group_id=df2.index, group_total=df2['FlowAmount']) else: df2 = df.aggregate_flowby( columns_to_group_by=df.groupby_cols + ['group_id'] # type: ignore[operator] From 6b48d30295b9362124eb1c93c63e1c0319d17c32 Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Fri, 4 Sep 2026 08:37:27 -0600 Subject: [PATCH 15/18] format --- bedrock/extract/eia/EIA_MECS.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bedrock/extract/eia/EIA_MECS.py b/bedrock/extract/eia/EIA_MECS.py index 792981757..7fba8df2b 100644 --- a/bedrock/extract/eia/EIA_MECS.py +++ b/bedrock/extract/eia/EIA_MECS.py @@ -398,9 +398,7 @@ def _eia_clean_mecs_energy( name.split(' | ', 2)[0] for name in table_dict[year][table]['col_names'] ] - skip_cols = [ - c for c in df_data_region.columns if str(c).startswith('SKIP') - ] + skip_cols = [c for c in df_data_region.columns if str(c).startswith('SKIP')] if skip_cols: df_data_region = df_data_region.drop(columns=skip_cols) df_rse_region = df_rse_region.drop(columns=skip_cols) From 7c71e380bf2f0dc6de2f132365751c61d9bb8bea Mon Sep 17 00:00:00 2001 From: Ben Young Date: Fri, 4 Sep 2026 21:50:38 -0400 Subject: [PATCH 16/18] fix additional naics -> schema --- bedrock/extract/eia/EIA_MECS.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bedrock/extract/eia/EIA_MECS.py b/bedrock/extract/eia/EIA_MECS.py index 722953a7f..616204cd4 100644 --- a/bedrock/extract/eia/EIA_MECS.py +++ b/bedrock/extract/eia/EIA_MECS.py @@ -715,6 +715,7 @@ def multiply_bea_by_mecs_petroleum_energy_fraction( else: ((name, src_config),) = clean_source.items() year = int(src_config['year']) + target_year = fba.config['target_schema_year'] mecs = FlowByActivity( getFlowByActivity(name, year), full_name=name, @@ -723,7 +724,7 @@ def multiply_bea_by_mecs_petroleum_energy_fraction( **src_config, 'year': year, 'activity_to_sector_mapping': fba.config['activity_to_sector_mapping'], - 'target_naics_year': fba.config['target_naics_year'], + 'target_schema_year': target_year, 'industry_spec': fba.config['industry_spec'], }, ) @@ -735,7 +736,6 @@ def multiply_bea_by_mecs_petroleum_energy_fraction( .reset_index(names='group_id') .assign(group_total=lambda x: x.FlowAmount) ) - target_year = fba.config['target_naics_year'] sector_col = 'SectorConsumedBy' t21 = ( mecs.query("Description == 'Table 2.1'") From 7606a7e0ca8d5bad73ffb704e714d58aa795ef95 Mon Sep 17 00:00:00 2001 From: Ben Young Date: Fri, 4 Sep 2026 22:21:19 -0400 Subject: [PATCH 17/18] test(mapping): set group_total in equally_attribute fixtures equally_attribute reweights by group_total (same contract as prepare_fbs); fixtures that only set group_id fail after the MECS merge. --- .../utils/mapping/__tests__/test_mixed_bea_naics_assignment.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bedrock/utils/mapping/__tests__/test_mixed_bea_naics_assignment.py b/bedrock/utils/mapping/__tests__/test_mixed_bea_naics_assignment.py index 654bbab79..080bfe985 100644 --- a/bedrock/utils/mapping/__tests__/test_mixed_bea_naics_assignment.py +++ b/bedrock/utils/mapping/__tests__/test_mixed_bea_naics_assignment.py @@ -113,6 +113,8 @@ def _mapped_fba( 'group_id': 0, } df = pd.DataFrame([{**base, **row} for row in rows]) + # equally_attribute reweights by group_total (prepare_fbs sets both). + df = df.assign(group_total=df['FlowAmount']) return FlowByActivity( df, convert_df_to_flowby=True, From 796a6ca0dba9bf1e0e4e240753d81630e9e8f8ab Mon Sep 17 00:00:00 2001 From: Ben Young Date: Fri, 4 Sep 2026 23:18:11 -0400 Subject: [PATCH 18/18] fix(mapping): preserve MECS mass across #854 map/convert ordering After merging main, target-year industry_spec_key matching drops source-year-only MECS leaves unless year convert runs before subset_sector_key. Convert-before-map then interacts with #847 group_total weighting and #854 denominator dedupe, and with aggregate_flowby dropping FlowAmount==0 leaves used for parent-incompleteChild descendant checks. - Convert NAICS activities before subset_sector_key (once) - Include Activity* in proportional denominator uniqueness - retain_zeros on parent-incompleteChild convert aggregate --- bedrock/extract/flowbyactivity.py | 33 +++++++++++++++++++++++++++++-- bedrock/transform/flowby.py | 10 +++++++++- bedrock/utils/mapping/sector.py | 6 +++++- 3 files changed, 45 insertions(+), 4 deletions(-) diff --git a/bedrock/extract/flowbyactivity.py b/bedrock/extract/flowbyactivity.py index 75cac44de..3c705fbc3 100644 --- a/bedrock/extract/flowbyactivity.py +++ b/bedrock/extract/flowbyactivity.py @@ -705,7 +705,34 @@ def _mapping_jobs_for_dict( ) return jobs + # Sector-like activities whose schema vintage differs from + # target_schema_year must be converted before subset_sector_key: the + # industry_spec_key is built for the target year, so mapping first + # drops source-year-only leaves (e.g. MECS 322120 vs 2017 key). + # Convert at most once; the post-map block below is for callers that + # did not need a pre-map convert. fba_w_sectors = self.copy() + converted_naics_pre_map = False + if sector_like: + naics_year = source_years.get('naics') + target_schema_year = int(self.config['target_schema_year']) + if naics_year is not None and int(naics_year) != target_schema_year: + fba_w_sectors = cast( + FlowByActivity, + convert_naics_year( + fba_w_sectors, + sector_source_name('naics', target_schema_year), + sector_source_name('naics', int(naics_year)), + self.full_name, + ), + ) + source_years['naics'] = target_schema_year + converted_naics_pre_map = True + if default_sec_source_name is not None and 'naics' in activity_schemas: + default_sec_source_name = sector_source_name( + 'naics', target_schema_year + ) + for direction in ['ProducedBy', 'ConsumedBy']: if fba_w_sectors[f'Activity{direction}'].isna().all(): fba_w_sectors = fba_w_sectors.assign( @@ -843,12 +870,14 @@ def _mapping_jobs_for_dict( } ) - # NAICS vintage conversion only when sector-like NAICS years differ + # NAICS vintage conversion only when sector-like NAICS years differ and + # we did not already convert before mapping (see converted_naics_pre_map). naics_year = source_years.get('naics') if ( sector_like + and not converted_naics_pre_map and naics_year is not None - and naics_year != self.config['target_schema_year'] + and int(naics_year) != int(self.config['target_schema_year']) ): fba_w_sectors = cast( FlowByActivity, diff --git a/bedrock/transform/flowby.py b/bedrock/transform/flowby.py index e5b2efadf..abd459792 100644 --- a/bedrock/transform/flowby.py +++ b/bedrock/transform/flowby.py @@ -1271,7 +1271,13 @@ def proportionally_attribute( ).fillna({'FlowAmount_other': 0}) # Unmatched peers (other==0) must not occupy the uniqueness - # slot for a code that a matched schema peer also uses. + # slot for a code that a matched schema peer also uses + # (SectorSourceName distinguishes mixed BEA/NAICS peers). + # Activity* distinguishes NAICS year-convert siblings that share + # group_id and land on the same target sector with split + # FlowAmounts (e.g. 322120 -> 322121/322122 both -> 32212); + # without them, duplicated() keeps one slot and each sibling + # receives full group_total. matched = merged['FlowAmount_other'] != 0 denominator_flag = pd.Series(False, index=merged.index) if matched.any(): @@ -1281,6 +1287,8 @@ def proportionally_attribute( *groupby_cols, f'{rank}Sector', f'{rank}SectorSourceName', + 'ActivityProducedBy', + 'ActivityConsumedBy', ] ) with_denominator = merged.assign( diff --git a/bedrock/utils/mapping/sector.py b/bedrock/utils/mapping/sector.py index 0f66d6cba..db62cfac4 100644 --- a/bedrock/utils/mapping/sector.py +++ b/bedrock/utils/mapping/sector.py @@ -1398,8 +1398,12 @@ def convert_naics_year( # year conversion so proportional BEA attribution conserves mass # across crosswalk fan-out rows. QCEW and other NAICS-like sources # use the default branch below (many:1 merge, new group_id per row). + # Retain FlowAmount==0 leaves: they still mark published children for + # parent-incompleteChild descendant drops after mapping (e.g. MECS + # Table 2.1 Other 325120=0 must keep parent 325 from mapping to 32512). df2 = df.aggregate_flowby( - columns_to_group_by=df.groupby_cols + ['group_id'] # type: ignore[operator] + columns_to_group_by=df.groupby_cols + ['group_id'], # type: ignore[operator] + retain_zeros=True, ) if pre_group_totals is not None: # Convert already scaled FlowAmount by allocation_ratio.