From d8f7eafe6cf1ed25c6a2c3ba55c06b50c0d4b59e Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Tue, 10 Jun 2025 08:18:27 -0600 Subject: [PATCH 01/48] update pandas package requirements --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 5c7ede698..393a5b5d5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ kaleido == 0.2.0; platform_system=="Linux" or platform_system=="Darwin" matplotlib >= 3.4.3 # Plotting numpy>=1.20.1, <2.0.0 # Library used for arrays openpyxl >= 3.0.7 # Extract data from xlsm files -pandas>=1.4.0, <2.1.0 # Powerful data structures for data analysis, time series, and statistics. +pandas>=2.2.0 # Powerful data structures for data analysis, time series, and statistics. pip>=9 # The PyPA recommended tool for installing Python packages. plotly >= 5.10.0 # Plotting pycountry >= 19.8.18 # ISO country codes diff --git a/setup.py b/setup.py index 9b21e943e..61e65dc91 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ 'matplotlib>=3.4.3', 'numpy>=1.20.1, <2.0.0', 'openpyxl>=3.0.7', - 'pandas>=1.4.0, <2.1.0', + 'pandas>=2.2.0', 'pip>=9', 'plotly>=5.10.0 ', 'pycountry>=19.8.18', From 02484a972db81ccb3d60d95fa2f14934434421ae Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Tue, 10 Jun 2025 08:21:11 -0600 Subject: [PATCH 02/48] update pandas package requirements for google collab --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 393a5b5d5..d8c36836d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ kaleido == 0.2.0; platform_system=="Linux" or platform_system=="Darwin" matplotlib >= 3.4.3 # Plotting numpy>=1.20.1, <2.0.0 # Library used for arrays openpyxl >= 3.0.7 # Extract data from xlsm files -pandas>=2.2.0 # Powerful data structures for data analysis, time series, and statistics. +pandas==2.2.2 # Powerful data structures for data analysis, time series, and statistics. pip>=9 # The PyPA recommended tool for installing Python packages. plotly >= 5.10.0 # Plotting pycountry >= 19.8.18 # ISO country codes diff --git a/setup.py b/setup.py index 61e65dc91..20dbf9f65 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ 'matplotlib>=3.4.3', 'numpy>=1.20.1, <2.0.0', 'openpyxl>=3.0.7', - 'pandas>=2.2.0', + 'pandas==2.2.2', 'pip>=9', 'plotly>=5.10.0 ', 'pycountry>=19.8.18', From 95ac2fcfb36f0c9d521702f426d8ee86b4312ba6 Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Tue, 10 Jun 2025 08:47:48 -0600 Subject: [PATCH 03/48] update additional package requirements for google collab --- requirements.txt | 28 ++++++++++++++-------------- setup.py | 2 +- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/requirements.txt b/requirements.txt index d8c36836d..01c1ff9a4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,22 +1,22 @@ git+https://github.com/USEPA/fedelemflowlist.git@develop#egg=fedelemflowlist git+https://github.com/USEPA/esupy.git@develop#egg=esupy git+https://github.com/USEPA/standardizedinventories.git@develop#egg=StEWI -appdirs >= 1.4.3 # Storing user data -bibtexparser>=1.2.0 # Generates bibtex +appdirs >= 1.4.4 # Storing user data +bibtexparser>=1.4.3 # Generates bibtex kaleido == 0.1.0.post1; platform_system == "Windows" # export plotly.py figures as static images kaleido == 0.2.0; platform_system=="Linux" or platform_system=="Darwin" -matplotlib >= 3.4.3 # Plotting -numpy>=1.20.1, <2.0.0 # Library used for arrays -openpyxl >= 3.0.7 # Extract data from xlsm files +matplotlib >= 3.10.3 # Plotting +numpy>=2.3.0 # Library used for arrays +openpyxl >= 3.1.5 # Extract data from xlsm files pandas==2.2.2 # Powerful data structures for data analysis, time series, and statistics. -pip>=9 # The PyPA recommended tool for installing Python packages. -plotly >= 5.10.0 # Plotting -pycountry >= 19.8.18 # ISO country codes -python-dotenv >= 0.19.1 # Reads .env files -pyyaml>=5.3 # Yaml for python -requests >=2.22.0 # Web service calls +pip>=25.1.1 # The PyPA recommended tool for installing Python packages. +plotly >= 6.1.2 # Plotting +pycountry >= 24.6.1 # ISO country codes +python-dotenv >= 1.1.0 # Reads .env files +pyyaml>=6.0.2 # Yaml for python +requests >=2.32.4 # Web service calls requests_ftp == 0.3.1 # Requests implementation for FTP -seaborn >= 0.11.2 # Plotting -setuptools>=41 # Fully-featured library designed to facilitate packaging Python projects. -tabula-py >= 2.1.1 # PDF reader +seaborn >= 0.13.2 # Plotting +setuptools>=80.9.0 # Fully-featured library designed to facilitate packaging Python projects. +tabula-py >= 2.10.0 # PDF reader xlrd >= 2.0.1 # Extract data from xls files diff --git a/setup.py b/setup.py index 20dbf9f65..757b7ee67 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ "kaleido==0.1.0.post1;platform_system=='Windows'", "kaleido==0.2.0;platform_system=='Linux' or platform_system=='Darwin'", 'matplotlib>=3.4.3', - 'numpy>=1.20.1, <2.0.0', + 'numpy>=2.3.0', 'openpyxl>=3.0.7', 'pandas==2.2.2', 'pip>=9', From d3684454cfc24aea495771b98da41c02a93c0475 Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Tue, 10 Jun 2025 09:35:45 -0600 Subject: [PATCH 04/48] change numpy version to align with colab requirements --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 01c1ff9a4..e9de33abd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ bibtexparser>=1.4.3 # Generates bibtex kaleido == 0.1.0.post1; platform_system == "Windows" # export plotly.py figures as static images kaleido == 0.2.0; platform_system=="Linux" or platform_system=="Darwin" matplotlib >= 3.10.3 # Plotting -numpy>=2.3.0 # Library used for arrays +numpy<2.1.0 # Library used for arrays openpyxl >= 3.1.5 # Extract data from xlsm files pandas==2.2.2 # Powerful data structures for data analysis, time series, and statistics. pip>=25.1.1 # The PyPA recommended tool for installing Python packages. diff --git a/setup.py b/setup.py index 757b7ee67..c2afd0901 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ "kaleido==0.1.0.post1;platform_system=='Windows'", "kaleido==0.2.0;platform_system=='Linux' or platform_system=='Darwin'", 'matplotlib>=3.4.3', - 'numpy>=2.3.0', + 'numpy<2.1.0', 'openpyxl>=3.0.7', 'pandas==2.2.2', 'pip>=9', From 8f6a0885e26bb713ce2006f37d941414111ca888 Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Sat, 14 Jun 2025 22:36:16 -0500 Subject: [PATCH 05/48] avoid warning: "Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version" --- flowsa/flowby.py | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/flowsa/flowby.py b/flowsa/flowby.py index 70fc7f616..dd44dec84 100644 --- a/flowsa/flowby.py +++ b/flowsa/flowby.py @@ -144,10 +144,13 @@ def __init__( for field, dtype in fields.items() if dtype == 'object' } - data = (data - .fillna(fill_na_dict) - .replace(null_string_dict) - .astype(fields)) + # avoid warning: "Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated + # and will change in a future version" + with pd.option_context('future.no_silent_downcasting', True): + data = (data + .fillna(fill_na_dict) + .replace(null_string_dict) + .astype(fields)) if isinstance(data, pd.DataFrame) and column_order is not None: data = data[[c for c in column_order if c in data.columns] @@ -293,17 +296,19 @@ def standardize_units(self: FB, year: int = None) -> FB: 'new_unit': 'USD', 'conversion_factor': 1 / exchange_rate}).to_frame().T ]) - - standardized = ( - self - .assign(Unit=self.Unit.str.strip()) - .merge(conversion_table, how='left', - left_on='Unit', right_on='old_unit') - .assign(Unit=lambda x: x.new_unit.mask(x.new_unit.isna(), x.Unit), - conversion_factor=lambda x: x.conversion_factor.fillna(1), - FlowAmount=lambda x: x.FlowAmount * x.conversion_factor) - .drop(columns=['old_unit', 'new_unit', 'conversion_factor']) - ) + # avoid warning: "Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated + # and will change in a future version" + with pd.option_context('future.no_silent_downcasting', True): + standardized = ( + self + .assign(Unit=self.Unit.str.strip()) + .merge(conversion_table, how='left', + left_on='Unit', right_on='old_unit') + .assign(Unit=lambda x: x.new_unit.mask(x.new_unit.isna(), x.Unit), + conversion_factor=lambda x: x.conversion_factor.fillna(1), + FlowAmount=lambda x: x.FlowAmount * x.conversion_factor) + .drop(columns=['old_unit', 'new_unit', 'conversion_factor']) + ) standardized_units = list(conversion_table.new_unit.unique()) From d9985650b07f19d11239d181aadf93732a5a6361 Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Sat, 14 Jun 2025 22:37:45 -0500 Subject: [PATCH 06/48] modify how index is assigned to a column name for new pkg versions --- flowsa/flowby.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flowsa/flowby.py b/flowsa/flowby.py index dd44dec84..fae044905 100644 --- a/flowsa/flowby.py +++ b/flowsa/flowby.py @@ -662,8 +662,8 @@ def attribute_flows_to_sectors( validate = True grouped: 'FB' = ( self - .reset_index(drop=True).reset_index() - .rename(columns={'index': 'group_id'}) + .reset_index(drop=True) + .reset_index(names='group_id') .assign(group_total=lambda x: x.FlowAmount) ) if len(grouped) == 0: From 8d426de5c31a95e50c783f988a415c204faa2090 Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Sat, 14 Jun 2025 22:44:26 -0500 Subject: [PATCH 07/48] updates to convert_to_geoscale() for updated package versions --- flowsa/flowbyactivity.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/flowsa/flowbyactivity.py b/flowsa/flowbyactivity.py index 61e283d69..8842f8df8 100644 --- a/flowsa/flowbyactivity.py +++ b/flowsa/flowbyactivity.py @@ -284,19 +284,19 @@ def convert_to_geoscale( f'{self.full_name}; target geoscale is ' f'{target_geoscale.name.lower()}') + # convert to df, otherwise assumes it is an FBA which requires 23 columns and we are outputting a df with 5 cols highest_reporting_level_by_geoscale = [ - (self + (pd.DataFrame(self .merge(geoscale_by_fips, how='inner') .query('geoscale <= @scale') + # use as_index=False to avoid errors when anActivity column is entirely np.nan .groupby(['ActivityProducedBy', 'ActivityConsumedBy'] + [s.name.title() for s in geo.scale if s.has_fips_level and s >= scale], - dropna=False) - .agg({'geoscale': 'max'}) - .reset_index() - .rename(columns={ - 'geoscale': f'highest_reporting_level_by_{scale.name.title()}' - })) + dropna=False, as_index=False) + .agg({'geoscale': 'max'})).rename(columns={ + 'geoscale': f'highest_reporting_level_by_{scale.name.title()}' + }, errors='ignore')) # ignore error when df is empty for scale in geo.scale if scale.has_fips_level and scale <= target_geoscale ] @@ -320,12 +320,20 @@ def convert_to_geoscale( for s in geo.scale if s.has_fips_level and s <= target_geoscale ] + + # drop rows where all reporting level columns are null + fba_with_reporting_levels = ( + fba_with_reporting_levels[fba_with_reporting_levels['geoscale'] + .apply(lambda s: s.has_fips_level and s <= target_geoscale)] + .reset_index(drop=True) + ) + fba_at_source_geoscale = ( fba_with_reporting_levels - .assign(source_geoscale=( - fba_with_reporting_levels[reporting_level_columns] - .max(axis='columns'))) - # ^^^ max() with axis='columns' takes max along rows + .assign(source_geoscale= + fba_with_reporting_levels[reporting_level_columns].apply( + lambda row: max((v for v in row if isinstance(v, geo.scale)), default=np.nan), + axis=1)) .query('geoscale == source_geoscale') .drop(columns=(['geoscale', *geoscale_name_columns, From 2f2a0b7b97972fa925b7e53f7a8a142242d25361 Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Sun, 15 Jun 2025 08:23:43 -0500 Subject: [PATCH 08/48] update how indexes are assigned to column name --- flowsa/data_source_scripts/EIA_MECS.py | 4 ++-- flowsa/data_source_scripts/USDA_ERS_MLU.py | 8 ++++---- flowsa/flowby.py | 5 +++-- flowsa/flowbyactivity.py | 4 +++- flowsa/flowbyclean.py | 16 ++++++++-------- 5 files changed, 20 insertions(+), 17 deletions(-) diff --git a/flowsa/data_source_scripts/EIA_MECS.py b/flowsa/data_source_scripts/EIA_MECS.py index a7b654e4c..bbfcfcf0d 100644 --- a/flowsa/data_source_scripts/EIA_MECS.py +++ b/flowsa/data_source_scripts/EIA_MECS.py @@ -509,8 +509,8 @@ def update_regions_to_states(fba: FlowByActivity, fba = ( fba .drop(columns=['group_id', 'group_total']) - .reset_index(drop=True).reset_index() - .rename(columns={'index': 'group_id'}) + .reset_index(drop=True) + .reset_index(names='group_id') .assign(group_total=fba.FlowAmount) ) diff --git a/flowsa/data_source_scripts/USDA_ERS_MLU.py b/flowsa/data_source_scripts/USDA_ERS_MLU.py index 5246646c5..4e1b70ef7 100644 --- a/flowsa/data_source_scripts/USDA_ERS_MLU.py +++ b/flowsa/data_source_scripts/USDA_ERS_MLU.py @@ -256,8 +256,8 @@ def allocate_usda_ers_mlu_land_in_urban_areas( allocated_urban_areas_df .drop(columns=['group_id', 'group_total', 'rurl_res', 'total_area'], errors='ignore') - .reset_index(drop=True).reset_index() - .rename(columns={'index': 'group_id'}) + .reset_index(drop=True) + .reset_index(names='group_id') .assign(group_total=allocated_urban_areas_df.FlowAmount) ) @@ -341,8 +341,8 @@ def allocate_usda_ers_mlu_land_in_rural_transportation_areas( allocated_rural_trans = ( allocated_rural_trans .drop(columns=['group_id', 'group_total']) - .reset_index(drop=True).reset_index() - .rename(columns={'index': 'group_id'}) + .reset_index(drop=True) + .reset_index(names='group_id') .assign(group_total=allocated_rural_trans.FlowAmount) ) diff --git a/flowsa/flowby.py b/flowsa/flowby.py index fae044905..8dbbbcbbc 100644 --- a/flowsa/flowby.py +++ b/flowsa/flowby.py @@ -105,7 +105,7 @@ def __init__( ) -> None: ''' Extends pandas DataFrame. Attaches metadata if provided as kwargs and - ensures that all columns described in flowby_config.yaml are present + ensures that all columns described in flowby_config.yaml are present and of the correct datatype. All args and kwargs not specified above or in FBA/FBS metadata are @@ -853,7 +853,8 @@ def activity_sets(self) -> List['FB']: if k not in ['activity_sets', 'clean_fba_before_activity_sets'] and not k.startswith('_')} - parent_df = self.reset_index().rename(columns={'index': 'row'}) + parent_df = (self + .reset_index(names='row')) child_df_list = [] assigned_rows = set() diff --git a/flowsa/flowbyactivity.py b/flowsa/flowbyactivity.py index 8842f8df8..0b33ea44d 100644 --- a/flowsa/flowbyactivity.py +++ b/flowsa/flowbyactivity.py @@ -634,7 +634,9 @@ def activity_sets(self) -> List['FlowByActivity']: if k not in ['activity_sets', 'clean_fba_before_activity_sets'] and not k.startswith('_')} - parent_fba = self.reset_index().rename(columns={'index': 'row'}) + parent_fba = (self + .reset_index(names='row') + ) child_fba_list = [] assigned_rows = set() diff --git a/flowsa/flowbyclean.py b/flowsa/flowbyclean.py index fd35d22c4..b4e946a09 100644 --- a/flowsa/flowbyclean.py +++ b/flowsa/flowbyclean.py @@ -120,8 +120,8 @@ def weighted_average( # reset dropped information wt_flow = (wt_flow - .reset_index(drop=True).reset_index() - .rename(columns={'index': 'group_id'}) + .reset_index(drop=True) + .reset_index(names='group_id') .assign(group_total=wt_flow.FlowAmount) ) @@ -179,8 +179,8 @@ def substitute_nonexistent_values( merged = (merged .drop(merged.filter(regex='_y').columns, axis=1) .drop(columns=['group_id']) - .reset_index(drop=True).reset_index() - .rename(columns={'index': 'group_id'}) + .reset_index(drop=True) + .reset_index(names='group_id') .assign(group_total=merged.FlowAmount) ) @@ -394,8 +394,8 @@ def attribute_national_to_states(fba: FlowByActivity, **_) -> FlowByActivity: fba = ( fba .drop(columns=['group_id', 'group_total']) - .reset_index(drop=True).reset_index() - .rename(columns={'index': 'group_id'}) + .reset_index(drop=True) + .reset_index(names='group_id') .assign(group_total=fba.FlowAmount) ) @@ -612,8 +612,8 @@ def proxy_sector_data( # break each sector into separate line fba3 = (fba2 .explode(col) - .reset_index(drop=True).reset_index() - .rename(columns={'index': 'group_id'}) + .reset_index(drop=True) + .reset_index(names='group_id') ) return fba3 From 6150e24b230943863a245fcb4980fb6aac90f9f9 Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Sun, 15 Jun 2025 16:44:15 -0500 Subject: [PATCH 09/48] address 'future.no_silent_downcasting' issue --- flowsa/dataclean.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/flowsa/dataclean.py b/flowsa/dataclean.py index 976609f85..3443c3587 100644 --- a/flowsa/dataclean.py +++ b/flowsa/dataclean.py @@ -61,9 +61,11 @@ def add_missing_flow_by_fields(flowby_partial_df, flowbyfields): else: flowby_partial_df[k] = flowby_partial_df[k].fillna(0) # convert all None, 'nan' to np.nan - flowby_partial_df = (flowby_partial_df - .replace('None', np.nan) - .replace('nan', np.nan)) + with pd.option_context('future.no_silent_downcasting', True): + flowby_partial_df = (flowby_partial_df + .replace({'None': np.nan, 'nan': np.nan}) + .infer_objects(copy=False) + ) # Resort it so order is correct cols = [e for e in flowbyfields.keys() if e in flowby_partial_df.columns] flowby_df = flowby_partial_df[cols] From 87dfa555f064e930c20553175f4beb21cd47938c Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Sun, 15 Jun 2025 22:44:01 -0500 Subject: [PATCH 10/48] put sum in quotes - avoid error from package update --- flowsa/data_source_scripts/EIA_CBECS_Land.py | 2 +- flowsa/data_source_scripts/USDA_ERS_MLU.py | 6 +++--- flowsa/flowby.py | 2 +- flowsa/flowbyclean.py | 2 +- flowsa/validation.py | 4 ++-- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/flowsa/data_source_scripts/EIA_CBECS_Land.py b/flowsa/data_source_scripts/EIA_CBECS_Land.py index b9c9ad753..e8e740ed4 100644 --- a/flowsa/data_source_scripts/EIA_CBECS_Land.py +++ b/flowsa/data_source_scripts/EIA_CBECS_Land.py @@ -405,7 +405,7 @@ def disaggregate_eia_cbecs_vacant_and_other(df_load): df_nvno = df_nvno[df_nvno['ActivityConsumedBy'] != 'All buildings'] df_nvno = df_nvno.groupby( ['Unit', 'Location', 'LocationSystem', 'Year', 'Description'], - as_index=False).agg({'FlowAmount': sum}) + as_index=False).agg({'FlowAmount': "sum"}) df_act = df_nvno.rename(columns={'FlowAmount': 'NonVacantNonOther'}) # merge df and subtract to determine FlowAmount to allocate to diff --git a/flowsa/data_source_scripts/USDA_ERS_MLU.py b/flowsa/data_source_scripts/USDA_ERS_MLU.py index 4e1b70ef7..34d9d7981 100644 --- a/flowsa/data_source_scripts/USDA_ERS_MLU.py +++ b/flowsa/data_source_scripts/USDA_ERS_MLU.py @@ -201,7 +201,7 @@ def allocate_usda_ers_mlu_land_in_urban_areas( df_non_urban_transport_area[['Location', 'Unit', 'FlowAmount']] non_urban_transport_area_sum = df_non_urban_transport_area.groupby( ['Location', 'Unit'], as_index=False).agg( - {'FlowAmount': sum}).rename(columns={'FlowAmount': 'NonTransport'}) + {'FlowAmount': "sum"}).rename(columns={'FlowAmount': 'NonTransport'}) # compare units compare_df_units(fba, df_non_urban_transport_area) # calculate total urban transportation by subtracting @@ -232,7 +232,7 @@ def allocate_usda_ers_mlu_land_in_urban_areas( air_rail_area = air_rail_area[['Location', 'Unit', 'FlowAmount']] air_rail_area_sum = air_rail_area.groupby( ['Location', 'Unit'], as_index=False).agg( - {'FlowAmount': sum}).rename(columns={'FlowAmount': 'AirRail'}) + {'FlowAmount': "sum"}).rename(columns={'FlowAmount': 'AirRail'}) df_highway = df_transport.merge(air_rail_area_sum, how='left') df_highway = df_highway.assign( @@ -317,7 +317,7 @@ def allocate_usda_ers_mlu_land_in_rural_transportation_areas( air_rail_area = air_rail_area[['Location', 'Unit', 'FlowAmount']] air_rail_area_sum = air_rail_area.groupby( ['Location', 'Unit'], as_index=False).agg( - {'FlowAmount': sum}).rename(columns={'FlowAmount': 'AirRail'}) + {'FlowAmount': "sum"}).rename(columns={'FlowAmount': 'AirRail'}) # compare units compare_df_units(fba, air_rail_area) diff --git a/flowsa/flowby.py b/flowsa/flowby.py index 8dbbbcbbc..6908de229 100644 --- a/flowsa/flowby.py +++ b/flowsa/flowby.py @@ -596,7 +596,7 @@ def aggregate_flowby( **{f'_{c}_weights': fb.FlowAmount * fb[c].notnull() for c in columns_to_average}) .groupby(columns_to_group_by, dropna=False) - .agg(sum) + .agg("sum") .reset_index() ) aggregated = ( diff --git a/flowsa/flowbyclean.py b/flowsa/flowbyclean.py index b4e946a09..52174ecb1 100644 --- a/flowsa/flowbyclean.py +++ b/flowsa/flowbyclean.py @@ -541,7 +541,7 @@ def define_parentincompletechild_descendants( fba = (fba .drop(columns='group_total') .merge((fba.groupby('group_id') - .agg({'FlowAmount':sum}) + .agg({'FlowAmount': "sum"}) .rename(columns={'FlowAmount': 'group_total'}) ), on='group_id', how='left', validate='m:1') diff --git a/flowsa/validation.py b/flowsa/validation.py index 912eeaa15..779940943 100644 --- a/flowsa/validation.py +++ b/flowsa/validation.py @@ -77,7 +77,7 @@ def calculate_flowamount_diff_between_dfs(dfa_load, dfb_load): 'Unit', 'geoscale'] dfagg = dfs.groupby( agg_cols, dropna=False, as_index=False).agg( - {'FlowAmount_Original': sum, 'FlowAmount_Modified': sum}) + {'FlowAmount_Original': "sum", 'FlowAmount_Modified': "sum"}) # column calculating difference dfagg['FlowAmount_Difference'] = \ dfagg['FlowAmount_Modified'] - dfagg['FlowAmount_Original'] @@ -95,7 +95,7 @@ def calculate_flowamount_diff_between_dfs(dfa_load, dfb_load): 'FlowAmount_Difference', 'Percent_Increase']) dfagg4 = dfagg3.groupby(flowcols + ['Unit', 'geoscale'], dropna=False, as_index=False).agg( - {'FlowAmount_Original': sum, 'FlowAmount_Modified': sum}) + {'FlowAmount_Original': "sum", 'FlowAmount_Modified': "sum"}) # column calculating difference dfagg4['FlowAmount_Difference'] = \ dfagg4['FlowAmount_Modified'] - dfagg4['FlowAmount_Original'] From e2a887a4d9ac1104cf24b3bba9a28c3b8e99bad0 Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Sun, 15 Jun 2025 23:13:36 -0500 Subject: [PATCH 11/48] use "convert_df_to_flowby" to indicate when certain 'flowby' code should be run - update due to pandas version update --- flowsa/flowby.py | 14 ++++++++------ flowsa/flowbyactivity.py | 5 +++-- flowsa/flowbysector.py | 2 +- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/flowsa/flowby.py b/flowsa/flowby.py index 6908de229..a9fbb7b9f 100644 --- a/flowsa/flowby.py +++ b/flowsa/flowby.py @@ -67,7 +67,7 @@ def get_flowby_from_config( external_data_path=external_data_path ) elif config.get('data_format') == 'FBS_outside_flowsa': - return FlowBySector( + return FlowBySector( # todo: add convert_df_to_flowby=True? config['FBS_datapull_fxn']( config=config, external_config_path=external_config_path, @@ -97,6 +97,8 @@ def __init__( self, data: pd.DataFrame or '_FlowBy' = None, *args, + # needs to initially be false bc otherwise all internal pandas fxns (.assign(), .copy()) trigger the code to run + convert_df_to_flowby: bool = False, add_missing_columns: bool = True, fields: dict = None, column_order: List[str] = None, @@ -124,9 +126,9 @@ def __init__( self.__annotations__.get(attribute, None)())) ) - + # only runs if truly a pandas df and not retriggered due to .copy() or .assign() if isinstance(data, pd.DataFrame) and fields is not None: - if add_missing_columns: + if convert_df_to_flowby and add_missing_columns: data = data.assign(**{field: None for field in fields if field not in data.columns}) @@ -152,7 +154,7 @@ def __init__( .replace(null_string_dict) .astype(fields)) - if isinstance(data, pd.DataFrame) and column_order is not None: + if isinstance(data, pd.DataFrame) and column_order is not None and convert_df_to_flowby: data = data[[c for c in column_order if c in data.columns] + [c for c in data.columns if c not in column_order]] super().__init__(data, *args, **kwargs) @@ -267,7 +269,7 @@ def _getFlowBy( else: log.error(f'{file_metadata.name_data} {file_metadata.category} ' f'could not be found locally, downloaded, or generated') - fb = cls(df, full_name=full_name or '', config=config or {}) + fb = cls(df, full_name=full_name or '', config=config or {}, convert_df_to_flowby=True) return fb def convert_daily_to_annual(self: FB) -> FB: @@ -1606,7 +1608,7 @@ def astype(self: FB, *args, **kwargs) -> FB: metadata = {attribute: self.__getattr__(attribute) for attribute in self._metadata} df = pd.DataFrame(self).astype(*args, **kwargs) - fb = type(self)(df, add_missing_columns=False, **metadata) + fb = type(self)(df, convert_df_to_flowby=True, add_missing_columns=False, **metadata) return fb diff --git a/flowsa/flowbyactivity.py b/flowsa/flowbyactivity.py index 0b33ea44d..3aa276e61 100644 --- a/flowsa/flowbyactivity.py +++ b/flowsa/flowbyactivity.py @@ -592,7 +592,7 @@ def prepare_fbs( .reset_index(drop=True) ) except ValueError: - return FlowBySector(pd.DataFrame()) + return FlowBySector(pd.DataFrame(), convert_df_to_flowby=True) log.info(f'Processing FlowBySector for {self.full_name}') # Primary FlowBySector generation approach: return FlowBySector( @@ -611,7 +611,8 @@ def prepare_fbs( download_sources_ok=download_sources_ok) # recursive call to prepare_fbs .drop(columns=drop_cols) .aggregate_flowby() - .function_socket('clean_fbs_after_aggregation') + .function_socket('clean_fbs_after_aggregation'), + convert_df_to_flowby=True ) def activity_sets(self) -> List['FlowByActivity']: diff --git a/flowsa/flowbysector.py b/flowsa/flowbysector.py index e122401df..d340983a6 100644 --- a/flowsa/flowbysector.py +++ b/flowsa/flowbysector.py @@ -298,7 +298,7 @@ def prepare_fbs( .reset_index(drop=True) ) except ValueError: - return FlowBySector(pd.DataFrame()) + return FlowBySector(pd.DataFrame(), convert_df_to_flowby=True) return ( self .function_socket('clean_fbs') From 6feefbe91f112d645a3f00a7a641a0ba4d6d11b0 Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Sun, 15 Jun 2025 23:28:42 -0500 Subject: [PATCH 12/48] revert changes made when determining issues caused by pandas 2.2 upgrade --- flowsa/flowby.py | 2 +- flowsa/flowbyactivity.py | 26 +++++++++----------------- flowsa/flowbyclean.py | 2 +- 3 files changed, 11 insertions(+), 19 deletions(-) diff --git a/flowsa/flowby.py b/flowsa/flowby.py index a9fbb7b9f..e591b2a0f 100644 --- a/flowsa/flowby.py +++ b/flowsa/flowby.py @@ -499,7 +499,7 @@ def select_by_fields( if k not in ['Activity', 'Sector'] } - filtered_fb = self + filtered_fb = self.copy() for field, values in special_fields.items(): check_values = ([*values.keys(), *values.values()] if isinstance(values, dict) else values) diff --git a/flowsa/flowbyactivity.py b/flowsa/flowbyactivity.py index 3aa276e61..1e2707f14 100644 --- a/flowsa/flowbyactivity.py +++ b/flowsa/flowbyactivity.py @@ -284,19 +284,19 @@ def convert_to_geoscale( f'{self.full_name}; target geoscale is ' f'{target_geoscale.name.lower()}') - # convert to df, otherwise assumes it is an FBA which requires 23 columns and we are outputting a df with 5 cols highest_reporting_level_by_geoscale = [ - (pd.DataFrame(self + (self .merge(geoscale_by_fips, how='inner') .query('geoscale <= @scale') - # use as_index=False to avoid errors when anActivity column is entirely np.nan .groupby(['ActivityProducedBy', 'ActivityConsumedBy'] + [s.name.title() for s in geo.scale if s.has_fips_level and s >= scale], - dropna=False, as_index=False) - .agg({'geoscale': 'max'})).rename(columns={ - 'geoscale': f'highest_reporting_level_by_{scale.name.title()}' - }, errors='ignore')) # ignore error when df is empty + dropna=False) + .agg({'geoscale': 'max'}) + .reset_index() + .rename(columns={ + 'geoscale': f'highest_reporting_level_by_{scale.name.title()}' + })) for scale in geo.scale if scale.has_fips_level and scale <= target_geoscale ] @@ -320,14 +320,6 @@ def convert_to_geoscale( for s in geo.scale if s.has_fips_level and s <= target_geoscale ] - - # drop rows where all reporting level columns are null - fba_with_reporting_levels = ( - fba_with_reporting_levels[fba_with_reporting_levels['geoscale'] - .apply(lambda s: s.has_fips_level and s <= target_geoscale)] - .reset_index(drop=True) - ) - fba_at_source_geoscale = ( fba_with_reporting_levels .assign(source_geoscale= @@ -338,7 +330,7 @@ def convert_to_geoscale( .drop(columns=(['geoscale', *geoscale_name_columns, *reporting_level_columns])) - ) + ).reset_index(drop=True) if len(fba_at_source_geoscale.source_geoscale.unique()) > 1: log.warning(f"{fba_at_source_geoscale.full_name} has multiple " @@ -377,7 +369,7 @@ def convert_to_geoscale( # ^^^ TODO: Rewrite validation to use fb metadata ) - return fba_at_target_geoscale + return fba_at_target_geoscale.reset_index(drop=True) def map_to_sectors( self: 'FlowByActivity', diff --git a/flowsa/flowbyclean.py b/flowsa/flowbyclean.py index 52174ecb1..26a400ae8 100644 --- a/flowsa/flowbyclean.py +++ b/flowsa/flowbyclean.py @@ -56,7 +56,7 @@ def load_prepare_clean_source( **config}, download_sources_ok=download_sources_ok ).prepare_fbs(download_sources_ok=download_sources_ok) - return clean_fbs + return clean_fbs.reset_index(drop=True) def weighted_average( From 81c38d15dd329fb44d905f7b436dfbb244cb994a Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Sun, 15 Jun 2025 23:29:15 -0500 Subject: [PATCH 13/48] changes for package upgrades --- flowsa/flowbyclean.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/flowsa/flowbyclean.py b/flowsa/flowbyclean.py index 26a400ae8..99f6b985e 100644 --- a/flowsa/flowbyclean.py +++ b/flowsa/flowbyclean.py @@ -86,14 +86,17 @@ def weighted_average( else 'Location'], right_on=['PrimarySector', 'Location'], suffixes=[None, '_other']) - .fillna({'FlowAmount_other': fba['FlowAmount']}) ) + merged['FlowAmount_other'] = merged['FlowAmount_other'].mask( + merged['FlowAmount_other'] == 0, merged['FlowAmount']) + # drop rows where flow is 0 merged = merged[merged['FlowAmount'] != 0] # replace terms for original, replacement in fba.config.get( 'replacement_dictionary').items(): - merged = merged.replace({original: replacement}) + with pd.option_context('future.no_silent_downcasting', True): + merged = merged.replace({original: replacement}) wt_flow = (merged .groupby(['Class', 'Flowable', 'Unit', @@ -147,19 +150,23 @@ def substitute_nonexistent_values( state_geo = pd.concat([ (geo.filtered_fips(fb.config['geoscale'])[['FIPS']] .assign(Location=location.US_FIPS)) - ]) + ]).reset_index(drop=True) - other = (other + other2 = pd.DataFrame(other .merge(state_geo) .drop(columns=['Location', 'FlowUUID']) .rename(columns={'FIPS': 'Location'}) ) + # todo: revise these check merge cols, expand merged = (fb - .merge(other, - on=list(other.select_dtypes( - include=['object', 'int']).columns), - how='outer', + .merge(other2[['Year', 'Location', 'SectorProducedBy', 'SectorConsumedBy', + 'SectorSourceName', 'Context', 'FlowAmount']], + # on=list(other2.select_dtypes( + # include=['object', 'int']).columns), + on = ['Year', 'Location', 'SectorProducedBy', 'SectorConsumedBy', + 'SectorSourceName', 'Context'], + how='left', suffixes=(None, '_y')) .fillna({'FlowAmount': 0}) ) From a59c632a05476c245c05d7bddb801887c16bdfee Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Thu, 4 Sep 2025 09:58:29 -0600 Subject: [PATCH 14/48] readd dev branch package requirements --- requirements.txt | 6 +++--- setup.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/requirements.txt b/requirements.txt index 28435d096..5c7ede698 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ -git+https://github.com/USEPA/fedelemflowlist.git#egg=fedelemflowlist -git+https://github.com/USEPA/esupy.git#egg=esupy -git+https://github.com/USEPA/standardizedinventories.git#egg=StEWI +git+https://github.com/USEPA/fedelemflowlist.git@develop#egg=fedelemflowlist +git+https://github.com/USEPA/esupy.git@develop#egg=esupy +git+https://github.com/USEPA/standardizedinventories.git@develop#egg=StEWI appdirs >= 1.4.3 # Storing user data bibtexparser>=1.2.0 # Generates bibtex kaleido == 0.1.0.post1; platform_system == "Windows" # export plotly.py figures as static images diff --git a/setup.py b/setup.py index 9035ca385..1b250b1e5 100644 --- a/setup.py +++ b/setup.py @@ -12,9 +12,9 @@ include_package_data=True, python_requires=">=3.9", install_requires=[ - 'fedelemflowlist @ git+https://github.com/USEPA/fedelemflowlist.git#egg=fedelemflowlist', - 'esupy @ git+https://github.com/USEPA/esupy.git#egg=esupy', - 'StEWI @ git+https://github.com/USEPA/standardizedinventories.git#egg=StEWI', + 'fedelemflowlist @ git+https://github.com/USEPA/fedelemflowlist.git@develop#egg=fedelemflowlist', + 'esupy @ git+https://github.com/USEPA/esupy.git@develop#egg=esupy', + 'StEWI @ git+https://github.com/USEPA/standardizedinventories.git@develop#egg=StEWI', 'appdirs>=1.4.3', 'bibtexparser>=1.2.0', "kaleido==0.1.0.post1;platform_system=='Windows'", From 9792657766c8cacc23becc723da83463d2a2bd19 Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Wed, 8 Oct 2025 15:59:05 -0600 Subject: [PATCH 15/48] update package versions --- requirements.txt | 4 ++-- setup.py | 30 +++++++++++++++--------------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/requirements.txt b/requirements.txt index e9de33abd..5e8d38ffa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,9 +6,9 @@ bibtexparser>=1.4.3 # Generates bibtex kaleido == 0.1.0.post1; platform_system == "Windows" # export plotly.py figures as static images kaleido == 0.2.0; platform_system=="Linux" or platform_system=="Darwin" matplotlib >= 3.10.3 # Plotting -numpy<2.1.0 # Library used for arrays +numpy>=2.3.0 # Library used for arrays openpyxl >= 3.1.5 # Extract data from xlsm files -pandas==2.2.2 # Powerful data structures for data analysis, time series, and statistics. +pandas>=2.3.3 # Powerful data structures for data analysis, time series, and statistics. pip>=25.1.1 # The PyPA recommended tool for installing Python packages. plotly >= 6.1.2 # Plotting pycountry >= 24.6.1 # ISO country codes diff --git a/setup.py b/setup.py index 60a51a69d..84e2dd565 100644 --- a/setup.py +++ b/setup.py @@ -15,24 +15,24 @@ 'fedelemflowlist @ git+https://github.com/USEPA/fedelemflowlist.git@develop#egg=fedelemflowlist', 'esupy @ git+https://github.com/USEPA/esupy.git@develop#egg=esupy', 'StEWI @ git+https://github.com/USEPA/standardizedinventories.git@develop#egg=StEWI', - 'appdirs>=1.4.3', - 'bibtexparser>=1.2.0', + 'appdirs>=1.4.4', + 'bibtexparser>=1.4.3', "kaleido==0.1.0.post1;platform_system=='Windows'", "kaleido==0.2.0;platform_system=='Linux' or platform_system=='Darwin'", - 'matplotlib>=3.4.3', - 'numpy<2.1.0', - 'openpyxl>=3.0.7', - 'pandas==2.2.2', - 'pip>=9', - 'plotly>=5.10.0 ', - 'pycountry>=19.8.18', - 'python-dotenv >= 0.19.1', - 'pyyaml>=5.3', - 'requests>=2.22.0', + 'matplotlib>=3.10.3', + 'numpy>=2.3.0', + 'openpyxl>=3.1.5', + 'pandas>=2.3.3', + 'pip>=25.1.1', + 'plotly>= 6.1.2', + 'pycountry>=24.6.1', + 'python-dotenv >= 1.1.0', + 'pyyaml>=6.0.2', + 'requests>=2.32.4', 'requests_ftp==0.3.1', - 'seaborn>=0.11.2', - 'setuptools>=41', - 'tabula-py>=2.1.1', + 'seaborn>=0.13.2', + 'setuptools>=80.9.0', + 'tabula-py>=2.10.0', 'xlrd>=2.0.1' ], url='https://github.com/USEPA/FLOWSA', From d9851fc4d4062ee2c2ee98c4b0f88c59696f79a6 Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Wed, 8 Oct 2025 18:05:13 -0600 Subject: [PATCH 16/48] modify apply() use to address futurewarning dataframegroupby.apply() --- flowsa/flowbyclean.py | 3 ++- flowsa/naics.py | 15 +++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/flowsa/flowbyclean.py b/flowsa/flowbyclean.py index 99f6b985e..c83c11cda 100644 --- a/flowsa/flowbyclean.py +++ b/flowsa/flowbyclean.py @@ -98,6 +98,7 @@ def weighted_average( with pd.option_context('future.no_silent_downcasting', True): merged = merged.replace({original: replacement}) + # todo: check modification for futurewarning dataframegroupby.apply works as expected wt_flow = (merged .groupby(['Class', 'Flowable', 'Unit', 'FlowType', 'ActivityProducedBy', @@ -108,7 +109,7 @@ def weighted_average( 'SectorProducedBy', 'ProducedBySectorType', 'SectorConsumedBy', 'ConsumedBySectorType', 'SectorSourceName'], - dropna=False) + dropna=False)[merged.columns.tolist()] .apply(lambda x: np.average(x['FlowAmount'], weights=x['FlowAmount_other'])) .drop(columns='FlowAmount') # original flowamounts diff --git a/flowsa/naics.py b/flowsa/naics.py index 145fd5233..6ffe2ef81 100644 --- a/flowsa/naics.py +++ b/flowsa/naics.py @@ -172,10 +172,12 @@ def drop_parent_sectors(sector_key): is_parent = lambda x: any(sector != x and sector.startswith(x) for sector in sector_list) return sector_key[~sector_key['source_naics'].astype(str).apply(is_parent)] - primary_sector_key_2 = primary_sector_key_2.groupby(['Class', 'Flowable', 'Context'], - group_keys=False, - dropna=False - ).apply(drop_parent_sectors) + # todo: check futurewarning dataframegroupby.apply fix working as expected + primary_sector_key_2 = (primary_sector_key_2 + .groupby(['Class', 'Flowable', 'Context'], + group_keys=False, dropna=False)[primary_sector_key_2.columns.tolist()] + .apply(drop_parent_sectors) + ) # modify dqi scores for data reliability and collection based on mapping if "DataReliability" in flowbyactivity.columns: @@ -224,10 +226,11 @@ def subset_target_sectors_by_source_sectors(group): if flowbyactivity.config.get('sector_hierarchy') == 'parent-incompleteChild': df_remaining_mapped = df_remaining.copy() else: + # todo: check impact of changing code to remove future warning df_remaining_mapped = (df_remaining - .groupby(group_cols, dropna=False) + .groupby(group_cols, dropna=False, group_keys=False)[df_remaining.columns.tolist()] .apply(subset_target_sectors_by_source_sectors) - .reset_index(drop=True) + # .reset_index(drop=True) ) mapping = pd.concat([df_keep, df_remaining_mapped], ignore_index=True) From 8439092ba3f88e35f22f58f62fe2f3fd2cac80f4 Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Tue, 14 Oct 2025 14:49:05 -0600 Subject: [PATCH 17/48] update how data is merged to correctly pull in national values --- flowsa/flowbyclean.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/flowsa/flowbyclean.py b/flowsa/flowbyclean.py index c83c11cda..87246ab73 100644 --- a/flowsa/flowbyclean.py +++ b/flowsa/flowbyclean.py @@ -98,7 +98,6 @@ def weighted_average( with pd.option_context('future.no_silent_downcasting', True): merged = merged.replace({original: replacement}) - # todo: check modification for futurewarning dataframegroupby.apply works as expected wt_flow = (merged .groupby(['Class', 'Flowable', 'Unit', 'FlowType', 'ActivityProducedBy', @@ -153,7 +152,7 @@ def substitute_nonexistent_values( .assign(Location=location.US_FIPS)) ]).reset_index(drop=True) - other2 = pd.DataFrame(other + other = (other .merge(state_geo) .drop(columns=['Location', 'FlowUUID']) .rename(columns={'FIPS': 'Location'}) @@ -161,13 +160,10 @@ def substitute_nonexistent_values( # todo: revise these check merge cols, expand merged = (fb - .merge(other2[['Year', 'Location', 'SectorProducedBy', 'SectorConsumedBy', - 'SectorSourceName', 'Context', 'FlowAmount']], - # on=list(other2.select_dtypes( - # include=['object', 'int']).columns), - on = ['Year', 'Location', 'SectorProducedBy', 'SectorConsumedBy', - 'SectorSourceName', 'Context'], - how='left', + .merge(other, + on=list(other.select_dtypes( + include=['object', 'int']).columns), + how='outer', suffixes=(None, '_y')) .fillna({'FlowAmount': 0}) ) From 29e65ab38782e743c6ee9773e72d9ecb34184be7 Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Tue, 14 Oct 2025 14:49:23 -0600 Subject: [PATCH 18/48] typo --- docs/user_guide/user_guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user_guide/user_guide.md b/docs/user_guide/user_guide.md index 35a4e5294..04aa34d67 100644 --- a/docs/user_guide/user_guide.md +++ b/docs/user_guide/user_guide.md @@ -327,7 +327,7 @@ To attribute a primary data source to target level sectors, identify the primary specified attribution data source - equal: Equally attribute parent values to child values until reach target sector length -- inheritance: Assign parent values to all child values. Usefull in +- inheritance: Assign parent values to all child values. Useful in situations where value is a rate, such as kg/m2. ### Recursive vs Sequential Attribution From cd3f5c438550e6368d83a5d1b561ec3a772089bb Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Fri, 24 Oct 2025 13:33:26 -0600 Subject: [PATCH 19/48] update min python test version to 3.10 with updates to numpy/pandas pkg versions --- .github/workflows/python-app.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index b97edbea1..00994d125 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -31,7 +31,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macos-latest] - py-version: ['3.9', '3.10', '3.11'] + py-version: ['3.10', '3.11', '3.12'] steps: - uses: actions/checkout@v3 From 986bfaddbbfc93130b38db9cda650f86824852ec Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Fri, 24 Oct 2025 13:34:36 -0600 Subject: [PATCH 20/48] run gh workflows with py-3.12 bc updates to numpy/pandas pkg versions --- .github/workflows/compare_single_FBA.yml | 2 +- .github/workflows/compare_single_FBS.yml | 2 +- .github/workflows/generate_FBA.yml | 2 +- .github/workflows/python-app.yml | 2 +- .github/workflows/test_methods.yml | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/compare_single_FBA.yml b/.github/workflows/compare_single_FBA.yml index d1f91fe3c..36377a232 100644 --- a/.github/workflows/compare_single_FBA.yml +++ b/.github/workflows/compare_single_FBA.yml @@ -25,7 +25,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v3 with: - python-version: "3.10" + python-version: "3.12" - name: Update pip run: | diff --git a/.github/workflows/compare_single_FBS.yml b/.github/workflows/compare_single_FBS.yml index bf6c26608..2110cd5e1 100644 --- a/.github/workflows/compare_single_FBS.yml +++ b/.github/workflows/compare_single_FBS.yml @@ -21,7 +21,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v3 with: - python-version: "3.10" + python-version: "3.12" - name: Update pip run: | diff --git a/.github/workflows/generate_FBA.yml b/.github/workflows/generate_FBA.yml index 41b92ce1a..d6f621f71 100644 --- a/.github/workflows/generate_FBA.yml +++ b/.github/workflows/generate_FBA.yml @@ -25,7 +25,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v3 with: - python-version: "3.10" + python-version: "3.12" - name: Update pip run: | diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 00994d125..13da50277 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -74,7 +74,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v3 with: - python-version: "3.10" + python-version: "3.12" - name: Update pip & install testing pkgs run: | diff --git a/.github/workflows/test_methods.yml b/.github/workflows/test_methods.yml index 2f0e32ac7..424bc8970 100644 --- a/.github/workflows/test_methods.yml +++ b/.github/workflows/test_methods.yml @@ -19,7 +19,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v3 with: - python-version: "3.10" + python-version: "3.12" - name: Update pip & install testing pkgs run: | @@ -47,7 +47,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v3 with: - python-version: "3.10" + python-version: "3.12" - name: Update pip & install testing pkgs run: | From a21a4940ef1bec5eef1090b7001185cffce0f2d8 Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Mon, 27 Oct 2025 15:24:16 -0600 Subject: [PATCH 21/48] update estimate_suppressed_sectors_equal_attribution() for updated pandas - .apply() resets flowamount col type within class, resetting np.nan to 0 --- flowsa/flowbyclean.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/flowsa/flowbyclean.py b/flowsa/flowbyclean.py index 87246ab73..a2113e704 100644 --- a/flowsa/flowbyclean.py +++ b/flowsa/flowbyclean.py @@ -295,7 +295,7 @@ def estimate_suppressed_sectors_equal_attribution( right_on='source_naics') .assign(location=fba3.Location, category=fba3.FlowName) - .replace({'FlowAmount': {0: np.nan} #, + # .replace({'FlowAmount': {0: np.nan} #, # col: {'1125 & 1129': '112X', # '11193 & 11194 & 11199': '1119X', # '31-33': '3X', @@ -306,7 +306,7 @@ def estimate_suppressed_sectors_equal_attribution( # '48': '4Y', '49': '4Y'}, # 'n4': {'1125': '112X', '1129': '112X'}, # 'n5': {'11193': '1119X', '11194': '1119X', '11199': '1119X'} - }) + # }) .dropna(subset='source_naics') .drop(columns='source_naics') ) @@ -315,16 +315,16 @@ def estimate_suppressed_sectors_equal_attribution( 'location', 'category'], verify_integrity=True) def fill_suppressed( - flows, level: int, activity + flows, level: int, activity ): parent = flows[flows[activity].str.len() == level] children = flows[flows[activity].str.len() == level + 1] - null_children = children[children['FlowAmount'].isna()] + null_children = children[children['flow_suppressed']] if null_children.empty or parent.empty: return flows else: - value = max(parent['Unattributed'][0] / len(null_children), 0) + value = max(parent['Unattributed'].iloc[0] / len(null_children), 0) # update the null children by adding the unattributed data to # the attributed data null_children = ( @@ -332,10 +332,15 @@ def fill_suppressed( .assign(FlowAmount=value+null_children['Attributed']) .assign(Unattributed=value) ) - flows.update(null_children) + flows.loc[null_children.index, ['FlowAmount', 'Unattributed']] = null_children[['FlowAmount', 'Unattributed']] return flows unsuppressed = indexed.copy() + # replace 0 values with np.nan for suppressed data to be estimated + unsuppressed['FlowAmount'] = unsuppressed['FlowAmount'].mask(unsuppressed['FlowAmount'] == 0) + unsuppressed['flow_suppressed'] = unsuppressed['FlowAmount'].isna() + + # loop through sector lengths, estimating suppressed data for level in [2, 3, 4, 5, 6]: groupcols = (["{}{}".format("n", i) for i in range(2, level+1)] + ['location', 'category']) @@ -344,11 +349,12 @@ def fill_suppressed( .apply(fill_suppressed, level, col) ) unsuppressed['Year'] = unsuppressed['Year'].astype('int') + aggregated = ( unsuppressed .reset_index(drop=True) .fillna({'FlowAmount': 0}) - .drop(columns=['Unattributed', 'Attributed']) + .drop(columns=['Unattributed', 'Attributed', 'flow_suppressed']) # .replace({col: {'3X': '31-33', # '4X': '44-45', # '4Y': '48-49'}}) From e5583aabcba653cb87be032a9a9c6599d57c9086 Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Mon, 27 Oct 2025 18:18:06 -0600 Subject: [PATCH 22/48] drop numpy/pandas pkg requirements to maintain support for python 3.10 (numpy dependent) --- requirements.txt | 4 ++-- setup.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index 5e8d38ffa..442cea6d1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,9 +6,9 @@ bibtexparser>=1.4.3 # Generates bibtex kaleido == 0.1.0.post1; platform_system == "Windows" # export plotly.py figures as static images kaleido == 0.2.0; platform_system=="Linux" or platform_system=="Darwin" matplotlib >= 3.10.3 # Plotting -numpy>=2.3.0 # Library used for arrays +numpy>=2.2.6 # Library used for arrays openpyxl >= 3.1.5 # Extract data from xlsm files -pandas>=2.3.3 # Powerful data structures for data analysis, time series, and statistics. +pandas>=2.3.0 # Powerful data structures for data analysis, time series, and statistics. pip>=25.1.1 # The PyPA recommended tool for installing Python packages. plotly >= 6.1.2 # Plotting pycountry >= 24.6.1 # ISO country codes diff --git a/setup.py b/setup.py index 84e2dd565..a5d92cfaf 100644 --- a/setup.py +++ b/setup.py @@ -20,9 +20,9 @@ "kaleido==0.1.0.post1;platform_system=='Windows'", "kaleido==0.2.0;platform_system=='Linux' or platform_system=='Darwin'", 'matplotlib>=3.10.3', - 'numpy>=2.3.0', + 'numpy>=2.2.6', 'openpyxl>=3.1.5', - 'pandas>=2.3.3', + 'pandas>=2.3.0', 'pip>=25.1.1', 'plotly>= 6.1.2', 'pycountry>=24.6.1', From c8dac7372db0bee0f237e940cf9bf619a3f9a5ce Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Tue, 28 Oct 2025 14:44:31 -0600 Subject: [PATCH 23/48] for data pulled from stewi, use convert_df_to_flowby=True to ensure required columns included --- flowsa/data_source_scripts/stewiFBS.py | 1 + 1 file changed, 1 insertion(+) diff --git a/flowsa/data_source_scripts/stewiFBS.py b/flowsa/data_source_scripts/stewiFBS.py index 9a9bb36c6..eb9020fc4 100644 --- a/flowsa/data_source_scripts/stewiFBS.py +++ b/flowsa/data_source_scripts/stewiFBS.py @@ -338,6 +338,7 @@ def prepare_stewi_fbs(df_load, config) -> 'FlowBySector': .reset_index(drop=True), full_name=config.get('full_name'), config=config, + convert_df_to_flowby=True ).prepare_fbs() fbs.config.update({'data_format': 'FBS'}) From 1834345f0ccf550483577fda216831fe803f948e Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Tue, 28 Oct 2025 14:47:13 -0600 Subject: [PATCH 24/48] option to modify subset cols and rounding cols if not all DQI columns are in df --- flowsa/naics.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/flowsa/naics.py b/flowsa/naics.py index 6ffe2ef81..52b13911d 100644 --- a/flowsa/naics.py +++ b/flowsa/naics.py @@ -149,12 +149,14 @@ def subset_sector_key(flowbyactivity, activitycol, sector_source_year, primary_s # for situations where an activity can be listed in both columns for different circumstances subset_cols = ['Class', 'Flowable', 'Context', 'ActivityProducedBy', 'ActivityConsumedBy', 'DataReliability', 'DataCollection'] - if "DataReliability" not in flowbyactivity.columns: - subset_cols = ['Class', 'Flowable', 'Context', 'ActivityProducedBy', 'ActivityConsumedBy'] + # list DQI columns in df + dqi = [col for col in ['DataReliability', 'DataCollection'] if col in flowbyactivity.columns] + # Drop missing DQI columns from subset list + subset_cols = [col for col in subset_cols if col not in ['DataReliability', 'DataCollection'] or col in dqi] # ensure dq column decimals do not cause errors with dropping duplicates, without this statement, rows often # duplicated - flowbyactivity.loc[:, ['DataReliability', 'DataCollection']] = ( - flowbyactivity.loc[:, ['DataReliability', 'DataCollection']].round(decimals=5)) + if dqi: + flowbyactivity[dqi] = flowbyactivity[dqi].round(decimals=5) flowbyactivity = flowbyactivity[subset_cols].drop_duplicates() primary_sector_key_2 = pd.DataFrame(flowbyactivity.merge( From 1ec098c44136dfbfbd9718b18b234fc2af78ea47 Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Tue, 28 Oct 2025 14:47:36 -0600 Subject: [PATCH 25/48] address issue of stewi NAICS data imported with decimals --- flowsa/naics.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/flowsa/naics.py b/flowsa/naics.py index 52b13911d..8bc61a30b 100644 --- a/flowsa/naics.py +++ b/flowsa/naics.py @@ -588,6 +588,13 @@ def convert_naics_year(df_load, targetsectorsourcename, sectorsourcename, # methods, so assigning schema as None activity_schema = "None" + # however, need to ensure that these NAICS are formatted correly - stewi data are at times imported + # with some NAICS values including decimals that do not get mapped correctly (ex. '311712.0') + for col in column_headers: + if col in df_load.columns: + df_load[col] = (df_load[col] + .apply(lambda x: x.split(".")[0] if isinstance(x, str) else x)) + if "NAICS" in activity_schema and "ActivityProducedBy" in df_load.columns: column_headers += ['ActivityProducedBy', 'ActivityConsumedBy'] From a93032b4d0b400705cbb799aec68d40d731204f2 Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Tue, 28 Oct 2025 16:17:10 -0600 Subject: [PATCH 26/48] address pandas future warnings on .fillna and .replace and concating dfs with all nan col --- flowsa/data_source_scripts/EPA_GHGI.py | 4 ++-- flowsa/dataclean.py | 4 ++-- flowsa/flowby.py | 9 ++++++--- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/flowsa/data_source_scripts/EPA_GHGI.py b/flowsa/data_source_scripts/EPA_GHGI.py index 5fc0d926a..41a0e783d 100644 --- a/flowsa/data_source_scripts/EPA_GHGI.py +++ b/flowsa/data_source_scripts/EPA_GHGI.py @@ -40,9 +40,9 @@ ANNEX_ENERGY_TABLES = ["A-" + str(x) for x in list(range(4,16))] DROP_COLS = ["Unnamed: 0"] + list(pd.date_range( - start="1990", end="2010", freq='Y').year.astype(str)) + start="1990", end="2010", freq='YE').year.astype(str)) -YEARS = list(pd.date_range(start="2010", end="2024", freq='Y').year.astype(str)) +YEARS = list(pd.date_range(start="2010", end="2024", freq='YE').year.astype(str)) def ghg_url_helper(*, build_url, config, **_): diff --git a/flowsa/dataclean.py b/flowsa/dataclean.py index 3443c3587..042756158 100644 --- a/flowsa/dataclean.py +++ b/flowsa/dataclean.py @@ -57,7 +57,7 @@ def add_missing_flow_by_fields(flowby_partial_df, flowbyfields): flowby_partial_df[k] = \ flowby_partial_df[k].astype(v[0]['dtype']) if v[0]['dtype'] in ['string', 'str', 'object']: - flowby_partial_df[k] = flowby_partial_df[k].fillna(np.nan) + flowby_partial_df[k] = flowby_partial_df[k].fillna(np.nan).infer_objects(copy=False) else: flowby_partial_df[k] = flowby_partial_df[k].fillna(0) # convert all None, 'nan' to np.nan @@ -101,7 +101,7 @@ def standardize_units(df): .merge(conversion_table, how='left', left_on='Unit', right_on='old_unit') .assign(Unit=lambda x: x.new_unit.mask(x.new_unit.isna(), x.Unit), - conversion_factor=lambda x: x.conversion_factor.fillna(1), + conversion_factor=lambda x: x.conversion_factor.fillna(1).infer_objects(copy=False), FlowAmount=lambda x: x.FlowAmount * x.conversion_factor) .drop(columns=['old_unit', 'new_unit', 'conversion_factor']) ) diff --git a/flowsa/flowby.py b/flowsa/flowby.py index 23cf6c9d3..98e6c1594 100644 --- a/flowsa/flowby.py +++ b/flowsa/flowby.py @@ -294,9 +294,11 @@ def standardize_units(self: FB, year: int = None) -> FB: conversion_table = pd.concat([ pd.read_csv(settings.datapath / 'unit_conversion.csv'), - pd.Series({'old_unit': 'Canadian Dollar', - 'new_unit': 'USD', - 'conversion_factor': 1 / exchange_rate}).to_frame().T + pd.DataFrame([{ + 'old_unit': 'Canadian Dollar', + 'new_unit': 'USD', + 'conversion_factor': 1 / exchange_rate + }]) ]) # avoid warning: "Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated # and will change in a future version" @@ -534,6 +536,7 @@ def select_by_fields( replaced_fb = ( filtered_fb .replace(replace_dict) + .infer_objects(copy=False) .drop(columns=['PrimaryActivity', 'PrimarySector'], errors='ignore') .reset_index(drop=True) From 9ec5db45d9e6bd6e4cdb3cf7cd57e0fd2fc78975 Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Tue, 28 Oct 2025 16:17:27 -0600 Subject: [PATCH 27/48] update canadian exchange rates --- flowsa/literature_values.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/flowsa/literature_values.py b/flowsa/literature_values.py index ee18e6f7d..8550aeec1 100644 --- a/flowsa/literature_values.py +++ b/flowsa/literature_values.py @@ -15,7 +15,7 @@ def get_Canadian_to_USD_exchange_rate(year): """ Return exchange rate (Canadian $/USD) - From https://www.federalreserve.gov/releases/h10/current/ on 09/07/2020 + From https://www.federalreserve.gov/releases/h10/current/ on 10/28/2025 :param year: str, year of exchange rate to return :return: number, value of exchange rate for year """ @@ -38,7 +38,12 @@ def get_Canadian_to_USD_exchange_rate(year): 2016: 1.3243, 2017: 1.2984, 2018: 1.2957, - 2019: 1.3269 + 2019: 1.3269, + 2020: 1.3422, + 2021: 1.2533, + 2022: 1.3014, + 2023: 1.3494, + 2024: 1.3699 }) exchange_rate = er.get(year, np.nan) From ac808d86b1424924dc0e218c581414d090d56b02 Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Tue, 28 Oct 2025 16:18:33 -0600 Subject: [PATCH 28/48] add to unit conversions --- flowsa/data/unit_conversion.csv | 3 +++ 1 file changed, 3 insertions(+) diff --git a/flowsa/data/unit_conversion.csv b/flowsa/data/unit_conversion.csv index 9376be144..b3db8ea59 100644 --- a/flowsa/data/unit_conversion.csv +++ b/flowsa/data/unit_conversion.csv @@ -13,6 +13,7 @@ TON,kg,907.185 tons,kg,907.185 short tons,kg,907.185 Thousands of Tons,kg,907185 +million short tons,kg,907184740 MT,kg,1000 Metric Tons,kg,1000 LB,kg,0.45359 @@ -20,7 +21,9 @@ ACRE FEET / ACRE,kg/m2,304.8000006 gal/USD,kg/USD,3.79 Quadrillion Btu,MJ,1.06E+12 Trillion Btu,MJ,1055055900 +trillion Btu,MJ,1055055900 Tbtu,MJ,1055055900 +million kWh,MJ,3600000 MT CO2e,MMT CO2e,0.000001 employees,p,1 Thousand USD,USD,1000 From 4de2ba3c052873a4e398bcf68544ca62d5b4af55 Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Tue, 28 Oct 2025 20:38:49 -0600 Subject: [PATCH 29/48] address FutureWarning: Downcasting behavior in replace is deprecated --- flowsa/dataclean.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/flowsa/dataclean.py b/flowsa/dataclean.py index 042756158..ecedb0ed3 100644 --- a/flowsa/dataclean.py +++ b/flowsa/dataclean.py @@ -49,8 +49,9 @@ def add_missing_flow_by_fields(flowby_partial_df, flowbyfields): flowby_partial_df[col] = np.nan # convert all None, 'nan' to np.nan flowby_partial_df = (flowby_partial_df - .replace('None', np.nan) - .replace('nan', np.nan)) + .replace({'None': np.nan, 'nan': np.nan}) + .infer_objects(copy=False) + ) # convert data types to match those defined in flow_by_activity_fields for k, v in flowbyfields.items(): if k in flowby_partial_df.columns: @@ -59,7 +60,7 @@ def add_missing_flow_by_fields(flowby_partial_df, flowbyfields): if v[0]['dtype'] in ['string', 'str', 'object']: flowby_partial_df[k] = flowby_partial_df[k].fillna(np.nan).infer_objects(copy=False) else: - flowby_partial_df[k] = flowby_partial_df[k].fillna(0) + flowby_partial_df[k] = flowby_partial_df[k].fillna(0).infer_objects(copy=False) # convert all None, 'nan' to np.nan with pd.option_context('future.no_silent_downcasting', True): flowby_partial_df = (flowby_partial_df From 3ac8e1eff27322a05df334b5e1b459eb72e203b0 Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Tue, 28 Oct 2025 20:39:30 -0600 Subject: [PATCH 30/48] address numpy.exceptions.DTypePromotionError: The DType could not be promoted by --- flowsa/data_source_scripts/EPA_GHGI.py | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/flowsa/data_source_scripts/EPA_GHGI.py b/flowsa/data_source_scripts/EPA_GHGI.py index 41a0e783d..6b73f0594 100644 --- a/flowsa/data_source_scripts/EPA_GHGI.py +++ b/flowsa/data_source_scripts/EPA_GHGI.py @@ -496,20 +496,15 @@ def ghg_parse(*, df_list, year, config, **_): # set suppressed values to 0 but mark as suppressed # otherwise set non-numeric to nan try: - df = (df.assign( - Suppressed = np.where(df.FlowAmount.str.strip() == "+", "+", - np.nan), - FlowAmount = pd.Series( - np.where(df.FlowAmount.str.strip() == "+", 0, - df.FlowAmount.str.replace(',','')))) - ) - df = (df.assign( - FlowAmount = np.where(pd.to_numeric( - df.FlowAmount, errors='coerce').isnull(), - np.nan, pd.to_numeric( - df.FlowAmount, errors='coerce'))) - .dropna(subset='FlowAmount') - ) + df['Suppressed'] = (df['FlowAmount'] + .astype(str).str.strip().eq('+') + .replace({True: '+', False: np.nan}) + .infer_objects(copy=False) + ) + df['FlowAmount'] = df['FlowAmount'].astype(str).str.replace(',', '').infer_objects(copy=False) + df['FlowAmount'] = df['FlowAmount'].replace('+', '0').infer_objects(copy=False) + df['FlowAmount'] = pd.to_numeric(df['FlowAmount'], errors='coerce') + df = df.dropna(subset='FlowAmount') except AttributeError: # if no string in FlowAmount, then proceed df = df.dropna(subset='FlowAmount') From a03ea1ae5b7dab0efd7b69984da3629a9a48f7e2 Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Wed, 29 Oct 2025 07:39:11 -0600 Subject: [PATCH 31/48] update python version requirement due to numpy version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a5d92cfaf..cf6742f9f 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ packages=find_packages(), package_dir={'flowsa': 'flowsa'}, include_package_data=True, - python_requires=">=3.9", + python_requires=">=3.10", install_requires=[ 'fedelemflowlist @ git+https://github.com/USEPA/fedelemflowlist.git@develop#egg=fedelemflowlist', 'esupy @ git+https://github.com/USEPA/esupy.git@develop#egg=esupy', From e1ab626c450eb6316cc3b3dc5848a9bac7f3ddbf Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Wed, 29 Oct 2025 13:48:02 -0600 Subject: [PATCH 32/48] address future warning Downcasting object dtype arrays on .fillna --- flowsa/flowby.py | 7 ++++--- flowsa/flowbyclean.py | 5 ++++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/flowsa/flowby.py b/flowsa/flowby.py index 98e6c1594..b8715f2d3 100644 --- a/flowsa/flowby.py +++ b/flowsa/flowby.py @@ -150,9 +150,10 @@ def __init__( # and will change in a future version" with pd.option_context('future.no_silent_downcasting', True): data = (data - .fillna(fill_na_dict) - .replace(null_string_dict) - .astype(fields)) + .fillna(fill_na_dict).infer_objects(copy=False) + .replace(null_string_dict).infer_objects(copy=False) + .astype(fields) + ) if isinstance(data, pd.DataFrame) and column_order is not None and convert_df_to_flowby: data = data[[c for c in column_order if c in data.columns] diff --git a/flowsa/flowbyclean.py b/flowsa/flowbyclean.py index a2113e704..258610705 100644 --- a/flowsa/flowbyclean.py +++ b/flowsa/flowbyclean.py @@ -96,7 +96,10 @@ def weighted_average( for original, replacement in fba.config.get( 'replacement_dictionary').items(): with pd.option_context('future.no_silent_downcasting', True): - merged = merged.replace({original: replacement}) + merged = (merged + .replace({original: replacement}) + .infer_objects(copy=False) + ) wt_flow = (merged .groupby(['Class', 'Flowable', 'Unit', From fbc0e35b3c863effc44f95993ece3ea0684763b0 Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Wed, 29 Oct 2025 13:48:46 -0600 Subject: [PATCH 33/48] spelling --- flowsa/naics.py | 2 +- flowsa/validation.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/flowsa/naics.py b/flowsa/naics.py index 8bc61a30b..75c4a3295 100644 --- a/flowsa/naics.py +++ b/flowsa/naics.py @@ -588,7 +588,7 @@ def convert_naics_year(df_load, targetsectorsourcename, sectorsourcename, # methods, so assigning schema as None activity_schema = "None" - # however, need to ensure that these NAICS are formatted correly - stewi data are at times imported + # however, need to ensure that these NAICS are formatted correctly - stewi data are at times imported # with some NAICS values including decimals that do not get mapped correctly (ex. '311712.0') for col in column_headers: if col in df_load.columns: diff --git a/flowsa/validation.py b/flowsa/validation.py index 779940943..15906e9bb 100644 --- a/flowsa/validation.py +++ b/flowsa/validation.py @@ -296,7 +296,7 @@ def compare_FBS_results(fbs1, fbs2, ignore_metasources=False, def compare_FBS(df1, df2, ignore_metasources=False): - "Assess differences between two FBS dataframes." + """Assess differences between two FBS dataframes.""" df1 = df1.rename(columns={'FlowAmount': 'FlowAmount_fbs1'}) df2 = df2.rename(columns={'FlowAmount': 'FlowAmount_fbs2'}) merge_cols = [c for c in df2.select_dtypes(include=[ From 7450eee19157fdb0a4a7752933963b4ee7b2cc60 Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Wed, 29 Oct 2025 13:49:20 -0600 Subject: [PATCH 34/48] address np.nan not being dropped in map_to_sectors() --- flowsa/flowbyactivity.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/flowsa/flowbyactivity.py b/flowsa/flowbyactivity.py index 1e2707f14..8349be9e2 100644 --- a/flowsa/flowbyactivity.py +++ b/flowsa/flowbyactivity.py @@ -533,10 +533,13 @@ def map_to_sectors( log.warning('Activities in %s are not mapped to sectors: %s', not_mapped.full_name, sorted( set(not_mapped.ActivityProducedBy.dropna()).union(set(not_mapped.ActivityConsumedBy.dropna())))) - # drop all NA data - fba_w_naics = (fba_w_naics - .dropna(subset=['SectorProducedBy', 'SectorConsumedBy'], how='all') - .assign(SectorSourceName=f'NAICS_{target_year}_Code') + # drop all NA data and clean up df + fba_w_naics = fba_w_naics[ + ~(fba_w_naics['SectorProducedBy'].isna() & fba_w_naics['SectorConsumedBy'].isna()) + ] + + fba_w_naics2 = (fba_w_naics + .assign(SectorSourceName=f'NAICS_{target_year}_Code') .drop(columns=['TechnologicalCorrelation_x', 'TechnologicalCorrelation_y', 'DataReliability_x', 'DataReliability_y', 'DataCollection_x', 'DataCollection_y'], @@ -544,7 +547,7 @@ def map_to_sectors( .reset_index(drop=True) ) - return fba_w_naics + return fba_w_naics2 def prepare_fbs( From d3db58e53a73f8b6e9829653425678a438971042 Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Wed, 29 Oct 2025 15:31:32 -0600 Subject: [PATCH 35/48] skip equal attribution on columns that are all np.nan --- flowsa/flowby.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/flowsa/flowby.py b/flowsa/flowby.py index b8715f2d3..50d4f0ba4 100644 --- a/flowsa/flowby.py +++ b/flowsa/flowby.py @@ -1391,6 +1391,11 @@ def equally_attribute(self: 'FB') -> 'FB': groupby_cols = ['group_id', 'Location'] for rank in ['Primary', 'Secondary']: + # continue if values are all np.nan + if fba[f'{rank}Sector'].isna().all(): + groupby_cols.append(f'{rank}Sector') + continue + fba = ( fba .merge(naics_key, how='left', left_on=f'{rank}Sector', From 6b9f9bba645dce04a2deea73b03b2f1cf489f49b Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Wed, 29 Oct 2025 15:32:14 -0600 Subject: [PATCH 36/48] at log statement for level currently being estimated --- flowsa/flowbyclean.py | 1 + 1 file changed, 1 insertion(+) diff --git a/flowsa/flowbyclean.py b/flowsa/flowbyclean.py index 258610705..1a342ba46 100644 --- a/flowsa/flowbyclean.py +++ b/flowsa/flowbyclean.py @@ -345,6 +345,7 @@ def fill_suppressed( # loop through sector lengths, estimating suppressed data for level in [2, 3, 4, 5, 6]: + log.info(f"Estimating suppressed data at sector level {level}") groupcols = (["{}{}".format("n", i) for i in range(2, level+1)] + ['location', 'category']) unsuppressed = (unsuppressed From 814dda82e0d2f070ef4713341e12806f563e70a2 Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Wed, 29 Oct 2025 15:32:38 -0600 Subject: [PATCH 37/48] address .loc warning --- flowsa/naics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flowsa/naics.py b/flowsa/naics.py index 75c4a3295..4f5901db5 100644 --- a/flowsa/naics.py +++ b/flowsa/naics.py @@ -156,7 +156,7 @@ def subset_sector_key(flowbyactivity, activitycol, sector_source_year, primary_s # ensure dq column decimals do not cause errors with dropping duplicates, without this statement, rows often # duplicated if dqi: - flowbyactivity[dqi] = flowbyactivity[dqi].round(decimals=5) + flowbyactivity.loc[:, dqi] = (flowbyactivity.loc[:, dqi].round(decimals=5)) flowbyactivity = flowbyactivity[subset_cols].drop_duplicates() primary_sector_key_2 = pd.DataFrame(flowbyactivity.merge( From 5bb0176bb59f170e1a8ad2fc7eb0ed540429b326 Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Wed, 29 Oct 2025 21:43:34 -0600 Subject: [PATCH 38/48] replace np.nan in sector columns with empty string when comparing 2 fbs to ensure they merge correctly --- flowsa/validation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flowsa/validation.py b/flowsa/validation.py index 15906e9bb..6b814515c 100644 --- a/flowsa/validation.py +++ b/flowsa/validation.py @@ -321,8 +321,8 @@ def compare_FBS(df1, df2, ignore_metasources=False): # convert sector columns to object to avoid valueErrors cols = ['SectorProducedBy', 'SectorConsumedBy'] for c in cols: - df1[c] = df1[c].astype(str) - df2[c] = df2[c].astype(str) + df1[c] = df1[c].where(df1[c].notna(), '').astype(str) + df2[c] = df2[c].where(df2[c].notna(), '').astype(str) for c in ['SectorSourceName']: df1 = df1.drop(columns=c, errors='ignore') df2 = df2.drop(columns=c, errors='ignore') From f6abba8527958638f8981635539bf7f8c0b3fac1 Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Thu, 30 Oct 2025 09:06:43 -0600 Subject: [PATCH 39/48] in compare fbs fillna for all string merge cols to ensure correct merge --- flowsa/validation.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/flowsa/validation.py b/flowsa/validation.py index 6b814515c..bd967d465 100644 --- a/flowsa/validation.py +++ b/flowsa/validation.py @@ -312,6 +312,10 @@ def compare_FBS(df1, df2, ignore_metasources=False): except ValueError: pass + # convert all np.nan in the string type merge cols to empty strings, to ensure correct merge + fill_cols = [c for c in merge_cols if df2[c].dtype == 'object'] + df1[fill_cols] = df1[fill_cols].fillna("") + df2[fill_cols] = df2[fill_cols].fillna("") # aggregate dfs before merge - might have duplicate sectors due to # dropping metasources/attribution sources df1 = (df1.groupby(merge_cols, dropna=False) @@ -321,8 +325,8 @@ def compare_FBS(df1, df2, ignore_metasources=False): # convert sector columns to object to avoid valueErrors cols = ['SectorProducedBy', 'SectorConsumedBy'] for c in cols: - df1[c] = df1[c].where(df1[c].notna(), '').astype(str) - df2[c] = df2[c].where(df2[c].notna(), '').astype(str) + df1[c] = df1[c].astype(str) + df2[c] = df2[c].astype(str) for c in ['SectorSourceName']: df1 = df1.drop(columns=c, errors='ignore') df2 = df2.drop(columns=c, errors='ignore') From 58ebe68d21af05438d3ae251d57797f6fe51d374 Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Thu, 30 Oct 2025 09:59:05 -0600 Subject: [PATCH 40/48] move where np.nans are converted to string and maintain leading 0s in location in excel files --- flowsa/validation.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/flowsa/validation.py b/flowsa/validation.py index bd967d465..3261376d6 100644 --- a/flowsa/validation.py +++ b/flowsa/validation.py @@ -312,10 +312,6 @@ def compare_FBS(df1, df2, ignore_metasources=False): except ValueError: pass - # convert all np.nan in the string type merge cols to empty strings, to ensure correct merge - fill_cols = [c for c in merge_cols if df2[c].dtype == 'object'] - df1[fill_cols] = df1[fill_cols].fillna("") - df2[fill_cols] = df2[fill_cols].fillna("") # aggregate dfs before merge - might have duplicate sectors due to # dropping metasources/attribution sources df1 = (df1.groupby(merge_cols, dropna=False) @@ -331,6 +327,10 @@ def compare_FBS(df1, df2, ignore_metasources=False): df1 = df1.drop(columns=c, errors='ignore') df2 = df2.drop(columns=c, errors='ignore') merge_cols = [x for x in merge_cols if x != c] + # convert all np.nan in the string type merge cols to empty strings, to ensure correct merge + fill_cols = [c for c in merge_cols if df2[c].dtype == 'object'] + df1[fill_cols] = df1[fill_cols].replace(['nan', np.nan], '') + df2[fill_cols] = df2[fill_cols].replace(['nan', np.nan], '') # check units # compare_df_units(df1, df2) df_m = pd.DataFrame( @@ -377,6 +377,8 @@ def compare_single_FBS_against_remote(m, outdir=diffpath, inplace=True) if len(df) > 0: print(f"Saving differences in {m} to csv") + # maintain leading 0s in location col + df.Location = df.Location.apply('="{}"'.format) df.to_csv(f"{outdir}/{m}_diff.csv", index=False) else: print(f"***No differences found in {m}***") @@ -406,6 +408,8 @@ def compare_single_FBA_against_remote(source, year, outdir=diffpath, inplace=True) if len(df) > 0: print(f"Saving differences in {source} {year} to csv") + # maintain leading 0s in location col + df.Location = df.Location.apply('="{}"'.format) df.to_csv(f"{outdir}/{source}_{year}_diff.csv", index=False) else: print(f"***No differences found in {source} {year}***") From e3832384420b49c5c5625261922812bf345b7ffd Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Thu, 30 Oct 2025 11:02:44 -0600 Subject: [PATCH 41/48] reorder code in compare_FBS() --- flowsa/validation.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/flowsa/validation.py b/flowsa/validation.py index 3261376d6..17ff55ac1 100644 --- a/flowsa/validation.py +++ b/flowsa/validation.py @@ -312,13 +312,7 @@ def compare_FBS(df1, df2, ignore_metasources=False): except ValueError: pass - # aggregate dfs before merge - might have duplicate sectors due to - # dropping metasources/attribution sources - df1 = (df1.groupby(merge_cols, dropna=False) - .agg({'FlowAmount_fbs1': 'sum'}).reset_index()) - df2 = (df2.groupby(merge_cols, dropna=False) - .agg({'FlowAmount_fbs2': 'sum'}).reset_index()) - # convert sector columns to object to avoid valueErrors + # convert sector columns to object to avoid valueErrors and df clean up cols = ['SectorProducedBy', 'SectorConsumedBy'] for c in cols: df1[c] = df1[c].astype(str) @@ -331,12 +325,21 @@ def compare_FBS(df1, df2, ignore_metasources=False): fill_cols = [c for c in merge_cols if df2[c].dtype == 'object'] df1[fill_cols] = df1[fill_cols].replace(['nan', np.nan], '') df2[fill_cols] = df2[fill_cols].replace(['nan', np.nan], '') + + # subset dfs + df1_sub = df1[merge_cols + ['FlowAmount_fbs1']] + df2_sub = df2[merge_cols + ['FlowAmount_fbs2']] + + # aggregate dfs before merge - might have duplicate sectors due to + # dropping metasources/attribution sources + df1_sub = (df1_sub.groupby(merge_cols, dropna=False) + .agg({'FlowAmount_fbs1': 'sum'}).reset_index()) + df2_sub = (df2_sub.groupby(merge_cols, dropna=False) + .agg({'FlowAmount_fbs2': 'sum'}).reset_index()) + # check units - # compare_df_units(df1, df2) - df_m = pd.DataFrame( - pd.merge(df1[merge_cols + ['FlowAmount_fbs1']], - df2[merge_cols + ['FlowAmount_fbs2']], - how='outer')) + # compare_df_units(df1_sub, df2_sub) + df_m = pd.DataFrame(pd.merge(df1_sub, df2_sub,how='outer')) df_m = df_m.assign(FlowAmount_diff=df_m['FlowAmount_fbs2'] .fillna(0) - df_m['FlowAmount_fbs1'].fillna(0)) df_m = df_m.assign( From 534b3502359d254fc823c0522546faaca13fa0b2 Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Thu, 30 Oct 2025 11:18:18 -0600 Subject: [PATCH 42/48] convert FBS to dfs in compare_fbs() --- flowsa/validation.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/flowsa/validation.py b/flowsa/validation.py index 17ff55ac1..93a9e4311 100644 --- a/flowsa/validation.py +++ b/flowsa/validation.py @@ -295,10 +295,10 @@ def compare_FBS_results(fbs1, fbs2, ignore_metasources=False, return df_m -def compare_FBS(df1, df2, ignore_metasources=False): +def compare_FBS(df1_load, df2_load, ignore_metasources=False): """Assess differences between two FBS dataframes.""" - df1 = df1.rename(columns={'FlowAmount': 'FlowAmount_fbs1'}) - df2 = df2.rename(columns={'FlowAmount': 'FlowAmount_fbs2'}) + df1 = pd.DataFrame(df1_load.rename(columns={'FlowAmount': 'FlowAmount_fbs1'})) + df2 = pd.DataFrame(df2_load.rename(columns={'FlowAmount': 'FlowAmount_fbs2'})) merge_cols = [c for c in df2.select_dtypes(include=[ 'object', 'int']).columns if c not in dq_fields] if ignore_metasources: @@ -332,14 +332,12 @@ def compare_FBS(df1, df2, ignore_metasources=False): # aggregate dfs before merge - might have duplicate sectors due to # dropping metasources/attribution sources - df1_sub = (df1_sub.groupby(merge_cols, dropna=False) - .agg({'FlowAmount_fbs1': 'sum'}).reset_index()) - df2_sub = (df2_sub.groupby(merge_cols, dropna=False) - .agg({'FlowAmount_fbs2': 'sum'}).reset_index()) + df1_sub = df1_sub.groupby(merge_cols, dropna=False).agg({'FlowAmount_fbs1': 'sum'}).reset_index() + df2_sub = df2_sub.groupby(merge_cols, dropna=False).agg({'FlowAmount_fbs2': 'sum'}).reset_index() # check units # compare_df_units(df1_sub, df2_sub) - df_m = pd.DataFrame(pd.merge(df1_sub, df2_sub,how='outer')) + df_m = pd.merge(df1_sub, df2_sub,how='outer') df_m = df_m.assign(FlowAmount_diff=df_m['FlowAmount_fbs2'] .fillna(0) - df_m['FlowAmount_fbs1'].fillna(0)) df_m = df_m.assign( From 975a1f5de524b51dca3a300c8fba7ee9d29296e3 Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Thu, 6 Nov 2025 12:18:57 -0700 Subject: [PATCH 43/48] speed up weighted_average() --- flowsa/flowbyclean.py | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/flowsa/flowbyclean.py b/flowsa/flowbyclean.py index 1a342ba46..8015458cb 100644 --- a/flowsa/flowbyclean.py +++ b/flowsa/flowbyclean.py @@ -101,22 +101,18 @@ def weighted_average( .infer_objects(copy=False) ) - wt_flow = (merged - .groupby(['Class', 'Flowable', 'Unit', - 'FlowType', 'ActivityProducedBy', - 'ActivityConsumedBy', 'Context', 'Location', - 'LocationSystem', 'Year', 'MeasureofSpread', - 'Spread', 'DistributionType', 'Min', 'Max', - 'DataReliability', 'DataCollection', - 'SectorProducedBy', 'ProducedBySectorType', - 'SectorConsumedBy', 'ConsumedBySectorType', - 'SectorSourceName'], - dropna=False)[merged.columns.tolist()] - .apply(lambda x: np.average(x['FlowAmount'], - weights=x['FlowAmount_other'])) - .drop(columns='FlowAmount') # original flowamounts - .reset_index(name='FlowAmount') # new, weighted flows - ) + wt_flow = ( + merged + .groupby(['Class', 'Flowable', 'Unit', 'FlowType', 'ActivityProducedBy', + 'ActivityConsumedBy', 'Context', 'Location', 'LocationSystem', + 'Year', 'MeasureofSpread', 'Spread', 'DistributionType', 'Min', + 'Max', 'DataReliability', 'DataCollection', 'SectorProducedBy', + 'ProducedBySectorType', 'SectorConsumedBy', 'ConsumedBySectorType', + 'SectorSourceName'], dropna=False)[['FlowAmount', 'FlowAmount_other']] + .apply(lambda x: np.average(x['FlowAmount'], weights=x['FlowAmount_other'])) + .reset_index(name='FlowAmount') + ) + # set attributes todo: revise above code so don't lose attributes attributes_to_save = { attr: getattr(fba, attr) for attr in fba._metadata + ['_metadata'] From 9e88ac02d5302d5fd03286a9902786a418ba1cf1 Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Fri, 7 Nov 2025 07:33:37 -0700 Subject: [PATCH 44/48] when cacluatling golf/crop irrigation ensure activity cols are object, not float, so not converted to 0 --- flowsa/data_source_scripts/USGS_NWIS_WU.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/flowsa/data_source_scripts/USGS_NWIS_WU.py b/flowsa/data_source_scripts/USGS_NWIS_WU.py index b5db767d3..a86c50df6 100644 --- a/flowsa/data_source_scripts/USGS_NWIS_WU.py +++ b/flowsa/data_source_scripts/USGS_NWIS_WU.py @@ -610,6 +610,13 @@ def subset_and_merge_irrigation_types(df: FlowByActivity): "ActivityConsumedBy_y": "Crop_ACB", "Description_x": 'Description', "Description_y": "Crop_Description"}) + + # ensure activity cols are object, not float, so not converted to 0 + col = ['ActivityProducedBy', 'Golf_APB', 'Crop_APB'] + for c in col: + df_m2[c] = df_m2[c].astype(str) + + # fill na and sum crop and golf for col in df_m2: if df_m2[col].dtype in ("int", "float"): From 876262080d0d2c8ed31ab0b48660c01573ab71ed Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Fri, 7 Nov 2025 12:08:43 -0500 Subject: [PATCH 45/48] include FBS example to generate FBS, overwriting local copy --- examples/get_flows_by_sector.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/examples/get_flows_by_sector.py b/examples/get_flows_by_sector.py index 84f54f2c6..4439369dd 100644 --- a/examples/get_flows_by_sector.py +++ b/examples/get_flows_by_sector.py @@ -36,6 +36,7 @@ :return: dataframe in flow by sector format """ +import flowsa from flowsa import getFlowBySector, collapse_FlowBySector, \ seeAvailableFlowByModels @@ -54,3 +55,10 @@ # `Water_national_2015_m1` FBS will have 2 fewer rows, as the df is aggregated # after dropping "SectorProducedBy" information fbs_water_collapsed = collapse_FlowBySector('Water_national_2015_m1') + +# alternatively, you can generate an FBS dataset even if you have a local copy. Your local copy will be overwritten +# if the same git hash. +flowsa.flowbysector.FlowBySector.generateFlowBySector( + method='Land_national_2012', + download_sources_ok=True # optionally download FBA data used to generate FBS +) From 99ca25c48e20284291ed7c97dac171a48dc8429d Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Fri, 7 Nov 2025 12:12:09 -0500 Subject: [PATCH 46/48] drop develop branch from requirements --- requirements.txt | 6 +++--- setup.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/requirements.txt b/requirements.txt index 442cea6d1..1a1a7ade5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ -git+https://github.com/USEPA/fedelemflowlist.git@develop#egg=fedelemflowlist -git+https://github.com/USEPA/esupy.git@develop#egg=esupy -git+https://github.com/USEPA/standardizedinventories.git@develop#egg=StEWI +git+https://github.com/USEPA/fedelemflowlist.git#egg=fedelemflowlist +git+https://github.com/USEPA/esupy.git#egg=esupy +git+https://github.com/USEPA/standardizedinventories.git#egg=StEWI appdirs >= 1.4.4 # Storing user data bibtexparser>=1.4.3 # Generates bibtex kaleido == 0.1.0.post1; platform_system == "Windows" # export plotly.py figures as static images diff --git a/setup.py b/setup.py index cf6742f9f..43ba3cfe2 100644 --- a/setup.py +++ b/setup.py @@ -12,9 +12,9 @@ include_package_data=True, python_requires=">=3.10", install_requires=[ - 'fedelemflowlist @ git+https://github.com/USEPA/fedelemflowlist.git@develop#egg=fedelemflowlist', - 'esupy @ git+https://github.com/USEPA/esupy.git@develop#egg=esupy', - 'StEWI @ git+https://github.com/USEPA/standardizedinventories.git@develop#egg=StEWI', + 'fedelemflowlist @ git+https://github.com/USEPA/fedelemflowlist.git#egg=fedelemflowlist', + 'esupy @ git+https://github.com/USEPA/esupy.git#egg=esupy', + 'StEWI @ git+https://github.com/USEPA/standardizedinventories.git#egg=StEWI', 'appdirs>=1.4.4', 'bibtexparser>=1.4.3', "kaleido==0.1.0.post1;platform_system=='Windows'", From f5d4b4a98018a61c489c7a13e5c9aec56040fe41 Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Fri, 7 Nov 2025 12:12:29 -0500 Subject: [PATCH 47/48] bump flowsa package version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 43ba3cfe2..686b7a637 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name='flowsa', - version='2.1.0', + version='2.2.0', packages=find_packages(), package_dir={'flowsa': 'flowsa'}, include_package_data=True, From acc56f6a1a1803982cfeb88a617f97eb8f3c0ba6 Mon Sep 17 00:00:00 2001 From: catherinebirney <60186515+catherinebirney@users.noreply.github.com> Date: Mon, 10 Nov 2025 13:29:03 -0500 Subject: [PATCH 48/48] update estimate suppressed qcew fxn to work with updated pandas --- flowsa/data_source_scripts/BLS_QCEW.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/flowsa/data_source_scripts/BLS_QCEW.py b/flowsa/data_source_scripts/BLS_QCEW.py index a302f7beb..901a9ade4 100644 --- a/flowsa/data_source_scripts/BLS_QCEW.py +++ b/flowsa/data_source_scripts/BLS_QCEW.py @@ -292,12 +292,12 @@ def fill_suppressed( ): parent = flows[flows[activity].str.len() == level] children = flows[flows[activity].str.len() == level + 1] - null_children = children[children['FlowAmount'].isna()] + null_children = children[children['flow_suppressed']] if null_children.empty or parent.empty: return flows else: - value = max(parent['Unattributed'][0] / len(null_children), 0) + value = max(parent['Unattributed'].iloc[0] / len(null_children), 0) # update the null children by adding the unattributed data to # the attributed data null_children = ( @@ -305,11 +305,15 @@ def fill_suppressed( .assign(FlowAmount=value+null_children['Attributed']) .assign(Unattributed=value) ) - flows.update(null_children) + flows.loc[null_children.index, ['FlowAmount', 'Unattributed']] = ( + null_children)[['FlowAmount', 'Unattributed']] return flows unsuppressed = indexed.copy() + # replace 0 values with np.nan for suppressed data to be estimated + unsuppressed['FlowAmount'] = unsuppressed['FlowAmount'].mask(unsuppressed['FlowAmount'] == 0) + unsuppressed['flow_suppressed'] = unsuppressed['FlowAmount'].isna() for level in range(2, max_level, 1): log.info(f"Estimating suppressed NAICS {level + 1}") groupcols = ["{}{}".format("n", i) for i in range(2, level+1)] + [ @@ -322,7 +326,7 @@ def fill_suppressed( unsuppressed .reset_index(drop=True) .fillna({'FlowAmount': 0}) - .drop(columns=['Unattributed', 'Attributed']) + .drop(columns=['Unattributed', 'Attributed', 'flow_suppressed']) .assign(FlowName='Number of employees') .replace({'ActivityProducedBy': {'3X': '31-33', '4X': '44-45',