V2.2 release

.github/workflows/compare_single_FBA.yml

@@ -25,7 +25,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v3
       with:
-        python-version: "3.10"
+        python-version: "3.12"
 
     - name: Update pip
       run: |

.github/workflows/compare_single_FBS.yml

@@ -21,7 +21,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v3
       with:
-        python-version: "3.10"
+        python-version: "3.12"
 
     - name: Update pip
       run: |

.github/workflows/generate_FBA.yml

@@ -25,7 +25,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v3
       with:
-        python-version: "3.10"
+        python-version: "3.12"
 
     - name: Update pip
       run: |

.github/workflows/python-app.yml

@@ -31,7 +31,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        py-version: ['3.9', '3.10', '3.11']
+        py-version: ['3.10', '3.11', '3.12']
 
     steps:
     - uses: actions/checkout@v3
@@ -74,7 +74,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v3
       with:
-        python-version: "3.10"
+        python-version: "3.12"
 
     - name: Update pip & install testing pkgs
       run: |

.github/workflows/test_methods.yml

@@ -19,7 +19,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v3
       with:
-        python-version: "3.10"
+        python-version: "3.12"
 
     - name: Update pip & install testing pkgs
       run: |
@@ -47,7 +47,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v3
       with:
-        python-version: "3.10"
+        python-version: "3.12"
 
     - name: Update pip & install testing pkgs
       run: |

docs/user_guide/user_guide.md

@@ -327,7 +327,7 @@ To attribute a primary data source to target level sectors, identify the primary
   specified attribution data source
 - equal: Equally attribute parent values to child values until reach target 
   sector length
-- inheritance: Assign parent values to all child values. Usefull in 
+- inheritance: Assign parent values to all child values. Useful in 
   situations where value is a rate, such as kg/m2.
 
 ### Recursive vs Sequential Attribution

examples/get_flows_by_sector.py

@@ -36,6 +36,7 @@
     :return: dataframe in flow by sector format
 
 """
+import flowsa
 from flowsa import getFlowBySector, collapse_FlowBySector, \
     seeAvailableFlowByModels
 
@@ -54,3 +55,10 @@
 # `Water_national_2015_m1` FBS will have 2 fewer rows, as the df is aggregated
 # after dropping "SectorProducedBy" information
 fbs_water_collapsed = collapse_FlowBySector('Water_national_2015_m1')
+
+# alternatively, you can generate an FBS dataset even if you have a local copy. Your local copy will be overwritten
+# if the same git hash.
+flowsa.flowbysector.FlowBySector.generateFlowBySector(
+            method='Land_national_2012',
+            download_sources_ok=True  # optionally download FBA data used to generate FBS
+)

flowsa/data/unit_conversion.csv

@@ -13,14 +13,17 @@ TON,kg,907.185
 tons,kg,907.185
 short tons,kg,907.185
 Thousands of Tons,kg,907185
+million short tons,kg,907184740
 MT,kg,1000
 Metric Tons,kg,1000
 LB,kg,0.45359
 ACRE FEET / ACRE,kg/m2,304.8000006
 gal/USD,kg/USD,3.79
 Quadrillion Btu,MJ,1.06E+12
 Trillion Btu,MJ,1055055900
+trillion Btu,MJ,1055055900
 Tbtu,MJ,1055055900
+million kWh,MJ,3600000
 MT CO2e,MMT CO2e,0.000001
 employees,p,1
 Thousand USD,USD,1000

flowsa/data_source_scripts/EIA_CBECS_Land.py

@@ -405,7 +405,7 @@ def disaggregate_eia_cbecs_vacant_and_other(df_load):
     df_nvno = df_nvno[df_nvno['ActivityConsumedBy'] != 'All buildings']
     df_nvno = df_nvno.groupby(
         ['Unit', 'Location', 'LocationSystem', 'Year', 'Description'],
-        as_index=False).agg({'FlowAmount': sum})
+        as_index=False).agg({'FlowAmount': "sum"})
     df_act = df_nvno.rename(columns={'FlowAmount': 'NonVacantNonOther'})
 
     # merge df and subtract to determine FlowAmount to allocate to

flowsa/data_source_scripts/EIA_MECS.py

@@ -509,8 +509,8 @@ def update_regions_to_states(fba: FlowByActivity,
     fba = (
         fba
         .drop(columns=['group_id', 'group_total'])
-        .reset_index(drop=True).reset_index()
-        .rename(columns={'index': 'group_id'})
+        .reset_index(drop=True)
+        .reset_index(names='group_id')
         .assign(group_total=fba.FlowAmount)
     )
 

flowsa/data_source_scripts/EPA_GHGI.py

@@ -40,9 +40,9 @@
 ANNEX_ENERGY_TABLES = ["A-" + str(x) for x in list(range(4,16))]
 
 DROP_COLS = ["Unnamed: 0"] + list(pd.date_range(
-    start="1990", end="2010", freq='Y').year.astype(str))
+    start="1990", end="2010", freq='YE').year.astype(str))
 
-YEARS = list(pd.date_range(start="2010", end="2024", freq='Y').year.astype(str))
+YEARS = list(pd.date_range(start="2010", end="2024", freq='YE').year.astype(str))
 
 
 def ghg_url_helper(*, build_url, config, **_):
@@ -496,20 +496,15 @@ def ghg_parse(*, df_list, year, config, **_):
         # set suppressed values to 0 but mark as suppressed
         # otherwise set non-numeric to nan
         try:
-            df = (df.assign(
-                    Suppressed = np.where(df.FlowAmount.str.strip() == "+", "+",
-                                          np.nan),
-                    FlowAmount = pd.Series(
-                        np.where(df.FlowAmount.str.strip() == "+", 0,
-                                 df.FlowAmount.str.replace(',',''))))
-                )
-            df = (df.assign(
-                    FlowAmount = np.where(pd.to_numeric(
-                        df.FlowAmount, errors='coerce').isnull(),
-                                          np.nan, pd.to_numeric(
-                                              df.FlowAmount, errors='coerce')))
-                .dropna(subset='FlowAmount')
-                )
+            df['Suppressed'] = (df['FlowAmount']
+                                .astype(str).str.strip().eq('+')
+                                .replace({True: '+', False: np.nan})
+                                .infer_objects(copy=False)
+                                )
+            df['FlowAmount'] = df['FlowAmount'].astype(str).str.replace(',', '').infer_objects(copy=False)
+            df['FlowAmount'] = df['FlowAmount'].replace('+', '0').infer_objects(copy=False)
+            df['FlowAmount'] = pd.to_numeric(df['FlowAmount'], errors='coerce')
+            df = df.dropna(subset='FlowAmount')
         except AttributeError:
             # if no string in FlowAmount, then proceed
             df = df.dropna(subset='FlowAmount')

flowsa/data_source_scripts/USDA_ERS_MLU.py

@@ -201,7 +201,7 @@ def allocate_usda_ers_mlu_land_in_urban_areas(
         df_non_urban_transport_area[['Location', 'Unit', 'FlowAmount']]
     non_urban_transport_area_sum = df_non_urban_transport_area.groupby(
             ['Location', 'Unit'], as_index=False).agg(
-        {'FlowAmount': sum}).rename(columns={'FlowAmount': 'NonTransport'})
+        {'FlowAmount': "sum"}).rename(columns={'FlowAmount': 'NonTransport'})
     # compare units
     compare_df_units(fba, df_non_urban_transport_area)
     # calculate total urban transportation by subtracting
@@ -232,7 +232,7 @@ def allocate_usda_ers_mlu_land_in_urban_areas(
     air_rail_area = air_rail_area[['Location', 'Unit', 'FlowAmount']]
     air_rail_area_sum = air_rail_area.groupby(
         ['Location', 'Unit'], as_index=False).agg(
-        {'FlowAmount': sum}).rename(columns={'FlowAmount': 'AirRail'})
+        {'FlowAmount': "sum"}).rename(columns={'FlowAmount': 'AirRail'})
 
     df_highway = df_transport.merge(air_rail_area_sum, how='left')
     df_highway = df_highway.assign(
@@ -256,8 +256,8 @@ def allocate_usda_ers_mlu_land_in_urban_areas(
         allocated_urban_areas_df
         .drop(columns=['group_id', 'group_total', 'rurl_res', 'total_area'],
               errors='ignore')
-        .reset_index(drop=True).reset_index()
-        .rename(columns={'index': 'group_id'})
+        .reset_index(drop=True)
+        .reset_index(names='group_id')
         .assign(group_total=allocated_urban_areas_df.FlowAmount)
     )
 
@@ -317,7 +317,7 @@ def allocate_usda_ers_mlu_land_in_rural_transportation_areas(
     air_rail_area = air_rail_area[['Location', 'Unit', 'FlowAmount']]
     air_rail_area_sum = air_rail_area.groupby(
         ['Location', 'Unit'], as_index=False).agg(
-        {'FlowAmount': sum}).rename(columns={'FlowAmount': 'AirRail'})
+        {'FlowAmount': "sum"}).rename(columns={'FlowAmount': 'AirRail'})
 
     # compare units
     compare_df_units(fba, air_rail_area)
@@ -341,8 +341,8 @@ def allocate_usda_ers_mlu_land_in_rural_transportation_areas(
     allocated_rural_trans = (
         allocated_rural_trans
         .drop(columns=['group_id', 'group_total'])
-        .reset_index(drop=True).reset_index()
-        .rename(columns={'index': 'group_id'})
+        .reset_index(drop=True)
+        .reset_index(names='group_id')
         .assign(group_total=allocated_rural_trans.FlowAmount)
     )
 

flowsa/data_source_scripts/USGS_NWIS_WU.py

@@ -610,6 +610,13 @@ def subset_and_merge_irrigation_types(df: FlowByActivity):
                                   "ActivityConsumedBy_y": "Crop_ACB",
                                   "Description_x": 'Description',
                                   "Description_y": "Crop_Description"})
+
+    # ensure activity cols are object, not float, so not converted to 0
+    col = ['ActivityProducedBy', 'Golf_APB', 'Crop_APB']
+    for c in col:
+        df_m2[c] = df_m2[c].astype(str)
+
+
     # fill na and sum crop and golf
     for col in df_m2:
         if df_m2[col].dtype in ("int", "float"):

flowsa/data_source_scripts/stewiFBS.py

@@ -338,6 +338,7 @@ def prepare_stewi_fbs(df_load, config) -> 'FlowBySector':
             .reset_index(drop=True),
             full_name=config.get('full_name'),
             config=config,
+        convert_df_to_flowby=True
             ).prepare_fbs()
 
     fbs.config.update({'data_format': 'FBS'})

flowsa/dataclean.py

@@ -49,21 +49,24 @@ def add_missing_flow_by_fields(flowby_partial_df, flowbyfields):
                 flowby_partial_df[col] = np.nan
     # convert all None, 'nan' to np.nan
     flowby_partial_df = (flowby_partial_df
-                         .replace('None', np.nan)
-                         .replace('nan', np.nan))
+                         .replace({'None': np.nan, 'nan': np.nan})
+                         .infer_objects(copy=False)
+                         )
     # convert data types to match those defined in flow_by_activity_fields
     for k, v in flowbyfields.items():
         if k in flowby_partial_df.columns:
             flowby_partial_df[k] = \
                 flowby_partial_df[k].astype(v[0]['dtype'])
             if v[0]['dtype'] in ['string', 'str', 'object']:
-                flowby_partial_df[k] = flowby_partial_df[k].fillna(np.nan)
+                flowby_partial_df[k] = flowby_partial_df[k].fillna(np.nan).infer_objects(copy=False)
             else:
-                flowby_partial_df[k] = flowby_partial_df[k].fillna(0)
+                flowby_partial_df[k] = flowby_partial_df[k].fillna(0).infer_objects(copy=False)
     # convert all None, 'nan' to np.nan
-    flowby_partial_df = (flowby_partial_df
-                         .replace('None', np.nan)
-                         .replace('nan', np.nan))
+    with pd.option_context('future.no_silent_downcasting', True):
+        flowby_partial_df = (flowby_partial_df
+                             .replace({'None': np.nan, 'nan': np.nan})
+                             .infer_objects(copy=False)
+                             )
     # Resort it so order is correct
     cols = [e for e in flowbyfields.keys() if e in flowby_partial_df.columns]
     flowby_df = flowby_partial_df[cols]
@@ -99,7 +102,7 @@ def standardize_units(df):
         .merge(conversion_table, how='left',
                left_on='Unit', right_on='old_unit')
         .assign(Unit=lambda x: x.new_unit.mask(x.new_unit.isna(), x.Unit),
-                conversion_factor=lambda x: x.conversion_factor.fillna(1),
+                conversion_factor=lambda x: x.conversion_factor.fillna(1).infer_objects(copy=False),
                 FlowAmount=lambda x: x.FlowAmount * x.conversion_factor)
         .drop(columns=['old_unit', 'new_unit', 'conversion_factor'])
     )