diff --git a/flowsa/data/BEA_country_names.csv b/flowsa/data/BEA_country_names.csv new file mode 100644 index 000000000..7c9d1adac --- /dev/null +++ b/flowsa/data/BEA_country_names.csv @@ -0,0 +1,91 @@ +BEA_AREAORCOUNTRY,country +Africa,Africa +AllCountries,All countries +Argentina,Argentina +AsiaAndPac,Asia and Pacific +Australia,Australia +Austria,Austria +Bahrain,Bahrain +Belgium,Belgium +Bermuda,Bermuda +Brazil,Brazil +Brunei,Brunei +Bulgaria,Bulgaria +CaftaDrCountries,CAFTA-DR countries +Canada,Canada +Chile,Chile +China,China +Colombia,Colombia +CostaRica,Costa Rica +Croatia,Croatia +Cyprus,Cyprus +CzechRep,Czech Republic +Denmark,Denmark +DominicanRep,Dominican Republic +ElSalvador,El Salvador +Estonia,Estonia +EU,European Union +EuroArea,Euro area +Europe,Europe +Finland,Finland +France,France +Germany,Germany +Greece,Greece +Guatemala,Guatemala +Honduras,Honduras +HongKong,Hong Kong +Hungary,Hungary +India,India +Indonesia,Indonesia +IntOrgAndUnalloc,International organizations and unallocated +Ireland,Ireland +Israel,Israel +Italy,Italy +Japan,Japan +Jordan,Jordan +LatAmAndOthWestHem,Latin America and Other Western Hemisphere +Latvia,Latvia +Lithuania,Lithuania +Luxembourg,Luxembourg +Malaysia,Malaysia +Malta,Malta +Mexico,Mexico +MiddleEast,Middle East +Morocco,Morocco +Netherlands,Netherlands +NewZealand,New Zealand +Nicaragua,Nicaragua +Nigeria,Nigeria +Norway,Norway +Oman,Oman +OthAfricaIst,Other Africa (International Services Trade) +OthAsiaAndPacIst,Other Asia and Pacific (International Services Trade) +OthEuropeIst,Other Europe (International Services Trade) +OthMiddleEastIst,Other Middle East (International Services Trade) +OthSouthAndCenAmIst,Other South And Central America (International Services Trade) +OthWestHem,Other Western Hemisphere +OthWestHemOthIst,"Other Western Hemisphere, Other (International Services Trade)" +Panama,Panama +Peru,Peru +Philippines,Philippines +Poland,Poland +Portugal,Portugal +Romania,Romania +Russia,Russia +SaudiArabia,Saudi Arabia +Singapore,Singapore +Slovakia,Slovakia +Slovenia,Slovenia +SouthAfrica,South Africa +SouthAndCenAm,South and Central America +SouthKorea,South Korea +Spain,Spain +Sweden,Sweden +Switzerland,Switzerland +Taiwan,Taiwan +Thailand,Thailand +Turkey,Turkey +UkIslandsCarib,"United Kingdom Islands, Caribbean" +UnitedKingdom,United Kingdom +Venezuela,Venezuela +Vietnam,Vietnam diff --git a/flowsa/data/Census_country_codes.txt b/flowsa/data/Census_country_codes.txt new file mode 100644 index 000000000..1240543cb --- /dev/null +++ b/flowsa/data/Census_country_codes.txt @@ -0,0 +1,249 @@ +---------------------------------------------------------------------- +Schedule C - Country List (by code) [Produced: 27JAN21] +---------------------------------------------------------------------- +Code | Name | ISO Code +---------------------------------------------------------------------- +1000 | United States of America | US +1010 | Greenland | GL +1220 | Canada | CA +1610 | Saint Pierre and Miquelon | PM +2010 | Mexico | MX +2050 | Guatemala | GT +2080 | Belize | BZ +2110 | El Salvador | SV +2150 | Honduras | HN +2190 | Nicaragua | NI +2230 | Costa Rica | CR +2250 | Panama | PA +2320 | Bermuda | BM +2360 | Bahamas | BS +2390 | Cuba | CU +2410 | Jamaica | JM +2430 | Turks and Caicos Islands | TC +2440 | Cayman Islands | KY +2450 | Haiti | HT +2470 | Dominican Republic | DO +2481 | Anguilla | AI +2482 | British Virgin Islands | VG +2483 | Saint Kitts and Nevis | KN +2484 | Antigua and Barbuda | AG +2485 | Montserrat | MS +2486 | Dominica | DM +2487 | Saint Lucia | LC +2488 | Saint Vincent and the Grenadines | VC +2489 | Grenada | GD +2720 | Barbados | BB +2740 | Trinidad and Tobago | TT +2774 | Sint Maarten | SX +2777 | Curacao | CW +2779 | Aruba | AW +2831 | Guadeloupe | GP +2839 | Martinique | MQ +3010 | Colombia | CO +3070 | Venezuela | VE +3120 | Guyana | GY +3150 | Suriname | SR +3170 | French Guiana | GF +3310 | Ecuador | EC +3330 | Peru | PE +3350 | Bolivia | BO +3370 | Chile | CL +3510 | Brazil | BR +3530 | Paraguay | PY +3550 | Uruguay | UY +3570 | Argentina | AR +3720 | Falkland Islands (Islas Malvinas) | FK +4000 | Iceland | IS +4010 | Sweden | SE +4031 | Svalbard and Jan Mayen | SJ +4039 | Norway | NO +4050 | Finland | FI +4091 | Faroe Islands | FO +4099 | Denmark, except Greenland | DK +4120 | United Kingdom | GB +4190 | Ireland | IE +4210 | Netherlands | NL +4231 | Belgium | BE +4239 | Luxembourg | LU +4271 | Andorra | AD +4272 | Monaco | MC +4279 | France | FR +4280 | Germany (Federal Republic of Germany) | DE +4330 | Austria | AT +4351 | Czech Republic | CZ +4359 | Slovakia | SK +4370 | Hungary | HU +4411 | Liechtenstein | LI +4419 | Switzerland | CH +4470 | Estonia | EE +4490 | Latvia | LV +4510 | Lithuania | LT +4550 | Poland | PL +4621 | Russia | RU +4622 | Belarus | BY +4623 | Ukraine | UA +4631 | Armenia | AM +4632 | Azerbaijan | AZ +4633 | Georgia | GE +4634 | Kazakhstan | KZ +4635 | Kyrgyzstan | KG +4641 | Moldova (Republic of Moldova) | MD +4642 | Tajikistan | TJ +4643 | Turkmenistan | TM +4644 | Uzbekistan | UZ +4700 | Spain | ES +4710 | Portugal | PT +4720 | Gibraltar | GI +4730 | Malta | MT +4751 | San Marino | SM +4752 | Holy See (Vatican City) | VA +4759 | Italy | IT +4791 | Croatia | HR +4792 | Slovenia | SI +4793 | Bosnia and Herzegovina | BA +4794 | North Macedonia | MK +4801 | Serbia | RS +4803 | Kosovo | KV +4804 | Montenegro | ME +4810 | Albania | AL +4840 | Greece | GR +4850 | Romania | RO +4870 | Bulgaria | BG +4890 | Turkey | TR +4910 | Cyprus | CY +5020 | Syria (Syrian Arab Republic) | SY +5040 | Lebanon | LB +5050 | Iraq | IQ +5070 | Iran | IR +5081 | Israel | IL +5082 | Gaza Strip administered by Israel | GZ +5083 | West Bank administered by Israel | WE +5110 | Jordan | JO +5130 | Kuwait | KW +5170 | Saudi Arabia | SA +5180 | Qatar | QA +5200 | United Arab Emirates | AE +5210 | Yemen (Republic of Yemen) | YE +5230 | Oman | OM +5250 | Bahrain | BH +5310 | Afghanistan | AF +5330 | India | IN +5350 | Pakistan | PK +5360 | Nepal | NP +5380 | Bangladesh | BD +5420 | Sri Lanka | LK +5460 | Burma (Myanmar) | MM +5490 | Thailand | TH +5520 | Vietnam | VN +5530 | Laos (Lao People's Democratic Republic) | LA +5550 | Cambodia | KH +5570 | Malaysia | MY +5590 | Singapore | SG +5600 | Indonesia | ID +5601 | Timor-Leste | TL +5610 | Brunei | BN +5650 | Philippines | PH +5660 | Macao | MO +5682 | Bhutan | BT +5683 | Maldives | MV +5700 | China | CN +5740 | Mongolia | MN +5790 | North Korea (Democratic People's Republic of Korea) | KP +5800 | South Korea (Republic of Korea) | KR +5820 | Hong Kong | HK +5830 | Taiwan | TW +5880 | Japan | JP +6021 | Australia | AU +6022 | Norfolk Island | NF +6023 | Cocos (Keeling) Islands | CC +6024 | Christmas Island (in the Indian Ocean) | CX +6029 | Heard Island and McDonald Islands | HM +6040 | Papua New Guinea | PG +6141 | New Zealand | NZ +6142 | Cook Islands | CK +6143 | Tokelau | TK +6144 | Niue | NU +6150 | Samoa (Western Samoa) | WS +6223 | Solomon Islands | SB +6224 | Vanuatu | VU +6225 | Pitcairn Islands | PN +6226 | Kiribati | KI +6227 | Tuvalu | TV +6412 | New Caledonia | NC +6413 | Wallis and Futuna | WF +6414 | French Polynesia | PF +6810 | Marshall Islands | MH +6820 | Micronesia, Federated States of | FM +6830 | Palau | PW +6862 | Nauru | NR +6863 | Fiji | FJ +6864 | Tonga | TO +7140 | Morocco | MA +7210 | Algeria | DZ +7230 | Tunisia | TN +7250 | Libya | LY +7290 | Egypt | EG +7321 | Sudan | SD +7323 | South Sudan | SS +7380 | Equatorial Guinea | GQ +7410 | Mauritania | MR +7420 | Cameroon | CM +7440 | Senegal | SN +7450 | Mali | ML +7460 | Guinea | GN +7470 | Sierra Leone | SL +7480 | Cote d'Ivoire | CI +7490 | Ghana | GH +7500 | Gambia | GM +7510 | Niger | NE +7520 | Togo | TG +7530 | Nigeria | NG +7540 | Central African Republic | CF +7550 | Gabon | GA +7560 | Chad | TD +7580 | Saint Helena | SH +7600 | Burkina Faso | BF +7610 | Benin | BJ +7620 | Angola | AO +7630 | Congo, Republic of the Congo | CG +7642 | Guinea-Bissau | GW +7643 | Cabo Verde | CV +7644 | Sao Tome and Principe | ST +7650 | Liberia | LR +7660 | Congo, Democratic Republic of the Congo (formerly Za|rCD +7670 | Burundi | BI +7690 | Rwanda | RW +7700 | Somalia | SO +7741 | Eritrea | ER +7749 | Ethiopia | ET +7770 | Djibouti | DJ +7780 | Uganda | UG +7790 | Kenya | KE +7800 | Seychelles | SC +7810 | British Indian Ocean Territory | IO +7830 | Tanzania (United Republic of Tanzania) | TZ +7850 | Mauritius | MU +7870 | Mozambique | MZ +7880 | Madagascar | MG +7881 | Mayotte | YT +7890 | Comoros | KM +7904 | Reunion | RE +7905 | French Southern and Antarctic Lands | TF +7910 | South Africa | ZA +7920 | Namibia | NA +7930 | Botswana | BW +7940 | Zambia | ZM +7950 | Eswatini | SZ +7960 | Zimbabwe | ZW +7970 | Malawi | MW +7990 | Lesotho | LS +9030 | Puerto Rico | PR +9110 | Virgin Islands of the United States | VI +9350 | Guam | GU +9510 | American Samoa | AS +9610 | Northern Mariana Islands | MP +9800 | United States Minor Outlying Islands | UM +---------------------------------------------------------------------- +Source: Foreign Trade Division, U.S. Census Bureau, Washington, D.C. 20233 +This listing produced...31JAN14 +---------------------------------------------------------------------- diff --git a/flowsa/data_source_scripts/BEA_IEA.py b/flowsa/data_source_scripts/BEA_IEA.py new file mode 100644 index 000000000..d6b612cd6 --- /dev/null +++ b/flowsa/data_source_scripts/BEA_IEA.py @@ -0,0 +1,111 @@ +# BEA_IEA.py (flowsa) +# !/usr/bin/env python3 +# coding=utf-8 +""" +Data for imports and exports from BEA International Economic Accounts +""" + +import json +import pandas as pd +from flowsa.common import datapath + + +def bea_iea_url_helper(*, build_url, config, year, **_): + """ + This helper function uses the "build_url" input from generateflowbyactivity.py, + which is a base url for data imports that requires parts of the url text + string to be replaced with info specific to the data year. This function + does not parse the data, only modifies the urls from which data is + obtained. + :param build_url: string, base url + :param config: dictionary, items in FBA method yaml + :return: list, urls to call, concat, parse, format into Flow-By-Activity + format + """ + urls = [] + country_dict = get_country_schema() + ctys = [value for key, value in country_dict.items() if value != '1000'] + for cty in ctys: + request_url = build_url.replace('__areaorcountry__', cty) + urls.append(request_url) + + return urls + + +def get_country_schema(): + """ + Acquires the concordance between countries across ISO country codes and + BEA service imports countries (strings with their API name equivalents) + """ + country_dict = (pd.read_csv(datapath / 'BEA_country_names.csv') + .filter(['BEA_AREAORCOUNTRY', 'country']) + .drop_duplicates() + .set_index('country')['BEA_AREAORCOUNTRY'] + .to_dict() + ) + return country_dict + + +def bea_iea_call(*, resp, **_): + """ + Convert response for calling url to pandas dataframe, + begin parsing df into FBA format + :param resp: df, response from url call + :return: pandas dataframe of original source data + """ + try: + json_load = json.loads(resp.text) + df = pd.DataFrame(data=json_load['BEAAPI']['Results']['Data']) + except: + df = pd.DataFrame() + finally: + return df + + +def bea_iea_parse(*, df_list, year, **_): + """ + Combine, parse, and format the provided dataframes + :param df_list: list of dataframes to concat and format + :param args: dictionary, used to run generateflowbyactivity.py + ('year' and 'source') + :return: df, parsed and partially formatted to flowbyactivity + specifications + """ + # Concat dataframes + df = pd.concat(df_list, ignore_index=True) + country_dict0 = get_country_schema() + country_dict = {v:k for k,v in country_dict0.items()} + + df = (df. + rename(columns={'AreaOrCountry': 'Location', + 'CL_UNIT': 'Unit', + 'TypeOfService': 'ActivityProducedBy', + 'TimeSeriesDescription': 'Description', + 'TradeDirection': 'FlowName', + }) + .assign(FlowAmount = lambda x: pd.to_numeric(x['DataValue']).fillna(0) * 1000000) + .assign(Location = lambda x: x['Location'].map(country_dict)) + .drop(columns=['UNIT_MULT', 'Affiliation', 'DataValue', + 'TimeSeriesId', 'TimePeriod'], errors='ignore') + ) + + # add hard code data + df['SourceName'] = 'BEA_IEA' + df['Class'] = 'Money' + df['ActivityConsumdBy'] = '' + df['LocationSystem'] = 'BEA Countries' + # Add tmp DQ scores + df['DataReliability'] = 5 + df['DataCollection'] = 5 + df['Compartment'] = None + df['FlowType'] = "TECHNOSPHERE_FLOW" + + return df + +if __name__ == "__main__": + import flowsa + flowsa.generateflowbyactivity.main(source='BEA_IEA', year=2023) + fba = pd.DataFrame() + for y in range(2023, 2024): + fba = pd.concat([fba, flowsa.getFlowByActivity('BEA_IEA', y)], + ignore_index=True) diff --git a/flowsa/data_source_scripts/Census_USATrade.py b/flowsa/data_source_scripts/Census_USATrade.py new file mode 100644 index 000000000..a298c65a1 --- /dev/null +++ b/flowsa/data_source_scripts/Census_USATrade.py @@ -0,0 +1,139 @@ +# Census_USATrade.py (flowsa) +# !/usr/bin/env python3 +# coding=utf-8 +""" +Pulls Census USA Trade data for imports ande exports by NAICS + +https://www.census.gov/foreign-trade/reference/guides/Guide_to_International_Trade_Datasets.pdf +https://www.census.gov/data/developers/data-sets/international-trade.html + +""" +import json +import pandas as pd +from flowsa.common import datapath +from flowsa.flowsa_log import log + + +def census_url_helper(*, build_url, year, config, **_): + """ + This helper function uses the "build_url" input from generateflowbyactivity.py, + which is a base url for data imports that requires parts of the url text + string to be replaced with info specific to the data year. This function + does not parse the data, only modifies the urls from which data + is obtained. + :param build_url: string, base url + :param year: year + :return: list, urls to call, concat, parse, format into + Flow-By-Activity format + """ + urls = [] + country_dict = get_country_schema() + ctys = [value for key, value in country_dict.items() if value != '1000'] + for cty in ctys: + request_url = build_url.replace('__areaorcountry__', cty) + for flow in config['url'].get('flows'): + if flow == 'exports': + urls.append(request_url + .replace('__flows__', flow) + .replace('GEN_CIF_YR', 'ALL_VAL_YR')) + elif flow == 'imports': + urls.append(request_url.replace('__flows__', flow)) + return urls + + +def get_country_schema(): + """ + Generates a a concordance between ISO codes and Census country codes (4-digit) + """ + l = [] + with open(datapath / 'Census_country_codes.txt') as f: + for line in f: + a = line.split('|') + l2 = [] + for item in a: + l2.append(item.strip()) + if len(l2)>=3: + l.append(l2) + headers = l[0] + df = pd.DataFrame(l, columns=headers) + df = df.iloc[1:,:] + df = df.rename(columns={'Code':'Census Code'}) + + country_dict = (df + .set_index('Name')['Census Code'] + .to_dict() + ) + return country_dict + + +def census_usatrade_call(*, resp, url, **_): + """ + Convert response for calling url to pandas dataframe, begin + parsing df into FBA format + :param resp: df, response from url call + :return: pandas dataframe of original source data + """ + if resp.status_code == 204: + # No content warning, return empty dataframe + log.warning(f"No content found for {resp.url}") + return pd.DataFrame() + census_json = json.loads(resp.text) + # convert response to dataframe + df_census = pd.DataFrame( + data=census_json[1:len(census_json)], columns=census_json[0]) + + df_census = (df_census + .assign(Type = 'exports' if 'exports' in url else 'imports')) + + return df_census + + +def census_usatrade_parse(*, df_list, year, **_): + """ + Combine, parse, and format the provided dataframes + :param df_list: list of dataframes to concat and format + :param year: year + :return: df, parsed and partially formatted to + flowbyactivity specifications + """ + # concat dataframes + df = pd.concat(df_list, sort=False) + + country_dict0 = get_country_schema() + country_dict = {v:k for k,v in country_dict0.items()} + + df = (df + .assign(FlowAmount = lambda x: x['GEN_CIF_YR'].astype(float) + .fillna(x['ALL_VAL_YR'].astype(float))) + .assign(Location = lambda x: x['CTY_CODE'].map(country_dict)) + .assign(FlowName = lambda x: x['Type']) + .rename(columns={'YEAR': 'Year', + 'NAICS': 'ActivityProducedBy'}) + .drop(columns=['MONTH', 'COMM_LVL', 'CTY_CODE', 'GEN_CIF_YR', 'Type', + 'ALL_VAL_YR'], errors='ignore') + .assign(Unit='USD') + .assign(SourceName='Census_USATrade') + ) + + x = df.drop_duplicates(subset=['ActivityProducedBy', 'Year', + 'Location', 'FlowName']) + if len(x) < len(df): + print('ERROR check duplicates') + + # add hard code data + df['Class'] = 'Money' + df['ActivityConsumdBy'] = '' + df['LocationSystem'] = 'Census Countries' + # Add tmp DQ scores + df['DataReliability'] = 5 + df['DataCollection'] = 5 + df['Compartment'] = None + df['FlowType'] = "TECHNOSPHERE_FLOW" + + return df + + +if __name__ == "__main__": + import flowsa + flowsa.generateflowbyactivity.main(source='Census_USATrade', year=2022) + fba = flowsa.getFlowByActivity('Census_USATrade', 2022) diff --git a/flowsa/data_source_scripts/Census_USATrade_Construction.py b/flowsa/data_source_scripts/Census_USATrade_Construction.py new file mode 100644 index 000000000..2e7472558 --- /dev/null +++ b/flowsa/data_source_scripts/Census_USATrade_Construction.py @@ -0,0 +1,149 @@ +# Census_USATrade.py (flowsa) +# !/usr/bin/env python3 +# coding=utf-8 +""" +Pulls Census USA Trade data for imports and exports by U.S. state for select +construction materials and their precursors + +https://www.census.gov/foreign-trade/reference/guides/Guide_to_International_Trade_Datasets.pdf +https://www.census.gov/data/developers/data-sets/international-trade.html + +""" +import json +import pandas as pd +import numpy as np +from flowsa.common import datapath +from flowsa.flowsa_log import log +from flowsa.location import apply_county_FIPS, US_FIPS +from flowsa.flowbyfunctions import assign_fips_location_system + + +def census_url_helper(*, build_url, year, config, **_): + """ + This helper function uses the 'build_url' input from generateflowbyactivity.py, + which is a base url for data imports/exports that requires parts of the url text + string to be replaced with info specific to the data request. This function + does not parse the data, only modifies the urls from which data + is obtained. + :param build_url: string, base url + :param year: year + :return: list, urls to call, concat, parse, format into Flow-By-Activity format + """ + urls = [] + dataset = config['url'].get('dataset') + for flow in config['url'].get('flows'): + for codes in config['url'].get(dataset.replace('state', '')): + if flow == 'imports': + get_params = config['url']['get_params'].get('import_params') + elif flow == 'exports': + get_params = config['url']['get_params'].get('export_params') + get_params_list = '%2C'.join(get_params) + replacements = { + '__dataset__': dataset, + '__flows__': flow, + '__codes__': codes, + '__get_params__': get_params_list + } + request_url = build_url + for placeholder, value in replacements.items(): + request_url = request_url.replace(placeholder, value) + request_url = request_url.replace('CODES', dataset.replace('state', '').upper()) + if dataset == 'statehs': + request_url = request_url.replace('HS', ('I_COMMODITY' if flow == 'imports' else 'E_COMMODITY')) + urls.append(request_url) + return urls + + +def census_usatrade_call(*, resp, url, **_): + """ + Convert response for calling url to pandas dataframe, begin + parsing df into FBA format + :param resp: df, response from url call + :return: pandas dataframe of original source data + """ + if resp.status_code == 204: + # No content warning, return empty dataframe + log.warning(f"No content found for {resp.url}") + return pd.DataFrame() + census_json = json.loads(resp.text) + # convert response to dataframe + df_census = pd.DataFrame( + data=census_json[1:len(census_json)], columns=census_json[0]) + + df_census = (df_census + .assign(Type = 'Exports' if 'exports' in url else 'Imports')) + + return df_census + + +def census_usatrade_parse(*, df_list, year, **_): + """ + Combine, parse, and format the provided dataframes + :param df_list: list of dataframes to concat and format + :param year: year + :return: df, parsed and partially formatted to + flowbyactivity specifications + """ + # concat dataframes + df = pd.concat(df_list, sort=False) + + # exclude country grouping codes and 'total' rows; only retain data for + # individual countries based on Schedule C codes + # remove rows where CTY_CODE starts with '0', contains 'X', or equals '-' + df = df[~( + df['CTY_CODE'].str.startswith('0') | + df['CTY_CODE'].str.contains('X') | + (df['CTY_CODE'] == '-') + )] + + # if dataset is statenaics, rename column containing NAICS code + if 'NAICS' in df.columns: + df = df.rename(columns={'NAICS': 'CODE'}) + # if dataset is statehs, combine columns containing HS code and rename + elif 'I_COMMODITY' in df.columns or 'E_COMMODITY' in df.columns: + df['CODE'] = df['I_COMMODITY'].combine_first(df['E_COMMODITY']).fillna('') + + # rename columns to FBA format and drop unnecessary columns + df = (df + .assign(FlowAmount = lambda x: x['CON_VAL_YR'].astype(float) + .fillna(x['ALL_VAL_YR'].astype(float))) + .rename(columns={'CODE':'FlowName', + 'Type':'ActivityProducedBy', + 'STATE':'State', + 'YEAR':'Year', + 'CTY_NAME':'Description'}) + .drop(columns=['CTY_CODE', 'CON_VAL_YR', 'MONTH', 'ALL_VAL_YR'], errors='ignore') + .assign(State = lambda x: x['State'].replace('-', 'US')) + ) + + # replace the 2-state abbreviations with FIPS code + df = apply_county_FIPS(df) # defaults to FIPS 2015 + df = (df + .assign(Location = lambda x: np.where(x['State'] == "Us", US_FIPS, x['Location'])) + .drop(columns=['State', 'County']) + .dropna(subset='Location') # drops data from territories + ) + + # check for duplicate rows + x = df.drop_duplicates(subset=['FlowName', 'Year', 'ActivityProducedBy', + 'Location', 'Description']) + if len(x) < len(df): + print('ERROR check duplicates') + + # add hard coded data + df['Class'] = 'Money' + df['SourceName'] = 'Census_USATrade' + df['Compartment'] = None + df['Unit'] = 'USD' + df['FlowType'] = 'TECHNOSPHERE_FLOW' + df['LocationSystem'] = 'FIPS_2015' + df['DataReliability'] = 5 + df['DataCollection'] = 5 + + return df + + +if __name__ == "__main__": + import flowsa + flowsa.generateflowbyactivity.main(source='Census_USATrade_Construction', year=2024) + fba = flowsa.getFlowByActivity('Census_USATrade_Construction', 2024) diff --git a/flowsa/generateflowbyactivity.py b/flowsa/generateflowbyactivity.py index dc4a4a978..dccc6bc0b 100644 --- a/flowsa/generateflowbyactivity.py +++ b/flowsa/generateflowbyactivity.py @@ -66,7 +66,7 @@ def assemble_urls_for_query(*, source, year, config): if 'url_params' in urlinfo: params = parse.urlencode(urlinfo['url_params'], safe='=&%', quote_via=parse.quote) - build_url = urlinfo['base_url'] + urlinfo['api_path'] + params + build_url = urlinfo['base_url'] + urlinfo.get('api_path','') + params else: build_url = urlinfo['base_url'] diff --git a/flowsa/methods/flowbyactivitymethods/BEA_IEA.yaml b/flowsa/methods/flowbyactivitymethods/BEA_IEA.yaml new file mode 100644 index 000000000..fb54f6730 --- /dev/null +++ b/flowsa/methods/flowbyactivitymethods/BEA_IEA.yaml @@ -0,0 +1,37 @@ +author: US Bureau of Economic Analysis +source_name: International Economic Accounts +source_url: https://www.bea.gov/data/intl-trade-investment/international-trade-goods-and-services +bib_id: BEA +api_name: BEA +api_key_required: True +format: [json,xml] +url: + base_url: https://apps.bea.gov/ + api_path: 'api/data/?' + url_params: + Affiliation: AllAffiliations + method: GetData + DataSetName: IntlServTrade + TradeDirection: Imports,Exports + ResultFormat: json + AreaOrCountry: __areaorcountry__ + Year: __year__ + UserID: __apiKey__ + +url_replace_fxn: !script_function:BEA_IEA bea_iea_url_helper +call_response_fxn: !script_function:BEA_IEA bea_iea_call +parse_response_fxn: !script_function:BEA_IEA bea_iea_parse + +years: +- 2012 +- 2013 +- 2014 +- 2015 +- 2016 +- 2017 +- 2018 +- 2019 +- 2020 +- 2021 +- 2022 +- 2023 diff --git a/flowsa/methods/flowbyactivitymethods/Census_USATrade.yaml b/flowsa/methods/flowbyactivitymethods/Census_USATrade.yaml new file mode 100644 index 000000000..8d4dfc88b --- /dev/null +++ b/flowsa/methods/flowbyactivitymethods/Census_USATrade.yaml @@ -0,0 +1,35 @@ +author: US Census Bureau +source_name: USA Trade +source_url: https://usatrade.census.gov/ +bib_id: Census +api_key_required: False +format: json +url: + base_url: http://api.census.gov/data/timeseries/intltrade/__flows__/naics? + flows: + - imports + - exports + url_params: + get: NAICS,GEN_CIF_YR # ALL_VAL_YR for exports + COMM_LVL: NA6 + CTY_CODE: __areaorcountry__ + YEAR: __year__ + MONTH: "12" + +url_replace_fxn: !script_function:Census_USATrade census_url_helper +call_response_fxn: !script_function:Census_USATrade census_usatrade_call +parse_response_fxn: !script_function:Census_USATrade census_usatrade_parse + +years: +- 2012 +- 2013 +- 2014 +- 2015 +- 2016 +- 2017 +- 2018 +- 2019 +- 2020 +- 2021 +- 2022 +- 2023 diff --git a/flowsa/methods/flowbyactivitymethods/Census_USATrade_Construction.yaml b/flowsa/methods/flowbyactivitymethods/Census_USATrade_Construction.yaml new file mode 100644 index 000000000..fa274a204 --- /dev/null +++ b/flowsa/methods/flowbyactivitymethods/Census_USATrade_Construction.yaml @@ -0,0 +1,85 @@ +author: US Census Bureau +source_name: USA Trade +source_url: https://usatrade.census.gov/ +bib_id: Census +api_key_required: False +format: json +url: + base_url: http://api.census.gov/data/timeseries/intltrade/__flows__/__dataset__? + dataset: 'statenaics' # choose either 'statenaics' or 'statehs' (see below) + # 'statenaics'= North American Industry Classification System (NAICS) by State + # 'statehs' = Harmonized System (HS) by State + flows: + - imports + - exports + naics: # NAICS by state data available for 2-, 3-, or 4-digit NAICS + - '3273' # Cement and Concrete Product Manufacturing + - '3272' # Glass & Glass Products + - '3311' # Iron & Steel & Ferroalloy + hs: # HS by state data available for 2-, 4-, or 6-digit HS + # Concrete + - '2523' # Portland cement, aluminous cement, slag cement, etc. + - '3816' # Refractory cements, mortars, concretes, and similar compositions + - '6810' # Articles of cement, concrete or artificial stone + - '382450' # Nonrefractory mortars and concretes + # Steel + - '72' # Iron and steel + - '73' # Articles of iron or steel + - '2618' # Granulated slag (slag sand) from the manufacture of iron or steel + - '820231' # Hand saws and blades with a working part made of steel + - '830710' # Flexible tubing made of iron or steel + - '940620' # Modular building units of steel + # Glass + - '70' # Glass and glassware + - '320740' # Glass frit and other glass, in the form of powder, granules or flakes + - '854610' # Electrical insulators of glass + - '900140' # Spectacle lenses of glass + - '940591' # Parts for lamps and light fittings made of glass + # Asphalt + - '6807' # Articles of asphalt or similar material + + url_params: + get: __get_params__ + YEAR: __year__ # 4-character YEAR + MONTH: "12" # 2-character MONTH + CODES: __codes__ # either NAICS or HS codes + # if NAICS, must be 2-, 3-, or 4-digit + # if HS, must be 2-, 4-, or 6-digit + get_params: + import_params: + - CTY_CODE # 4-character COUNTRY CODE + - CTY_NAME # 50-character COUNTRY NAME + - CON_VAL_YR # YEAR-TO-DATE imports for CONSUMPTION, TOTAL VALUE + - STATE # 2-character STATE of destination, "XX" = unidentified + export_params: + - CTY_CODE # 4-character COUNTRY CODE + - CTY_NAME # 50-character COUNTRY NAME + - ALL_VAL_YR # YEAR-TO-DATE TOTAL VALUE + - STATE # 2-character STATE of origin of movement + +url_replace_fxn: !script_function:Census_USATrade_Construction census_url_helper +call_response_fxn: !script_function:Census_USATrade_Construction census_usatrade_call +parse_response_fxn: !script_function:Census_USATrade_Construction census_usatrade_parse + +# the most recent 10 years for which data are available +years: +- 2015 +- 2013 +- 2014 +- 2015 +- 2016 +- 2017 +- 2018 +- 2019 +- 2020 +- 2021 +- 2022 +- 2023 +- 2024 + + + + +# Start with imports +# http://api.census.gov/data/timeseries/intltrade/imports/statenaics?get=CTY_CODE%2CCTY_NAME%2CCON_VAL_YR%2CSTATE&YEAR=2024&MONTH=12&NAICS=3273 +# Do this for all years, 2015 - 2024