From 07986b090f6c212085a462ecd41a9c4ed29651ab Mon Sep 17 00:00:00 2001 From: "Daniel.Martin" Date: Wed, 8 Apr 2026 11:37:05 -0400 Subject: [PATCH 1/8] update data dictionary s3 bucket path --- usaspending_api/settings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/usaspending_api/settings.py b/usaspending_api/settings.py index c8267d77ad..465da45a9f 100644 --- a/usaspending_api/settings.py +++ b/usaspending_api/settings.py @@ -168,8 +168,8 @@ ) # S3 Bucket and Key to retrieve the Data Dictionary -DATA_DICTIONARY_S3_BUCKET_NAME = f"dti-da-public-files-{'nonprod' if CONFIG.ENV_CODE not in ('prd', 'stg') else 'prod'}" -DATA_DICTIONARY_S3_KEY = f"user_reference_docs/{DATA_DICTIONARY_FILE_NAME}" +DATA_DICTIONARY_S3_BUCKET_NAME = os.environ.get("DATA_DICTIONARY_S3_BUCKET_NAME", f"dti-da-public-files-{'nonprod' if CONFIG.ENV_CODE not in ('prd', 'stg') else 'prod'}") +DATA_DICTIONARY_S3_KEY = os.environ.get("DATA_DICTIONARY_S3_BUCKET_NAME", f"user_reference_docs/{DATA_DICTIONARY_FILE_NAME}") # Local download files IDV_DOWNLOAD_README_FILE_PATH = str(APP_DIR / "data" / "idv_download_readme.txt") From 038711538e87be70df255ec8fea08f88c5ba4e90 Mon Sep 17 00:00:00 2001 From: "Daniel.Martin" Date: Wed, 8 Apr 2026 17:45:21 -0400 Subject: [PATCH 2/8] add logging settings --- usaspending_api/settings.py | 45 +++++++++++++++++++++++++++---------- 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/usaspending_api/settings.py b/usaspending_api/settings.py index 465da45a9f..dd5bff7423 100644 --- a/usaspending_api/settings.py +++ b/usaspending_api/settings.py @@ -5,6 +5,7 @@ import json import os +import sys from pathlib import Path import dj_database_url @@ -436,6 +437,36 @@ def _configure_database_connection( logs_dir.mkdir(parents=True, exist_ok=True) console_log_file_path.touch(exist_ok=True) +## Logging settings adjusted based on env +fapc = os.environ.get("FAPC", "") + +if fapc: + server_handler = { + "level": "DEBUG", + "class": "logging.StreamHandler", + "stream": "ext://sys.stdout", + "formatter": "user_readable" + } + console_file_handler = { + "level": "DEBUG", + "class": "logging.StreamHandler", + "stream": "ext://sys.stdout", + "formatter": "specifics" + } +else: + server_handler = { + "level": "DEBUG", + "class": "logging.handlers.WatchedFileHandler", + "filename": str(APP_DIR / "logs" / "server.log"), + "formatter": "user_readable", + } + console_file_handler = { + "level": "DEBUG", + "class": "logging.handlers.WatchedFileHandler", + "filename": str(APP_DIR / "logs" / "console.log"), + "formatter": "specifics", + } + LOGGING = { "version": 1, "disable_existing_loggers": False, @@ -462,18 +493,8 @@ def _configure_database_connection( }, }, "handlers": { - "server": { - "level": "DEBUG", - "class": "logging.handlers.WatchedFileHandler", - "filename": str(APP_DIR / "logs" / "server.log"), - "formatter": "user_readable", - }, - "console_file": { - "level": "DEBUG", - "class": "logging.handlers.WatchedFileHandler", - "filename": str(APP_DIR / "logs" / "console.log"), - "formatter": "specifics", - }, + "server": server_handler, + "console_file": console_file_handler, "console": { "level": "DEBUG", "class": "logging.StreamHandler", From 59c47432911d24bfa4678532fa7664b17ac6cafc Mon Sep 17 00:00:00 2001 From: aguest-kc Date: Tue, 14 Apr 2026 08:13:46 -0500 Subject: [PATCH 3/8] Ruff formatter fixes --- usaspending_api/settings.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/usaspending_api/settings.py b/usaspending_api/settings.py index dd5bff7423..20e6ef43ce 100644 --- a/usaspending_api/settings.py +++ b/usaspending_api/settings.py @@ -5,7 +5,6 @@ import json import os -import sys from pathlib import Path import dj_database_url @@ -169,8 +168,14 @@ ) # S3 Bucket and Key to retrieve the Data Dictionary -DATA_DICTIONARY_S3_BUCKET_NAME = os.environ.get("DATA_DICTIONARY_S3_BUCKET_NAME", f"dti-da-public-files-{'nonprod' if CONFIG.ENV_CODE not in ('prd', 'stg') else 'prod'}") -DATA_DICTIONARY_S3_KEY = os.environ.get("DATA_DICTIONARY_S3_BUCKET_NAME", f"user_reference_docs/{DATA_DICTIONARY_FILE_NAME}") +DATA_DICTIONARY_S3_BUCKET_NAME = os.environ.get( + "DATA_DICTIONARY_S3_BUCKET_NAME", + f"dti-da-public-files-{'nonprod' if CONFIG.ENV_CODE not in ('prd', 'stg') else 'prod'}" +) +DATA_DICTIONARY_S3_KEY = os.environ.get( + "DATA_DICTIONARY_S3_BUCKET_NAME", + f"user_reference_docs/{DATA_DICTIONARY_FILE_NAME}" +) # Local download files IDV_DOWNLOAD_README_FILE_PATH = str(APP_DIR / "data" / "idv_download_readme.txt") @@ -437,7 +442,7 @@ def _configure_database_connection( logs_dir.mkdir(parents=True, exist_ok=True) console_log_file_path.touch(exist_ok=True) -## Logging settings adjusted based on env +# Logging settings adjusted based on env fapc = os.environ.get("FAPC", "") if fapc: From 91d82677a01227e9226742322ccbc9368bfd5bd2 Mon Sep 17 00:00:00 2001 From: "Daniel.Martin" Date: Mon, 20 Apr 2026 10:52:21 -0400 Subject: [PATCH 4/8] update s3 get_simple_url --- usaspending_api/download/filestreaming/s3_handler.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/usaspending_api/download/filestreaming/s3_handler.py b/usaspending_api/download/filestreaming/s3_handler.py index 703307049b..c69a0d2749 100644 --- a/usaspending_api/download/filestreaming/s3_handler.py +++ b/usaspending_api/download/filestreaming/s3_handler.py @@ -30,9 +30,5 @@ def get_simple_url(self, file_name): Gets URL for read """ bucket_url = f"{settings.FILES_SERVER_BASE_URL}/{self.redirect_dir}/" - env_dir = "" - if self.redirect_dir == settings.BULK_DOWNLOAD_S3_REDIRECT_DIR and self.environment != "production": - # currently only downloads have a bucket per environment - env_dir = f"{self.environment}/" - generated = f"{bucket_url}{env_dir}{file_name}" + generated = f"{bucket_url}/{file_name}" return generated From f780cd24d5bcdcb24b525865c6b917621fd88b2e Mon Sep 17 00:00:00 2001 From: "Daniel.Martin" Date: Mon, 20 Apr 2026 11:02:48 -0400 Subject: [PATCH 5/8] remove mid slash --- usaspending_api/download/filestreaming/s3_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usaspending_api/download/filestreaming/s3_handler.py b/usaspending_api/download/filestreaming/s3_handler.py index c69a0d2749..489621a70a 100644 --- a/usaspending_api/download/filestreaming/s3_handler.py +++ b/usaspending_api/download/filestreaming/s3_handler.py @@ -30,5 +30,5 @@ def get_simple_url(self, file_name): Gets URL for read """ bucket_url = f"{settings.FILES_SERVER_BASE_URL}/{self.redirect_dir}/" - generated = f"{bucket_url}/{file_name}" + generated = f"{bucket_url}{file_name}" return generated From 1984f73428730d64409443f2735932eb26a6d128 Mon Sep 17 00:00:00 2001 From: "Daniel.Martin" Date: Fri, 15 May 2026 13:35:48 -0400 Subject: [PATCH 6/8] update data dictionary s3 key --- usaspending_api/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usaspending_api/settings.py b/usaspending_api/settings.py index f3863bdadb..3bd6467cdf 100644 --- a/usaspending_api/settings.py +++ b/usaspending_api/settings.py @@ -172,7 +172,7 @@ f"dti-da-public-files-{'nonprod' if CONFIG.ENV_CODE not in ('prd', 'stg') else 'prod'}" ) DATA_DICTIONARY_S3_KEY = os.environ.get( - "DATA_DICTIONARY_S3_BUCKET_NAME", + "DATA_DICTIONARY_S3_KEY", f"user_reference_docs/{DATA_DICTIONARY_FILE_NAME}" ) From 59a286c91fed72c28a531afe135ac217f690b554 Mon Sep 17 00:00:00 2001 From: Wil Collins Date: Tue, 30 Jun 2026 12:32:42 -0400 Subject: [PATCH 7/8] Adds logging for gtas_command --- .../references/management/commands/load_gtas.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/usaspending_api/references/management/commands/load_gtas.py b/usaspending_api/references/management/commands/load_gtas.py index 6d467bcae9..4d3ef14434 100644 --- a/usaspending_api/references/management/commands/load_gtas.py +++ b/usaspending_api/references/management/commands/load_gtas.py @@ -68,8 +68,12 @@ def process_data(self) -> None: total_obligation_objs = [GTASSF133Balances(**values) for values in total_obligation_values] logger.info("Loading new GTAS records into database") - new_rec_count = len(GTASSF133Balances.objects.bulk_create(total_obligation_objs)) - logger.info(f"Loaded: {new_rec_count:,} records") + try: + new_rec_count = len(GTASSF133Balances.objects.bulk_create(total_obligation_objs)) + logger.info(f"Loaded: {new_rec_count:,} records") + except Exception as e: + logger.error(f"Error during bulk_create: {type(e).__name__}: {str(e)}") + raise load_rec = self._execute_dml_sql(self.tas_fk_sql, "Populating TAS foreign keys") logger.info(f"Set {load_rec:,} TAS FKs in GTAS table, {new_rec_count - load_rec:,} NULLs") From 992edcd3d57434835e14c2ed08c3fa3cfaff051d Mon Sep 17 00:00:00 2001 From: "Daniel.Martin" Date: Thu, 16 Jul 2026 14:35:11 -0400 Subject: [PATCH 8/8] remove master branch --- usaspending_api/common/spark/jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usaspending_api/common/spark/jobs.py b/usaspending_api/common/spark/jobs.py index 5e3f95e23e..30f1f8b268 100644 --- a/usaspending_api/common/spark/jobs.py +++ b/usaspending_api/common/spark/jobs.py @@ -166,7 +166,7 @@ def handle_start(self, job_name: str, command_name: str, command_options: list[s mode="BATCH", jobDriver={ "sparkSubmit": { - "entryPoint": f"s3://{CONFIG.SPARK_S3_BUCKET}/master/manage.py", + "entryPoint": f"s3://{CONFIG.SPARK_S3_BUCKET}/manage.py", "entryPointArguments": [command_name, *command_options], } },