Mod/settings update

usaspending_api/common/spark/jobs.py

@@ -166,7 +166,7 @@ def handle_start(self, job_name: str, command_name: str, command_options: list[s
             mode="BATCH",
             jobDriver={
                 "sparkSubmit": {
-                    "entryPoint": f"s3://{CONFIG.SPARK_S3_BUCKET}/master/manage.py",
+                    "entryPoint": f"s3://{CONFIG.SPARK_S3_BUCKET}/manage.py",
                     "entryPointArguments": [command_name, *command_options],
                 }
             },

usaspending_api/download/filestreaming/s3_handler.py

@@ -30,9 +30,5 @@ def get_simple_url(self, file_name):
         Gets URL for read
         """
         bucket_url = f"{settings.FILES_SERVER_BASE_URL}/{self.redirect_dir}/"
-        env_dir = ""
-        if self.redirect_dir == settings.BULK_DOWNLOAD_S3_REDIRECT_DIR and self.environment != "production":
-            # currently only downloads have a bucket per environment
-            env_dir = f"{self.environment}/"
-        generated = f"{bucket_url}{env_dir}{file_name}"
+        generated = f"{bucket_url}{file_name}"
         return generated

usaspending_api/references/management/commands/load_gtas.py

@@ -68,8 +68,12 @@ def process_data(self) -> None:
         total_obligation_objs = [GTASSF133Balances(**values) for values in total_obligation_values]
 
         logger.info("Loading new GTAS records into database")
-        new_rec_count = len(GTASSF133Balances.objects.bulk_create(total_obligation_objs))
-        logger.info(f"Loaded: {new_rec_count:,} records")
+        try:
+            new_rec_count = len(GTASSF133Balances.objects.bulk_create(total_obligation_objs))
+            logger.info(f"Loaded: {new_rec_count:,} records")
+        except Exception as e:
+            logger.error(f"Error during bulk_create: {type(e).__name__}: {str(e)}")
+            raise
 
         load_rec = self._execute_dml_sql(self.tas_fk_sql, "Populating TAS foreign keys")
         logger.info(f"Set {load_rec:,} TAS FKs in GTAS table, {new_rec_count - load_rec:,} NULLs")

usaspending_api/settings.py

@@ -167,8 +167,14 @@
 )
 
 # S3 Bucket and Key to retrieve the Data Dictionary
-DATA_DICTIONARY_S3_BUCKET_NAME = f"dti-da-public-files-{'nonprod' if CONFIG.ENV_CODE not in ('prd', 'stg') else 'prod'}"
-DATA_DICTIONARY_S3_KEY = f"user_reference_docs/{DATA_DICTIONARY_FILE_NAME}"
+DATA_DICTIONARY_S3_BUCKET_NAME = os.environ.get(
+    "DATA_DICTIONARY_S3_BUCKET_NAME",
+    f"dti-da-public-files-{'nonprod' if CONFIG.ENV_CODE not in ('prd', 'stg') else 'prod'}"
+)
+DATA_DICTIONARY_S3_KEY = os.environ.get(
+    "DATA_DICTIONARY_S3_KEY",
+    f"user_reference_docs/{DATA_DICTIONARY_FILE_NAME}"
+)
 
 # Local download files
 IDV_DOWNLOAD_README_FILE_PATH = str(APP_DIR / "data" / "idv_download_readme.txt")
@@ -434,6 +440,36 @@ def _configure_database_connection(
 logs_dir.mkdir(parents=True, exist_ok=True)
 console_log_file_path.touch(exist_ok=True)
 
+# Logging settings adjusted based on env
+fapc = os.environ.get("FAPC", "")
+
+if fapc:
+    server_handler = {
+        "level": "DEBUG",
+        "class": "logging.StreamHandler",
+        "stream": "ext://sys.stdout",
+        "formatter": "user_readable"
+    }
+    console_file_handler = {
+        "level": "DEBUG",
+        "class": "logging.StreamHandler",
+        "stream": "ext://sys.stdout",
+        "formatter": "specifics"
+    }
+else:
+    server_handler = {
+        "level": "DEBUG",
+        "class": "logging.handlers.WatchedFileHandler",
+        "filename": str(APP_DIR / "logs" / "server.log"),
+        "formatter": "user_readable",
+    }
+    console_file_handler = {
+        "level": "DEBUG",
+        "class": "logging.handlers.WatchedFileHandler",
+        "filename": str(APP_DIR / "logs" / "console.log"),
+        "formatter": "specifics",
+    }
+
 LOGGING = {
     "version": 1,
     "disable_existing_loggers": False,
@@ -460,18 +496,8 @@ def _configure_database_connection(
         },
     },
     "handlers": {
-        "server": {
-            "level": "DEBUG",
-            "class": "logging.handlers.WatchedFileHandler",
-            "filename": str(APP_DIR / "logs" / "server.log"),
-            "formatter": "user_readable",
-        },
-        "console_file": {
-            "level": "DEBUG",
-            "class": "logging.handlers.WatchedFileHandler",
-            "filename": str(APP_DIR / "logs" / "console.log"),
-            "formatter": "specifics",
-        },
+        "server": server_handler,
+        "console_file": console_file_handler,
         "console": {
             "level": "DEBUG",
             "class": "logging.StreamHandler",