diff --git a/usaspending_api/common/spark/jobs.py b/usaspending_api/common/spark/jobs.py index 5e3f95e23e..30f1f8b268 100644 --- a/usaspending_api/common/spark/jobs.py +++ b/usaspending_api/common/spark/jobs.py @@ -166,7 +166,7 @@ def handle_start(self, job_name: str, command_name: str, command_options: list[s mode="BATCH", jobDriver={ "sparkSubmit": { - "entryPoint": f"s3://{CONFIG.SPARK_S3_BUCKET}/master/manage.py", + "entryPoint": f"s3://{CONFIG.SPARK_S3_BUCKET}/manage.py", "entryPointArguments": [command_name, *command_options], } }, diff --git a/usaspending_api/download/filestreaming/s3_handler.py b/usaspending_api/download/filestreaming/s3_handler.py index 703307049b..489621a70a 100644 --- a/usaspending_api/download/filestreaming/s3_handler.py +++ b/usaspending_api/download/filestreaming/s3_handler.py @@ -30,9 +30,5 @@ def get_simple_url(self, file_name): Gets URL for read """ bucket_url = f"{settings.FILES_SERVER_BASE_URL}/{self.redirect_dir}/" - env_dir = "" - if self.redirect_dir == settings.BULK_DOWNLOAD_S3_REDIRECT_DIR and self.environment != "production": - # currently only downloads have a bucket per environment - env_dir = f"{self.environment}/" - generated = f"{bucket_url}{env_dir}{file_name}" + generated = f"{bucket_url}{file_name}" return generated diff --git a/usaspending_api/references/management/commands/load_gtas.py b/usaspending_api/references/management/commands/load_gtas.py index 67f58ba145..7dc67ccf7b 100644 --- a/usaspending_api/references/management/commands/load_gtas.py +++ b/usaspending_api/references/management/commands/load_gtas.py @@ -68,8 +68,12 @@ def process_data(self) -> None: total_obligation_objs = [GTASSF133Balances(**values) for values in total_obligation_values] logger.info("Loading new GTAS records into database") - new_rec_count = len(GTASSF133Balances.objects.bulk_create(total_obligation_objs)) - logger.info(f"Loaded: {new_rec_count:,} records") + try: + new_rec_count = len(GTASSF133Balances.objects.bulk_create(total_obligation_objs)) + logger.info(f"Loaded: {new_rec_count:,} records") + except Exception as e: + logger.error(f"Error during bulk_create: {type(e).__name__}: {str(e)}") + raise load_rec = self._execute_dml_sql(self.tas_fk_sql, "Populating TAS foreign keys") logger.info(f"Set {load_rec:,} TAS FKs in GTAS table, {new_rec_count - load_rec:,} NULLs") diff --git a/usaspending_api/settings.py b/usaspending_api/settings.py index 69a71444da..3bd6467cdf 100644 --- a/usaspending_api/settings.py +++ b/usaspending_api/settings.py @@ -167,8 +167,14 @@ ) # S3 Bucket and Key to retrieve the Data Dictionary -DATA_DICTIONARY_S3_BUCKET_NAME = f"dti-da-public-files-{'nonprod' if CONFIG.ENV_CODE not in ('prd', 'stg') else 'prod'}" -DATA_DICTIONARY_S3_KEY = f"user_reference_docs/{DATA_DICTIONARY_FILE_NAME}" +DATA_DICTIONARY_S3_BUCKET_NAME = os.environ.get( + "DATA_DICTIONARY_S3_BUCKET_NAME", + f"dti-da-public-files-{'nonprod' if CONFIG.ENV_CODE not in ('prd', 'stg') else 'prod'}" +) +DATA_DICTIONARY_S3_KEY = os.environ.get( + "DATA_DICTIONARY_S3_KEY", + f"user_reference_docs/{DATA_DICTIONARY_FILE_NAME}" +) # Local download files IDV_DOWNLOAD_README_FILE_PATH = str(APP_DIR / "data" / "idv_download_readme.txt") @@ -434,6 +440,36 @@ def _configure_database_connection( logs_dir.mkdir(parents=True, exist_ok=True) console_log_file_path.touch(exist_ok=True) +# Logging settings adjusted based on env +fapc = os.environ.get("FAPC", "") + +if fapc: + server_handler = { + "level": "DEBUG", + "class": "logging.StreamHandler", + "stream": "ext://sys.stdout", + "formatter": "user_readable" + } + console_file_handler = { + "level": "DEBUG", + "class": "logging.StreamHandler", + "stream": "ext://sys.stdout", + "formatter": "specifics" + } +else: + server_handler = { + "level": "DEBUG", + "class": "logging.handlers.WatchedFileHandler", + "filename": str(APP_DIR / "logs" / "server.log"), + "formatter": "user_readable", + } + console_file_handler = { + "level": "DEBUG", + "class": "logging.handlers.WatchedFileHandler", + "filename": str(APP_DIR / "logs" / "console.log"), + "formatter": "specifics", + } + LOGGING = { "version": 1, "disable_existing_loggers": False, @@ -460,18 +496,8 @@ def _configure_database_connection( }, }, "handlers": { - "server": { - "level": "DEBUG", - "class": "logging.handlers.WatchedFileHandler", - "filename": str(APP_DIR / "logs" / "server.log"), - "formatter": "user_readable", - }, - "console_file": { - "level": "DEBUG", - "class": "logging.handlers.WatchedFileHandler", - "filename": str(APP_DIR / "logs" / "console.log"), - "formatter": "specifics", - }, + "server": server_handler, + "console_file": console_file_handler, "console": { "level": "DEBUG", "class": "logging.StreamHandler",