From 23d90a83d5f498a2e19bd537922e826b5f525989 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Thu, 13 Mar 2025 23:16:44 +0000 Subject: [PATCH 1/3] workaround fork --- boto3/crt.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/boto3/crt.py b/boto3/crt.py index 4b8df3140e..4c35a1380a 100644 --- a/boto3/crt.py +++ b/boto3/crt.py @@ -29,6 +29,7 @@ CRTTransferManager, acquire_crt_s3_process_lock, create_s3_crt_client, + clean_up_s3_crt_client ) # Singletons for CRT-backed transfers @@ -40,6 +41,7 @@ def _create_crt_client(session, config, region_name, cred_provider): + print("_create_crt_client") """Create a CRT S3 Client for file transfer. Instantiating many of these may lead to degraded performance or @@ -74,6 +76,7 @@ def _create_crt_s3_client( def _initialize_crt_transfer_primatives(client, config): + print(f"before _initialize_crt_transfer_primatives is {CRT_S3_CLIENT}") lock = acquire_crt_s3_process_lock(PROCESS_LOCK_NAME) if lock is None: # If we're unable to acquire the lock, we cannot @@ -96,6 +99,28 @@ def get_crt_s3_client(client, config): global CRT_S3_CLIENT global BOTOCORE_CRT_SERIALIZER + import os + import gc + from awscrt.common import join_all_native_threads + + def before_fork(): + global CRT_S3_CLIENT + try: + if CRT_S3_CLIENT is not None: + # The client is not safe to use after fork, so we need to release it. + # make sure the client is shutdown properly before fork + # also wait for every thread to be joined, incase of some thread is in the middle of cleanup. + CRT_S3_CLIENT = None + gc.collect() + if join_all_native_threads(timeout_sec=0.5) is False: + # generate warning about the background threads still runnning. + # It's possible when CRT used spearately from user, but it's user's responsbility to clean those up. + print("WARNING: Background threads still running, dead lock or crash could happen. Check for CRT resources held before forking.") + + except: + exit(-1) + + with CLIENT_CREATION_LOCK: if CRT_S3_CLIENT is None: serializer, s3_client = _initialize_crt_transfer_primatives( @@ -104,6 +129,10 @@ def get_crt_s3_client(client, config): BOTOCORE_CRT_SERIALIZER = serializer CRT_S3_CLIENT = s3_client + os.register_at_fork(before=before_fork) + # Only register the handler once. + if not getattr(get_crt_s3_client, '_fork_handler_registered', False): + get_crt_s3_client._fork_handler_registered = True return CRT_S3_CLIENT From 4e42b0e5279288b31bb5d134be2e5eedbc4182ea Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Thu, 13 Mar 2025 23:21:14 +0000 Subject: [PATCH 2/3] remove prints --- boto3/crt.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/boto3/crt.py b/boto3/crt.py index 4c35a1380a..be1b309740 100644 --- a/boto3/crt.py +++ b/boto3/crt.py @@ -41,7 +41,6 @@ def _create_crt_client(session, config, region_name, cred_provider): - print("_create_crt_client") """Create a CRT S3 Client for file transfer. Instantiating many of these may lead to degraded performance or @@ -76,7 +75,6 @@ def _create_crt_s3_client( def _initialize_crt_transfer_primatives(client, config): - print(f"before _initialize_crt_transfer_primatives is {CRT_S3_CLIENT}") lock = acquire_crt_s3_process_lock(PROCESS_LOCK_NAME) if lock is None: # If we're unable to acquire the lock, we cannot From 0dccc2bd427b928e61c73d852213a5c530d20413 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Thu, 13 Mar 2025 23:31:18 +0000 Subject: [PATCH 3/3] remove my previous attampt --- boto3/crt.py | 1 - 1 file changed, 1 deletion(-) diff --git a/boto3/crt.py b/boto3/crt.py index be1b309740..186d788434 100644 --- a/boto3/crt.py +++ b/boto3/crt.py @@ -29,7 +29,6 @@ CRTTransferManager, acquire_crt_s3_process_lock, create_s3_crt_client, - clean_up_s3_crt_client ) # Singletons for CRT-backed transfers