[v2] Auto-detect account regional namespace buckets in s3 mb

.changes/next-release/enhancement-s3-61261.json

@@ -0,0 +1,5 @@
+{
+  "type": "enhancement",
+  "category": "s3",
+  "description": "Add support for creating S3 account regional namespace buckets with aws s3 mb. The command now automatically detects bucket names matching the account-regional naming pattern and sets the required x-amz-bucket-namespace header."
+}

awscli/customizations/s3/subcommands.py

@@ -50,6 +50,7 @@
     find_bucket_key,
     find_dest_path_comp_key,
     human_readable_size,
+    is_account_regional_namespace_bucket,
     split_s3_bucket_key,
 )
 from awscli.customizations.utils import uni_print
@@ -1210,24 +1211,27 @@ class SyncCommand(S3TransferCommand):
             }
         ]
         + TRANSFER_ARGS
-        + [METADATA, COPY_PROPS, METADATA_DIRECTIVE, CASE_CONFLICT, NO_OVERWRITE]
+        + [
+            METADATA,
+            COPY_PROPS,
+            METADATA_DIRECTIVE,
+            CASE_CONFLICT,
+            NO_OVERWRITE,
+        ]
     )
 
 
 class MbCommand(S3Command):
     NAME = 'mb'
     DESCRIPTION = "Creates an S3 bucket."
     USAGE = "<S3Uri>"
-    ARG_TABLE = (
-        [
-            {
-                'name': 'path',
-                'positional_arg': True,
-                'synopsis': USAGE,
-            }
-        ]
-        + [TAGS]
-    )
+    ARG_TABLE = [
+        {
+            'name': 'path',
+            'positional_arg': True,
+            'synopsis': USAGE,
+        }
+    ] + [TAGS]
 
     def _run_main(self, parsed_args, parsed_globals):
         super(MbCommand, self)._run_main(parsed_args, parsed_globals)
@@ -1247,6 +1251,9 @@ def _run_main(self, parsed_args, parsed_globals):
         bucket_config = {}
         bucket_tags = self._create_bucket_tags(parsed_args)
 
+        if is_account_regional_namespace_bucket(bucket):
+            params['BucketNamespace'] = 'account-regional'
+
         # Only set LocationConstraint when the region name is not us-east-1.
         # Sending LocationConstraint with value us-east-1 results in an error.
         if self.client.meta.region_name != 'us-east-1':
@@ -1270,7 +1277,9 @@ def _run_main(self, parsed_args, parsed_globals):
 
     def _create_bucket_tags(self, parsed_args):
         if parsed_args.tags is not None:
-            return [{'Key': tag[0], 'Value': tag[1]} for tag in parsed_args.tags]
+            return [
+                {'Key': tag[0], 'Value': tag[1]} for tag in parsed_args.tags
+            ]
         return []
 
 
@@ -1621,12 +1630,13 @@ def _map_sse_c_params(self, request_parameters, paths_type):
 
     def _get_s3_handler_params(self):
         params = self.parameters.copy()
-        
+
         # Removing no-overwrite params from sync since file to be synced are
         # already separated out using sync strategy
         if self.cmd == 'sync':
             params.pop('no_overwrite', None)
         return params
+
     def _should_handle_case_conflicts(self):
         return (
             self.cmd in {'sync', 'cp', 'mv'}

awscli/customizations/s3/utils.py

@@ -56,12 +56,20 @@
     r'[a-zA-Z0-9\-]{1,63})[/:]?(?P<key>.*)$'
 )
 
+_S3_ACCOUNT_REGIONAL_NAMESPACE_REGEX = re.compile(
+    r'^.+-\d{12}-[a-z]{2}(-[a-z]+-\d+)?-an$'
+)
+
 _S3_OBJECT_LAMBDA_TO_BUCKET_KEY_REGEX = re.compile(
     r'^(?P<bucket>arn:(aws).*:s3-object-lambda:[a-z\-0-9]+:[0-9]{12}:'
     r'accesspoint[/:][a-zA-Z0-9\-]{1,63})[/:]?(?P<key>.*)$'
 )
 
 
+def is_account_regional_namespace_bucket(bucket):
+    return bool(_S3_ACCOUNT_REGIONAL_NAMESPACE_REGEX.match(bucket))
+
+
 def human_readable_size(value):
     """Convert a size in bytes into a human readable format.
 

tests/functional/s3/test_mb_command.py

@@ -47,48 +47,74 @@ def test_incompatible_with_express_directory_bucket(self):
         self.assertIn('Cannot use mb command with a directory bucket.', stderr)
 
     def test_make_bucket_with_single_tag(self):
-        command = self.prefix + 's3://bucket --tags Key1 Value1 --region us-west-2'
+        command = (
+            self.prefix + 's3://bucket --tags Key1 Value1 --region us-west-2'
+        )
         expected_params = {
             'Bucket': 'bucket',
             'CreateBucketConfiguration': {
                 'LocationConstraint': 'us-west-2',
-                'Tags': [
-                    {'Key': 'Key1', 'Value': 'Value1'}
-                ]
-            }
+                'Tags': [{'Key': 'Key1', 'Value': 'Value1'}],
+            },
         }
         self.assert_params_for_cmd(command, expected_params)
 
     def test_make_bucket_with_single_tag_us_east_1(self):
-        command = self.prefix + 's3://bucket --tags Key1 Value1 --region us-east-1'
+        command = (
+            self.prefix + 's3://bucket --tags Key1 Value1 --region us-east-1'
+        )
         expected_params = {
             'Bucket': 'bucket',
             'CreateBucketConfiguration': {
-                'Tags': [
-                    {'Key': 'Key1', 'Value': 'Value1'}
-                ]
-            }
+                'Tags': [{'Key': 'Key1', 'Value': 'Value1'}]
+            },
         }
         self.assert_params_for_cmd(command, expected_params)
 
     def test_make_bucket_with_multiple_tags(self):
-        command = self.prefix + 's3://bucket --tags Key1 Value1 --tags Key2 Value2 --region us-west-2'
+        command = (
+            self.prefix
+            + 's3://bucket --tags Key1 Value1 --tags Key2 Value2 --region us-west-2'
+        )
         expected_params = {
             'Bucket': 'bucket',
             'CreateBucketConfiguration': {
                 'LocationConstraint': 'us-west-2',
                 'Tags': [
                     {'Key': 'Key1', 'Value': 'Value1'},
-                    {'Key': 'Key2', 'Value': 'Value2'}
-                ]
-            }
+                    {'Key': 'Key2', 'Value': 'Value2'},
+                ],
+            },
         }
         self.assert_params_for_cmd(command, expected_params)
 
+    def test_account_regional_namespace_bucket(self):
+        bucket = 'amzn-s3-demo-bucket-111122223333-us-west-2-an'
+        command = self.prefix + 's3://%s --region us-west-2' % bucket
+        self.parsed_responses = [{'Location': 'us-west-2'}]
+        expected_params = {
+            'Bucket': bucket,
+            'BucketNamespace': 'account-regional',
+            'CreateBucketConfiguration': {'LocationConstraint': 'us-west-2'},
+        }
+        self.assert_params_for_cmd(command, expected_params)
+
+    def test_account_regional_namespace_bucket_us_east_1(self):
+        bucket = 'my-bucket-111122223333-us-east-1-an'
+        command = self.prefix + 's3://%s --region us-east-1' % bucket
+        expected_params = {
+            'Bucket': bucket,
+            'BucketNamespace': 'account-regional',
+        }
+        self.assert_params_for_cmd(command, expected_params)
+
+    def test_regular_bucket_no_namespace(self):
+        command = self.prefix + 's3://my-regular-bucket --region us-east-1'
+        expected_params = {'Bucket': 'my-regular-bucket'}
+        self.assert_params_for_cmd(command, expected_params)
+
     def test_tags_with_three_arguments_fails(self):
         command = self.prefix + 's3://bucket --tags Key1 Value1 ExtraArg'
         self.assert_params_for_cmd(
-            command, 
-            expected_rc=252, 
-            stderr_contains='ParamValidation'
+            command, expected_rc=252, stderr_contains='ParamValidation'
         )

tests/unit/customizations/s3/test_utils.py

@@ -39,6 +39,7 @@
     guess_content_type,
     human_readable_size,
     human_readable_to_int,
+    is_account_regional_namespace_bucket,
     relative_path,
     set_file_utime,
 )
@@ -368,6 +369,44 @@ def test_outpost_bucket_arn_with_slash_raises_exception(self):
             )
 
 
+class TestIsAccountRegionalNamespaceBucket(unittest.TestCase):
+    def test_matches_standard_pattern(self):
+        self.assertTrue(
+            is_account_regional_namespace_bucket(
+                'amzn-s3-demo-bucket-111122223333-us-west-2-an'
+            )
+        )
+
+    def test_matches_different_region(self):
+        self.assertTrue(
+            is_account_regional_namespace_bucket(
+                'my-bucket-123456789012-eu-central-1-an'
+            )
+        )
+
+    def test_no_match_regular_bucket(self):
+        self.assertFalse(
+            is_account_regional_namespace_bucket('my-regular-bucket')
+        )
+
+    def test_no_match_missing_an_suffix(self):
+        self.assertFalse(
+            is_account_regional_namespace_bucket(
+                'bucket-111122223333-us-west-2'
+            )
+        )
+
+    def test_no_match_wrong_account_id_length(self):
+        self.assertFalse(
+            is_account_regional_namespace_bucket('bucket-12345-us-west-2-an')
+        )
+
+    def test_no_match_express_directory_bucket(self):
+        self.assertFalse(
+            is_account_regional_namespace_bucket('bucket--usw2-az1--x-s3')
+        )
+
+
 class TestCreateWarning(unittest.TestCase):
     def test_create_warning(self):
         path = '/foo/'