diff --git a/.changes/next-release/enhancement-s3-61261.json b/.changes/next-release/enhancement-s3-61261.json new file mode 100644 index 000000000000..646099011468 --- /dev/null +++ b/.changes/next-release/enhancement-s3-61261.json @@ -0,0 +1,5 @@ +{ + "type": "enhancement", + "category": "``S3``", + "description": "Add support for creating S3 account regional namespace buckets with ``aws s3 mb``. The command now automatically detects bucket names matching the account-regional naming pattern and sets the required ``x-amz-bucket-namespace`` header." +} diff --git a/awscli/customizations/s3/subcommands.py b/awscli/customizations/s3/subcommands.py index 2946a4482e33..54c40060ef6a 100644 --- a/awscli/customizations/s3/subcommands.py +++ b/awscli/customizations/s3/subcommands.py @@ -50,6 +50,7 @@ find_bucket_key, find_dest_path_comp_key, human_readable_size, + is_account_regional_namespace_bucket, split_s3_bucket_key, ) from awscli.customizations.utils import uni_print @@ -1210,7 +1211,13 @@ class SyncCommand(S3TransferCommand): } ] + TRANSFER_ARGS - + [METADATA, COPY_PROPS, METADATA_DIRECTIVE, CASE_CONFLICT, NO_OVERWRITE] + + [ + METADATA, + COPY_PROPS, + METADATA_DIRECTIVE, + CASE_CONFLICT, + NO_OVERWRITE, + ] ) @@ -1218,16 +1225,13 @@ class MbCommand(S3Command): NAME = 'mb' DESCRIPTION = "Creates an S3 bucket." USAGE = "" - ARG_TABLE = ( - [ - { - 'name': 'path', - 'positional_arg': True, - 'synopsis': USAGE, - } - ] - + [TAGS] - ) + ARG_TABLE = [ + { + 'name': 'path', + 'positional_arg': True, + 'synopsis': USAGE, + } + ] + [TAGS] def _run_main(self, parsed_args, parsed_globals): super(MbCommand, self)._run_main(parsed_args, parsed_globals) @@ -1247,6 +1251,9 @@ def _run_main(self, parsed_args, parsed_globals): bucket_config = {} bucket_tags = self._create_bucket_tags(parsed_args) + if is_account_regional_namespace_bucket(bucket): + params['BucketNamespace'] = 'account-regional' + # Only set LocationConstraint when the region name is not us-east-1. # Sending LocationConstraint with value us-east-1 results in an error. if self.client.meta.region_name != 'us-east-1': @@ -1270,7 +1277,9 @@ def _run_main(self, parsed_args, parsed_globals): def _create_bucket_tags(self, parsed_args): if parsed_args.tags is not None: - return [{'Key': tag[0], 'Value': tag[1]} for tag in parsed_args.tags] + return [ + {'Key': tag[0], 'Value': tag[1]} for tag in parsed_args.tags + ] return [] @@ -1621,12 +1630,13 @@ def _map_sse_c_params(self, request_parameters, paths_type): def _get_s3_handler_params(self): params = self.parameters.copy() - + # Removing no-overwrite params from sync since file to be synced are # already separated out using sync strategy if self.cmd == 'sync': params.pop('no_overwrite', None) return params + def _should_handle_case_conflicts(self): return ( self.cmd in {'sync', 'cp', 'mv'} diff --git a/awscli/customizations/s3/utils.py b/awscli/customizations/s3/utils.py index c90c421e8337..c6679d7f516a 100644 --- a/awscli/customizations/s3/utils.py +++ b/awscli/customizations/s3/utils.py @@ -62,6 +62,10 @@ ) +def is_account_regional_namespace_bucket(bucket): + return bucket.endswith('-an') + + def human_readable_size(value): """Convert a size in bytes into a human readable format. diff --git a/tests/functional/s3/test_mb_command.py b/tests/functional/s3/test_mb_command.py index bb115ab60569..c466835461ae 100644 --- a/tests/functional/s3/test_mb_command.py +++ b/tests/functional/s3/test_mb_command.py @@ -47,48 +47,83 @@ def test_incompatible_with_express_directory_bucket(self): self.assertIn('Cannot use mb command with a directory bucket.', stderr) def test_make_bucket_with_single_tag(self): - command = self.prefix + 's3://bucket --tags Key1 Value1 --region us-west-2' + command = ( + self.prefix + 's3://bucket --tags Key1 Value1 --region us-west-2' + ) expected_params = { 'Bucket': 'bucket', 'CreateBucketConfiguration': { 'LocationConstraint': 'us-west-2', - 'Tags': [ - {'Key': 'Key1', 'Value': 'Value1'} - ] - } + 'Tags': [{'Key': 'Key1', 'Value': 'Value1'}], + }, } self.assert_params_for_cmd(command, expected_params) def test_make_bucket_with_single_tag_us_east_1(self): - command = self.prefix + 's3://bucket --tags Key1 Value1 --region us-east-1' + command = ( + self.prefix + 's3://bucket --tags Key1 Value1 --region us-east-1' + ) expected_params = { 'Bucket': 'bucket', 'CreateBucketConfiguration': { - 'Tags': [ - {'Key': 'Key1', 'Value': 'Value1'} - ] - } + 'Tags': [{'Key': 'Key1', 'Value': 'Value1'}] + }, } self.assert_params_for_cmd(command, expected_params) def test_make_bucket_with_multiple_tags(self): - command = self.prefix + 's3://bucket --tags Key1 Value1 --tags Key2 Value2 --region us-west-2' + command = ( + self.prefix + + 's3://bucket --tags Key1 Value1 --tags Key2 Value2 --region us-west-2' + ) expected_params = { 'Bucket': 'bucket', 'CreateBucketConfiguration': { 'LocationConstraint': 'us-west-2', 'Tags': [ {'Key': 'Key1', 'Value': 'Value1'}, - {'Key': 'Key2', 'Value': 'Value2'} - ] - } + {'Key': 'Key2', 'Value': 'Value2'}, + ], + }, + } + self.assert_params_for_cmd(command, expected_params) + + def test_account_regional_namespace_bucket(self): + bucket = 'amzn-s3-demo-bucket-111122223333-us-west-2-an' + command = self.prefix + f's3://{bucket} --region us-west-2' + self.parsed_responses = [{'Location': 'us-west-2'}] + expected_params = { + 'Bucket': bucket, + 'BucketNamespace': 'account-regional', + 'CreateBucketConfiguration': {'LocationConstraint': 'us-west-2'}, } self.assert_params_for_cmd(command, expected_params) + def test_account_regional_namespace_bucket_us_east_1(self): + bucket = 'my-bucket-111122223333-us-east-1-an' + command = self.prefix + f's3://{bucket} --region us-east-1' + expected_params = { + 'Bucket': bucket, + 'BucketNamespace': 'account-regional', + } + self.assert_params_for_cmd(command, expected_params) + + def test_account_regional_namespace_short_bucket_name(self): + bucket = 'xyz-an' + command = self.prefix + f's3://{bucket} --region us-east-1' + expected_params = { + 'Bucket': bucket, + 'BucketNamespace': 'account-regional', + } + self.assert_params_for_cmd(command, expected_params) + + def test_regular_bucket_no_namespace(self): + command = self.prefix + 's3://my-regular-bucket --region us-east-1' + expected_params = {'Bucket': 'my-regular-bucket'} + self.assert_params_for_cmd(command, expected_params) + def test_tags_with_three_arguments_fails(self): command = self.prefix + 's3://bucket --tags Key1 Value1 ExtraArg' self.assert_params_for_cmd( - command, - expected_rc=252, - stderr_contains='ParamValidation' + command, expected_rc=252, stderr_contains='ParamValidation' ) diff --git a/tests/integration/customizations/s3/test_plugin.py b/tests/integration/customizations/s3/test_plugin.py index 8a20ec49c9ea..0536bb1d179a 100644 --- a/tests/integration/customizations/s3/test_plugin.py +++ b/tests/integration/customizations/s3/test_plugin.py @@ -171,6 +171,13 @@ def _running_on_rhel(): ) +class TestMakeBucketAccountRegionalNamespace(BaseS3IntegrationTest): + def test_short_an_suffix_sends_namespace_header(self): + p = aws('s3 mb s3://xyz-an') + assert p.rc != 0 + assert 'InvalidBucketNamespace' in p.stderr + + @pytest.mark.parametrize( 'preferred_transfer_client', ['classic', 'crt'], indirect=True ) diff --git a/tests/unit/customizations/s3/test_utils.py b/tests/unit/customizations/s3/test_utils.py index 0c79dc0ea913..3f1fe1ff7615 100644 --- a/tests/unit/customizations/s3/test_utils.py +++ b/tests/unit/customizations/s3/test_utils.py @@ -39,6 +39,7 @@ guess_content_type, human_readable_size, human_readable_to_int, + is_account_regional_namespace_bucket, relative_path, set_file_utime, ) @@ -368,6 +369,44 @@ def test_outpost_bucket_arn_with_slash_raises_exception(self): ) +class TestIsAccountRegionalNamespaceBucket(unittest.TestCase): + def test_matches_standard_pattern(self): + self.assertTrue( + is_account_regional_namespace_bucket( + 'amzn-s3-demo-bucket-111122223333-us-west-2-an' + ) + ) + + def test_matches_different_region(self): + self.assertTrue( + is_account_regional_namespace_bucket( + 'my-bucket-123456789012-eu-central-1-an' + ) + ) + + def test_no_match_regular_bucket(self): + self.assertFalse( + is_account_regional_namespace_bucket('my-regular-bucket') + ) + + def test_no_match_missing_an_suffix(self): + self.assertFalse( + is_account_regional_namespace_bucket( + 'bucket-111122223333-us-west-2' + ) + ) + + def test_matches_short_bucket_name(self): + self.assertTrue( + is_account_regional_namespace_bucket('xyz-an') + ) + + def test_no_match_express_directory_bucket(self): + self.assertFalse( + is_account_regional_namespace_bucket('bucket--usw2-az1--x-s3') + ) + + class TestCreateWarning(unittest.TestCase): def test_create_warning(self): path = '/foo/'