diff --git a/.changes/next-release/enhancement-s3-43539.json b/.changes/next-release/enhancement-s3-43539.json new file mode 100644 index 000000000000..12c1a139cb14 --- /dev/null +++ b/.changes/next-release/enhancement-s3-43539.json @@ -0,0 +1,5 @@ +{ + "type": "enhancement", + "category": "``s3``", + "description": "Added support for opting out of Amazon S3 Express session authentication via the new ``AWS_S3_DISABLE_EXPRESS_SESSION_AUTH`` environment variable, or the ``s3_disable_express_session_auth`` shared configuration setting." +} diff --git a/awscli/botocore/args.py b/awscli/botocore/args.py index 5897ab99eb0b..522a4ee4189e 100644 --- a/awscli/botocore/args.py +++ b/awscli/botocore/args.py @@ -116,6 +116,9 @@ def get_client_args( signing_region = endpoint_config['signing_region'] endpoint_region_name = endpoint_config['region_name'] account_id_endpoint_mode = config_kwargs['account_id_endpoint_mode'] + s3_disable_express_session_auth = config_kwargs[ + 's3_disable_express_session_auth' + ] event_emitter = copy.copy(self._event_emitter) signer = RequestSigner( @@ -165,6 +168,7 @@ def get_client_args( event_emitter, credentials, account_id_endpoint_mode, + s3_disable_express_session_auth, ) # Copy the session's user agent factory and adds client configuration. @@ -280,6 +284,11 @@ def compute_client_args( ), account_id_endpoint_mode=client_config.account_id_endpoint_mode, auth_scheme_preference=client_config.auth_scheme_preference, + s3_disable_express_session_auth=( + client_config.s3.get('disable_s3_express_session_auth') + if client_config.s3 is not None + else None + ), ) self._compute_retry_config(config_kwargs) self._compute_request_compression_config(config_kwargs) @@ -292,6 +301,7 @@ def compute_client_args( client_config, config_kwargs ) self._compute_signature_version_config(client_config, config_kwargs) + self._compute_s3_disable_express_session_auth(config_kwargs) s3_config = self.compute_s3_config(client_config) is_s3_service = self._is_s3_service(service_name) @@ -412,6 +422,19 @@ def _compute_s3_endpoint_config( ) return endpoint_config + def _validate_s3_disable_express_session_auth(self, config_val): + string_bool = isinstance(config_val, str) and config_val.lower() in [ + 'true', + 'false', + ] + if not isinstance(config_val, bool) and not string_bool: + raise botocore.exceptions.InvalidConfigError( + error_msg=( + f'Invalid value "{config_val}" for ' + 's3_disable_express_session_auth. Value must be a boolean' + ) + ) + def _set_region_if_custom_s3_endpoint( self, endpoint_config, endpoint_bridge ): @@ -506,6 +529,20 @@ def _compute_request_compression_config(self, config_kwargs): disabled = ensure_boolean(disabled) config_kwargs['disable_request_compression'] = disabled + def _compute_s3_disable_express_session_auth(self, config_kwargs): + disable_express = config_kwargs.get('s3_disable_express_session_auth') + if disable_express is None: + disable_express = self._config_store.get_config_variable( + 's3_disable_express_session_auth' + ) + + # Raise an error if the value does not represent a boolean. + if disable_express is not None: + self._validate_s3_disable_express_session_auth(disable_express) + config_kwargs['s3_disable_express_session_auth'] = ensure_boolean( + disable_express + ) + def _validate_min_compression_size(self, min_size): min_allowed_min_size = 1 max_allowed_min_size = 1048576 @@ -551,6 +588,7 @@ def _build_endpoint_resolver( event_emitter, credentials, account_id_endpoint_mode, + s3_disable_express_session_auth, ): if endpoints_ruleset_data is None: return None @@ -577,6 +615,7 @@ def _build_endpoint_resolver( legacy_endpoint_url=endpoint.host, credentials=credentials, account_id_endpoint_mode=account_id_endpoint_mode, + s3_disable_express_session_auth=s3_disable_express_session_auth, ) # Client context params for s3 conflict with the available settings # in the `s3` parameter on the `Config` object. If the same parameter @@ -587,6 +626,10 @@ def _build_endpoint_resolver( client_context = {} if self._is_s3_service(service_name_raw): client_context.update(s3_config_raw) + if s3_disable_express_session_auth is not None: + client_context['disable_s3_express_session_auth'] = ( + s3_disable_express_session_auth + ) sig_version = ( client_config.signature_version @@ -614,6 +657,7 @@ def compute_endpoint_resolver_builtin_defaults( legacy_endpoint_url, credentials, account_id_endpoint_mode, + s3_disable_express_session_auth, ): # EndpointRulesetResolver rulesets may accept an "SDK::Endpoint" as # input. If the endpoint_url argument of create_client() is set, it @@ -679,6 +723,9 @@ def compute_endpoint_resolver_builtin_defaults( EPRBuiltins.AWS_S3_DISABLE_MRAP: s3_config.get( 's3_disable_multiregion_access_points', False ), + EPRBuiltins.AWS_S3_DISABLE_EXPRESS_SESSION_AUTH: ( + s3_disable_express_session_auth + ), EPRBuiltins.SDK_ENDPOINT: given_endpoint, EPRBuiltins.ACCOUNT_ID: credentials.get_deferred_property( 'account_id' diff --git a/awscli/botocore/config.py b/awscli/botocore/config.py index 9362e0138ee9..92e060a92a03 100644 --- a/awscli/botocore/config.py +++ b/awscli/botocore/config.py @@ -120,6 +120,10 @@ class Config: * path -- Addressing style is always by path. Endpoints will be addressed as such: s3.amazonaws.com/mybucket + * ``disable_s3_express_session_auth`` -- Refers to whether to use S3 + Express session authentication. The value must be a boolean. If True, the + client will NOT use S3 Express session authentication. + :type retries: dict :param retries: A dictionary for retry specific configurations. Valid keys are: @@ -272,6 +276,7 @@ class Config: ('proxies', None), ('proxies_config', None), ('s3', None), + ('s3_disable_express_session_auth', None), ('retries', None), ('client_cert', None), ('inject_host_prefix', None), diff --git a/awscli/botocore/configprovider.py b/awscli/botocore/configprovider.py index 427d2ed2cdec..9e40dd58695a 100644 --- a/awscli/botocore/configprovider.py +++ b/awscli/botocore/configprovider.py @@ -192,6 +192,12 @@ None, None, ), + 's3_disable_express_session_auth': ( + 's3_disable_express_session_auth', + 'AWS_S3_DISABLE_EXPRESS_SESSION_AUTH', + None, + None, + ), } # A mapping for the s3 specific configuration vars. These are the configuration # vars that typically go in the s3 section of the config file. This mapping diff --git a/awscli/botocore/regions.py b/awscli/botocore/regions.py index 32367ef8dcc3..4ed4145b6c88 100644 --- a/awscli/botocore/regions.py +++ b/awscli/botocore/regions.py @@ -440,6 +440,11 @@ class EndpointResolverBuiltins(str, Enum): # Whether to use the ARN region or raise an error when ARN and client # region differ (for s3 service only, bool) AWS_S3_USE_ARN_REGION = "AWS::S3::UseArnRegion" + # Whether to use S3 Express session authentication, or fallback to default + # authentication (for s3 service only, bool). + AWS_S3_DISABLE_EXPRESS_SESSION_AUTH = ( + "AWS::S3::DisableS3ExpressSessionAuth" + ) # Whether to use the ARN region or raise an error when ARN and client # region differ (for s3-control service only, bool) AWS_S3CONTROL_USE_ARN_REGION = 'AWS::S3Control::UseArnRegion' diff --git a/awscli/topics/s3-config.rst b/awscli/topics/s3-config.rst index ad127f776ec9..945c4d1e1535 100644 --- a/awscli/topics/s3-config.rst +++ b/awscli/topics/s3-config.rst @@ -59,6 +59,10 @@ and ``aws s3api``: payloads. By default, this is disabled for streaming uploads (UploadPart and PutObject) when using https. +Additionally, the ``disable_s3_express_session_auth`` configuration value +can be set for both ``aws s3`` and ``aws s3api`` commands. Unlike the other +S3 configuration values, this value is set at the profile level and not under +the ``s3`` key. These values must be set under the top level ``s3`` key in the AWS Config File, which has a default location of ``~/.aws/config``. Below is an example @@ -295,6 +299,30 @@ but only if a ContentMD5 is present (it is generated by default) and the endpoint uses HTTPS. +disable_s3_express_session_auth +------------------------------- + +**Default** - ``false`` + +If set to ``true``, the client will not use S3 Express session authentication +for S3 Express One Zone directory buckets. By default, S3 Express session +authentication is enabled for requests to S3 Express One Zone directory buckets. +S3 Express session authentication provides optimized authentication for high +performance workloads with S3 Express One Zone. + +Unlike other S3 configuration values, this value must be set at the profile +level in the AWS Config File and not under the ``s3`` key. For example:: + + [profile development] + aws_access_key_id=foo + aws_secret_access_key=bar + disable_s3_express_session_auth = true + +To set this value programmatically using the ``aws configure set`` command:: + + $ aws configure set default.disable_s3_express_session_auth true + + preferred_transfer_client ------------------------- diff --git a/tests/functional/botocore/test_disable_s3_express_auth.py b/tests/functional/botocore/test_disable_s3_express_auth.py new file mode 100644 index 000000000000..f711d2f258d0 --- /dev/null +++ b/tests/functional/botocore/test_disable_s3_express_auth.py @@ -0,0 +1,104 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +import datetime +import os +from unittest import mock + +import pytest +from dateutil.tz import tzutc + +from botocore.config import Config +from tests import ClientHTTPStubber, temporary_file + + +class TestDisableS3ExpressAuth: + DATE = datetime.datetime(2024, 11, 30, 23, 59, 59, tzinfo=tzutc()) + BUCKET_NAME = 'mybucket--usw2-az1--x-s3' + + CREATE_SESSION_RESPONSE = b'\ntest-key2024-12-31T23:59:59Ztest-secrettest-token' + LIST_OBJECTS_RESPONSE = b'\nmybucket--usw2-az1--x-s301000urlfalse' + + @pytest.fixture + def mock_datetime(self): + with mock.patch('datetime.datetime', spec=True) as mock_dt: + mock_dt.now.return_value = self.DATE + mock_dt.utcnow.return_value = self.DATE + yield mock_dt + + def test_disable_s3_express_auth_enabled( + self, patched_session, mock_datetime + ): + config = Config(s3={'disable_s3_express_session_auth': True}) + s3_client = patched_session.create_client( + 's3', + config=config, + region_name='us-west-2', + ) + + with ClientHTTPStubber(s3_client, strict=True) as stubber: + stubber.add_response(body=self.LIST_OBJECTS_RESPONSE) + s3_client.list_objects_v2(Bucket=self.BUCKET_NAME) + + assert len(stubber.requests) == 1 + + def test_disable_s3_express_auth_enabled_env_var( + self, patched_session, mock_datetime + ): + os.environ['AWS_S3_DISABLE_EXPRESS_SESSION_AUTH'] = 'true' + s3_client = patched_session.create_client( + 's3', + region_name='us-west-2', + ) + + with ClientHTTPStubber(s3_client, strict=True) as stubber: + stubber.add_response(body=self.LIST_OBJECTS_RESPONSE) + s3_client.list_objects_v2(Bucket=self.BUCKET_NAME) + + assert len(stubber.requests) == 1 + + def test_disable_s3_express_auth_enabled_shared_config( + self, patched_session, mock_datetime + ): + with temporary_file('w') as f: + os.environ['AWS_CONFIG_FILE'] = f.name + f.write('[default]\n') + f.write('s3_disable_express_session_auth = true\n') + f.flush() + + s3_client = patched_session.create_client( + 's3', + region_name='us-west-2', + ) + + with ClientHTTPStubber(s3_client, strict=True) as stubber: + stubber.add_response(body=self.LIST_OBJECTS_RESPONSE) + s3_client.list_objects_v2(Bucket=self.BUCKET_NAME) + + assert len(stubber.requests) == 1 + + def test_disable_s3_express_auth_disabled( + self, patched_session, mock_datetime + ): + config = Config(s3={'disable_s3_express_session_auth': False}) + s3_client = patched_session.create_client( + 's3', + config=config, + region_name='us-west-2', + ) + + with ClientHTTPStubber(s3_client, strict=True) as stubber: + stubber.add_response(body=self.CREATE_SESSION_RESPONSE) + stubber.add_response(body=self.LIST_OBJECTS_RESPONSE) + s3_client.list_objects_v2(Bucket=self.BUCKET_NAME) + + assert len(stubber.requests) == 2 diff --git a/tests/unit/botocore/test_args.py b/tests/unit/botocore/test_args.py index 8a789e5f6699..3ac73389a797 100644 --- a/tests/unit/botocore/test_args.py +++ b/tests/unit/botocore/test_args.py @@ -695,6 +695,7 @@ def call_compute_endpoint_resolver_builtin_defaults(self, **overrides): defaults = { 'region_name': 'ca-central-1', 'service_name': 'fooservice', + 's3_disable_express_session_auth': False, 's3_config': {}, 'endpoint_bridge': self.bridge, 'client_endpoint_url': None, @@ -722,6 +723,7 @@ def test_builtins_defaults(self): self.assertEqual( bins['AWS::S3::DisableMultiRegionAccessPoints'], False ) + self.assertEqual(bins['AWS::S3::DisableS3ExpressSessionAuth'], False) self.assertEqual(bins['SDK::Endpoint'], None) self.assertEqual(bins['AWS::Auth::AccountId'], None) self.assertEqual(bins['AWS::Auth::AccountIdEndpointMode'], 'preferred') @@ -888,6 +890,22 @@ def test_account_id_endpoint_mode_set_to_disabled(self): ) self.assertEqual(bins['AWS::Auth::AccountIdEndpointMode'], 'disabled') + def test_disable_s3_express_session_auth_default(self): + bins = self.call_compute_endpoint_resolver_builtin_defaults() + self.assertEqual(bins['AWS::S3::DisableS3ExpressSessionAuth'], False) + + def test_disable_s3_express_session_auth_set_to_false(self): + bins = self.call_compute_endpoint_resolver_builtin_defaults( + s3_disable_express_session_auth=False, + ) + self.assertEqual(bins['AWS::S3::DisableS3ExpressSessionAuth'], False) + + def test_disable_s3_express_session_auth_set_to_true(self): + bins = self.call_compute_endpoint_resolver_builtin_defaults( + s3_disable_express_session_auth=True, + ) + self.assertEqual(bins['AWS::S3::DisableS3ExpressSessionAuth'], True) + class TestProtocolPriorityList: def test_all_parsers_accounted_for(self):