[v2] Implement opting out of S3 Express session auth via config variables.

.changes/next-release/enhancement-s3-43539.json

@@ -0,0 +1,5 @@
+{
+  "type": "enhancement",
+  "category": "``s3``",
+  "description": "Added support for opting out of Amazon S3 Express session authentication via the new ``AWS_S3_DISABLE_EXPRESS_SESSION_AUTH`` environment variable, or the ``s3_disable_express_session_auth`` shared configuration setting."
+}

awscli/botocore/args.py

@@ -116,6 +116,9 @@ def get_client_args(
         signing_region = endpoint_config['signing_region']
         endpoint_region_name = endpoint_config['region_name']
         account_id_endpoint_mode = config_kwargs['account_id_endpoint_mode']
+        s3_disable_express_session_auth = config_kwargs[
+            's3_disable_express_session_auth'
+        ]
 
         event_emitter = copy.copy(self._event_emitter)
         signer = RequestSigner(
@@ -165,6 +168,7 @@ def get_client_args(
             event_emitter,
             credentials,
             account_id_endpoint_mode,
+            s3_disable_express_session_auth,
         )
 
         # Copy the session's user agent factory and adds client configuration.
@@ -280,6 +284,11 @@ def compute_client_args(
                 ),
                 account_id_endpoint_mode=client_config.account_id_endpoint_mode,
                 auth_scheme_preference=client_config.auth_scheme_preference,
+                s3_disable_express_session_auth=(
+                    client_config.s3.get('disable_s3_express_session_auth')
+                    if client_config.s3 is not None
+                    else None
+                ),
             )
         self._compute_retry_config(config_kwargs)
         self._compute_request_compression_config(config_kwargs)
@@ -292,6 +301,7 @@ def compute_client_args(
             client_config, config_kwargs
         )
         self._compute_signature_version_config(client_config, config_kwargs)
+        self._compute_s3_disable_express_session_auth(config_kwargs)
         s3_config = self.compute_s3_config(client_config)
 
         is_s3_service = self._is_s3_service(service_name)
@@ -412,6 +422,19 @@ def _compute_s3_endpoint_config(
         )
         return endpoint_config
 
+    def _validate_s3_disable_express_session_auth(self, config_val):
+        string_bool = isinstance(config_val, str) and config_val.lower() in [
+            'true',
+            'false',
+        ]
+        if not isinstance(config_val, bool) and not string_bool:
+            raise botocore.exceptions.InvalidConfigError(
+                error_msg=(
+                    f'Invalid value "{config_val}" for '
+                    's3_disable_express_session_auth. Value must be a boolean'
+                )
+            )
+
     def _set_region_if_custom_s3_endpoint(
         self, endpoint_config, endpoint_bridge
     ):
@@ -506,6 +529,20 @@ def _compute_request_compression_config(self, config_kwargs):
             disabled = ensure_boolean(disabled)
         config_kwargs['disable_request_compression'] = disabled
 
+    def _compute_s3_disable_express_session_auth(self, config_kwargs):
+        disable_express = config_kwargs.get('s3_disable_express_session_auth')
+        if disable_express is None:
+            disable_express = self._config_store.get_config_variable(
+                's3_disable_express_session_auth'
+            )
+
+        # Raise an error if the value does not represent a boolean.
+        if disable_express is not None:
+            self._validate_s3_disable_express_session_auth(disable_express)
+        config_kwargs['s3_disable_express_session_auth'] = ensure_boolean(
+            disable_express
+        )
+
     def _validate_min_compression_size(self, min_size):
         min_allowed_min_size = 1
         max_allowed_min_size = 1048576
@@ -551,6 +588,7 @@ def _build_endpoint_resolver(
         event_emitter,
         credentials,
         account_id_endpoint_mode,
+        s3_disable_express_session_auth,
     ):
         if endpoints_ruleset_data is None:
             return None
@@ -577,6 +615,7 @@ def _build_endpoint_resolver(
             legacy_endpoint_url=endpoint.host,
             credentials=credentials,
             account_id_endpoint_mode=account_id_endpoint_mode,
+            s3_disable_express_session_auth=s3_disable_express_session_auth,
         )
         # Client context params for s3 conflict with the available settings
         # in the `s3` parameter on the `Config` object. If the same parameter
@@ -587,6 +626,10 @@ def _build_endpoint_resolver(
             client_context = {}
         if self._is_s3_service(service_name_raw):
             client_context.update(s3_config_raw)
+            if s3_disable_express_session_auth is not None:
+                client_context['disable_s3_express_session_auth'] = (
+                    s3_disable_express_session_auth
+                )
 
         sig_version = (
             client_config.signature_version
@@ -614,6 +657,7 @@ def compute_endpoint_resolver_builtin_defaults(
         legacy_endpoint_url,
         credentials,
         account_id_endpoint_mode,
+        s3_disable_express_session_auth,
     ):
         # EndpointRulesetResolver rulesets may accept an "SDK::Endpoint" as
         # input. If the endpoint_url argument of create_client() is set, it
@@ -679,6 +723,9 @@ def compute_endpoint_resolver_builtin_defaults(
             EPRBuiltins.AWS_S3_DISABLE_MRAP: s3_config.get(
                 's3_disable_multiregion_access_points', False
             ),
+            EPRBuiltins.AWS_S3_DISABLE_EXPRESS_SESSION_AUTH: (
+                s3_disable_express_session_auth
+            ),
             EPRBuiltins.SDK_ENDPOINT: given_endpoint,
             EPRBuiltins.ACCOUNT_ID: credentials.get_deferred_property(
                 'account_id'

awscli/botocore/config.py

@@ -120,6 +120,10 @@ class Config:
           * path -- Addressing style is always by path. Endpoints will be
             addressed as such: s3.amazonaws.com/mybucket
 
+        * ``disable_s3_express_session_auth`` -- Refers to whether to use S3
+          Express session authentication. The value must be a boolean. If True, the
+          client will NOT use S3 Express session authentication.
+
     :type retries: dict
     :param retries: A dictionary for retry specific configurations.
         Valid keys are:
@@ -272,6 +276,7 @@ class Config:
             ('proxies', None),
             ('proxies_config', None),
             ('s3', None),
+            ('s3_disable_express_session_auth', None),
             ('retries', None),
             ('client_cert', None),
             ('inject_host_prefix', None),

awscli/botocore/configprovider.py

@@ -192,6 +192,12 @@
         None,
         None,
     ),
+    's3_disable_express_session_auth': (
+        's3_disable_express_session_auth',
+        'AWS_S3_DISABLE_EXPRESS_SESSION_AUTH',
+        None,
+        None,
+    ),
 }
 # A mapping for the s3 specific configuration vars. These are the configuration
 # vars that typically go in the s3 section of the config file. This mapping

awscli/botocore/regions.py

@@ -440,6 +440,11 @@ class EndpointResolverBuiltins(str, Enum):
     # Whether to use the ARN region or raise an error when ARN and client
     # region differ (for s3 service only, bool)
     AWS_S3_USE_ARN_REGION = "AWS::S3::UseArnRegion"
+    # Whether to use S3 Express session authentication, or fallback to default
+    # authentication (for s3 service only, bool).
+    AWS_S3_DISABLE_EXPRESS_SESSION_AUTH = (
+        "AWS::S3::DisableS3ExpressSessionAuth"
+    )
     # Whether to use the ARN region or raise an error when ARN and client
     # region differ (for s3-control service only, bool)
     AWS_S3CONTROL_USE_ARN_REGION = 'AWS::S3Control::UseArnRegion'

awscli/topics/s3-config.rst

@@ -59,6 +59,10 @@ and ``aws s3api``:
   payloads. By default, this is disabled for streaming uploads (UploadPart
   and PutObject) when using https.
 
+Additionally, the ``disable_s3_express_session_auth`` configuration value
+can be set for both ``aws s3`` and ``aws s3api`` commands. Unlike the other
+S3 configuration values, this value is set at the profile level and not under
+the ``s3`` key.
 
 These values must be set under the top level ``s3`` key in the AWS Config File,
 which has a default location of ``~/.aws/config``.  Below is an example
@@ -295,6 +299,30 @@ but only if a ContentMD5 is present (it is generated by default) and the
 endpoint uses HTTPS.
 
 
+disable_s3_express_session_auth
+-------------------------------
+
+**Default** - ``false``
+
+If set to ``true``, the client will not use S3 Express session authentication
+for S3 Express One Zone directory buckets. By default, S3 Express session
+authentication is enabled for requests to S3 Express One Zone directory buckets.
+S3 Express session authentication provides optimized authentication for high
+performance workloads with S3 Express One Zone.
+
+Unlike other S3 configuration values, this value must be set at the profile
+level in the AWS Config File and not under the ``s3`` key. For example::
+
+    [profile development]
+    aws_access_key_id=foo
+    aws_secret_access_key=bar
+    disable_s3_express_session_auth = true
+
+To set this value programmatically using the ``aws configure set`` command::
+
+    $ aws configure set default.disable_s3_express_session_auth true
+
+
 preferred_transfer_client
 -------------------------
 

tests/functional/botocore/test_disable_s3_express_auth.py

@@ -0,0 +1,104 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+import datetime
+import os
+from unittest import mock
+
+import pytest
+from dateutil.tz import tzutc
+
+from botocore.config import Config
+from tests import ClientHTTPStubber, temporary_file
+
+
+class TestDisableS3ExpressAuth:
+    DATE = datetime.datetime(2024, 11, 30, 23, 59, 59, tzinfo=tzutc())
+    BUCKET_NAME = 'mybucket--usw2-az1--x-s3'
+
+    CREATE_SESSION_RESPONSE = b'<?xml version="1.0" encoding="UTF-8"?>\n<CreateSessionResult><Credentials><AccessKeyId>test-key</AccessKeyId><Expiration>2024-12-31T23:59:59Z</Expiration><SecretAccessKey>test-secret</SecretAccessKey><SessionToken>test-token</SessionToken></Credentials></CreateSessionResult>'
+    LIST_OBJECTS_RESPONSE = b'<?xml version="1.0" encoding="UTF-8"?>\n<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Name>mybucket--usw2-az1--x-s3</Name><Prefix/><KeyCount>0</KeyCount><MaxKeys>1000</MaxKeys><EncodingType>url</EncodingType><IsTruncated>false</IsTruncated></ListBucketResult>'
+
+    @pytest.fixture
+    def mock_datetime(self):
+        with mock.patch('datetime.datetime', spec=True) as mock_dt:
+            mock_dt.now.return_value = self.DATE
+            mock_dt.utcnow.return_value = self.DATE
+            yield mock_dt
+
+    def test_disable_s3_express_auth_enabled(
+            self, patched_session, mock_datetime
+    ):
+        config = Config(s3={'disable_s3_express_session_auth': True})
+        s3_client = patched_session.create_client(
+            's3',
+            config=config,
+            region_name='us-west-2',
+        )
+
+        with ClientHTTPStubber(s3_client, strict=True) as stubber:
+            stubber.add_response(body=self.LIST_OBJECTS_RESPONSE)
+            s3_client.list_objects_v2(Bucket=self.BUCKET_NAME)
+
+        assert len(stubber.requests) == 1
+
+    def test_disable_s3_express_auth_enabled_env_var(
+            self, patched_session, mock_datetime
+    ):
+        os.environ['AWS_S3_DISABLE_EXPRESS_SESSION_AUTH'] = 'true'
+        s3_client = patched_session.create_client(
+            's3',
+            region_name='us-west-2',
+        )
+
+        with ClientHTTPStubber(s3_client, strict=True) as stubber:
+            stubber.add_response(body=self.LIST_OBJECTS_RESPONSE)
+            s3_client.list_objects_v2(Bucket=self.BUCKET_NAME)
+
+        assert len(stubber.requests) == 1
+
+    def test_disable_s3_express_auth_enabled_shared_config(
+            self, patched_session, mock_datetime
+    ):
+        with temporary_file('w') as f:
+            os.environ['AWS_CONFIG_FILE'] = f.name
+            f.write('[default]\n')
+            f.write('s3_disable_express_session_auth = true\n')
+            f.flush()
+
+            s3_client = patched_session.create_client(
+                's3',
+                region_name='us-west-2',
+            )
+
+        with ClientHTTPStubber(s3_client, strict=True) as stubber:
+            stubber.add_response(body=self.LIST_OBJECTS_RESPONSE)
+            s3_client.list_objects_v2(Bucket=self.BUCKET_NAME)
+
+        assert len(stubber.requests) == 1
+
+    def test_disable_s3_express_auth_disabled(
+            self, patched_session, mock_datetime
+    ):
+        config = Config(s3={'disable_s3_express_session_auth': False})
+        s3_client = patched_session.create_client(
+            's3',
+            config=config,
+            region_name='us-west-2',
+        )
+
+        with ClientHTTPStubber(s3_client, strict=True) as stubber:
+            stubber.add_response(body=self.CREATE_SESSION_RESPONSE)
+            stubber.add_response(body=self.LIST_OBJECTS_RESPONSE)
+            s3_client.list_objects_v2(Bucket=self.BUCKET_NAME)
+
+        assert len(stubber.requests) == 2

tests/unit/botocore/test_args.py

@@ -695,6 +695,7 @@ def call_compute_endpoint_resolver_builtin_defaults(self, **overrides):
         defaults = {
             'region_name': 'ca-central-1',
             'service_name': 'fooservice',
+            's3_disable_express_session_auth': False,
             's3_config': {},
             'endpoint_bridge': self.bridge,
             'client_endpoint_url': None,
@@ -722,6 +723,7 @@ def test_builtins_defaults(self):
         self.assertEqual(
             bins['AWS::S3::DisableMultiRegionAccessPoints'], False
         )
+        self.assertEqual(bins['AWS::S3::DisableS3ExpressSessionAuth'], False)
         self.assertEqual(bins['SDK::Endpoint'], None)
         self.assertEqual(bins['AWS::Auth::AccountId'], None)
         self.assertEqual(bins['AWS::Auth::AccountIdEndpointMode'], 'preferred')
@@ -888,6 +890,22 @@ def test_account_id_endpoint_mode_set_to_disabled(self):
         )
         self.assertEqual(bins['AWS::Auth::AccountIdEndpointMode'], 'disabled')
 
+    def test_disable_s3_express_session_auth_default(self):
+        bins = self.call_compute_endpoint_resolver_builtin_defaults()
+        self.assertEqual(bins['AWS::S3::DisableS3ExpressSessionAuth'], False)
+
+    def test_disable_s3_express_session_auth_set_to_false(self):
+        bins = self.call_compute_endpoint_resolver_builtin_defaults(
+            s3_disable_express_session_auth=False,
+        )
+        self.assertEqual(bins['AWS::S3::DisableS3ExpressSessionAuth'], False)
+
+    def test_disable_s3_express_session_auth_set_to_true(self):
+        bins = self.call_compute_endpoint_resolver_builtin_defaults(
+            s3_disable_express_session_auth=True,
+        )
+        self.assertEqual(bins['AWS::S3::DisableS3ExpressSessionAuth'], True)
+
 
 class TestProtocolPriorityList:
     def test_all_parsers_accounted_for(self):