From 5fc128392038809153267c95888fcf7e71b4b634 Mon Sep 17 00:00:00 2001 From: Alex Eitzman Date: Mon, 19 Aug 2024 11:00:47 -0700 Subject: [PATCH 1/4] Improve AWS supplier samples --- docs/user-guide.rst | 29 ++++++--- .../authenticate_with_aws_supplier.py | 64 +++++++++++++++++++ 2 files changed, 83 insertions(+), 10 deletions(-) create mode 100644 samples/cloud-client/snippets/authenticate_with_aws_supplier.py diff --git a/docs/user-guide.rst b/docs/user-guide.rst index e9ad000e5..6d0f36e27 100644 --- a/docs/user-guide.rst +++ b/docs/user-guide.rst @@ -552,36 +552,45 @@ whether the credential retrieval is retryable. Any call to the supplier from the Identity Pool credential will send a :class:`google.auth.external_account.SupplierContext` object, which contains the requested audience and subject type. Additionally, the credential will send the :class:`google.auth.transport.requests.Request` passed in the credential refresh call which -can be used to make HTTP requests.:: +can be used to make HTTP requests. Using a custom supplier allows workload or workforce identity federation to be used +with other AWS credential sources such as EKS or ECS in addition to the EC2 metadata endpoint which is already natively supported by ADC.:: + import boto3 from google.auth import aws from google.auth import exceptions + from google.cloud import storage class CustomAwsSecurityCredentialsSupplier(aws.AwsSecurityCredentialsSupplier): + def __init__(self, region): + self._region = region + def get_aws_security_credentials(self, context, request): - audience = context.audience + aws_credentials = boto3.Session(region_name=self._region).get_credentials().get_frozen_credentials() + try: - # Return valid AWS security credentials. These credentials are not cached by - # the google credential, so caching should be implemented in the supplier. - return aws.AwsSecurityCredentials(ACCESS_KEY_ID, SECRET_ACCESS_KEY, SESSION_TOKEN) + return aws.AwsSecurityCredentials(aws_credentials.access_key, aws_credentials.secret_key, aws_credentials.token) except Exception as e: - # If credentials retrieval fails, raise a refresh error, setting retryable to true if the client should - # attempt to retrieve the subject token again. raise exceptions.RefreshError(e, retryable=True) def get_aws_region(self, context, request): - # Return active AWS region. + return self._region - supplier = CustomAwsSecurityCredentialsSupplier() + aws_region = "AWS_REGION" # Set the current AWS region (i.e. us-east-2) + supplier = CustomAwsSecurityCredentialsSupplier(aws_region) + # Create credentials using the custom supplier. credentials = aws.Credentials( AUDIENCE, # Set GCP Audience. "urn:ietf:params:aws:token-type:aws4_request", # Set AWS subject token type. aws_security_credentials_supplier=supplier, # Set supplier. - scopes=SCOPES # Set desired scopes. + scopes=['https://www.googleapis.com/auth/cloud-platform'] # Set scopes. ) + # Create service client and use the credentials you just created. + # The GCP project id must also be set. + storage_client = storage.Client(credentials=credentials, project=GCP_PROJECT_ID) + Where the `audience`_ is: ``///iam.googleapis.com/projects/PROJECT_NUMBER/locations/global/workloadIdentityPools/POOL_ID/providers/PROVIDER_ID`` Where the following variables need to be substituted: diff --git a/samples/cloud-client/snippets/authenticate_with_aws_supplier.py b/samples/cloud-client/snippets/authenticate_with_aws_supplier.py new file mode 100644 index 000000000..76ff65854 --- /dev/null +++ b/samples/cloud-client/snippets/authenticate_with_aws_supplier.py @@ -0,0 +1,64 @@ +# Copyright 2024 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START auth_cloud_aws_supplier] + +from google.cloud import storage + +from google.auth import aws +from google.auth import exceptions +import boto3 + +class CustomAwsSecurityCredentialsSupplier(aws.AwsSecurityCredentialsSupplier): + + def __init__(self, region): + self._region = region + + def get_aws_security_credentials(self, context, request): + aws_credentials = boto3.Session(region_name=self._region).get_credentials().get_frozen_credentials() + + try: + return aws.AwsSecurityCredentials(aws_credentials.access_key, aws_credentials.secret_key, aws_credentials.token) + except Exception as e: + raise exceptions.RefreshError(e, retryable=True) + + def get_aws_region(self, context, request): + return self._region + +def authenticate_with_aws_supplier(project_id="your-google-cloud-project-id", aws_region="your_aws_region", audience="your_federation_audience"): + """ + List storage buckets by authenticating with a custom AWS supplier. + + // TODO(Developer) Before running this sample,: + // 1. Replace the project, region, and audience variables. + // 2. Make sure you have the necessary permission to list storage buckets: "storage.buckets.list" + """ + + + credentials = aws.Credentials( + audience, + "urn:ietf:params:aws:token-type:aws4_request", + aws_security_credentials_supplier=CustomAwsSecurityCredentialsSupplier(aws_region), + scopes=['https://www.googleapis.com/auth/cloud-platform'] + ) + + # Construct the Storage client. + storage_client = storage.Client(credentials=credentials, project=project_id) + buckets = storage_client.list_buckets() + print("Buckets:") + for bucket in buckets: + print(bucket.name) + print("Listed all storage buckets.") + +# [END auth_cloud_aws_supplier] From 665a9ffd815bcd56beca6901e24417dabb3cd183 Mon Sep 17 00:00:00 2001 From: Alex Eitzman Date: Mon, 19 Aug 2024 11:12:39 -0700 Subject: [PATCH 2/4] fix lint --- .../cloud-client/snippets/authenticate_with_aws_supplier.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/samples/cloud-client/snippets/authenticate_with_aws_supplier.py b/samples/cloud-client/snippets/authenticate_with_aws_supplier.py index 76ff65854..fe8bf7c79 100644 --- a/samples/cloud-client/snippets/authenticate_with_aws_supplier.py +++ b/samples/cloud-client/snippets/authenticate_with_aws_supplier.py @@ -20,6 +20,7 @@ from google.auth import exceptions import boto3 + class CustomAwsSecurityCredentialsSupplier(aws.AwsSecurityCredentialsSupplier): def __init__(self, region): @@ -36,6 +37,7 @@ def get_aws_security_credentials(self, context, request): def get_aws_region(self, context, request): return self._region + def authenticate_with_aws_supplier(project_id="your-google-cloud-project-id", aws_region="your_aws_region", audience="your_federation_audience"): """ List storage buckets by authenticating with a custom AWS supplier. @@ -45,7 +47,6 @@ def authenticate_with_aws_supplier(project_id="your-google-cloud-project-id", aw // 2. Make sure you have the necessary permission to list storage buckets: "storage.buckets.list" """ - credentials = aws.Credentials( audience, "urn:ietf:params:aws:token-type:aws4_request", From 0a7ee6c405de6e83dc4c2fadd0abb8c716a796d7 Mon Sep 17 00:00:00 2001 From: Alex Eitzman Date: Wed, 21 Aug 2024 14:44:59 -0700 Subject: [PATCH 3/4] applying code review comments, also fixes typo in non-aws supplier sample. --- docs/user-guide.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/user-guide.rst b/docs/user-guide.rst index 6d0f36e27..84dffcfbb 100644 --- a/docs/user-guide.rst +++ b/docs/user-guide.rst @@ -520,7 +520,7 @@ can be used to make HTTP requests.:: credentials = identity_pool.Credentials( AUDIENCE, # Set GCP Audience. - "urn:ietf:params:aws:token-type:jwt", # Set subject token type. + "urn:ietf:params:oauth:token-type:jwt", # Set subject token type. subject_token_supplier=supplier, # Set supplier. scopes=SCOPES # Set desired scopes. ) @@ -552,8 +552,9 @@ whether the credential retrieval is retryable. Any call to the supplier from the Identity Pool credential will send a :class:`google.auth.external_account.SupplierContext` object, which contains the requested audience and subject type. Additionally, the credential will send the :class:`google.auth.transport.requests.Request` passed in the credential refresh call which -can be used to make HTTP requests. Using a custom supplier allows workload or workforce identity federation to be used -with other AWS credential sources such as EKS or ECS in addition to the EC2 metadata endpoint which is already natively supported by ADC.:: +can be used to make HTTP requests. Currently, using ADC with your AWS workloads is only supported with EC2. +An example of a good use case for using a custom credential suppliers is when your workloads are running +in other AWS environments, such as ECS, EKS, Fargate, etc.:: import boto3 from google.auth import aws From 2141cf65f702140f8f6af0925b1eb407a267eb77 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Thu, 15 Jan 2026 12:29:07 -0500 Subject: [PATCH 4/4] updates the snippet in the pre-existing file custom_aws_supplier.py --- .../snippets/custom_aws_supplier.py | 122 ++++++++++-------- 1 file changed, 68 insertions(+), 54 deletions(-) diff --git a/samples/cloud-client/snippets/custom_aws_supplier.py b/samples/cloud-client/snippets/custom_aws_supplier.py index ec5bf8a10..ba1b392a6 100644 --- a/samples/cloud-client/snippets/custom_aws_supplier.py +++ b/samples/cloud-client/snippets/custom_aws_supplier.py @@ -11,36 +11,55 @@ # See the License for the specific language governing permissions and # limitations under the License. +# [START auth_cloud_aws_supplier] + import json import os import sys +from typing import Optional import boto3 from dotenv import load_dotenv -from google.auth.aws import Credentials as AwsCredentials -from google.auth.aws import AwsSecurityCredentials, AwsSecurityCredentialsSupplier -from google.auth.exceptions import GoogleAuthError -from google.auth.transport.requests import AuthorizedSession + +from google.auth import aws +from google.auth import exceptions +from google.cloud import storage load_dotenv() -class CustomAwsSupplier(AwsSecurityCredentialsSupplier): +class CustomAwsSupplier(aws.AwsSecurityCredentialsSupplier): """Custom AWS Security Credentials Supplier.""" - def __init__(self): - """Initializes the Boto3 session, prioritizing environment variables for region.""" + def __init__(self, region=None): + """Initializes the Boto3 session, prioritizing environment variables for region. + + Args: + region Optional[str]: The AWS region name. If None, it will be + sourced from environment variables or Boto3's default discovery. + """ + # Explicitly read the region from the environment first. This ensures that # a value from a .env file is picked up reliably for local testing. - region = os.getenv("AWS_REGION") or os.getenv("AWS_DEFAULT_REGION") + self._region = region or os.getenv("AWS_REGION") or os.getenv("AWS_DEFAULT_REGION") # If region is None, Boto3's discovery chain will be used when needed. - self.session = boto3.Session(region_name=region) + self.session = boto3.Session(region_name=self._region) self._cached_region = None - print(f"[INFO] CustomAwsSupplier initialized. Region from env: {region}") + + print(f"[INFO] CustomAwsSupplier initialized. Region: {self._region}") def get_aws_region(self, context, request) -> str: - """Returns the AWS region using Boto3's default provider chain.""" + """Returns the AWS region using Boto3's default provider chain. + + Args: + context (google.auth.transport.requests.Request): The context. + request Optional[google.auth.transport.Request]: The request. + + Returns: + str: The AWS region name. + """ + if self._cached_region: return self._cached_region @@ -50,68 +69,63 @@ def get_aws_region(self, context, request) -> str: if not self._cached_region: print("[ERROR] Boto3 was unable to resolve an AWS region.", file=sys.stderr) - raise GoogleAuthError("Boto3 was unable to resolve an AWS region.") + raise exceptions.GoogleAuthError("Boto3 was unable to resolve an AWS region.") print(f"[INFO] Boto3 resolved AWS Region: {self._cached_region}") return self._cached_region - def get_aws_security_credentials(self, context, request=None) -> AwsSecurityCredentials: - """Retrieves AWS security credentials using Boto3's default provider chain.""" + def get_aws_security_credentials(self, context, request=None) -> aws.AwsSecurityCredentials: + """Retrieves AWS security credentials using Boto3's default provider chain. + + Args: + context (google.auth.transport.requests.Request): The context. + request Optional[google.auth.transport.Request]: The request. + + Returns: + aws.AwsSecurityCredentials: The AWS security credentials. + """ + aws_credentials = self.session.get_credentials() if not aws_credentials: print("[ERROR] Unable to resolve AWS credentials.", file=sys.stderr) - raise GoogleAuthError("Unable to resolve AWS credentials from the provider chain.") + raise exceptions.GoogleAuthError("Unable to resolve AWS credentials from the provider chain.") print(f"[INFO] Resolved AWS Access Key ID: {aws_credentials.access_key}") - return AwsSecurityCredentials( + return aws.AwsSecurityCredentials( access_key_id=aws_credentials.access_key, secret_access_key=aws_credentials.secret_key, session_token=aws_credentials.token, ) -def main(): - """Main function to demonstrate the custom AWS supplier.""" - print("--- Starting Script ---") - - gcp_audience = os.getenv("GCP_WORKLOAD_AUDIENCE") - sa_impersonation_url = os.getenv("GCP_SERVICE_ACCOUNT_IMPERSONATION_URL") - gcs_bucket_name = os.getenv("GCS_BUCKET_NAME") - - print(f"GCP_WORKLOAD_AUDIENCE: {gcp_audience}") - print(f"GCS_BUCKET_NAME: {gcs_bucket_name}") - - if not all([gcp_audience, sa_impersonation_url, gcs_bucket_name]): - print("[ERROR] Missing required environment variables.", file=sys.stderr) - raise GoogleAuthError("Missing required environment variables.") +def authenticate_with_aws_supplier(project_id, aws_region, audience, service_account_impersonation_url=None): + """ + List storage buckets by authenticating with a custom AWS supplier. + + Args: + project_id (str): The Google Cloud project ID. + aws_region Optional[str]: The AWS region name. + audience (str): The audience for the OIDC token. + service_account_impersonation_url Optional[str]: The URL for service account + impersonation. + """ - custom_supplier = CustomAwsSupplier() + custom_supplier = CustomAwsSupplier(region=aws_region) - credentials = AwsCredentials( - audience=gcp_audience, + credentials = aws.Credentials( + audience=audience, subject_token_type="urn:ietf:params:aws:token-type:aws4_request", - service_account_impersonation_url=sa_impersonation_url, + service_account_impersonation_url=service_account_impersonation_url, aws_security_credentials_supplier=custom_supplier, - scopes=['https://www.googleapis.com/auth/devstorage.read_write'], + scopes=['https://www.googleapis.com/auth/cloud-platform'], ) - bucket_url = f"https://storage.googleapis.com/storage/v1/b/{gcs_bucket_name}" - print(f"Request URL: {bucket_url}") - - authed_session = AuthorizedSession(credentials) - try: - print("Attempting to make authenticated request to Google Cloud Storage...") - res = authed_session.get(bucket_url) - res.raise_for_status() - print("\n--- SUCCESS! ---") - print("Successfully authenticated and retrieved bucket data:") - print(json.dumps(res.json(), indent=2)) - except Exception as e: - print("--- FAILED --- ", file=sys.stderr) - print(e, file=sys.stderr) - exit(1) - - -if __name__ == "__main__": - main() + # Construct the Storage client. + storage_client = storage.Client(credentials=credentials, project=project_id) + buckets = storage_client.list_buckets() + print("Buckets:") + for bucket in buckets: + print(bucket.name) + print("Listed all storage buckets.") +# [END auth_cloud_aws_supplier]