Skip to content

[Feature] New Retry Behavior #3836

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
kai-ion wants to merge 5 commits into
base: main
Choose a base branch
from
+586 −13
Open

[Feature] New Retry Behavior #3836

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
8416718
Implement Retry Behavior 2.1 core logic gated behind AWS_NEW_RETRIES_…
kai-ion May 28, 2026
965a186
removed transient back off
kai-ion Jun 2, 2026
a2f5d4a
moved implementation to pimple
kai-ion Jun 2, 2026
7246957
added a new function to support dynamodb transient backoff
kai-ion Jun 3, 2026
adcbc4f
added a new function to support dynamodb transient backoff
kai-ion Jun 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions src/aws-cpp-sdk-core/include/aws/core/client/RetryStrategy.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
#pragma once

#include <aws/core/Core_EXPORTS.h>
#include <aws/core/utils/memory/AWSMemory.h>
#include <aws/core/utils/threading/ReaderWriterLock.h>
#include <memory>

Expand Down Expand Up @@ -123,6 +124,7 @@ namespace Aws
public:
StandardRetryStrategy(long maxAttempts = 3);
StandardRetryStrategy(std::shared_ptr<RetryQuotaContainer> retryQuotaContainer, long maxAttempts = 3);
virtual ~StandardRetryStrategy();

virtual void RequestBookkeeping(const HttpResponseOutcome& httpResponseOutcome) override;
virtual void RequestBookkeeping(const HttpResponseOutcome& httpResponseOutcome, const AWSError<CoreErrors>& lastError) override;
Expand All @@ -138,6 +140,10 @@ namespace Aws
protected:
std::shared_ptr<RetryQuotaContainer> m_retryQuotaContainer;
long m_maxAttempts;

private:
struct RetryImpl;
Aws::UniquePtr<RetryImpl> m_impl;
};
} // namespace Client
} // namespace Aws
72 changes: 64 additions & 8 deletions src/aws-cpp-sdk-core/source/client/ClientConfiguration.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -543,7 +543,52 @@ ClientConfiguration::ClientConfiguration(bool /*useSmartDefaults*/, const char*
Aws::Config::Defaults::SetSmartDefaultsConfigurationParameters(*this, defaultMode, hasEc2MetadataRegion, ec2MetadataRegion);
}

namespace {
class ThrottleBasedRetryQuotaContainer : public RetryQuotaContainer
{
public:
ThrottleBasedRetryQuotaContainer(int retryCost = 14, int throttlingRetryCost = 5)
: m_retryQuota(500), m_retryCost(retryCost), m_throttlingRetryCost(throttlingRetryCost) {}

bool AcquireRetryQuota(int capacityAmount) override
{
Aws::Utils::Threading::WriterLockGuard guard(m_retryQuotaLock);
if (capacityAmount > m_retryQuota) return false;
m_retryQuota -= capacityAmount;
return true;
}

bool AcquireRetryQuota(const AWSError<CoreErrors>& error) override
{
int capacityAmount = error.ShouldThrottle() ? m_throttlingRetryCost : m_retryCost;
return AcquireRetryQuota(capacityAmount);
}

void ReleaseRetryQuota(int capacityAmount) override
{
Aws::Utils::Threading::WriterLockGuard guard(m_retryQuotaLock);
m_retryQuota = (std::min)(m_retryQuota + capacityAmount, 500);
}

void ReleaseRetryQuota(const AWSError<CoreErrors>& error) override
{
int capacityAmount = error.ShouldThrottle() ? m_throttlingRetryCost : m_retryCost;
ReleaseRetryQuota(capacityAmount);
}

int GetRetryQuota() const override { return m_retryQuota; }

private:
mutable Aws::Utils::Threading::ReaderWriterLock m_retryQuotaLock;
int m_retryQuota;
int m_retryCost;
int m_throttlingRetryCost;
};
} // anonymous namespace

std::shared_ptr<RetryStrategy> InitRetryStrategy(int maxAttempts, Aws::String retryMode) {
const bool newRetriesEnabled = Aws::Environment::GetEnv("AWS_NEW_RETRIES_2026") == "true";

if (retryMode.empty())
{
retryMode = Aws::Environment::GetEnv("AWS_RETRY_MODE");
Expand All @@ -552,37 +597,48 @@ std::shared_ptr<RetryStrategy> InitRetryStrategy(int maxAttempts, Aws::String re
{
retryMode = Aws::Config::GetCachedConfigValue("retry_mode");
}
if (newRetriesEnabled && retryMode.empty())
{
retryMode = "standard";
}

std::shared_ptr<RetryStrategy> retryStrategy;
if (retryMode == "standard")
{
if (maxAttempts < 0)
long attempts = (maxAttempts < 0) ? 3 : maxAttempts;
if (newRetriesEnabled)
{
// negative value set above force usage of default max attempts
retryStrategy = Aws::MakeShared<StandardRetryStrategy>(CLIENT_CONFIG_TAG);
auto quota = Aws::MakeShared<ThrottleBasedRetryQuotaContainer>(CLIENT_CONFIG_TAG);
retryStrategy = Aws::MakeShared<StandardRetryStrategy>(CLIENT_CONFIG_TAG, quota, attempts);
}
else
{
retryStrategy = Aws::MakeShared<StandardRetryStrategy>(CLIENT_CONFIG_TAG, maxAttempts);
retryStrategy = Aws::MakeShared<StandardRetryStrategy>(CLIENT_CONFIG_TAG, attempts);
}
}
else if (retryMode == "adaptive")
{
if (maxAttempts < 0)
long attempts = (maxAttempts < 0) ? 3 : maxAttempts;
if (newRetriesEnabled)
{
// negative value set above force usage of default max attempts
retryStrategy = Aws::MakeShared<AdaptiveRetryStrategy>(CLIENT_CONFIG_TAG);
auto quota = Aws::MakeShared<ThrottleBasedRetryQuotaContainer>(CLIENT_CONFIG_TAG);
retryStrategy = Aws::MakeShared<AdaptiveRetryStrategy>(CLIENT_CONFIG_TAG, quota, attempts);
}
else
{
retryStrategy = Aws::MakeShared<AdaptiveRetryStrategy>(CLIENT_CONFIG_TAG, maxAttempts);
retryStrategy = Aws::MakeShared<AdaptiveRetryStrategy>(CLIENT_CONFIG_TAG, attempts);
}
}
else
{
retryStrategy = Aws::MakeShared<DefaultRetryStrategy>(CLIENT_CONFIG_TAG);
}

if (newRetriesEnabled)
{
AWS_LOGSTREAM_INFO(CLIENT_CONFIG_TAG, "Retry Behavior 2.1 active (AWS_NEW_RETRIES_2026=true), mode=" << retryMode);
}

return retryStrategy;
}

Expand Down
49 changes: 44 additions & 5 deletions src/aws-cpp-sdk-core/source/client/RetryStrategy.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
#include <aws/core/client/CoreErrors.h>
#include <aws/core/client/RetryStrategy.h>
#include <aws/core/utils/Outcome.h>
#include <aws/core/utils/StringUtils.h>
#include <aws/core/utils/local/Random.h>

using namespace Aws::Utils::Threading;
Expand All @@ -19,11 +20,51 @@ namespace Aws
static const int RETRY_COST = 5;
static const int TIMEOUT_RETRY_COST = 10;

struct StandardRetryStrategy::RetryImpl
{
bool newRetriesEnabled = false;

long CalculateDelay(const AWSError<CoreErrors>& error, long attemptedRetries) const
{
if (!newRetriesEnabled)
{
AWS_UNREFERENCED_PARAM(error);
// Maximum left shift factor is capped by ceil(log2(max_delay)), to avoid wrap-around and overflow into negative values:
return std::min(static_cast<int>(Aws::Utils::GetRandomValue() % 1000) * (1 << std::min(attemptedRetries, 15L)), 20000);
}

double x = error.ShouldThrottle() ? 1.0 : 0.05;
double exponentialPart = x * static_cast<double>(1L << std::min(attemptedRetries, 30L));
double cappedPart = std::min(exponentialPart, 20.0);

double b = static_cast<double>(Aws::Utils::GetRandomValue() % 10000) / 10000.0;
double t_i = b * cappedPart;

const auto& headers = error.GetResponseHeaders();
auto it = headers.find("x-amz-retry-after");
if (it != headers.end())
{
double headerSec = static_cast<double>(Aws::Utils::StringUtils::ConvertToInt64(it->second.c_str())) / 1000.0;
double clamped = std::max(t_i, std::min(headerSec, 5.0 + t_i));
return static_cast<long>(clamped * 1000.0);
}

return static_cast<long>(t_i * 1000.0);
}
};

StandardRetryStrategy::StandardRetryStrategy(long maxAttempts)
: m_retryQuotaContainer(Aws::MakeShared<DefaultRetryQuotaContainer>("StandardRetryStrategy")), m_maxAttempts(maxAttempts) {}
: m_retryQuotaContainer(Aws::MakeShared<DefaultRetryQuotaContainer>("StandardRetryStrategy")), m_maxAttempts(maxAttempts),
m_impl(Aws::MakeUnique<RetryImpl>("StandardRetryStrategy")) {}

StandardRetryStrategy::StandardRetryStrategy(std::shared_ptr<RetryQuotaContainer> retryQuotaContainer, long maxAttempts)
: m_retryQuotaContainer(retryQuotaContainer), m_maxAttempts(maxAttempts) {}
: m_retryQuotaContainer(retryQuotaContainer), m_maxAttempts(maxAttempts),
m_impl(Aws::MakeUnique<RetryImpl>("StandardRetryStrategy"))
{
m_impl->newRetriesEnabled = true;
}

StandardRetryStrategy::~StandardRetryStrategy() = default;

void StandardRetryStrategy::RequestBookkeeping(const HttpResponseOutcome& httpResponseOutcome)
{
Expand Down Expand Up @@ -54,9 +95,7 @@ namespace Aws

long StandardRetryStrategy::CalculateDelayBeforeNextRetry(const AWSError<CoreErrors>& error, long attemptedRetries) const
{
AWS_UNREFERENCED_PARAM(error);
// Maximum left shift factor is capped by ceil(log2(max_delay)), to avoid wrap-around and overflow into negative values:
return std::min(static_cast<int>(Aws::Utils::GetRandomValue() % 1000) * (1 << std::min(attemptedRetries, 15L)), 20000);
return m_impl->CalculateDelay(error, attemptedRetries);
}

DefaultRetryQuotaContainer::DefaultRetryQuotaContainer() : m_retryQuota(INITIAL_RETRY_TOKENS)
Expand Down
Loading
Loading