Skip to content

Retry terraform init on transient 502/503 errors #1840

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
HuyPhanNguyen wants to merge 6 commits into
base: main
Choose a base branch
from
+110 −2
Open

Retry terraform init on transient 502/503 errors #1840

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
c8522f8
Refactor Template Step to use App and AppSource Updates (#1829)
rain-on Mar 19, 2026
33635f9
ArgoCD Steps: Replace directory-separator for Patches/Replaced files …
rain-on Mar 19, 2026
65a9b06
Retry terraform init on transient 502/503 errors
HuyPhanNguyen Mar 23, 2026
82254bc
Merge remote-tracking branch 'origin/main' into huy/hpy-1283-terrafor…
HuyPhanNguyen Mar 25, 2026
72e6de0
Remove accidentally committed build log
HuyPhanNguyen Mar 25, 2026
ff8c348
Merge branch 'main' into huy/hpy-1283-terraform-init-retry-502
HuyPhanNguyen Mar 26, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 86 additions & 0 deletions source/Calamari.Terraform.Tests/TerraformCliExecutorFixture.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,5 +51,91 @@ public void TerraformVariableFiles_MultiLine()
variables.Get(TerraformSpecialVariables.Action.Terraform.VarFiles).Returns("foo\nbar\r\nbaz");
cliExecutor.TerraformVariableFiles.Should().Be("-var-file=\"foo\" -var-file=\"bar\" -var-file=\"baz\"");
}

[TestCase("│ Error while installing dnsimple/dnsimple v2.0.1: unsuccessful request to\n│ https://github.com/...\n│ 502 Bad Gateway", true)]
[TestCase("│ Error while installing hashicorp/azurerm v4.63.0: unsuccessful request to\n│ https://releases.hashicorp.com/...\n│ 503 Service Unavailable", true)]
[TestCase("Error configuring backend: 502 Bad Gateway", true)]
[TestCase("Error: Invalid backend configuration", false)]
[TestCase("Error: No valid credential sources found", false)]
[TestCase("", false)]
[TestCase(null, false)]
public void IsTransientInitError_MatchesExpectedPatterns(string errorText, bool expected)
{
TerraformCliExecutor.IsTransientInitError(errorText).Should().Be(expected);
}

[Test]
public void InitializePlugins_RetriesOnTransient502Error()
{
var testVariables = Substitute.For<IVariables>();
testVariables.GetStrings(KnownVariables.EnabledFeatureToggles).Returns(new List<string>());
var commandLineRunner = Substitute.For<ICommandLineRunner>();
var callCount = 0;
commandLineRunner.Execute(Arg.Do<CommandLineInvocation>(invocation =>
{
callCount++;
if (callCount == 1)
invocation.AdditionalInvocationOutputSink.WriteInfo("Terraform v0.15.0");
})).Returns(_ =>
{
if (callCount == 2)
return new CommandResult("terraform init", 1, "│ Error while installing dnsimple/dnsimple v2.0.1: unsuccessful request to\n│ https://github.com/...\n│ 502 Bad Gateway");
return new CommandResult("foo", 0);
});

var executor = new TerraformCliExecutor(Substitute.For<ILog>(), Substitute.For<ICalamariFileSystem>(), commandLineRunner, new RunningDeployment("blah", testVariables), new Dictionary<string, string>());

commandLineRunner.Received(3).Execute(Arg.Any<CommandLineInvocation>());
}

[Test]
public void InitializePlugins_DoesNotRetryNonTransientError()
{
var testVariables = Substitute.For<IVariables>();
testVariables.GetStrings(KnownVariables.EnabledFeatureToggles).Returns(new List<string>());
var commandLineRunner = Substitute.For<ICommandLineRunner>();
var callCount = 0;
commandLineRunner.Execute(Arg.Do<CommandLineInvocation>(invocation =>
{
callCount++;
if (callCount == 1)
invocation.AdditionalInvocationOutputSink.WriteInfo("Terraform v0.15.0");
})).Returns(_ =>
{
if (callCount == 2)
return new CommandResult("terraform init", 1, "Error: Invalid backend configuration");
return new CommandResult("foo", 0);
});

var act = () => new TerraformCliExecutor(Substitute.For<ILog>(), Substitute.For<ICalamariFileSystem>(), commandLineRunner, new RunningDeployment("blah", testVariables), new Dictionary<string, string>());

act.Should().Throw<CommandLineException>();
commandLineRunner.Received(2).Execute(Arg.Any<CommandLineInvocation>());
}

[Test]
public void InitializePlugins_ThrowsAfterRetriesExhausted()
{
var testVariables = Substitute.For<IVariables>();
testVariables.GetStrings(KnownVariables.EnabledFeatureToggles).Returns(new List<string>());
var commandLineRunner = Substitute.For<ICommandLineRunner>();
var callCount = 0;
commandLineRunner.Execute(Arg.Do<CommandLineInvocation>(invocation =>
{
callCount++;
if (callCount == 1)
invocation.AdditionalInvocationOutputSink.WriteInfo("Terraform v0.15.0");
})).Returns(_ =>
{
if (callCount >= 2)
return new CommandResult("terraform init", 1, "│ Error while installing dnsimple/dnsimple v2.0.1: unsuccessful request to\n│ https://github.com/...\n│ 502 Bad Gateway");
return new CommandResult("foo", 0);
});

var act = () => new TerraformCliExecutor(Substitute.For<ILog>(), Substitute.For<ICalamariFileSystem>(), commandLineRunner, new RunningDeployment("blah", testVariables), new Dictionary<string, string>());

act.Should().Throw<CommandLineException>();
commandLineRunner.Received(5).Execute(Arg.Any<CommandLineInvocation>());
}
}
}
26 changes: 24 additions & 2 deletions source/Calamari.Terraform/TerraformCliExecutor.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading;
using Calamari.Common.Commands;
using Calamari.Common.Features.Processes;
using Calamari.Common.FeatureToggles;
Expand All @@ -12,6 +13,7 @@
using Calamari.Common.Plumbing.FileSystem;
using Calamari.Common.Plumbing.Logging;
using Calamari.Common.Plumbing.Proxies;
using Calamari.Common.Plumbing.Retry;
using Calamari.Common.Plumbing.Variables;
using Calamari.Terraform.Helpers;
using Newtonsoft.Json;
Expand Down Expand Up @@ -217,10 +219,30 @@ void InitializePlugins()
initCommand += " -no-color";
if (Version?.IsLessThan("0.15.0") == true)
initCommand += $" -get-plugins={allowPluginDownloads.ToString().ToLower()}";

initCommand += $" {initParams}";

ExecuteCommandAndVerifySuccess(new[] { initCommand }, out _, true);
var retry = new RetryTracker(maxRetries: 3, timeLimit: null, new LimitedExponentialRetryInterval(2000, 30000, 2));
while (retry.Try())
{
var commandResult = ExecuteCommandInternal(new[] { initCommand }, out _, true);
if (commandResult.ExitCode == 0)
return;

if (IsTransientInitError(commandResult.Errors) && retry.CanRetry())
{
log.Warn($"Terraform init failed with a transient error (attempt {retry.CurrentTry} of 4). Retrying after backoff...");
Thread.Sleep(retry.Sleep());
continue;
Comment on lines +224 to +235
Copy link

Copilot AI Mar 25, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The backoff settings here don't match the PR description (5s, 10s, 20s). With LimitedExponentialRetryInterval(2000, 30000, 2) and RetryTracker.Sleep() using the current try number, the sleeps are 4s, 8s, 16s. Either update the interval parameters to achieve the intended delays, or adjust the PR description/log messaging accordingly.

Copilot uses AI. Check for mistakes.
}
Comment on lines +224 to +236
Copy link

Copilot AI Mar 25, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This retry loop uses Thread.Sleep(retry.Sleep()), which means callers block for real time on every transient failure. It also makes the new unit tests (e.g., retries exhausted) take tens of seconds (currently ~4s + 8s + 16s) and likely become slow/flaky in CI. Consider injecting a delay/sleeper (or a RetryInterval) so tests can use a no-op/zero delay while production uses real backoff.

Copilot uses AI. Check for mistakes.

VerifySuccess(commandResult);
}
}

internal static bool IsTransientInitError(string errors)
{
if (string.IsNullOrEmpty(errors)) return false;
return Regex.IsMatch(errors, @"50[23] (Bad Gateway|Service Unavailable)", RegexOptions.IgnoreCase);
}

class TerraformVersionCommandOutput
Expand Down