From deb797f0be2a9a8a2e07b95eb26b216bfec6e329 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Fri, 22 May 2026 14:38:21 -0700 Subject: [PATCH 01/10] Route getStaleImages base lookups through staging mirror copyBaseImages already imports every base image into the internal staging ACR ('mirror/' prefix) immediately before getStaleImages runs, but getStaleImages was still resolving FROM tags against docker.io. That path is unreachable from the internal 1ES pool, so the command hangs for 30s on each Docker Hub base image and the job fails. Rewrite any non-MCR / non-*.azurecr.io FROM reference to the staging mirror via --base-override-regex/--base-override-sub. The job already authenticates to InternalMirrorRegistry via reference-service-connections, so no credential changes are needed. Also drops the buildtools-only override (which never matched the actual library/ FROM lines). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/pipelines/templates/jobs/check-base-image-updates.yml | 7 ++----- .../templates/stages/check-base-image-updates.yml | 1 - 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/eng/pipelines/templates/jobs/check-base-image-updates.yml b/eng/pipelines/templates/jobs/check-base-image-updates.yml index 372a44b83..ac5b34a28 100644 --- a/eng/pipelines/templates/jobs/check-base-image-updates.yml +++ b/eng/pipelines/templates/jobs/check-base-image-updates.yml @@ -20,10 +20,6 @@ parameters: # Schema is defined in src/ImageBuilder/Configuration/PublishConfiguration.cs. - name: publishConfig type: object -# Additional arguments to pass to the getStaleImages command (optional). -- name: customGetStaleImagesArgs - type: string - default: "" jobs: - job: ${{ parameters.jobName }} @@ -56,7 +52,8 @@ jobs: $(dotnetDockerBot.email) --gh-token $(BotAccount-dotnet-docker-bot-PAT) staleImagePaths - ${{ parameters.customGetStaleImagesArgs }} + --base-override-regex '^(?!mcr\.microsoft\.com/|[^/]+\.azurecr\.io/)' + --base-override-sub '${{ parameters.publishConfig.InternalMirrorRegistry.server }}/${{ parameters.publishConfig.InternalMirrorRegistry.repoPrefix }}' --subscriptions-path ${{ parameters.subscriptionsPath }} --os-type '*' --architecture '*' diff --git a/eng/pipelines/templates/stages/check-base-image-updates.yml b/eng/pipelines/templates/stages/check-base-image-updates.yml index d2c857247..9650a1d25 100644 --- a/eng/pipelines/templates/stages/check-base-image-updates.yml +++ b/eng/pipelines/templates/stages/check-base-image-updates.yml @@ -31,7 +31,6 @@ stages: parameters: jobName: CheckBaseImages_BuildTools subscriptionsPath: eng/check-base-image-subscriptions-buildtools.json - customGetStaleImagesArgs: --base-override-regex '^((centos|debian|ubuntu):.+)' --base-override-sub '$(overrideRegistry)/$1' publicProjectName: ${{ parameters.publicProjectName }} internalProjectName: ${{ parameters.internalProjectName }} publishConfig: ${{ parameters.publishConfig }} From d17f3e449c61e57d97a4e6d1dd7ef24404b7b877 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Fri, 22 May 2026 15:09:06 -0700 Subject: [PATCH 02/10] Set SYSTEM_ACCESSTOKEN --- eng/pipelines/templates/jobs/check-base-image-updates.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/templates/jobs/check-base-image-updates.yml b/eng/pipelines/templates/jobs/check-base-image-updates.yml index ac5b34a28..c5467395b 100644 --- a/eng/pipelines/templates/jobs/check-base-image-updates.yml +++ b/eng/pipelines/templates/jobs/check-base-image-updates.yml @@ -46,7 +46,7 @@ jobs: additionalOptions: "--subscriptions-path '${{ parameters.subscriptionsPath }}'" - script: > - $(runImageBuilderCmd) + $(runAuthedImageBuilderCmd) getStaleImages $(dotnetDockerBot.userName) $(dotnetDockerBot.email) @@ -60,6 +60,9 @@ jobs: $(dockerHubRegistryCreds) displayName: Get Stale Images name: GetStaleImages + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + SYSTEM_OIDCREQUESTURI: $(System.OidcRequestUri) - script: > $(runImageBuilderCmd) From 3925e841acb04819ece2390bd2b3483659218233 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Fri, 22 May 2026 15:58:16 -0700 Subject: [PATCH 03/10] Use ImageNameResolver for staging mirror lookups in getStaleImages The previous --base-override-regex/sub approach rewrote external FROM tags to point at the staging mirror, but the rewritten repo prefix also leaked into the digest comparison string. image-info.json stores the digest against the canonical (public) repo, so every rewritten image compared unequal and was reported stale on every run. Switch getStaleImages to the same mechanism the build/matrix flow already uses: - Add --registry-override and --source-repo-prefix options (mirroring what ManifestOptions exposes and what copyBaseImages consumes). - Construct ImageNameResolverForMatrix per subscription manifest. GetFromImagePullTag returns the staging mirror location for fetching the digest; GetFromImagePublicTag returns the canonical reference used to build the digest comparison string. The pipeline yml now passes --registry-override / --source-repo-prefix in place of the regex pair, matching how the copyBaseImages step in the same job is invoked. --base-override-regex/sub remains supported for genuine one-off overrides. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../jobs/check-base-image-updates.yml | 4 +- .../Commands/GetStaleImagesCommand.cs | 38 +++++++++++++++---- .../Commands/GetStaleImagesOptions.cs | 21 ++++++++++ 3 files changed, 53 insertions(+), 10 deletions(-) diff --git a/eng/pipelines/templates/jobs/check-base-image-updates.yml b/eng/pipelines/templates/jobs/check-base-image-updates.yml index c5467395b..8f743cd0c 100644 --- a/eng/pipelines/templates/jobs/check-base-image-updates.yml +++ b/eng/pipelines/templates/jobs/check-base-image-updates.yml @@ -52,8 +52,8 @@ jobs: $(dotnetDockerBot.email) --gh-token $(BotAccount-dotnet-docker-bot-PAT) staleImagePaths - --base-override-regex '^(?!mcr\.microsoft\.com/|[^/]+\.azurecr\.io/)' - --base-override-sub '${{ parameters.publishConfig.InternalMirrorRegistry.server }}/${{ parameters.publishConfig.InternalMirrorRegistry.repoPrefix }}' + --registry-override '${{ parameters.publishConfig.InternalMirrorRegistry.server }}' + --source-repo-prefix '${{ parameters.publishConfig.InternalMirrorRegistry.repoPrefix }}' --subscriptions-path ${{ parameters.subscriptionsPath }} --os-type '*' --architecture '*' diff --git a/src/ImageBuilder/Commands/GetStaleImagesCommand.cs b/src/ImageBuilder/Commands/GetStaleImagesCommand.cs index 4129afbc8..56f86732e 100644 --- a/src/ImageBuilder/Commands/GetStaleImagesCommand.cs +++ b/src/ImageBuilder/Commands/GetStaleImagesCommand.cs @@ -55,7 +55,11 @@ public override async Task ExecuteAsync() IEnumerable> getPathResults = SubscriptionHelper.GetSubscriptionManifests( - Options.SubscriptionOptions.SubscriptionsPath, Options.FilterOptions, _gitService, _manifestJsonService) + Options.SubscriptionOptions.SubscriptionsPath, + Options.FilterOptions, + _gitService, + _manifestJsonService, + manifestOptions => manifestOptions.RegistryOverride = Options.RegistryOverride) .Select(async subscriptionManifest => new SubscriptionImagePaths { @@ -87,6 +91,12 @@ private async Task> GetPathsToRebuildAsync(Models.Subscripti { ImageArtifactDetails imageArtifactDetails = await GetImageInfoForSubscriptionAsync(subscription, manifest); + ImageNameResolverForMatrix imageNameResolver = new( + Options.BaseImageOverrideOptions, + manifest, + repoPrefix: null, + sourceRepoPrefix: Options.SourceRepoPrefix); + List pathsToRebuild = new(); foreach (RepoInfo repo in manifest.FilteredRepos) @@ -97,7 +107,8 @@ private async Task> GetPathsToRebuildAsync(Models.Subscripti foreach (PlatformInfo platform in platforms) { - pathsToRebuild.AddRange(await GetPathsToRebuildAsync(manifest, platform, repo, imageArtifactDetails)); + pathsToRebuild.AddRange( + await GetPathsToRebuildAsync(manifest, platform, repo, imageArtifactDetails, imageNameResolver)); } } @@ -109,7 +120,11 @@ private static IEnumerable GetDescendants(PlatformInfo platform, M .Prepend(platform); private async Task> GetPathsToRebuildAsync( - ManifestInfo manifest, PlatformInfo platform, RepoInfo repo, ImageArtifactDetails imageArtifactDetails) + ManifestInfo manifest, + PlatformInfo platform, + RepoInfo repo, + ImageArtifactDetails imageArtifactDetails, + ImageNameResolverForMatrix imageNameResolver) { string? fromImage = platform.FinalStageFromImage; if (fromImage is null) @@ -129,19 +144,26 @@ private async Task> GetPathsToRebuildAsync( return dependentPlatforms.Select(p => p.Model.Dockerfile).ToList(); } - fromImage = Options.BaseImageOverrideOptions.ApplyBaseImageOverride(fromImage); + // Resolve where to actually fetch the digest from. For external base images this + // points to the mirror location in the staging registry; for internal images it is + // the original FROM tag. The "public" form is the canonical reference matching what + // gets recorded in image-info.json and so is the right repo to use in the digest + // comparison string below. + string pullTag = imageNameResolver.GetFromImagePullTag(fromImage); + string publicTag = imageNameResolver.GetFromImagePublicTag(fromImage); - string currentDigest = await LockHelper.DoubleCheckedLockLookupAsync(_imageDigestsLock, _imageDigests, fromImage, + string currentDigest = await LockHelper.DoubleCheckedLockLookupAsync(_imageDigestsLock, _imageDigests, pullTag, async () => { - string digest = await _manifestService.Value.GetManifestDigestShaAsync(fromImage, Options.IsDryRun); - return DockerHelper.GetDigestString(DockerHelper.GetRepo(fromImage), digest); + string digest = await _manifestService.Value.GetManifestDigestShaAsync(pullTag, Options.IsDryRun); + return DockerHelper.GetDigestString(DockerHelper.GetRepo(publicTag), digest); }); bool rebuildImage = matchingPlatform.Value.Platform.BaseImageDigest != currentDigest; _logger.LogInformation( - $"Checking base image '{fromImage}' from '{platform.DockerfilePath}'{Environment.NewLine}" + $"Checking base image '{publicTag}' from '{platform.DockerfilePath}'{Environment.NewLine}" + + $"\tPulled from: {pullTag}{Environment.NewLine}" + $"\tLast build digest: {matchingPlatform.Value.Platform.BaseImageDigest}{Environment.NewLine}" + $"\tCurrent digest: {currentDigest}{Environment.NewLine}" + $"\tImage is up-to-date: {!rebuildImage}{Environment.NewLine}"); diff --git a/src/ImageBuilder/Commands/GetStaleImagesOptions.cs b/src/ImageBuilder/Commands/GetStaleImagesOptions.cs index cb41145be..ede654504 100644 --- a/src/ImageBuilder/Commands/GetStaleImagesOptions.cs +++ b/src/ImageBuilder/Commands/GetStaleImagesOptions.cs @@ -23,6 +23,10 @@ public class GetStaleImagesOptions : Options, IFilterableOptions, IGitOptionsHos public BaseImageOverrideOptions BaseImageOverrideOptions { get; set; } = new(); + public string? RegistryOverride { get; set; } + + public string? SourceRepoPrefix { get; set; } + private static readonly GitOptionsBuilder GitBuilder = GitOptionsBuilder.BuildForRepositoryOperations(); private static readonly Argument VariableNameArgument = new(nameof(VariableName)) @@ -30,6 +34,19 @@ public class GetStaleImagesOptions : Options, IFilterableOptions, IGitOptionsHos Description = "The Azure Pipeline variable name to assign the image paths to" }; + private static readonly Option RegistryOverrideOption = new($"--{ManifestOptions.RegistryOverrideName}") + { + Description = "Alternative registry that overrides the registry defined in each subscription's manifest. " + + "Used together with --source-repo-prefix to redirect external base image lookups to a mirror location." + }; + + private static readonly Option SourceRepoPrefixOption = new("--source-repo-prefix") + { + Description = "Repo prefix used to locate mirrored external base images in the overridden registry " + + "(e.g. 'mirror/'). Combined with --registry-override, external FROM tags are resolved against " + + "'/:' instead of their public source." + }; + public override IEnumerable