diff --git a/eng/docker-tools/CHANGELOG.md b/eng/docker-tools/CHANGELOG.md index 99dbceb69..b2cc7d19f 100644 --- a/eng/docker-tools/CHANGELOG.md +++ b/eng/docker-tools/CHANGELOG.md @@ -4,6 +4,69 @@ All breaking changes and new features in `eng/docker-tools` will be documented i --- +## 2026-05-22: Mirror registry config refactor + +### PublishConfiguration: `InternalMirrorRegistry` and `PublicMirrorRegistry` replaced with `MirrorRegistry` + +`PublishConfiguration` no longer exposes separate `InternalMirrorRegistry` and +`PublicMirrorRegistry` properties. Instead, there is a single `MirrorRegistry` +property. + +The `--source-repo-prefix` CLI option has also been removed in favor of `PublishConfiguration.MirrorRegistry.RepoPrefix`. + +#### How to migrate: + +Replace `InternalMirrorRegistry` and `PublicMirrorRegistry` with a pipeline +template conditional: + +Before: + +```yaml +publishConfig: + ... + InternalMirrorRegistry: + server: $(acr-staging.server) + repoPrefix: $(internalMirrorRepoPrefix) + PublicMirrorRegistry: + server: $(public-mirror.server) + repoPrefix: $(publicMirrorRepoPrefix) + ... +``` + +After: + +```yaml +publishConfig: + ... + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + MirrorRegistry: + server: $(acr-staging.server) + repoPrefix: $(internalMirrorRepoPrefix) + ${{ else }}: + MirrorRegistry: + server: $(public-mirror.server) + repoPrefix: $(publicMirrorRepoPrefix) + ... +``` + +### Base image override options removed from CLI + +The `--base-override-regex` and `--base-override-sub` options have also been +removed from the `build`, `buildMatrix`, `copyBaseImages`, and `getStaleImages` +commands. They are superseded by the properties set in the +`PublishConfiguration`. + +#### How to migrate: + +Use `PublishConfiguration.MirrorRegistry` instead. + +An evaluation of downstream usage of these parameters found only one usage, in +[dotnet-buildtools-prereqs-docker](https://github.com/dotnet/dotnet-buildtools-prereqs-docker/blob/f2842d71c33c10fc5a736988e66911c13d59fb32/eng/pipelines/steps/set-base-image-override-options.yml), +and the usage does not match any current Dockerfiles in that repo, so its +removal will not cause any changes in behavior. + +--- + ## 2026-04-02: Extra Docker build options can be passed through ImageBuilder - Pull request: [#2063](https://github.com/dotnet/docker-tools/pull/2063) diff --git a/eng/docker-tools/templates/jobs/build-images.yml b/eng/docker-tools/templates/jobs/build-images.yml index 7327b6d69..8a35a66ec 100644 --- a/eng/docker-tools/templates/jobs/build-images.yml +++ b/eng/docker-tools/templates/jobs/build-images.yml @@ -115,7 +115,7 @@ jobs: - powershell: | $images = "$(BuildImages.builtImages)" if (-not $images) { return 0 } - $syftImageName = "${{ parameters.publishConfig.PublicMirrorRegistry.server }}/$(imageNames.syft)" + $syftImageName = "$(public-mirror.server)/$(imageNames.syft)" & $(engDockerToolsPath)/Pull-Image.ps1 $syftImageName $images -Split ',' | ForEach-Object { echo "Generating SBOM for $_"; diff --git a/eng/docker-tools/templates/jobs/copy-base-images-staging.yml b/eng/docker-tools/templates/jobs/copy-base-images-staging.yml index 02c6cc84e..6f957ea56 100644 --- a/eng/docker-tools/templates/jobs/copy-base-images-staging.yml +++ b/eng/docker-tools/templates/jobs/copy-base-images-staging.yml @@ -36,5 +36,5 @@ jobs: customInitSteps: ${{ parameters.customInitSteps }} customCopyBaseImagesInitSteps: ${{ parameters.customCopyBaseImagesInitSteps }} additionalOptions: ${{ parameters.additionalOptions }} - acr: ${{ parameters.publishConfig.InternalMirrorRegistry }} + acr: ${{ parameters.publishConfig.MirrorRegistry }} versionsRepoRef: ${{ parameters.versionsRepoRef }} diff --git a/eng/docker-tools/templates/stages/dotnet/publish-config-nonprod.yml b/eng/docker-tools/templates/stages/dotnet/publish-config-nonprod.yml index 0ce87d4d2..3d45fc63a 100644 --- a/eng/docker-tools/templates/stages/dotnet/publish-config-nonprod.yml +++ b/eng/docker-tools/templates/stages/dotnet/publish-config-nonprod.yml @@ -55,13 +55,14 @@ stages: # publishConfig schema is defined in src/ImageBuilder/Configuration/PublishConfiguration.cs. # This will get converted to JSON and placed in appsettings.json to be loaded by ImageBuilder at runtime. publishConfig: - InternalMirrorRegistry: - server: $(acr-staging-test.server) - repoPrefix: $(internalMirrorRepoPrefix) - - PublicMirrorRegistry: - server: $(public-mirror.server) - repoPrefix: $(publicMirrorRepoPrefix) + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + MirrorRegistry: + server: $(acr-staging-test.server) + repoPrefix: $(internalMirrorRepoPrefix) + ${{ else }}: + MirrorRegistry: + server: $(public-mirror.server) + repoPrefix: $(publicMirrorRepoPrefix) BuildRegistry: server: $(acr-staging-test.server) diff --git a/eng/docker-tools/templates/stages/dotnet/publish-config-prod.yml b/eng/docker-tools/templates/stages/dotnet/publish-config-prod.yml index f90e5e806..a57fd9923 100644 --- a/eng/docker-tools/templates/stages/dotnet/publish-config-prod.yml +++ b/eng/docker-tools/templates/stages/dotnet/publish-config-prod.yml @@ -55,13 +55,14 @@ stages: # publishConfig schema is defined in src/ImageBuilder/Configuration/PublishConfiguration.cs. # This will get converted to JSON and placed in appsettings.json to be loaded by ImageBuilder at runtime. publishConfig: - InternalMirrorRegistry: - server: $(acr-staging.server) - repoPrefix: $(internalMirrorRepoPrefix) - - PublicMirrorRegistry: - server: $(public-mirror.server) - repoPrefix: $(publicMirrorRepoPrefix) + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + MirrorRegistry: + server: $(acr-staging.server) + repoPrefix: $(internalMirrorRepoPrefix) + ${{ else }}: + MirrorRegistry: + server: $(public-mirror.server) + repoPrefix: $(publicMirrorRepoPrefix) BuildRegistry: server: $(acr-staging.server) diff --git a/eng/docker-tools/templates/steps/init-common.yml b/eng/docker-tools/templates/steps/init-common.yml index eda572618..99acb8361 100644 --- a/eng/docker-tools/templates/steps/init-common.yml +++ b/eng/docker-tools/templates/steps/init-common.yml @@ -86,20 +86,17 @@ steps: condition: and(succeeded(), ${{ parameters.condition }}) # Build Registry Configuration -# Extends commonMatrixAndBuildOptions with registry-specific settings: -# - Internal builds: Use internal mirror registry prefix and build registry -# server to pull/push from private ACR instead of public MCR -# - Public builds: Override non-MCR base images to use public mirror, reducing -# external dependencies and improving build reliability +# Extends commonMatrixAndBuildOptions with registry-specific settings for +# internal builds: +# - Internal builds push outputs to the build registry instead of MCR. +# - Source mirror selection (for pulling external base images) happens +# automatically inside ImageBuilder based on PublishConfiguration loaded +# from appsettings.json. - ${{ if parameters.publishConfig }}: - powershell: | $commonMatrixAndBuildOptions = "$(commonMatrixAndBuildOptions)" if ("$(System.TeamProject)" -eq "internal" -and "$(Build.Reason)" -ne "PullRequest") { - $commonMatrixAndBuildOptions = "$commonMatrixAndBuildOptions --source-repo-prefix ${{ parameters.publishConfig.InternalMirrorRegistry.repoPrefix }} --registry-override ${{ parameters.publishConfig.BuildRegistry.server }}" - } - - if ("$(System.TeamProject)" -eq "public" -and "$(public-mirror.server)" -ne "") { - $commonMatrixAndBuildOptions = "$commonMatrixAndBuildOptions --base-override-regex '^(?!mcr\.microsoft\.com)' --base-override-sub '$(public-mirror.server)/'" + $commonMatrixAndBuildOptions = "$commonMatrixAndBuildOptions --registry-override ${{ parameters.publishConfig.BuildRegistry.server }}" } Write-Host "Setting commonMatrixAndBuildOptions to '$commonMatrixAndBuildOptions'" diff --git a/eng/pipelines/check-base-image-updates.yml b/eng/pipelines/check-base-image-updates.yml index 23173848a..fa084cda6 100644 --- a/eng/pipelines/check-base-image-updates.yml +++ b/eng/pipelines/check-base-image-updates.yml @@ -1,6 +1,15 @@ trigger: none pr: none +parameters: +- name: bootstrapImageBuilder + displayName: > + Build ImageBuilder from source at the start of every job instead of pulling + the pre-built image from MCR. Use this option during development to + validate changes to ImageBuilder and pipeline templates at the same time. + type: boolean + default: false + schedules: - cron: "0 0,4,8,12,16,20 * * *" displayName: Daily build @@ -21,3 +30,5 @@ extends: - template: /eng/docker-tools/templates/stages/dotnet/publish-config-prod.yml@self parameters: stagesTemplate: /eng/pipelines/templates/stages/check-base-image-updates.yml@self + stagesTemplateParameters: + bootstrapImageBuilder: ${{ parameters.bootstrapImageBuilder }} diff --git a/eng/pipelines/templates/jobs/check-base-image-updates.yml b/eng/pipelines/templates/jobs/check-base-image-updates.yml index 372a44b83..d4ac0f239 100644 --- a/eng/pipelines/templates/jobs/check-base-image-updates.yml +++ b/eng/pipelines/templates/jobs/check-base-image-updates.yml @@ -20,10 +20,12 @@ parameters: # Schema is defined in src/ImageBuilder/Configuration/PublishConfiguration.cs. - name: publishConfig type: object -# Additional arguments to pass to the getStaleImages command (optional). -- name: customGetStaleImagesArgs - type: string - default: "" +# When true, build ImageBuilder from source instead of pulling from MCR. Used +# during development to validate ImageBuilder and pipeline template changes +# together before a new ImageBuilder image is published. +- name: bootstrapImageBuilder + type: boolean + default: false jobs: - job: ${{ parameters.jobName }} @@ -37,32 +39,37 @@ jobs: parameters: dockerClientOS: linux publishConfig: ${{ parameters.publishConfig }} + ${{ if eq(parameters.bootstrapImageBuilder, true) }}: + customInitSteps: + - template: /eng/pipelines/templates/steps/bootstrap-imagebuilder.yml@self - template: /eng/docker-tools/templates/steps/reference-service-connections.yml@self parameters: publishConfig: ${{ parameters.publishConfig }} usesRegistries: - - ${{ parameters.publishConfig.InternalMirrorRegistry.server }} + - ${{ parameters.publishConfig.MirrorRegistry.server }} - template: /eng/docker-tools/templates/steps/copy-base-images.yml@self parameters: - acr: ${{ parameters.publishConfig.InternalMirrorRegistry }} + acr: ${{ parameters.publishConfig.MirrorRegistry }} additionalOptions: "--subscriptions-path '${{ parameters.subscriptionsPath }}'" - script: > - $(runImageBuilderCmd) + $(runAuthedImageBuilderCmd) getStaleImages $(dotnetDockerBot.userName) $(dotnetDockerBot.email) --gh-token $(BotAccount-dotnet-docker-bot-PAT) staleImagePaths - ${{ parameters.customGetStaleImagesArgs }} --subscriptions-path ${{ parameters.subscriptionsPath }} --os-type '*' --architecture '*' $(dockerHubRegistryCreds) displayName: Get Stale Images name: GetStaleImages + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + SYSTEM_OIDCREQUESTURI: $(System.OidcRequestUri) - script: > $(runImageBuilderCmd) diff --git a/eng/pipelines/templates/jobs/copy-base-images-public-mirror.yml b/eng/pipelines/templates/jobs/copy-base-images-public-mirror.yml index 7db34130b..27dd8c942 100644 --- a/eng/pipelines/templates/jobs/copy-base-images-public-mirror.yml +++ b/eng/pipelines/templates/jobs/copy-base-images-public-mirror.yml @@ -22,7 +22,9 @@ jobs: image: $(default1ESInternalPoolImage) os: linux publishConfig: ${{ parameters.publishConfig }} - acr: ${{ parameters.publishConfig.PublicMirrorRegistry }} + acr: + server: $(public-mirror.server) + repoPrefix: $(publicMirrorRepoPrefix) customInitSteps: ${{ parameters.customInitSteps }} additionalOptions: '--subscriptions-path ${{ parameters.subscriptionsPath }}' forceDryRun: ${{ parameters.dryRun }} diff --git a/eng/pipelines/templates/stages/check-base-image-updates.yml b/eng/pipelines/templates/stages/check-base-image-updates.yml index d2c857247..e5c877d60 100644 --- a/eng/pipelines/templates/stages/check-base-image-updates.yml +++ b/eng/pipelines/templates/stages/check-base-image-updates.yml @@ -12,6 +12,12 @@ parameters: # The name of the public Azure DevOps project (e.g., "public"). - name: publicProjectName type: string +# When true, build ImageBuilder from source instead of pulling from MCR. Used +# during development to validate ImageBuilder and pipeline template changes +# together before a new ImageBuilder image is published. +- name: bootstrapImageBuilder + type: boolean + default: false stages: - stage: CheckBaseImages @@ -26,12 +32,13 @@ stages: publicProjectName: ${{ parameters.publicProjectName }} internalProjectName: ${{ parameters.internalProjectName }} publishConfig: ${{ parameters.publishConfig }} + bootstrapImageBuilder: ${{ parameters.bootstrapImageBuilder }} - template: /eng/pipelines/templates/jobs/check-base-image-updates.yml@self parameters: jobName: CheckBaseImages_BuildTools subscriptionsPath: eng/check-base-image-subscriptions-buildtools.json - customGetStaleImagesArgs: --base-override-regex '^((centos|debian|ubuntu):.+)' --base-override-sub '$(overrideRegistry)/$1' publicProjectName: ${{ parameters.publicProjectName }} internalProjectName: ${{ parameters.internalProjectName }} publishConfig: ${{ parameters.publishConfig }} + bootstrapImageBuilder: ${{ parameters.bootstrapImageBuilder }} diff --git a/eng/pipelines/templates/stages/cleanup-acr-images.yml b/eng/pipelines/templates/stages/cleanup-acr-images.yml index 1563f6833..fa3582da5 100644 --- a/eng/pipelines/templates/stages/cleanup-acr-images.yml +++ b/eng/pipelines/templates/stages/cleanup-acr-images.yml @@ -86,7 +86,7 @@ stages: publishConfig: ${{ parameters.publishConfig }} usesRegistries: - ${{ parameters.publishConfig.BuildRegistry.server }} - - ${{ parameters.publishConfig.PublicMirrorRegistry.server }} + - $(public-mirror.server) serviceConnections: - name: $(marStatus.serviceConnectionName) - script: mkdir -p $(Build.ArtifactStagingDirectory)/eol-annotation-data @@ -96,7 +96,9 @@ stages: acr: ${{ parameters.publishConfig.BuildRegistry }} - template: /eng/pipelines/templates/steps/set-eol-annotations.yml@self parameters: - acr: ${{ parameters.publishConfig.PublicMirrorRegistry }} + acr: + server: $(public-mirror.server) + repoPrefix: $(publicMirrorRepoPrefix) - template: /eng/docker-tools/templates/steps/publish-artifact.yml@self parameters: path: $(Build.ArtifactStagingDirectory)/eol-annotation-data diff --git a/src/ImageBuilder.Tests/BuildCommandTests.cs b/src/ImageBuilder.Tests/BuildCommandTests.cs index ba056b212..8098666ec 100644 --- a/src/ImageBuilder.Tests/BuildCommandTests.cs +++ b/src/ImageBuilder.Tests/BuildCommandTests.cs @@ -3100,13 +3100,20 @@ public async Task BuildCommand_MirroredImages(bool hasCachedImage, string srcBas gitService: gitServiceMock.Object, copyImageService: copyImageServiceMock.Object, manifestServiceFactory: CreateManifestServiceFactoryMock(manifestServiceMock).Object, - imageCacheService: new ImageCacheService(Mock.Of>(), gitServiceMock.Object)); + imageCacheService: new ImageCacheService(Mock.Of>(), gitServiceMock.Object), + publishConfiguration: new PublishConfiguration + { + MirrorRegistry = new RegistryEndpoint + { + Server = RegistryOverride, + RepoPrefix = SourceRepoPrefix, + }, + }); command.Options.Manifest = Path.Combine(tempFolderContext.Path, "manifest.json"); command.Options.ImageInfoOutputPath = Path.Combine(tempFolderContext.Path, "image-info.json"); command.Options.ImageInfoSourcePath = Path.Combine(tempFolderContext.Path, "src-image-info.json"); command.Options.IsPushEnabled = true; command.Options.SourceRepoUrl = "https://github.com/dotnet/test"; - command.Options.SourceRepoPrefix = SourceRepoPrefix; command.Options.RegistryOverride = RegistryOverride; command.Options.RepoPrefix = RepoPrefix; @@ -3443,13 +3450,20 @@ public async Task BuildCommand_MirroredImages_External(string baseImageRegistry, gitService: gitServiceMock.Object, copyImageService: Mock.Of(), manifestServiceFactory: CreateManifestServiceFactoryMock(manifestServiceMock).Object, - imageCacheService: new ImageCacheService(Mock.Of>(), gitServiceMock.Object)); + imageCacheService: new ImageCacheService(Mock.Of>(), gitServiceMock.Object), + publishConfiguration: new PublishConfiguration + { + MirrorRegistry = new RegistryEndpoint + { + Server = RegistryOverride, + RepoPrefix = SourceRepoPrefix, + }, + }); command.Options.Manifest = Path.Combine(tempFolderContext.Path, "manifest.json"); command.Options.ImageInfoOutputPath = Path.Combine(tempFolderContext.Path, "image-info.json"); command.Options.IsPushEnabled = true; command.Options.SourceRepoUrl = "https://github.com/dotnet/test"; command.Options.RegistryOverride = RegistryOverride; - command.Options.SourceRepoPrefix = SourceRepoPrefix; const string ProductVersion = "1.0.1"; @@ -3501,145 +3515,6 @@ public async Task BuildCommand_MirroredImages_External(string baseImageRegistry, dockerServiceMock.VerifyNoOtherCalls(); } - /// - /// Tests the logic of image mirroring when the base image tag is configured to be overriden. - /// - [Fact] - public async Task BuildCommand_MirroredImages_BaseImageTagOverride() - { - const string RegistryOverride = "dotnetdocker.azurecr.io"; - const string SamplesRepo = "samples"; - const string RuntimeDigest = "sha256:adc914a9f125ca612f9a67e4a0551937b7a37c82fabb46172c4867b73ed99227"; - const string SampleDigest = "sha256:781914a9f125ca612f9a67e4a0551937b7a37c82fabb46172c4867b73ed0045a"; - const string ImageTag = "tag"; - const string SourceRepoPrefix = "mirror/"; - const string CustomRegistry = "contoso.azurecr.io"; - const string SourceRepo = "amd64/os"; - const string SrcBaseTag = $"{SourceRepo}:tag"; - const string MirroredBaseTag = "os:tag"; - - using TempFolderContext tempFolderContext = TestHelper.UseTempFolder(); - Mock dockerServiceMock = CreateDockerServiceMock(); - - string baseImageRepoPrefix = $"{RegistryOverride}/{SourceRepoPrefix}{CustomRegistry}"; - - Mock manifestServiceMock = CreateManifestServiceMock([ - new($"{baseImageRepoPrefix}/{MirroredBaseTag}", $"{baseImageRepoPrefix}/{MirroredBaseTag}@{RuntimeDigest}"), - new($"{RegistryOverride}/{SamplesRepo}:{ImageTag}", $"{RegistryOverride}/{SamplesRepo}@{SampleDigest}"), - ], []); - - const string dockerfileCommitSha = "mycommit"; - Mock gitServiceMock = new(); - gitServiceMock - .Setup(o => o.GetCommitSha(It.IsAny(), It.IsAny())) - .Returns(dockerfileCommitSha); - - BuildCommand command = CreateBuildCommand( - dockerService: dockerServiceMock.Object, - gitService: gitServiceMock.Object, - copyImageService: Mock.Of(), - manifestServiceFactory: CreateManifestServiceFactoryMock(manifestServiceMock).Object, - imageCacheService: new ImageCacheService(Mock.Of>(), gitServiceMock.Object)); - command.Options.Manifest = Path.Combine(tempFolderContext.Path, "manifest.json"); - command.Options.ImageInfoOutputPath = Path.Combine(tempFolderContext.Path, "image-info.json"); - command.Options.IsPushEnabled = true; - command.Options.SourceRepoUrl = "https://github.com/dotnet/test"; - command.Options.RegistryOverride = RegistryOverride; - command.Options.SourceRepoPrefix = SourceRepoPrefix; - command.Options.BaseImageOverrideOptions.RegexPattern = @".*\/(.*)"; - command.Options.BaseImageOverrideOptions.Substitution = $"{CustomRegistry}/$1"; - - Manifest manifest = CreateManifest( - CreateRepo(SamplesRepo, - CreateImage( - new Platform[] - { - CreatePlatform( - DockerfileHelper.CreateDockerfile( - "1.0/samples/os", tempFolderContext, SrcBaseTag), - new string[] { ImageTag }) - }, - productVersion: "1.0.1")) - ); - manifest.Registry = "mcr.microsoft.com"; - - File.WriteAllText(Path.Combine(tempFolderContext.Path, command.Options.Manifest), JsonConvert.SerializeObject(manifest)); - - command.LoadManifest(); - await command.ExecuteAsync(); - - ImageArtifactDetails imageArtifactDetails = new() - { - Repos = - { - new RepoData - { - Repo = "samples", - Images = - { - new ImageData - { - ProductVersion = "1.0.1", - Platforms = - { - new PlatformData - { - Dockerfile = "1.0/samples/os/Dockerfile", - Architecture = "amd64", - OsType = "Linux", - OsVersion = "noble", - Digest = $"{manifest.Registry}/{SamplesRepo}@{SampleDigest}", - - // This is the key change. The digest should be referring to the image from the - // override, not what's defined in the Dockerfile. - BaseImageDigest = $"{CustomRegistry}/os@{RuntimeDigest}", - - Created = DateTime.MinValue.ToUniversalTime(), - CommitUrl = $"{command.Options.SourceRepoUrl}/blob/{dockerfileCommitSha}/1.0/samples/os/Dockerfile", - SimpleTags = - { - ImageTag - }, - } - } - } - } - } - } - }; - - string expectedOutput = JsonHelper.SerializeObject(imageArtifactDetails); - string actualOutput = File.ReadAllText(command.Options.ImageInfoOutputPath); - - Assert.Equal(expectedOutput, actualOutput); - - dockerServiceMock.Verify( - o => o.BuildImage( - It.IsAny(), - It.IsAny(), - It.IsAny(), - It.IsAny>(), - It.IsAny>(), - It.IsAny>(), - It.IsAny(), - It.IsAny())); - dockerServiceMock.Verify( - o => o.GetImageSize($"{RegistryOverride}/{SamplesRepo}:{ImageTag}", false)); - - dockerServiceMock.Verify(o => o.PullImage($"{baseImageRepoPrefix}/{MirroredBaseTag}", "linux/amd64", false)); - manifestServiceMock.Verify(o => o.GetLocalImageDigestAsync($"{baseImageRepoPrefix}/{MirroredBaseTag}", false)); - manifestServiceMock.Verify(o => o.GetLocalImageDigestAsync($"{RegistryOverride}/{SamplesRepo}:{ImageTag}", false)); - dockerServiceMock.Verify(o => o.PushImage($"{RegistryOverride}/{SamplesRepo}:{ImageTag}", false)); - dockerServiceMock.Verify(o => o.GetCreatedDate($"{RegistryOverride}/{SamplesRepo}:{ImageTag}", false)); - manifestServiceMock.Verify(o => o.GetImageLayersAsync($"{RegistryOverride}/{SamplesRepo}:{ImageTag}", false)); - - dockerServiceMock.Verify(o => - o.CreateTag($"{baseImageRepoPrefix}/{MirroredBaseTag}", SrcBaseTag, false)); - dockerServiceMock.Verify(o => o.GetImageArch($"{baseImageRepoPrefix}/{MirroredBaseTag}", false)); - - dockerServiceMock.VerifyNoOtherCalls(); - } - #nullable enable private static BuildCommand CreateBuildCommand( IManifestJsonService? manifestJsonService = null, @@ -3651,7 +3526,8 @@ private static BuildCommand CreateBuildCommand( IManifestServiceFactory? manifestServiceFactory = null, IRegistryCredentialsProvider? registryCredentialsProvider = null, IAzureTokenCredentialProvider? azureTokenCredentialProvider = null, - IImageCacheService? imageCacheService = null) + IImageCacheService? imageCacheService = null, + PublishConfiguration? publishConfiguration = null) { BuildCommand command = new( manifestJsonService ?? TestHelper.CreateManifestJsonService(), @@ -3663,7 +3539,8 @@ private static BuildCommand CreateBuildCommand( manifestServiceFactory ?? Mock.Of(), registryCredentialsProvider ?? Mock.Of(), azureTokenCredentialProvider ?? Mock.Of(), - imageCacheService ?? Mock.Of()); + imageCacheService ?? Mock.Of(), + Microsoft.Extensions.Options.Options.Create(publishConfiguration ?? new PublishConfiguration())); return command; } diff --git a/src/ImageBuilder.Tests/CopyBaseImagesCommandTests.cs b/src/ImageBuilder.Tests/CopyBaseImagesCommandTests.cs index e1525d39f..d2e0581ba 100644 --- a/src/ImageBuilder.Tests/CopyBaseImagesCommandTests.cs +++ b/src/ImageBuilder.Tests/CopyBaseImagesCommandTests.cs @@ -101,78 +101,5 @@ public async Task MultipleBaseTags() copyImageServiceMock.VerifyNoOtherCalls(); } - - /// - /// Verifies that we can dynamically override the base image tag that's defined in the Dockerfile with a custom one - /// that is used for the purposes of copying. - /// - /// - /// In this test, the Dockerfiles contains: - /// FROM arm32v7/os:tag - /// But we want to override that so we don't use that tag as the source of the copy but rather from a custom location. - /// So it's configured to be overriden to use contoso.azurecr.io/os:tag as the source tag. This ends up getting copied - /// to mcr.microsoft.com/custom-repo/contoso.azurecr.io/os:tag. - /// - [Fact] - public async Task OverridenBaseTag() - { - const string CustomRegistry = "contoso.azurecr.io"; - - using TempFolderContext tempFolderContext = TestHelper.UseTempFolder(); - - Mock copyImageServiceMock = new(); - - CopyBaseImagesCommand command = new( - TestHelper.CreateManifestJsonService(), - copyImageServiceMock.Object, - Mock.Of>(), - Mock.Of()); - command.Options.Manifest = Path.Combine(tempFolderContext.Path, "manifest.json"); - command.Options.RepoPrefix = "custom-repo/"; - command.Options.CredentialsOptions.Credentials.Add("docker.io", new RegistryCredentials("user", "pass")); - command.Options.BaseImageOverrideOptions.RegexPattern = @".*\/(os:.*)"; - command.Options.BaseImageOverrideOptions.Substitution = $"{CustomRegistry}/$1"; - - const string runtimeRelativeDir = "1.0/runtime/os"; - Directory.CreateDirectory(Path.Combine(tempFolderContext.Path, runtimeRelativeDir)); - string dockerfileRelativePath = Path.Combine(runtimeRelativeDir, "Dockerfile.custom"); - File.WriteAllText(Path.Combine(tempFolderContext.Path, dockerfileRelativePath), "FROM repo:tag"); - - Manifest manifest = CreateManifest( - CreateRepo("runtime", - CreateImage( - CreatePlatform( - CreateDockerfile("1.0/runtime/os/arm32v7", tempFolderContext, "arm32v7/os:tag"), - new string[] { "os" }), - CreatePlatform( - CreateDockerfile("1.0/runtime/os2/arm32v7", tempFolderContext, "arm32v7/os2:tag"), - new string[] { "os2" }))) - ); - manifest.Registry = DestinationRegistry; - - File.WriteAllText(Path.Combine(tempFolderContext.Path, command.Options.Manifest), JsonConvert.SerializeObject(manifest)); - - command.LoadManifest(); - await command.ExecuteAsync(); - - var expectedTagInfos = new(string SourceImage, string TargetTag, string Registry, string Username, string Password)[] - { - ( $"os:tag", $"{command.Options.RepoPrefix}{CustomRegistry}/os:tag", CustomRegistry, null, null), - ( "arm32v7/os2:tag", $"{command.Options.RepoPrefix}arm32v7/os2:tag", "docker.io", "user", "pass" ), - }; - - foreach (var expectedTagInfo in expectedTagInfos) - { - copyImageServiceMock.Verify(o => - o.ImportImageAsync( - new string[] { expectedTagInfo.TargetTag }, - manifest.Registry, - expectedTagInfo.SourceImage, - false, - expectedTagInfo.Registry, - It.Is(creds => (creds == null && expectedTagInfo.Username == null) || (creds.Username == expectedTagInfo.Username && creds.Password == expectedTagInfo.Password)), - false)); - } - } } } diff --git a/src/ImageBuilder.Tests/GenerateBuildMatrixCommandTests.cs b/src/ImageBuilder.Tests/GenerateBuildMatrixCommandTests.cs index d017fd0e1..8ca7cb889 100644 --- a/src/ImageBuilder.Tests/GenerateBuildMatrixCommandTests.cs +++ b/src/ImageBuilder.Tests/GenerateBuildMatrixCommandTests.cs @@ -10,10 +10,12 @@ using System.Net.Http; using System.Threading.Tasks; using Microsoft.DotNet.ImageBuilder.Commands; +using Microsoft.DotNet.ImageBuilder.Configuration; using Microsoft.DotNet.ImageBuilder.Models.Image; using Microsoft.DotNet.ImageBuilder.Models.Manifest; using Microsoft.DotNet.ImageBuilder.Tests.Helpers; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Moq; using Newtonsoft.Json; using Xunit; @@ -257,7 +259,7 @@ public async Task FilterOutCachedImages( SetCacheResult(imageCacheServiceMock, dockerfileRuntime2Path, ImageCacheState.NotCached); SetCacheResult(imageCacheServiceMock, dockerfileSdk2Path, ImageCacheState.NotCached); - GenerateBuildMatrixCommand command = new(TestHelper.CreateManifestJsonService(), imageCacheServiceMock.Object, Mock.Of(), Mock.Of>()); + GenerateBuildMatrixCommand command = new(TestHelper.CreateManifestJsonService(), imageCacheServiceMock.Object, Mock.Of(), Mock.Of>(), Microsoft.Extensions.Options.Options.Create(new PublishConfiguration())); command.Options.Manifest = Path.Combine(tempFolderContext.Path, "manifest.json"); command.Options.MatrixType = MatrixType.PlatformDependencyGraph; command.Options.ImageInfoPath = Path.Combine(tempFolderContext.Path, "imageinfo.json"); @@ -1677,14 +1679,14 @@ private static GenerateBuildMatrixCommand SetupTrimCacheTest( TestHelper.CreateManifestJsonService(), imageCacheService, manifestServiceFactoryMock.Object, - Mock.Of>()); + Mock.Of>(), + Microsoft.Extensions.Options.Options.Create(new PublishConfiguration())); command.Options.Manifest = Path.Combine(tempFolderContext.Path, "manifest.json"); command.Options.MatrixType = MatrixType.PlatformDependencyGraph; command.Options.ImageInfoPath = Path.Combine(tempFolderContext.Path, "imageinfo.json"); command.Options.TrimCachedImages = true; command.Options.SourceRepoUrl = sourceRepoUrl; - command.Options.SourceRepoPrefix = "mirror/"; File.WriteAllText( Path.Combine(tempFolderContext.Path, command.Options.Manifest), @@ -1720,6 +1722,6 @@ private static GenerateBuildMatrixCommand SetupTrimCacheTest( } private static GenerateBuildMatrixCommand CreateCommand() => - new(TestHelper.CreateManifestJsonService(), Mock.Of(), Mock.Of(), Mock.Of>()); + new(TestHelper.CreateManifestJsonService(), Mock.Of(), Mock.Of(), Mock.Of>(), Microsoft.Extensions.Options.Options.Create(new PublishConfiguration())); } } diff --git a/src/ImageBuilder.Tests/GetStaleImagesCommandTests.cs b/src/ImageBuilder.Tests/GetStaleImagesCommandTests.cs index 868cb53dc..b086f230c 100644 --- a/src/ImageBuilder.Tests/GetStaleImagesCommandTests.cs +++ b/src/ImageBuilder.Tests/GetStaleImagesCommandTests.cs @@ -13,6 +13,7 @@ using System.Threading.Tasks; using LibGit2Sharp; using Microsoft.DotNet.ImageBuilder.Commands; +using Microsoft.DotNet.ImageBuilder.Configuration; using Microsoft.DotNet.ImageBuilder.Models.Image; using Microsoft.DotNet.ImageBuilder.Models.Manifest; using Microsoft.DotNet.ImageBuilder.Models.Subscription; @@ -1521,98 +1522,6 @@ public async Task GetStaleImagesCommand_SharedDockerfile() } } - /// - /// Verifies that the check for a stale base image is done by targeting the tag override rather than the tag - /// defined in the Dockerfile. - /// - [Fact] - public async Task GetStaleImagesCommand_BaseImageTagOverride() - { - const string repo1 = "test-repo"; - const string dockerfile1Path = "dockerfile1/Dockerfile"; - const string dockerfile2Path = "dockerfile2/Dockerfile"; - const string CustomRegistry = "my-registry.io"; - - SubscriptionInfo[] subscriptionInfos = new SubscriptionInfo[] - { - new SubscriptionInfo( - CreateSubscription(repo1), - CreateManifest( - CreateRepo( - repo1, - CreateImage( - CreatePlatform(dockerfile1Path, new string[] { "tag1" }), - CreatePlatform(dockerfile2Path, new string[] { "tag2" })))), - new ImageArtifactDetails - { - Repos = - { - new RepoData - { - Repo = repo1, - Images = - { - new ImageData - { - Platforms = - { - CreatePlatform( - dockerfile1Path, - baseImageDigest: $"{CustomRegistry}/base1@base1digest", - simpleTags: new List { "tag1" }), - CreatePlatform( - dockerfile2Path, - baseImageDigest: $"{CustomRegistry}/base2@alternate-base2digest", - simpleTags: new List { "tag2" }) - } - } - } - } - } - } - ) - }; - - Dictionary> dockerfileInfos = new() - { - { - subscriptionInfos[0].Subscription.Manifest, - new List - { - new DockerfileInfo(dockerfile1Path, new FromImageInfo("base1", "base1digest")), - new DockerfileInfo(dockerfile2Path, new FromImageInfo("base2", "base2digest")) - } - } - }; - - using (TestContext context = new(subscriptionInfos, dockerfileInfos)) - { - context.ImageDigests.Add($"{CustomRegistry}/base1", "alternate-base1digest"); - context.ImageDigests.Add($"{CustomRegistry}/base2", "alternate-base2digest"); - - // Override the image tags to target a custom registry - context.Command.Options.BaseImageOverrideOptions.RegexPattern = "(base.*)"; - context.Command.Options.BaseImageOverrideOptions.Substitution = "my-registry.io/$1"; - - await context.ExecuteCommandAsync(); - - // Only one of the images has a changed digest - // It should be comparing against the digest of the image from the override. - Dictionary> expectedPathsBySubscription = new() - { - { - subscriptionInfos[0].Subscription, - new List - { - dockerfile1Path - } - } - }; - - context.Verify(expectedPathsBySubscription); - } - } - /// /// Use this method to generate a unique repo owner name for the tests. This ensures that each test /// uses a different name and prevents collisions when running the tests in parallel. This is because @@ -1768,7 +1677,12 @@ private string SerializeJsonObjectToTempFile(object jsonObject) private GetStaleImagesCommand CreateCommand() { GetStaleImagesCommand command = new( - this.ManifestServiceFactoryMock.Object, TestHelper.CreateManifestJsonService(), this.loggerServiceMock.Object, this.octokitClientFactory, this.gitService); + this.ManifestServiceFactoryMock.Object, + TestHelper.CreateManifestJsonService(), + this.loggerServiceMock.Object, + this.octokitClientFactory, + this.gitService, + Microsoft.Extensions.Options.Options.Create(new PublishConfiguration())); command.Options.SubscriptionOptions.SubscriptionsPath = this.subscriptionsPath; command.Options.VariableName = VariableName; command.Options.FilterOptions.Platform.OsType = this.osType; diff --git a/src/ImageBuilder.Tests/PublishConfigurationBindingTests.cs b/src/ImageBuilder.Tests/PublishConfigurationBindingTests.cs index 0220524e3..ffd47703a 100644 --- a/src/ImageBuilder.Tests/PublishConfigurationBindingTests.cs +++ b/src/ImageBuilder.Tests/PublishConfigurationBindingTests.cs @@ -32,11 +32,9 @@ public class PublishConfigurationBindingTests "PublishRegistry": { "Server": "publish.azurecr.io" }, - "InternalMirrorRegistry": { - "Server": "internal-mirror.azurecr.io" - }, - "PublicMirrorRegistry": { - "Server": "public-mirror.azurecr.io" + "MirrorRegistry": { + "Server": "mirror.azurecr.io", + "RepoPrefix": "mirror/" }, "RegistryAuthentication": [ { @@ -77,11 +75,9 @@ public void AddPublishConfiguration_BindsAllRegistryEndpoints() config.PublishRegistry.ShouldNotBeNull(); config.PublishRegistry.Server.ShouldBe("publish.azurecr.io"); - config.InternalMirrorRegistry.ShouldNotBeNull(); - config.InternalMirrorRegistry.Server.ShouldBe("internal-mirror.azurecr.io"); - - config.PublicMirrorRegistry.ShouldNotBeNull(); - config.PublicMirrorRegistry.Server.ShouldBe("public-mirror.azurecr.io"); + config.MirrorRegistry.ShouldNotBeNull(); + config.MirrorRegistry.Server.ShouldBe("mirror.azurecr.io"); + config.MirrorRegistry.RepoPrefix.ShouldBe("mirror/"); } [Fact] @@ -117,11 +113,10 @@ public void AddPublishConfiguration_GetKnownRegistries_ReturnsAllEndpoints() PublishConfiguration config = BuildConfiguration(FullConfigJson); var knownRegistries = config.GetKnownRegistries().ToList(); - knownRegistries.Count.ShouldBe(4); + knownRegistries.Count.ShouldBe(3); knownRegistries.ShouldContain(r => r.Server == "build.azurecr.io"); knownRegistries.ShouldContain(r => r.Server == "publish.azurecr.io"); - knownRegistries.ShouldContain(r => r.Server == "internal-mirror.azurecr.io"); - knownRegistries.ShouldContain(r => r.Server == "public-mirror.azurecr.io"); + knownRegistries.ShouldContain(r => r.Server == "mirror.azurecr.io"); } [Fact] @@ -174,8 +169,7 @@ public void AddPublishConfiguration_PartialConfig_HandlesNulls() config.BuildRegistry.ShouldNotBeNull(); config.BuildRegistry.Server.ShouldBe("build.azurecr.io"); config.PublishRegistry.ShouldBeNull(); - config.InternalMirrorRegistry.ShouldBeNull(); - config.PublicMirrorRegistry.ShouldBeNull(); + config.MirrorRegistry.ShouldBeNull(); config.RegistryAuthentication.ShouldBeEmpty(); } @@ -192,8 +186,7 @@ public void AddPublishConfiguration_EmptyConfig_DefaultsToEmpty() config.BuildRegistry.ShouldBeNull(); config.PublishRegistry.ShouldBeNull(); - config.InternalMirrorRegistry.ShouldBeNull(); - config.PublicMirrorRegistry.ShouldBeNull(); + config.MirrorRegistry.ShouldBeNull(); config.RegistryAuthentication.ShouldBeEmpty(); } @@ -206,7 +199,7 @@ public void AddPublishConfiguration_SharedAuthentication_MultipleRegistriesCanUs "BuildRegistry": { "Server": "shared.azurecr.io" }, - "InternalMirrorRegistry": { + "MirrorRegistry": { "Server": "shared.azurecr.io" }, "RegistryAuthentication": [ @@ -225,7 +218,7 @@ public void AddPublishConfiguration_SharedAuthentication_MultipleRegistriesCanUs PublishConfiguration config = BuildConfiguration(sharedAuthJson); var buildAuth = config.FindRegistryAuthentication(config.BuildRegistry!.Server!); - var mirrorAuth = config.FindRegistryAuthentication(config.InternalMirrorRegistry!.Server!); + var mirrorAuth = config.FindRegistryAuthentication(config.MirrorRegistry!.Server!); buildAuth.ShouldNotBeNull(); mirrorAuth.ShouldNotBeNull(); diff --git a/src/ImageBuilder/Commands/BaseImageOverrideOptions.cs b/src/ImageBuilder/Commands/BaseImageOverrideOptions.cs deleted file mode 100644 index 3f7d07f0c..000000000 --- a/src/ImageBuilder/Commands/BaseImageOverrideOptions.cs +++ /dev/null @@ -1,69 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.Collections.Generic; -using System.CommandLine; -using System.CommandLine.Parsing; -using System.Text.RegularExpressions; - -namespace Microsoft.DotNet.ImageBuilder.Commands; - -/// -/// Defines options that allow the caller to configure whether and how base image tags defined in a Dockerfile -/// are to be overriden. -/// -/// -/// This allows for images to be sourced from a different location than described in the Dockerfile. -/// For example, the Build command implements this by pulling an image from the overriden location, retagging it with the -/// tag used in the Dockerfile, and continue with the rest of the build. -/// -public class BaseImageOverrideOptions -{ - public const string BaseOverrideRegexName = "base-override-regex"; - public const string BaseOverrideSubName = "base-override-sub"; - - public string? RegexPattern { get; set; } - - public string? Substitution { get; set; } - - private static readonly Option RegexPatternOption = new($"--{BaseOverrideRegexName}") - { - Description = $"Regular expression identifying base image tags to apply string substitution to (requires {BaseOverrideSubName} to be set)" - }; - - private static readonly Option SubstitutionOption = new($"--{BaseOverrideSubName}") - { - Description = $"Regular expression substitution that overrides a matching base image tag (requires {BaseOverrideRegexName} to be set)" - }; - - public void Validate() - { - if ((RegexPattern is null) != (Substitution is null)) - { - throw new InvalidOperationException( - $"The '{BaseOverrideRegexName}' and '{BaseOverrideSubName}' options must both be set."); - } - } - - public string ApplyBaseImageOverride(string imageName) - { - if (RegexPattern is not null && Substitution is not null) - { - return Regex.Replace(imageName, RegexPattern, Substitution); - } - - return imageName; - } - - public IEnumerable