From 49aef844927380913e0c00201466d094de58084f Mon Sep 17 00:00:00 2001 From: Mathiyarasy <157102811+Mathiyarasy@users.noreply.github.com> Date: Wed, 13 May 2026 10:27:19 +0000 Subject: [PATCH 1/2] Include preprocessing file --- src/spec-node/dockerCompose.ts | 12 ++- src/spec-node/dockerfilePreprocess.ts | 84 +++++++++++++++++ src/spec-node/imageMetadata.ts | 7 +- src/spec-node/singleContainer.ts | 6 +- .../configs/podman-test/.devcontainer.json | 5 + .../configs/podman-test/cpp.Dockerfile.in | 2 + src/test/configs/podman-test/tools.Dockerfile | 2 + src/test/dockerfilePreprocess.test.ts | 93 +++++++++++++++++++ 8 files changed, 206 insertions(+), 5 deletions(-) create mode 100644 src/spec-node/dockerfilePreprocess.ts create mode 100644 src/test/configs/podman-test/.devcontainer.json create mode 100644 src/test/configs/podman-test/cpp.Dockerfile.in create mode 100644 src/test/configs/podman-test/tools.Dockerfile create mode 100644 src/test/dockerfilePreprocess.test.ts diff --git a/src/spec-node/dockerCompose.ts b/src/spec-node/dockerCompose.ts index 8093464cc..38201a188 100644 --- a/src/spec-node/dockerCompose.ts +++ b/src/spec-node/dockerCompose.ts @@ -19,6 +19,7 @@ import { Mount, parseMount } from '../spec-configuration/containerFeaturesConfig import path from 'path'; import { getDevcontainerMetadata, getImageBuildInfoFromDockerfile, getImageBuildInfoFromImage, getImageMetadataFromContainer, ImageBuildInfo, lifecycleCommandOriginMapFromMetadata, mergeConfiguration, MergedDevContainerConfig } from './imageMetadata'; import { ensureDockerfileHasFinalStageName } from './dockerfileUtils'; +import { resolveDockerfileIncludesIfNeeded } from './dockerfilePreprocess'; import { randomUUID } from 'crypto'; const projectLabel = 'com.docker.compose.project'; @@ -163,11 +164,16 @@ export async function buildAndExtendDockerCompose(configWithRaw: SubstitutedConf let baseName = 'dev_container_auto_added_stage_label'; let dockerfile: string | undefined; let imageBuildInfo: ImageBuildInfo; + let preprocessedDockerfilePathForComposeBuild: string | undefined; const serviceInfo = getBuildInfoForService(composeService, cliHost.path, localComposeFiles); if (serviceInfo.build) { const { context, dockerfilePath, target } = serviceInfo.build; const resolvedDockerfilePath = cliHost.path.isAbsolute(dockerfilePath) ? dockerfilePath : path.resolve(context, dockerfilePath); - const originalDockerfile = (await cliHost.readFile(resolvedDockerfilePath)).toString(); + const resolvedDockerfile = await resolveDockerfileIncludesIfNeeded(cliHost, resolvedDockerfilePath); + const originalDockerfile = resolvedDockerfile.effectiveDockerfileContent; + if (resolvedDockerfile.preprocessed) { + preprocessedDockerfilePathForComposeBuild = resolvedDockerfile.effectiveDockerfilePath; + } dockerfile = originalDockerfile; if (target) { // Explictly set build target for the dev container build features on that @@ -194,6 +200,10 @@ export async function buildAndExtendDockerCompose(configWithRaw: SubstitutedConf let overrideImageName: string | undefined; let buildOverrideContent = ''; + if (preprocessedDockerfilePathForComposeBuild && !extendImageBuildInfo?.featureBuildInfo) { + buildOverrideContent += ' build:\n'; + buildOverrideContent += ` dockerfile: ${preprocessedDockerfilePathForComposeBuild}\n`; + } if (extendImageBuildInfo?.featureBuildInfo) { // Avoid retagging a previously pulled image. if (!serviceInfo.build) { diff --git a/src/spec-node/dockerfilePreprocess.ts b/src/spec-node/dockerfilePreprocess.ts new file mode 100644 index 000000000..6c86e813e --- /dev/null +++ b/src/spec-node/dockerfilePreprocess.ts @@ -0,0 +1,84 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { randomUUID } from 'crypto'; +import { CLIHost } from '../spec-common/cliHost'; +import { ContainerError } from '../spec-common/errors'; + +const includeLine = /^\s*#include\s+"([^"]+)"\s*$/; + +export interface ResolvedDockerfile { + originalDockerfilePath: string; + effectiveDockerfilePath: string; + effectiveDockerfileContent: string; + preprocessed: boolean; +} + +export async function resolveDockerfileIncludesIfNeeded(cliHost: CLIHost, dockerfilePath: string): Promise { + const dockerfileText = (await cliHost.readFile(dockerfilePath)).toString(); + if (!dockerfilePath.toLowerCase().endsWith('.in')) { + return { + originalDockerfilePath: dockerfilePath, + effectiveDockerfilePath: dockerfilePath, + effectiveDockerfileContent: dockerfileText, + preprocessed: false, + }; + } + + const effectiveDockerfileContent = await preprocessDockerfileIncludes(cliHost, dockerfilePath, []); + const preprocessedDockerfilePath = await writePreprocessedDockerfile(cliHost, dockerfilePath, effectiveDockerfileContent); + + return { + originalDockerfilePath: dockerfilePath, + effectiveDockerfilePath: preprocessedDockerfilePath, + effectiveDockerfileContent, + preprocessed: true, + }; +} + +async function preprocessDockerfileIncludes(cliHost: CLIHost, currentPath: string, stack: string[]): Promise { + if (stack.includes(currentPath)) { + const chain = [...stack, currentPath].join(' -> '); + throw new ContainerError({ description: `Cyclic #include detected while preprocessing Dockerfile: ${chain}` }); + } + if (!(await cliHost.isFile(currentPath))) { + throw new ContainerError({ description: `Included Dockerfile not found: ${currentPath}` }); + } + + const currentText = (await cliHost.readFile(currentPath)).toString(); + const lines = currentText.split(/\r?\n/); + const expanded: string[] = []; + const nextStack = [...stack, currentPath]; + for (const line of lines) { + const match = includeLine.exec(line); + if (!match) { + expanded.push(line); + continue; + } + + const includePath = match[1]; + const resolvedIncludePath = cliHost.path.isAbsolute(includePath) + ? includePath + : cliHost.path.resolve(cliHost.path.dirname(currentPath), includePath); + expanded.push(await preprocessDockerfileIncludes(cliHost, resolvedIncludePath, nextStack)); + } + + return expanded.join('\n'); +} + +async function writePreprocessedDockerfile(cliHost: CLIHost, sourceDockerfilePath: string, content: string): Promise { + const cacheFolder = cliHost.path.join( + await cliHost.tmpdir(), + cliHost.platform === 'linux' ? `devcontainercli-${await cliHost.getUsername()}` : 'devcontainercli', + 'dockerfile-preprocess' + ); + await cliHost.mkdirp(cacheFolder); + + const sourceBasename = cliHost.path.basename(sourceDockerfilePath); + const targetBasename = sourceBasename.replace(/\.in$/i, '') || 'Dockerfile'; + const preprocessedDockerfilePath = cliHost.path.join(cacheFolder, `${Date.now()}-${randomUUID()}-${targetBasename}`); + await cliHost.writeFile(preprocessedDockerfilePath, Buffer.from(content)); + return preprocessedDockerfilePath; +} diff --git a/src/spec-node/imageMetadata.ts b/src/spec-node/imageMetadata.ts index 60884592e..b4839e2cc 100644 --- a/src/spec-node/imageMetadata.ts +++ b/src/spec-node/imageMetadata.ts @@ -12,6 +12,7 @@ import { ContainerDetails, DockerCLIParameters, ImageDetails } from '../spec-shu import { Log, LogLevel } from '../spec-utils/log'; import { getBuildInfoForService, readDockerComposeConfig } from './dockerCompose'; import { Dockerfile, extractDockerfile, findBaseImage, findUserStatement } from './dockerfileUtils'; +import { resolveDockerfileIncludesIfNeeded } from './dockerfilePreprocess'; import { SubstituteConfig, SubstitutedConfig, DockerResolverParameters, inspectDockerImage, uriToWSLFsPath, envListToObj } from './utils'; const pickConfigProperties: (keyof DevContainerConfig & keyof ImageMetadataEntry)[] = [ @@ -342,7 +343,8 @@ export async function getImageBuildInfo(params: DockerResolverParameters | Docke if (!cliHost.isFile(dockerfilePath)) { throw new ContainerError({ description: `Dockerfile (${dockerfilePath}) not found.` }); } - const dockerfile = (await cliHost.readFile(dockerfilePath)).toString(); + const resolvedDockerfile = await resolveDockerfileIncludesIfNeeded(cliHost, dockerfilePath); + const dockerfile = resolvedDockerfile.effectiveDockerfileContent; return getImageBuildInfoFromDockerfile(params, dockerfile, config.build?.args || {}, config.build?.target, configWithRaw.substitute); } else if ('dockerComposeFile' in config) { @@ -363,7 +365,8 @@ export async function getImageBuildInfo(params: DockerResolverParameters | Docke if (serviceInfo.build) { const { context, dockerfilePath } = serviceInfo.build; const resolvedDockerfilePath = cliHost.path.isAbsolute(dockerfilePath) ? dockerfilePath : cliHost.path.resolve(context, dockerfilePath); - const dockerfile = (await cliHost.readFile(resolvedDockerfilePath)).toString(); + const resolvedDockerfile = await resolveDockerfileIncludesIfNeeded(cliHost, resolvedDockerfilePath); + const dockerfile = resolvedDockerfile.effectiveDockerfileContent; return getImageBuildInfoFromDockerfile(params, dockerfile, serviceInfo.build.args || {}, serviceInfo.build.target, configWithRaw.substitute); } else { return getImageBuildInfoFromImage(params, composeService.image, configWithRaw.substitute); diff --git a/src/spec-node/singleContainer.ts b/src/spec-node/singleContainer.ts index 1c3669f74..ed926d91f 100644 --- a/src/spec-node/singleContainer.ts +++ b/src/spec-node/singleContainer.ts @@ -13,6 +13,7 @@ import { LogLevel, Log, makeLog } from '../spec-utils/log'; import { extendImage, getExtendImageBuildInfo, updateRemoteUserUID } from './containerFeatures'; import { getDevcontainerMetadata, getImageBuildInfoFromDockerfile, getImageMetadataFromContainer, ImageMetadataEntry, lifecycleCommandOriginMapFromMetadata, mergeConfiguration, MergedDevContainerConfig } from './imageMetadata'; import { ensureDockerfileHasFinalStageName, generateMountCommand } from './dockerfileUtils'; +import { resolveDockerfileIncludesIfNeeded } from './dockerfilePreprocess'; export const hostFolderLabel = 'devcontainer.local_folder'; // used to label containers created from a workspace/folder export const configFileLabel = 'devcontainer.config_file'; @@ -130,7 +131,8 @@ async function buildAndExtendImage(buildParams: DockerResolverParameters, config throw new ContainerError({ description: `Dockerfile (${dockerfilePath}) not found.` }); } - let dockerfile = (await cliHost.readFile(dockerfilePath)).toString(); + const resolvedDockerfile = await resolveDockerfileIncludesIfNeeded(cliHost, dockerfilePath); + let dockerfile = resolvedDockerfile.effectiveDockerfileContent; const originalDockerfile = dockerfile; let baseName = 'dev_container_auto_added_stage_label'; if (config.build?.target) { @@ -149,7 +151,7 @@ async function buildAndExtendImage(buildParams: DockerResolverParameters, config const imageBuildInfo = await getImageBuildInfoFromDockerfile(buildParams, originalDockerfile, config.build?.args || {}, config.build?.target, configWithRaw.substitute); const extendImageBuildInfo = await getExtendImageBuildInfo(buildParams, configWithRaw, baseName, imageBuildInfo, undefined, additionalFeatures, false); - let finalDockerfilePath = dockerfilePath; + let finalDockerfilePath = resolvedDockerfile.effectiveDockerfilePath; const additionalBuildArgs: string[] = []; if (extendImageBuildInfo?.featureBuildInfo) { const { featureBuildInfo } = extendImageBuildInfo; diff --git a/src/test/configs/podman-test/.devcontainer.json b/src/test/configs/podman-test/.devcontainer.json new file mode 100644 index 000000000..8bd3f065a --- /dev/null +++ b/src/test/configs/podman-test/.devcontainer.json @@ -0,0 +1,5 @@ +{ + "build": { + "dockerfile": "cpp.Dockerfile.in" + } +} diff --git a/src/test/configs/podman-test/cpp.Dockerfile.in b/src/test/configs/podman-test/cpp.Dockerfile.in new file mode 100644 index 000000000..fb7b042ed --- /dev/null +++ b/src/test/configs/podman-test/cpp.Dockerfile.in @@ -0,0 +1,2 @@ +#include "tools.Dockerfile" +RUN apt-get update && apt-get install -y clang \ No newline at end of file diff --git a/src/test/configs/podman-test/tools.Dockerfile b/src/test/configs/podman-test/tools.Dockerfile new file mode 100644 index 000000000..faccdf07a --- /dev/null +++ b/src/test/configs/podman-test/tools.Dockerfile @@ -0,0 +1,2 @@ +FROM docker.io/debian:latest +RUN apt-get update && apt-get install -y vim \ No newline at end of file diff --git a/src/test/dockerfilePreprocess.test.ts b/src/test/dockerfilePreprocess.test.ts new file mode 100644 index 000000000..e6a042093 --- /dev/null +++ b/src/test/dockerfilePreprocess.test.ts @@ -0,0 +1,93 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + *--------------------------------------------------------------------------------------------*/ + +import { assert } from 'chai'; +import * as fs from 'fs'; +import * as path from 'path'; +import { CLIHost } from '../spec-common/cliHost'; +import { resolveDockerfileIncludesIfNeeded } from '../spec-node/dockerfilePreprocess'; + +function createMockCLIHost(files: Record, platform: NodeJS.Platform = 'linux'): CLIHost { + const pathModule = platform === 'win32' ? path.win32 : path.posix; + return { + type: 'local', + platform, + arch: 'x64', + path: pathModule, + cwd: platform === 'win32' ? 'C:\\' : '/', + env: {}, + exec: () => { throw new Error('Not implemented'); }, + ptyExec: () => { throw new Error('Not implemented'); }, + homedir: async () => platform === 'win32' ? 'C:\\Users\\test' : '/home/test', + tmpdir: async () => platform === 'win32' ? 'C:\\tmp' : '/tmp', + isFile: async (filepath: string) => filepath in files, + isFolder: async () => false, + readFile: async (filepath: string) => { + if (!(filepath in files)) { + throw new Error(`File not found: ${filepath}`); + } + return Buffer.from(files[filepath]); + }, + writeFile: async (filepath: string, content: Buffer) => { + files[filepath] = content.toString(); + }, + rename: async () => { }, + mkdirp: async () => { }, + readDir: async () => [], + getUsername: async () => 'test', + toCommonURI: async () => undefined, + connect: () => { throw new Error('Not implemented'); }, + }; +} + +describe('resolveDockerfileIncludesIfNeeded', () => { + it('returns source Dockerfile unchanged when not using .in extension', async () => { + const files: Record = { + '/workspace/Dockerfile': 'FROM debian:latest\nRUN echo ok', + }; + const cliHost = createMockCLIHost(files); + const result = await resolveDockerfileIncludesIfNeeded(cliHost, '/workspace/Dockerfile'); + assert.isFalse(result.preprocessed); + assert.equal(result.effectiveDockerfilePath, '/workspace/Dockerfile'); + assert.equal(result.effectiveDockerfileContent, files['/workspace/Dockerfile']); + }); + + it('expands #include lines and writes a generated Dockerfile for .in files', async () => { + const podmanTestConfigPath = path.resolve(__dirname, 'configs', 'podman-test'); + const sourceDockerfilePath = path.join(podmanTestConfigPath, 'cpp.Dockerfile.in'); + const includedDockerfilePath = path.join(podmanTestConfigPath, 'tools.Dockerfile'); + const sourceDockerfileContent = fs.readFileSync(sourceDockerfilePath).toString(); + const includedDockerfileContent = fs.readFileSync(includedDockerfilePath).toString(); + const files: Record = { + [sourceDockerfilePath]: sourceDockerfileContent, + [includedDockerfilePath]: includedDockerfileContent, + }; + const cliHost = createMockCLIHost(files); + const result = await resolveDockerfileIncludesIfNeeded(cliHost, sourceDockerfilePath); + assert.isTrue(result.preprocessed); + assert.notEqual(result.effectiveDockerfilePath, sourceDockerfilePath); + assert.include(result.effectiveDockerfilePath, '/tmp/devcontainercli-test/dockerfile-preprocess/'); + assert.equal( + result.effectiveDockerfileContent, + 'FROM docker.io/debian:latest\nRUN apt-get update && apt-get install -y vim\nRUN apt-get update && apt-get install -y clang' + ); + assert.equal(files[result.effectiveDockerfilePath], result.effectiveDockerfileContent); + }); + + it('fails with a clear error when #include has a cycle', async () => { + const files: Record = { + '/workspace/a.Dockerfile.in': '#include "b.Dockerfile"\nRUN echo a', + '/workspace/b.Dockerfile': '#include "a.Dockerfile.in"\nRUN echo b', + }; + const cliHost = createMockCLIHost(files); + let err: any; + try { + await resolveDockerfileIncludesIfNeeded(cliHost, '/workspace/a.Dockerfile.in'); + } catch (e) { + err = e; + } + assert.ok(err); + assert.include(String(err.message || err), 'Cyclic #include detected while preprocessing Dockerfile'); + }); +}); From 4316f524e4acbae489bcb8196e814a49416c8919 Mon Sep 17 00:00:00 2001 From: Mathiyarasy <157102811+Mathiyarasy@users.noreply.github.com> Date: Thu, 14 May 2026 16:30:53 +0000 Subject: [PATCH 2/2] Dockerfile preprocessing --- src/spec-common/cliHost.ts | 4 +- src/spec-node/dockerCompose.ts | 19 +- src/spec-node/dockerfilePreprocess.ts | 373 ++++++++++++++++-- src/spec-node/singleContainer.ts | 11 +- .../.devcontainer/Dockerfile.in | 6 + .../.devcontainer/bootstrap.sh | 2 + .../.devcontainer/common.Dockerfile | 1 + .../.devcontainer/devcontainer.json | 7 + .../.devcontainer/docker-compose.yml | 10 + .../.devcontainer/tools.Dockerfile | 3 + .../bootstrap.sh | 2 + .../Dockerfile.in | 6 + .../bootstrap.sh | 2 + .../common.Dockerfile | 1 + .../tools.Dockerfile | 2 + .../configs/podman-test/.devcontainer.json | 7 +- src/test/configs/podman-test/Dockerfile.in | 16 + src/test/configs/podman-test/bootstrap.sh | 2 + .../configs/podman-test/common.Dockerfile | 4 + .../configs/podman-test/cpp.Dockerfile.in | 2 - src/test/configs/podman-test/tools.Dockerfile | 4 +- src/test/dockerfilePreprocess.test.ts | 188 ++++++++- 22 files changed, 620 insertions(+), 52 deletions(-) create mode 100644 src/test/configs/podman-preprocess-compose-test/.devcontainer/Dockerfile.in create mode 100644 src/test/configs/podman-preprocess-compose-test/.devcontainer/bootstrap.sh create mode 100644 src/test/configs/podman-preprocess-compose-test/.devcontainer/common.Dockerfile create mode 100644 src/test/configs/podman-preprocess-compose-test/.devcontainer/devcontainer.json create mode 100644 src/test/configs/podman-preprocess-compose-test/.devcontainer/docker-compose.yml create mode 100644 src/test/configs/podman-preprocess-compose-test/.devcontainer/tools.Dockerfile create mode 100644 src/test/configs/podman-preprocess-compose-test/bootstrap.sh create mode 100644 src/test/configs/podman-preprocess-without feature/Dockerfile.in create mode 100644 src/test/configs/podman-preprocess-without feature/bootstrap.sh create mode 100644 src/test/configs/podman-preprocess-without feature/common.Dockerfile create mode 100644 src/test/configs/podman-preprocess-without feature/tools.Dockerfile create mode 100644 src/test/configs/podman-test/Dockerfile.in create mode 100644 src/test/configs/podman-test/bootstrap.sh create mode 100644 src/test/configs/podman-test/common.Dockerfile delete mode 100644 src/test/configs/podman-test/cpp.Dockerfile.in diff --git a/src/spec-common/cliHost.ts b/src/spec-common/cliHost.ts index 294f8be4a..85d0d47fd 100644 --- a/src/spec-common/cliHost.ts +++ b/src/spec-common/cliHost.ts @@ -7,7 +7,7 @@ import * as path from 'path'; import * as net from 'net'; import * as os from 'os'; -import { readLocalFile, writeLocalFile, mkdirpLocal, isLocalFile, renameLocal, readLocalDir, isLocalFolder } from '../spec-utils/pfs'; +import { readLocalFile, writeLocalFile, mkdirpLocal, isLocalFile, renameLocal, readLocalDir, isLocalFolder, unlinkLocal } from '../spec-utils/pfs'; import { URI } from 'vscode-uri'; import { ExecFunction, getLocalUsername, plainExec, plainPtyExec, PtyExecFunction } from './commonUtils'; import { Abort, Duplex, Sink, Source, SourceCallback } from 'pull-stream'; @@ -32,6 +32,7 @@ export interface CLIHost { isFolder(filepath: string): Promise; readFile(filepath: string): Promise; writeFile(filepath: string, content: Buffer): Promise; + deleteFile?(filepath: string): Promise; rename(oldPath: string, newPath: string): Promise; mkdirp(dirpath: string): Promise; readDir(dirpath: string): Promise; @@ -76,6 +77,7 @@ function createLocalCLIHostFromExecFunctions(localCwd: string, exec: ExecFunctio isFolder: isLocalFolder, readFile: readLocalFile, writeFile: writeLocalFile, + deleteFile: unlinkLocal, rename: renameLocal, mkdirp: async (dirpath) => { await mkdirpLocal(dirpath); diff --git a/src/spec-node/dockerCompose.ts b/src/spec-node/dockerCompose.ts index 38201a188..6ceef7231 100644 --- a/src/spec-node/dockerCompose.ts +++ b/src/spec-node/dockerCompose.ts @@ -19,7 +19,7 @@ import { Mount, parseMount } from '../spec-configuration/containerFeaturesConfig import path from 'path'; import { getDevcontainerMetadata, getImageBuildInfoFromDockerfile, getImageBuildInfoFromImage, getImageMetadataFromContainer, ImageBuildInfo, lifecycleCommandOriginMapFromMetadata, mergeConfiguration, MergedDevContainerConfig } from './imageMetadata'; import { ensureDockerfileHasFinalStageName } from './dockerfileUtils'; -import { resolveDockerfileIncludesIfNeeded } from './dockerfilePreprocess'; +import { materializeResolvedDockerfileForBuild, resolveDockerfileIncludesIfNeeded } from './dockerfilePreprocess'; import { randomUUID } from 'crypto'; const projectLabel = 'com.docker.compose.project'; @@ -165,15 +165,15 @@ export async function buildAndExtendDockerCompose(configWithRaw: SubstitutedConf let dockerfile: string | undefined; let imageBuildInfo: ImageBuildInfo; let preprocessedDockerfilePathForComposeBuild: string | undefined; + let disposeMaterializedDockerfile = async () => { }; + let resolvedBuildDockerfile; const serviceInfo = getBuildInfoForService(composeService, cliHost.path, localComposeFiles); if (serviceInfo.build) { const { context, dockerfilePath, target } = serviceInfo.build; const resolvedDockerfilePath = cliHost.path.isAbsolute(dockerfilePath) ? dockerfilePath : path.resolve(context, dockerfilePath); const resolvedDockerfile = await resolveDockerfileIncludesIfNeeded(cliHost, resolvedDockerfilePath); + resolvedBuildDockerfile = resolvedDockerfile; const originalDockerfile = resolvedDockerfile.effectiveDockerfileContent; - if (resolvedDockerfile.preprocessed) { - preprocessedDockerfilePathForComposeBuild = resolvedDockerfile.effectiveDockerfilePath; - } dockerfile = originalDockerfile; if (target) { // Explictly set build target for the dev container build features on that @@ -197,6 +197,11 @@ export async function buildAndExtendDockerCompose(configWithRaw: SubstitutedConf const supportsAdditionalBuildContexts = !params.isPodman && version && !isEarlierVersion(version, [2, 17, 0]); const optionalBuildKitParams = supportsAdditionalBuildContexts ? params : { ...params, buildKitVersion: undefined }; const extendImageBuildInfo = await getExtendImageBuildInfo(optionalBuildKitParams, configWithRaw, baseName, imageBuildInfo, composeService.user, additionalFeatures, canAddLabelsToContainer); + if (resolvedBuildDockerfile && !extendImageBuildInfo?.featureBuildInfo) { + const materializedDockerfile = await materializeResolvedDockerfileForBuild(cliHost, resolvedBuildDockerfile); + preprocessedDockerfilePathForComposeBuild = materializedDockerfile.dockerfilePath; + disposeMaterializedDockerfile = materializedDockerfile.dispose; + } let overrideImageName: string | undefined; let buildOverrideContent = ''; @@ -274,7 +279,8 @@ ${cacheFromOverrideContent} args.push('-f', composeOverrideFile); } - if (!noBuild) { + try { + if (!noBuild) { args.push('build'); if (noCache) { args.push('--no-cache'); @@ -304,6 +310,9 @@ ${cacheFromOverrideContent} throw err instanceof ContainerError ? err : new ContainerError({ description: 'An error occurred building the Docker Compose images.', originalError: err, data: { fileWithError: localComposeFiles[0] } }); } + } + } finally { + await disposeMaterializedDockerfile(); } return { diff --git a/src/spec-node/dockerfilePreprocess.ts b/src/spec-node/dockerfilePreprocess.ts index 6c86e813e..c2dc5ac84 100644 --- a/src/spec-node/dockerfilePreprocess.ts +++ b/src/spec-node/dockerfilePreprocess.ts @@ -3,42 +3,84 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { randomUUID } from 'crypto'; import { CLIHost } from '../spec-common/cliHost'; import { ContainerError } from '../spec-common/errors'; +import { randomUUID } from 'crypto'; + +const preprocessorDirective = /^\s*#\s*(\w+)\b(.*)$/; +const includeLine = /^\s*"([^"]+)"\s*$/; +const defineLine = /^\s*([a-zA-Z_][a-zA-Z0-9_]*)(?:\s+(.*))?$/; +const undefLine = /^\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*$/; +const ifdefLine = /^\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*$/; +const fromLine = /^\s*FROM(?:\s|$)/mi; -const includeLine = /^\s*#include\s+"([^"]+)"\s*$/; +interface ConditionalState { + parentActive: boolean; + thisActive: boolean; + hasMatched: boolean; +} export interface ResolvedDockerfile { originalDockerfilePath: string; - effectiveDockerfilePath: string; effectiveDockerfileContent: string; preprocessed: boolean; } +export interface MaterializedDockerfile { + dockerfilePath: string; + dispose(): Promise; +} + + +/** + * Preprocesses a Dockerfile, simulating cpp -E style preprocessing for Podman compatibility. + * If the file ends with .in, it will resolve #include statements recursively and return the rewritten Dockerfile content. + * Otherwise, returns the original Dockerfile content. + * + * @param cliHost CLIHost for file operations + * @param dockerfilePath Path to the Dockerfile (may be .in) + * @returns { originalDockerfilePath, effectiveDockerfileContent, preprocessed } + */ export async function resolveDockerfileIncludesIfNeeded(cliHost: CLIHost, dockerfilePath: string): Promise { const dockerfileText = (await cliHost.readFile(dockerfilePath)).toString(); if (!dockerfilePath.toLowerCase().endsWith('.in')) { return { originalDockerfilePath: dockerfilePath, - effectiveDockerfilePath: dockerfilePath, effectiveDockerfileContent: dockerfileText, preprocessed: false, }; } - const effectiveDockerfileContent = await preprocessDockerfileIncludes(cliHost, dockerfilePath, []); - const preprocessedDockerfilePath = await writePreprocessedDockerfile(cliHost, dockerfilePath, effectiveDockerfileContent); + const rewrittenContent = await preprocessDockerfileIncludes(cliHost, dockerfilePath, [], new Map()); + validateResolvedFromInstruction(dockerfilePath, rewrittenContent); return { originalDockerfilePath: dockerfilePath, - effectiveDockerfilePath: preprocessedDockerfilePath, - effectiveDockerfileContent, + effectiveDockerfileContent: rewrittenContent, preprocessed: true, }; } -async function preprocessDockerfileIncludes(cliHost: CLIHost, currentPath: string, stack: string[]): Promise { +export async function materializeResolvedDockerfileForBuild(cliHost: CLIHost, resolvedDockerfile: ResolvedDockerfile): Promise { + if (!resolvedDockerfile.preprocessed) { + return { + dockerfilePath: resolvedDockerfile.originalDockerfilePath, + dispose: async () => { }, + }; + } + + const dockerfilePath = await writePreprocessedDockerfile(cliHost, resolvedDockerfile.originalDockerfilePath, resolvedDockerfile.effectiveDockerfileContent); + return { + dockerfilePath, + dispose: async () => { + if (cliHost.deleteFile) { + await cliHost.deleteFile(dockerfilePath); + } + }, + }; +} + +async function preprocessDockerfileIncludes(cliHost: CLIHost, currentPath: string, stack: string[], macros: Map): Promise { if (stack.includes(currentPath)) { const chain = [...stack, currentPath].join(' -> '); throw new ContainerError({ description: `Cyclic #include detected while preprocessing Dockerfile: ${chain}` }); @@ -51,34 +93,303 @@ async function preprocessDockerfileIncludes(cliHost: CLIHost, currentPath: strin const lines = currentText.split(/\r?\n/); const expanded: string[] = []; const nextStack = [...stack, currentPath]; - for (const line of lines) { - const match = includeLine.exec(line); - if (!match) { - expanded.push(line); + const conditionals: ConditionalState[] = []; + + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + const lineNumber = i + 1; + const currentActive = conditionals.length === 0 ? true : conditionals[conditionals.length - 1].thisActive; + const directive = preprocessorDirective.exec(line); + + if (!directive) { + if (currentActive) { + expanded.push(substituteMacros(line, macros)); + } + continue; + } + + const directiveName = directive[1].toLowerCase(); + const directiveBody = directive[2] ?? ''; + + if (directiveName === 'if' || directiveName === 'ifdef' || directiveName === 'ifndef') { + const parentActive = currentActive; + const condition = evaluateIfDirectiveCondition(currentPath, lineNumber, directiveName, directiveBody, macros); + const thisActive = parentActive && condition; + conditionals.push({ + parentActive, + thisActive, + hasMatched: thisActive, + }); + continue; + } + + if (directiveName === 'elif') { + const state = conditionals[conditionals.length - 1]; + if (!state) { + throw new ContainerError({ description: `#elif without matching #if in ${currentPath}:${lineNumber}` }); + } + if (!state.parentActive || state.hasMatched) { + state.thisActive = false; + continue; + } + const elifCondition = evaluateIfDirectiveCondition(currentPath, lineNumber, directiveName, directiveBody, macros); + state.thisActive = state.parentActive && elifCondition; + if (state.thisActive) { + state.hasMatched = true; + } + continue; + } + + if (directiveName === 'else') { + const state = conditionals[conditionals.length - 1]; + if (!state) { + throw new ContainerError({ description: `#else without matching #if in ${currentPath}:${lineNumber}` }); + } + state.thisActive = state.parentActive && !state.hasMatched; + state.hasMatched = true; + continue; + } + + if (directiveName === 'endif') { + const state = conditionals.pop(); + if (!state) { + throw new ContainerError({ description: `#endif without matching #if in ${currentPath}:${lineNumber}` }); + } + continue; + } + + if (!currentActive) { continue; } - const includePath = match[1]; - const resolvedIncludePath = cliHost.path.isAbsolute(includePath) - ? includePath - : cliHost.path.resolve(cliHost.path.dirname(currentPath), includePath); - expanded.push(await preprocessDockerfileIncludes(cliHost, resolvedIncludePath, nextStack)); + switch (directiveName) { + case 'include': { + const includeMatch = includeLine.exec(directiveBody); + if (!includeMatch) { + throw new ContainerError({ description: `Invalid #include directive in ${currentPath}:${lineNumber}. Use #include "path".` }); + } + const includePath = substituteMacros(includeMatch[1], macros); + const resolvedIncludePath = cliHost.path.isAbsolute(includePath) + ? includePath + : cliHost.path.resolve(cliHost.path.dirname(currentPath), includePath); + expanded.push(await preprocessDockerfileIncludes(cliHost, resolvedIncludePath, nextStack, macros)); + break; + } + case 'define': { + const defineMatch = defineLine.exec(directiveBody); + if (!defineMatch) { + throw new ContainerError({ description: `Invalid #define directive in ${currentPath}:${lineNumber}.` }); + } + macros.set(defineMatch[1], defineMatch[2] ?? '1'); + break; + } + case 'undef': { + const undefMatch = undefLine.exec(directiveBody); + if (!undefMatch) { + throw new ContainerError({ description: `Invalid #undef directive in ${currentPath}:${lineNumber}.` }); + } + macros.delete(undefMatch[1]); + break; + } + case 'error': { + const message = substituteMacros(directiveBody.trim(), macros); + throw new ContainerError({ description: `#error in ${currentPath}:${lineNumber}: ${message}` }); + } + case 'warning': { + const message = substituteMacros(directiveBody.trim(), macros); + expanded.push(`# warning: ${message}`); + break; + } + default: + expanded.push(substituteMacros(line, macros)); + break; + } + } + + if (conditionals.length > 0) { + throw new ContainerError({ description: `Unterminated preprocessor conditionals in ${currentPath}. Missing #endif.` }); } return expanded.join('\n'); } -async function writePreprocessedDockerfile(cliHost: CLIHost, sourceDockerfilePath: string, content: string): Promise { - const cacheFolder = cliHost.path.join( - await cliHost.tmpdir(), - cliHost.platform === 'linux' ? `devcontainercli-${await cliHost.getUsername()}` : 'devcontainercli', - 'dockerfile-preprocess' - ); - await cliHost.mkdirp(cacheFolder); - - const sourceBasename = cliHost.path.basename(sourceDockerfilePath); - const targetBasename = sourceBasename.replace(/\.in$/i, '') || 'Dockerfile'; - const preprocessedDockerfilePath = cliHost.path.join(cacheFolder, `${Date.now()}-${randomUUID()}-${targetBasename}`); - await cliHost.writeFile(preprocessedDockerfilePath, Buffer.from(content)); - return preprocessedDockerfilePath; +function evaluateIfDirectiveCondition(currentPath: string, lineNumber: number, directiveName: string, body: string, macros: Map): boolean { + if (directiveName === 'ifdef' || directiveName === 'ifndef') { + const ifdefMatch = ifdefLine.exec(body); + if (!ifdefMatch) { + throw new ContainerError({ description: `Invalid #${directiveName} directive in ${currentPath}:${lineNumber}.` }); + } + const isDefined = macros.has(ifdefMatch[1]); + return directiveName === 'ifdef' ? isDefined : !isDefined; + } + + return evaluateBooleanExpression(body.trim(), macros); } + +function evaluateBooleanExpression(expression: string, macros: Map): boolean { + if (!expression) { + return false; + } + + const tokens = tokenizeExpression(expression); + let index = 0; + + const parseExpression = (): boolean => { + let value = parseTerm(); + while (tokens[index] === '||') { + index++; + value = value || parseTerm(); + } + return value; + }; + + const parseTerm = (): boolean => { + let value = parseFactor(); + while (tokens[index] === '&&') { + index++; + value = value && parseFactor(); + } + return value; + }; + + const parseFactor = (): boolean => { + const token = tokens[index]; + if (token === '!') { + index++; + return !parseFactor(); + } + if (token === '(') { + index++; + const value = parseExpression(); + if (tokens[index] !== ')') { + throw new ContainerError({ description: `Invalid #if expression: missing ')' in '${expression}'` }); + } + index++; + return value; + } + if (token === 'defined') { + index++; + if (tokens[index] === '(') { + index++; + const name = tokens[index++]; + if (!name || !isIdentifier(name)) { + throw new ContainerError({ description: `Invalid #if expression: expected identifier after defined(` }); + } + if (tokens[index] !== ')') { + throw new ContainerError({ description: `Invalid #if expression: missing ')' after defined(${name}` }); + } + index++; + return macros.has(name); + } + const name = tokens[index++]; + if (!name || !isIdentifier(name)) { + throw new ContainerError({ description: 'Invalid #if expression: expected identifier after defined' }); + } + return macros.has(name); + } + + if (!token) { + throw new ContainerError({ description: `Invalid #if expression: unexpected end of expression '${expression}'` }); + } + index++; + if (/^[+-]?\d+$/.test(token)) { + return Number(token) !== 0; + } + if (isIdentifier(token)) { + return evaluateMacroTruthiness(token, macros, new Set()); + } + throw new ContainerError({ description: `Invalid #if expression token '${token}'` }); + }; + + const result = parseExpression(); + if (index !== tokens.length) { + throw new ContainerError({ description: `Invalid #if expression near '${tokens.slice(index).join(' ')}'` }); + } + return result; +} + +function tokenizeExpression(expression: string): string[] { + const tokens = expression.match(/\s+|\|\||&&|!|\(|\)|defined|[a-zA-Z_][a-zA-Z0-9_]*|[+-]?\d+/g) || []; + let reconstructed = ''; + const filtered = tokens.filter(token => token.trim().length > 0); + for (const token of tokens) { + reconstructed += token; + } + const normalizedExpression = expression.replace(/\s+/g, ''); + if (reconstructed.replace(/\s+/g, '') !== normalizedExpression) { + throw new ContainerError({ description: `Unsupported token in #if expression '${expression}'` }); + } + return filtered; +} + +function isIdentifier(token: string): boolean { + return /^[a-zA-Z_][a-zA-Z0-9_]*$/.test(token); +} + +function evaluateMacroTruthiness(name: string, macros: Map, seen: Set): boolean { + if (seen.has(name)) { + return false; + } + const value = macros.get(name); + if (typeof value !== 'string') { + return false; + } + const trimmed = value.trim(); + if (!trimmed || trimmed === '0' || trimmed.toLowerCase() === 'false') { + return false; + } + if (/^[+-]?\d+$/.test(trimmed)) { + return Number(trimmed) !== 0; + } + if (isIdentifier(trimmed)) { + seen.add(name); + return evaluateMacroTruthiness(trimmed, macros, seen); + } + return true; +} + +function substituteMacros(line: string, macros: Map): string { + if (!macros.size) { + return line; + } + + let result = line; + for (let i = 0; i < 10; i++) { + let changed = false; + const names = [...macros.keys()].sort((a, b) => b.length - a.length); + for (const name of names) { + const value = macros.get(name)!; + const pattern = new RegExp(`\\b${escapeRegExp(name)}\\b`, 'g'); + const replaced = result.replace(pattern, value); + if (replaced !== result) { + changed = true; + result = replaced; + } + } + if (!changed) { + break; + } + } + + return result; +} + +function escapeRegExp(str: string): string { + return str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +function validateResolvedFromInstruction(originalDockerfilePath: string, dockerfileContent: string) { + if (!fromLine.test(dockerfileContent)) { + throw new ContainerError({ + description: `Preprocessed Dockerfile '${originalDockerfilePath}' contains no resolved FROM instruction. Ensure preprocessing directives produce at least one final FROM line.`, + }); + } +} + +async function writePreprocessedDockerfile(cliHost: CLIHost, originalDockerfilePath: string, dockerfileContent: string): Promise { + const dockerfileFolder = cliHost.path.dirname(originalDockerfilePath); + const outputFileName = `${Date.now()}-${randomUUID()}-${cliHost.path.basename(originalDockerfilePath).replace(/\.in$/i, '')}`; + const effectiveDockerfilePath = cliHost.path.join(dockerfileFolder, outputFileName); + await cliHost.writeFile(effectiveDockerfilePath, Buffer.from(dockerfileContent)); + return effectiveDockerfilePath; +} \ No newline at end of file diff --git a/src/spec-node/singleContainer.ts b/src/spec-node/singleContainer.ts index ed926d91f..d6ad4749d 100644 --- a/src/spec-node/singleContainer.ts +++ b/src/spec-node/singleContainer.ts @@ -13,7 +13,7 @@ import { LogLevel, Log, makeLog } from '../spec-utils/log'; import { extendImage, getExtendImageBuildInfo, updateRemoteUserUID } from './containerFeatures'; import { getDevcontainerMetadata, getImageBuildInfoFromDockerfile, getImageMetadataFromContainer, ImageMetadataEntry, lifecycleCommandOriginMapFromMetadata, mergeConfiguration, MergedDevContainerConfig } from './imageMetadata'; import { ensureDockerfileHasFinalStageName, generateMountCommand } from './dockerfileUtils'; -import { resolveDockerfileIncludesIfNeeded } from './dockerfilePreprocess'; +import { materializeResolvedDockerfileForBuild, resolveDockerfileIncludesIfNeeded } from './dockerfilePreprocess'; export const hostFolderLabel = 'devcontainer.local_folder'; // used to label containers created from a workspace/folder export const configFileLabel = 'devcontainer.config_file'; @@ -151,7 +151,8 @@ async function buildAndExtendImage(buildParams: DockerResolverParameters, config const imageBuildInfo = await getImageBuildInfoFromDockerfile(buildParams, originalDockerfile, config.build?.args || {}, config.build?.target, configWithRaw.substitute); const extendImageBuildInfo = await getExtendImageBuildInfo(buildParams, configWithRaw, baseName, imageBuildInfo, undefined, additionalFeatures, false); - let finalDockerfilePath = resolvedDockerfile.effectiveDockerfilePath; + let finalDockerfilePath = dockerfilePath; + let disposeMaterializedDockerfile = async () => { }; const additionalBuildArgs: string[] = []; if (extendImageBuildInfo?.featureBuildInfo) { const { featureBuildInfo } = extendImageBuildInfo; @@ -175,6 +176,10 @@ async function buildAndExtendImage(buildParams: DockerResolverParameters, config for (const securityOpt of featureBuildInfo.securityOpts) { additionalBuildArgs.push('--security-opt', securityOpt); } + } else { + const materializedDockerfile = await materializeResolvedDockerfileForBuild(cliHost, resolvedDockerfile); + finalDockerfilePath = materializedDockerfile.dockerfilePath; + disposeMaterializedDockerfile = materializedDockerfile.dispose; } const args: string[] = []; @@ -263,6 +268,8 @@ async function buildAndExtendImage(buildParams: DockerResolverParameters, config } throw new ContainerError({ description: 'An error occurred building the image.', originalError: err, data: { fileWithError: dockerfilePath } }); + } finally { + await disposeMaterializedDockerfile(); } const imageDetails = () => inspectDockerImage(buildParams, baseImageNames[0], false); diff --git a/src/test/configs/podman-preprocess-compose-test/.devcontainer/Dockerfile.in b/src/test/configs/podman-preprocess-compose-test/.devcontainer/Dockerfile.in new file mode 100644 index 000000000..4eeb54dbe --- /dev/null +++ b/src/test/configs/podman-preprocess-compose-test/.devcontainer/Dockerfile.in @@ -0,0 +1,6 @@ +#define BASE_IMAGE debian:bookworm-slim + +FROM BASE_IMAGE + +#include "common.Dockerfile" +#include "tools.Dockerfile" diff --git a/src/test/configs/podman-preprocess-compose-test/.devcontainer/bootstrap.sh b/src/test/configs/podman-preprocess-compose-test/.devcontainer/bootstrap.sh new file mode 100644 index 000000000..a587077ef --- /dev/null +++ b/src/test/configs/podman-preprocess-compose-test/.devcontainer/bootstrap.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo compose-bootstrap diff --git a/src/test/configs/podman-preprocess-compose-test/.devcontainer/common.Dockerfile b/src/test/configs/podman-preprocess-compose-test/.devcontainer/common.Dockerfile new file mode 100644 index 000000000..67d4c72e0 --- /dev/null +++ b/src/test/configs/podman-preprocess-compose-test/.devcontainer/common.Dockerfile @@ -0,0 +1 @@ +RUN apt-get update && apt-get install -y curl wget diff --git a/src/test/configs/podman-preprocess-compose-test/.devcontainer/devcontainer.json b/src/test/configs/podman-preprocess-compose-test/.devcontainer/devcontainer.json new file mode 100644 index 000000000..61d00fbb9 --- /dev/null +++ b/src/test/configs/podman-preprocess-compose-test/.devcontainer/devcontainer.json @@ -0,0 +1,7 @@ +{ + "name": "Podman Preprocess Compose Test", + "dockerComposeFile": "docker-compose.yml", + "service": "app", + "workspaceFolder": "/workspace", + "remoteUser": "root" +} diff --git a/src/test/configs/podman-preprocess-compose-test/.devcontainer/docker-compose.yml b/src/test/configs/podman-preprocess-compose-test/.devcontainer/docker-compose.yml new file mode 100644 index 000000000..c8ab5e8ed --- /dev/null +++ b/src/test/configs/podman-preprocess-compose-test/.devcontainer/docker-compose.yml @@ -0,0 +1,10 @@ +version: '3.8' + +services: + app: + build: + context: .. + dockerfile: .devcontainer/Dockerfile.in + volumes: + - ..:/workspace:cached + command: sleep infinity diff --git a/src/test/configs/podman-preprocess-compose-test/.devcontainer/tools.Dockerfile b/src/test/configs/podman-preprocess-compose-test/.devcontainer/tools.Dockerfile new file mode 100644 index 000000000..9b03a9352 --- /dev/null +++ b/src/test/configs/podman-preprocess-compose-test/.devcontainer/tools.Dockerfile @@ -0,0 +1,3 @@ +RUN apt-get update && apt-get install -y vim +COPY ./bootstrap.sh /usr/local/bin/bootstrap.sh +RUN chmod +x /usr/local/bin/bootstrap.sh diff --git a/src/test/configs/podman-preprocess-compose-test/bootstrap.sh b/src/test/configs/podman-preprocess-compose-test/bootstrap.sh new file mode 100644 index 000000000..a587077ef --- /dev/null +++ b/src/test/configs/podman-preprocess-compose-test/bootstrap.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo compose-bootstrap diff --git a/src/test/configs/podman-preprocess-without feature/Dockerfile.in b/src/test/configs/podman-preprocess-without feature/Dockerfile.in new file mode 100644 index 000000000..4eeb54dbe --- /dev/null +++ b/src/test/configs/podman-preprocess-without feature/Dockerfile.in @@ -0,0 +1,6 @@ +#define BASE_IMAGE debian:bookworm-slim + +FROM BASE_IMAGE + +#include "common.Dockerfile" +#include "tools.Dockerfile" diff --git a/src/test/configs/podman-preprocess-without feature/bootstrap.sh b/src/test/configs/podman-preprocess-without feature/bootstrap.sh new file mode 100644 index 000000000..a9a0c42bc --- /dev/null +++ b/src/test/configs/podman-preprocess-without feature/bootstrap.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo without-feature-bootstrap diff --git a/src/test/configs/podman-preprocess-without feature/common.Dockerfile b/src/test/configs/podman-preprocess-without feature/common.Dockerfile new file mode 100644 index 000000000..67d4c72e0 --- /dev/null +++ b/src/test/configs/podman-preprocess-without feature/common.Dockerfile @@ -0,0 +1 @@ +RUN apt-get update && apt-get install -y curl wget diff --git a/src/test/configs/podman-preprocess-without feature/tools.Dockerfile b/src/test/configs/podman-preprocess-without feature/tools.Dockerfile new file mode 100644 index 000000000..92354cfbd --- /dev/null +++ b/src/test/configs/podman-preprocess-without feature/tools.Dockerfile @@ -0,0 +1,2 @@ +RUN apt-get update && apt-get install -y vim +COPY ./bootstrap.sh /usr/local/bin/bootstrap.sh diff --git a/src/test/configs/podman-test/.devcontainer.json b/src/test/configs/podman-test/.devcontainer.json index 8bd3f065a..51c0d1bea 100644 --- a/src/test/configs/podman-test/.devcontainer.json +++ b/src/test/configs/podman-test/.devcontainer.json @@ -1,5 +1,10 @@ { "build": { - "dockerfile": "cpp.Dockerfile.in" + "dockerfile": "Dockerfile.in" + }, + "features": { + "ghcr.io/devcontainers/features/github-cli:1": { + "version": "latest" + } } } diff --git a/src/test/configs/podman-test/Dockerfile.in b/src/test/configs/podman-test/Dockerfile.in new file mode 100644 index 000000000..d7e4fcd28 --- /dev/null +++ b/src/test/configs/podman-test/Dockerfile.in @@ -0,0 +1,16 @@ +#define BASE_IMAGE ubuntu:20.04 +#define INSTALL_NODE +#define INSTALL_PYTHON + +FROM BASE_IMAGE + +#ifdef INSTALL_NODE +RUN apt-get update && apt-get install -y nodejs +#endif + +#ifdef INSTALL_PYTHON +RUN apt-get update && apt-get install -y python3 +#endif + +#include "common.Dockerfile" +#include "tools.Dockerfile" \ No newline at end of file diff --git a/src/test/configs/podman-test/bootstrap.sh b/src/test/configs/podman-test/bootstrap.sh new file mode 100644 index 000000000..1f4cc2df9 --- /dev/null +++ b/src/test/configs/podman-test/bootstrap.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo bootstrap diff --git a/src/test/configs/podman-test/common.Dockerfile b/src/test/configs/podman-test/common.Dockerfile new file mode 100644 index 000000000..97856b250 --- /dev/null +++ b/src/test/configs/podman-test/common.Dockerfile @@ -0,0 +1,4 @@ +RUN apt-get update && apt-get install -y curl wget + +ENV APP_ENV=development +ENV APP_DEBUG=true \ No newline at end of file diff --git a/src/test/configs/podman-test/cpp.Dockerfile.in b/src/test/configs/podman-test/cpp.Dockerfile.in deleted file mode 100644 index fb7b042ed..000000000 --- a/src/test/configs/podman-test/cpp.Dockerfile.in +++ /dev/null @@ -1,2 +0,0 @@ -#include "tools.Dockerfile" -RUN apt-get update && apt-get install -y clang \ No newline at end of file diff --git a/src/test/configs/podman-test/tools.Dockerfile b/src/test/configs/podman-test/tools.Dockerfile index faccdf07a..8d307bce3 100644 --- a/src/test/configs/podman-test/tools.Dockerfile +++ b/src/test/configs/podman-test/tools.Dockerfile @@ -1,2 +1,2 @@ -FROM docker.io/debian:latest -RUN apt-get update && apt-get install -y vim \ No newline at end of file +RUN apt-get update && apt-get install -y vim +COPY ./bootstrap.sh /usr/local/bin/bootstrap.sh \ No newline at end of file diff --git a/src/test/dockerfilePreprocess.test.ts b/src/test/dockerfilePreprocess.test.ts index e6a042093..97d584191 100644 --- a/src/test/dockerfilePreprocess.test.ts +++ b/src/test/dockerfilePreprocess.test.ts @@ -5,8 +5,10 @@ import { assert } from 'chai'; import * as fs from 'fs'; import * as path from 'path'; +import * as yaml from 'js-yaml'; import { CLIHost } from '../spec-common/cliHost'; -import { resolveDockerfileIncludesIfNeeded } from '../spec-node/dockerfilePreprocess'; +import { materializeResolvedDockerfileForBuild, resolveDockerfileIncludesIfNeeded } from '../spec-node/dockerfilePreprocess'; +import { getBuildInfoForService } from '../spec-node/dockerCompose'; function createMockCLIHost(files: Record, platform: NodeJS.Platform = 'linux'): CLIHost { const pathModule = platform === 'win32' ? path.win32 : path.posix; @@ -32,6 +34,9 @@ function createMockCLIHost(files: Record, platform: NodeJS.Platf writeFile: async (filepath: string, content: Buffer) => { files[filepath] = content.toString(); }, + deleteFile: async (filepath: string) => { + delete files[filepath]; + }, rename: async () => { }, mkdirp: async () => { }, readDir: async () => [], @@ -49,30 +54,156 @@ describe('resolveDockerfileIncludesIfNeeded', () => { const cliHost = createMockCLIHost(files); const result = await resolveDockerfileIncludesIfNeeded(cliHost, '/workspace/Dockerfile'); assert.isFalse(result.preprocessed); - assert.equal(result.effectiveDockerfilePath, '/workspace/Dockerfile'); assert.equal(result.effectiveDockerfileContent, files['/workspace/Dockerfile']); }); - it('expands #include lines and writes a generated Dockerfile for .in files', async () => { + it('expands #include lines for .in files without writing a generated Dockerfile yet', async () => { const podmanTestConfigPath = path.resolve(__dirname, 'configs', 'podman-test'); - const sourceDockerfilePath = path.join(podmanTestConfigPath, 'cpp.Dockerfile.in'); + const sourceDockerfilePath = path.join(podmanTestConfigPath, 'Dockerfile.in'); + const commonDockerfilePath = path.join(podmanTestConfigPath, 'common.Dockerfile'); const includedDockerfilePath = path.join(podmanTestConfigPath, 'tools.Dockerfile'); + const copiedFilePath = path.join(podmanTestConfigPath, 'bootstrap.sh'); const sourceDockerfileContent = fs.readFileSync(sourceDockerfilePath).toString(); + const commonDockerfileContent = fs.readFileSync(commonDockerfilePath).toString(); const includedDockerfileContent = fs.readFileSync(includedDockerfilePath).toString(); + const copiedFileContent = fs.readFileSync(copiedFilePath).toString(); const files: Record = { [sourceDockerfilePath]: sourceDockerfileContent, + [commonDockerfilePath]: commonDockerfileContent, [includedDockerfilePath]: includedDockerfileContent, + [copiedFilePath]: copiedFileContent, }; const cliHost = createMockCLIHost(files); const result = await resolveDockerfileIncludesIfNeeded(cliHost, sourceDockerfilePath); assert.isTrue(result.preprocessed); - assert.notEqual(result.effectiveDockerfilePath, sourceDockerfilePath); - assert.include(result.effectiveDockerfilePath, '/tmp/devcontainercli-test/dockerfile-preprocess/'); assert.equal( result.effectiveDockerfileContent, - 'FROM docker.io/debian:latest\nRUN apt-get update && apt-get install -y vim\nRUN apt-get update && apt-get install -y clang' + '\nFROM ubuntu:20.04\n\nRUN apt-get update && apt-get install -y nodejs\n\nRUN apt-get update && apt-get install -y python3\n\nRUN apt-get update && apt-get install -y curl wget\n\nENV APP_ENV=development\nENV APP_DEBUG=true\nRUN apt-get update && apt-get install -y vim\nCOPY ./bootstrap.sh /usr/local/bin/bootstrap.sh' ); - assert.equal(files[result.effectiveDockerfilePath], result.effectiveDockerfileContent); + assert.deepEqual(Object.keys(files).sort(), [sourceDockerfilePath, commonDockerfilePath, includedDockerfilePath, copiedFilePath].sort()); + }); + + it('materializes a preprocessed Dockerfile in the source directory and cleans it up', async () => { + const files: Record = { + '/workspace/.devcontainer/Dockerfile.in': [ + 'FROM docker.io/debian:latest', + '#include "scripts.Dockerfile"', + ].join('\n'), + '/workspace/.devcontainer/scripts.Dockerfile': [ + 'RUN echo preparing scripts', + 'COPY ./bootstrap.sh /usr/local/bin/bootstrap.sh', + ].join('\n'), + '/workspace/.devcontainer/bootstrap.sh': '#!/bin/sh\necho bootstrap', + }; + const cliHost = createMockCLIHost(files); + const result = await resolveDockerfileIncludesIfNeeded(cliHost, '/workspace/.devcontainer/Dockerfile.in'); + const materialized = await materializeResolvedDockerfileForBuild(cliHost, result); + + assert.isTrue(result.preprocessed); + assert.equal(path.dirname(materialized.dockerfilePath), '/workspace/.devcontainer'); + assert.include(result.effectiveDockerfileContent, 'RUN echo preparing scripts'); + assert.include(result.effectiveDockerfileContent, 'COPY ./bootstrap.sh /usr/local/bin/bootstrap.sh'); + assert.equal(files[materialized.dockerfilePath], result.effectiveDockerfileContent); + await materialized.dispose(); + assert.notProperty(files, materialized.dockerfilePath); + }); + + it('materializes compose-referenced preprocessed Dockerfiles beside the original Dockerfile to preserve relative COPY paths', async () => { + const composeFixturePath = path.resolve(__dirname, 'configs', 'podman-preprocess-compose-test'); + const composeFilePath = path.join(composeFixturePath, '.devcontainer', 'docker-compose.yml'); + const dockerfilePath = path.join(composeFixturePath, '.devcontainer', 'Dockerfile.in'); + const commonDockerfilePath = path.join(composeFixturePath, '.devcontainer', 'common.Dockerfile'); + const toolsDockerfilePath = path.join(composeFixturePath, '.devcontainer', 'tools.Dockerfile'); + const copiedFilePath = path.join(composeFixturePath, '.devcontainer', 'bootstrap.sh'); + const composeContent = fs.readFileSync(composeFilePath, 'utf8'); + const files: Record = { + [composeFilePath]: composeContent, + [dockerfilePath]: fs.readFileSync(dockerfilePath, 'utf8'), + [commonDockerfilePath]: fs.readFileSync(commonDockerfilePath, 'utf8'), + [toolsDockerfilePath]: fs.readFileSync(toolsDockerfilePath, 'utf8'), + [copiedFilePath]: fs.readFileSync(copiedFilePath, 'utf8'), + }; + const cliHost = createMockCLIHost(files); + const composeConfig = yaml.load(composeContent) as any; + const serviceInfo = getBuildInfoForService(composeConfig.services.app, cliHost.path, [composeFilePath]); + + assert.isDefined(serviceInfo.build); + serviceInfo.build!.context = cliHost.path.resolve(path.dirname(composeFilePath), serviceInfo.build!.context); + const resolvedDockerfilePath = cliHost.path.isAbsolute(serviceInfo.build!.dockerfilePath) + ? serviceInfo.build!.dockerfilePath + : cliHost.path.resolve(serviceInfo.build!.context, serviceInfo.build!.dockerfilePath); + const resolvedDockerfile = await resolveDockerfileIncludesIfNeeded(cliHost, resolvedDockerfilePath); + const materialized = await materializeResolvedDockerfileForBuild(cliHost, resolvedDockerfile); + + assert.equal(resolvedDockerfilePath, dockerfilePath); + assert.equal(path.dirname(materialized.dockerfilePath), path.dirname(dockerfilePath)); + assert.include(files[materialized.dockerfilePath], 'COPY ./bootstrap.sh /usr/local/bin/bootstrap.sh'); + assert.equal(path.resolve(path.dirname(materialized.dockerfilePath), 'bootstrap.sh'), copiedFilePath); + + await materialized.dispose(); + assert.notProperty(files, materialized.dockerfilePath); + }); + + it('expands includes for podman-preprocess-without feature without writing a generated Dockerfile yet', async () => { + const fixturePath = path.resolve(__dirname, 'configs', 'podman-preprocess-without feature'); + const sourceDockerfilePath = path.join(fixturePath, 'Dockerfile.in'); + const commonDockerfilePath = path.join(fixturePath, 'common.Dockerfile'); + const toolsDockerfilePath = path.join(fixturePath, 'tools.Dockerfile'); + const copiedFilePath = path.join(fixturePath, 'bootstrap.sh'); + const files: Record = { + [sourceDockerfilePath]: fs.readFileSync(sourceDockerfilePath, 'utf8'), + [commonDockerfilePath]: fs.readFileSync(commonDockerfilePath, 'utf8'), + [toolsDockerfilePath]: fs.readFileSync(toolsDockerfilePath, 'utf8'), + [copiedFilePath]: fs.readFileSync(copiedFilePath, 'utf8'), + }; + const cliHost = createMockCLIHost(files); + + const result = await resolveDockerfileIncludesIfNeeded(cliHost, sourceDockerfilePath); + + assert.isTrue(result.preprocessed); + assert.include(result.effectiveDockerfileContent, 'FROM debian:bookworm-slim'); + assert.include(result.effectiveDockerfileContent, 'RUN apt-get update && apt-get install -y curl wget'); + assert.include(result.effectiveDockerfileContent, 'COPY ./bootstrap.sh /usr/local/bin/bootstrap.sh'); + assert.deepEqual(Object.keys(files).sort(), [sourceDockerfilePath, commonDockerfilePath, toolsDockerfilePath, copiedFilePath].sort()); + }); + + it('materializes podman-preprocess-without feature beside the original Dockerfile so relative COPY keeps working', async () => { + const fixturePath = path.resolve(__dirname, 'configs', 'podman-preprocess-without feature'); + const sourceDockerfilePath = path.join(fixturePath, 'Dockerfile.in'); + const commonDockerfilePath = path.join(fixturePath, 'common.Dockerfile'); + const toolsDockerfilePath = path.join(fixturePath, 'tools.Dockerfile'); + const copiedFilePath = path.join(fixturePath, 'bootstrap.sh'); + const files: Record = { + [sourceDockerfilePath]: fs.readFileSync(sourceDockerfilePath, 'utf8'), + [commonDockerfilePath]: fs.readFileSync(commonDockerfilePath, 'utf8'), + [toolsDockerfilePath]: fs.readFileSync(toolsDockerfilePath, 'utf8'), + [copiedFilePath]: fs.readFileSync(copiedFilePath, 'utf8'), + }; + const cliHost = createMockCLIHost(files); + + const resolvedDockerfile = await resolveDockerfileIncludesIfNeeded(cliHost, sourceDockerfilePath); + const materialized = await materializeResolvedDockerfileForBuild(cliHost, resolvedDockerfile); + + assert.equal(path.dirname(materialized.dockerfilePath), fixturePath); + assert.include(files[materialized.dockerfilePath], 'COPY ./bootstrap.sh /usr/local/bin/bootstrap.sh'); + assert.equal(path.resolve(path.dirname(materialized.dockerfilePath), 'bootstrap.sh'), copiedFilePath); + + await materialized.dispose(); + assert.notProperty(files, materialized.dockerfilePath); + assert.property(files, copiedFilePath); + }); + + it('returns the original Dockerfile path when materialization is unnecessary', async () => { + const files: Record = { + '/workspace/Dockerfile': 'FROM debian:latest\nRUN echo ok', + }; + const cliHost = createMockCLIHost(files); + const result = await resolveDockerfileIncludesIfNeeded(cliHost, '/workspace/Dockerfile'); + const materialized = await materializeResolvedDockerfileForBuild(cliHost, result); + + assert.equal(materialized.dockerfilePath, '/workspace/Dockerfile'); + await materialized.dispose(); + assert.equal(files['/workspace/Dockerfile'], 'FROM debian:latest\nRUN echo ok'); }); it('fails with a clear error when #include has a cycle', async () => { @@ -90,4 +221,45 @@ describe('resolveDockerfileIncludesIfNeeded', () => { assert.ok(err); assert.include(String(err.message || err), 'Cyclic #include detected while preprocessing Dockerfile'); }); + + it('supports #define/#undef with conditionals and #warning', async () => { + const files: Record = { + '/workspace/Dockerfile.in': [ + '#define BASE_IMAGE docker.io/debian:bookworm', + '#if defined(BASE_IMAGE)', + 'FROM BASE_IMAGE', + '#else', + '#error BASE_IMAGE must be defined', + '#endif', + '#warning Using BASE_IMAGE', + '#undef BASE_IMAGE', + '#ifndef BASE_IMAGE', + 'RUN echo fallback-ok', + '#endif', + ].join('\n'), + }; + const cliHost = createMockCLIHost(files); + const result = await resolveDockerfileIncludesIfNeeded(cliHost, '/workspace/Dockerfile.in'); + + assert.isTrue(result.preprocessed); + assert.include(result.effectiveDockerfileContent, 'FROM docker.io/debian:bookworm'); + assert.include(result.effectiveDockerfileContent, '# warning: Using docker.io/debian:bookworm'); + assert.include(result.effectiveDockerfileContent, 'RUN echo fallback-ok'); + }); + + it('fails with a clear error when preprocessed output has no resolved FROM', async () => { + const files: Record = { + '/workspace/Dockerfile.in': '#if 0\nFROM docker.io/debian:latest\n#endif\nRUN echo missing-from', + }; + const cliHost = createMockCLIHost(files); + let err: any; + try { + await resolveDockerfileIncludesIfNeeded(cliHost, '/workspace/Dockerfile.in'); + } catch (e) { + err = e; + } + + assert.ok(err); + assert.include(String(err.message || err), 'contains no resolved FROM instruction'); + }); });