Support .in Dockerfiles via cpp when available

src/spec-node/dockerCompose.ts

@@ -18,7 +18,7 @@ import { getExtendImageBuildInfo, updateRemoteUserUID } from './containerFeature
 import { Mount, parseMount } from '../spec-configuration/containerFeaturesConfiguration';
 import path from 'path';
 import { getDevcontainerMetadata, getImageBuildInfoFromDockerfile, getImageBuildInfoFromImage, getImageMetadataFromContainer, ImageBuildInfo, lifecycleCommandOriginMapFromMetadata, mergeConfiguration, MergedDevContainerConfig } from './imageMetadata';
-import { ensureDockerfileHasFinalStageName } from './dockerfileUtils';
+import { ensureDockerfileHasFinalStageName, preprocessDockerfileIn } from './dockerfileUtils';
 import { randomUUID } from 'crypto';
 
 const projectLabel = 'com.docker.compose.project';
@@ -167,7 +167,10 @@ export async function buildAndExtendDockerCompose(configWithRaw: SubstitutedConf
 	if (serviceInfo.build) {
 		const { context, dockerfilePath, target } = serviceInfo.build;
 		const resolvedDockerfilePath = cliHost.path.isAbsolute(dockerfilePath) ? dockerfilePath : path.resolve(context, dockerfilePath);
-		const originalDockerfile = (await cliHost.readFile(resolvedDockerfilePath)).toString();
+		let originalDockerfile = (await cliHost.readFile(resolvedDockerfilePath)).toString();
+		if (resolvedDockerfilePath.endsWith('.in')) {
+			originalDockerfile = await preprocessDockerfileIn(resolvedDockerfilePath, cliHost.exec, output);
+		}
 		dockerfile = originalDockerfile;
 		if (target) {
 			// Explictly set build target for the dev container build features on that

src/spec-node/dockerfileUtils.ts

@@ -4,6 +4,8 @@
  *--------------------------------------------------------------------------------------------*/
 
 import * as semver from 'semver';
+import { ExecFunction, runCommandNoPty } from '../spec-common/commonUtils';
+import { Log } from '../spec-utils/log';
 import { Mount } from '../spec-configuration/containerFeaturesConfiguration';
 
 
@@ -260,6 +262,36 @@ export function ensureDockerfileHasFinalStageName(dockerfile: string, defaultLas
 	return { lastStageName: defaultLastStageName, modifiedDockerfile: modifiedDockerfile };
 }
 
+/**
+ * Preprocess a Dockerfile.in file using the host cpp tool.
+ * This is needed when using Podman, which supports cpp preprocessing of .in files natively.
+ * Throws a clear error if cpp is not available on the host.
+ */
+export async function preprocessDockerfileIn(dockerfilePath: string, exec: ExecFunction, output: Log): Promise<string> {
+	let result: { stdout: Buffer; stderr: Buffer };
+	try {
+		result = await runCommandNoPty({
+			exec,
+			cmd: 'cpp',
+			// -undef:               do not predefine platform/compiler macros
+			// -w:                   suppress warnings
+			// -P:                   suppress linemarker output lines
+			args: ['-P', dockerfilePath],
+			output,
+		});
+	} catch (err: any) {
+		if (err?.code === 'ENOENT' || err?.message?.includes('ENOENT')) {
+			throw new Error(
+				`Preprocessing '${dockerfilePath}' requires 'cpp', but it was not found on the host. ` +
+				`Please install cpp (e.g. "sudo apt-get install cpp") to use Dockerfile.in files with devcontainers.`
+			);
+		}
+		const stderrText = err?.stderr ? `\n${(err.stderr as Buffer).toString()}` : '';
+		throw new Error(`Failed to preprocess '${dockerfilePath}' using cpp: ${err?.message || String(err)}${stderrText}`);
+	}
+	return result.stdout.toString();
+}
+
 export function supportsBuildContexts(dockerfile: Dockerfile) {
 	const version = dockerfile.preamble.version;
 	if (!version) {

src/spec-node/singleContainer.ts

@@ -12,7 +12,7 @@ import { DevContainerConfig, DevContainerFromDockerfileConfig, DevContainerFromI
 import { LogLevel, Log, makeLog } from '../spec-utils/log';
 import { extendImage, getExtendImageBuildInfo, updateRemoteUserUID } from './containerFeatures';
 import { getDevcontainerMetadata, getImageBuildInfoFromDockerfile, getImageMetadataFromContainer, ImageMetadataEntry, lifecycleCommandOriginMapFromMetadata, mergeConfiguration, MergedDevContainerConfig } from './imageMetadata';
-import { ensureDockerfileHasFinalStageName, generateMountCommand } from './dockerfileUtils';
+import { ensureDockerfileHasFinalStageName, generateMountCommand, preprocessDockerfileIn } from './dockerfileUtils';
 
 export const hostFolderLabel = 'devcontainer.local_folder'; // used to label containers created from a workspace/folder
 export const configFileLabel = 'devcontainer.config_file';
@@ -131,6 +131,9 @@ async function buildAndExtendImage(buildParams: DockerResolverParameters, config
 	}
 
 	let dockerfile = (await cliHost.readFile(dockerfilePath)).toString();
+	if (dockerfilePath.endsWith('.in')) {
+		dockerfile = await preprocessDockerfileIn(dockerfilePath, cliHost.exec, output);
+	}
 	const originalDockerfile = dockerfile;
 	let baseName = 'dev_container_auto_added_stage_label';
 	if (config.build?.target) {

src/test/configs/preprocessdocker-with-cpp/.devcontainer.json

@@ -0,0 +1,10 @@
+{
+	"build": {
+		"dockerfile": "Dockerfile.in"
+	},
+	"features": {
+		"ghcr.io/devcontainers/features/github-cli:1": {
+			"version": "latest"
+		}
+	}
+}

src/test/configs/preprocessdocker-with-cpp/Dockerfile.in

@@ -0,0 +1,16 @@
+#define BASE_IMAGE ubuntu:20.04
+#define INSTALL_NODE
+#define INSTALL_PYTHON
+
+FROM BASE_IMAGE
+
+#ifdef INSTALL_NODE
+RUN apt-get update && apt-get install -y nodejs
+#endif
+
+#ifdef INSTALL_PYTHON
+RUN apt-get update && apt-get install -y python3
+#endif
+
+#include "common.Dockerfile"
+#include "tools.Dockerfile"

src/test/configs/preprocessdocker-with-cpp/common.Dockerfile

@@ -0,0 +1,4 @@
+RUN apt-get update && apt-get install -y curl wget
+
+ENV APP_ENV=development
+ENV APP_DEBUG=true

src/test/configs/preprocessdocker-with-cpp/test.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+echo "hello! podman with cpp test"

src/test/configs/preprocessdocker-with-cpp/tools.Dockerfile

@@ -0,0 +1,2 @@
+RUN apt-get update && apt-get install -y vim
+COPY ./test.sh /usr/local/bin/test.sh

src/test/configs/preprocessdockercompose-with-cpp/.devcontainer/Dockerfile.in

@@ -0,0 +1,15 @@
+#define BASE_IMAGE ubuntu:20.04
+#define INSTALL_NODE
+#define INSTALL_PYTHON
+
+FROM BASE_IMAGE
+
+#ifdef INSTALL_NODE
+RUN apt-get update && apt-get install -y nodejs
+#endif
+
+#ifdef INSTALL_PYTHON
+RUN apt-get update && apt-get install -y python3
+#endif
+
+#include "common.Dockerfile"

src/test/configs/preprocessdockercompose-with-cpp/.devcontainer/common.Dockerfile

@@ -0,0 +1,4 @@
+RUN apt-get update && apt-get install -y curl wget
+
+ENV APP_ENV=development
+ENV APP_DEBUG=true

src/test/configs/preprocessdockercompose-with-cpp/.devcontainer/devcontainer.json

@@ -0,0 +1,6 @@
+{
+	"name": "Node.js & Mongo DB",
+	"dockerComposeFile": "docker-compose.yml",
+	"service": "app",
+	"workspaceFolder": "/workspace"
+}

src/test/configs/preprocessdockercompose-with-cpp/.devcontainer/docker-compose.yml

@@ -0,0 +1,17 @@
+version: '3.8'
+
+services:
+  app:
+    build:
+      context: .
+      dockerfile: Dockerfile.in
+    # Overrides default command so things don't shut down after the process ends.
+    command: sleep infinity
+  db:
+    image: mongo:latest
+    restart: unless-stopped
+    volumes:
+      - mongodb-data:/data/db
+
+volumes:
+  mongodb-data: null

src/test/preprocess.test.ts

@@ -0,0 +1,165 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { assert } from 'chai';
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import { ExecFunction, plainExec } from '../spec-common/commonUtils';
+import { nullLog } from '../spec-utils/log';
+import { ensureDockerfileHasFinalStageName, preprocessDockerfileIn } from '../spec-node/dockerfileUtils';
+import { shellExec } from './testUtils';
+
+describe('preprocessDockerfileIn', () => {
+    // Use the actual Dockerfile.in from the podman-with-cpp config directory.
+    // It defines BASE_IMAGE, INSTALL_NODE, and INSTALL_PYTHON macros, uses
+    // #ifdef/#endif blocks, and #includes common.Dockerfile and tools.Dockerfile.
+    const configDir = path.join(__dirname, 'configs', 'preprocessdocker-with-cpp');
+    const dockerfileInPath = path.join(configDir, 'Dockerfile.in');
+
+    let tmpDir: string;
+
+    before(() => {
+        tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'devcontainer-preprocess-test-'));
+    });
+
+    after(() => {
+        fs.rmSync(tmpDir, { recursive: true, force: true });
+    });
+
+    describe('when cpp is available on the host', () => {
+        let exec: ExecFunction;
+
+        before(async () => {
+            exec = await plainExec(undefined);
+        });
+
+        it('should preprocess #ifdef / #endif conditional blocks', async () => {
+            // INSTALL_NODE and INSTALL_PYTHON are both #define'd in Dockerfile.in
+            const result = await preprocessDockerfileIn(dockerfileInPath, exec, nullLog);
+
+            assert.include(result, 'apt-get install -y nodejs');
+            assert.include(result, 'apt-get install -y python3');
+            assert.notInclude(result, '#ifdef');
+            assert.notInclude(result, '#endif');
+        });
+
+        it('should inline content from #include directives', async () => {
+            // Dockerfile.in #includes common.Dockerfile and tools.Dockerfile
+            const result = await preprocessDockerfileIn(dockerfileInPath, exec, nullLog);
+
+            assert.include(result, 'apt-get install -y curl wget');
+            assert.include(result, 'APP_ENV=development');
+            assert.include(result, 'apt-get install -y vim');
+        });
+
+        it('should substitute macros and pass through plain Dockerfile content', async () => {
+            // BASE_IMAGE is #define'd as ubuntu:20.04 in Dockerfile.in
+            const result = await preprocessDockerfileIn(dockerfileInPath, exec, nullLog);
+
+            assert.include(result, 'FROM ubuntu:20.04');
+            assert.notInclude(result, 'FROM BASE_IMAGE');
+        });
+
+        it('should produce output parseable by ensureDockerfileHasFinalStageName when stage is unnamed', async () => {
+            // Dockerfile.in has no AS-named final stage, so a fallback name is assigned
+            const preprocessed = await preprocessDockerfileIn(dockerfileInPath, exec, nullLog);
+            const { lastStageName, modifiedDockerfile } = ensureDockerfileHasFinalStageName(preprocessed, 'dev');
+
+            assert.equal(lastStageName, 'dev');
+            assert.isDefined(modifiedDockerfile);
+        });
+
+        it('should produce output where ensureDockerfileHasFinalStageName assigns a name to an unnamed final stage', async () => {
+            // Dockerfile.in has no AS-named final stage, so the auto-generated label is injected
+            const preprocessed = await preprocessDockerfileIn(dockerfileInPath, exec, nullLog);
+            const { lastStageName, modifiedDockerfile } = ensureDockerfileHasFinalStageName(preprocessed, 'dev_container_auto_added_stage_label');
+
+            assert.equal(lastStageName, 'dev_container_auto_added_stage_label');
+            assert.isDefined(modifiedDockerfile);
+            assert.include(modifiedDockerfile!, 'AS dev_container_auto_added_stage_label');
+        });
+
+        it('should throw an error when the input file does not exist', async () => {
+            const nonExistentPath = path.join(tmpDir, 'does-not-exist.in');
+
+            let caughtError: Error | undefined;
+            try {
+                await preprocessDockerfileIn(nonExistentPath, exec, nullLog);
+            } catch (err: any) {
+                caughtError = err;
+            }
+
+            assert.isDefined(caughtError);
+            assert.include(caughtError!.message, 'Failed to preprocess');
+        });
+    });
+
+    describe('when cpp is not available on the host', () => {
+        // Simulate ENOENT by making exec throw with code 'ENOENT',
+        // mirroring what happens when the binary is missing.
+        const cppNotFoundExec: ExecFunction = async (_params) => {
+            const err: any = new Error('spawn cpp ENOENT');
+            err.code = 'ENOENT';
+            throw err;
+        };
+
+        it('should throw a clear error message directing the user to install cpp', async () => {
+            let caughtError: Error | undefined;
+            try {
+                await preprocessDockerfileIn(dockerfileInPath, cppNotFoundExec, nullLog);
+            } catch (err: any) {
+                caughtError = err;
+            }
+
+            assert.isDefined(caughtError);
+            assert.include(caughtError!.message, 'cpp');
+        });
+    });
+});
+
+describe('preprocess Docker Compose config', function () {
+    this.timeout('240s');
+
+    const cli = 'node ./dist/spec-node/devContainersSpecCLI.js';
+    const containerEngine = 'docker';
+    const workspaceFolder = path.join(__dirname, 'configs', 'preprocessdockercompose-with-cpp');
+
+    let containerId: string | undefined;
+    let composeProjectName: string | undefined;
+    let outcome: string | undefined;
+
+    before(async () => {
+        const res = await shellExec(`${cli} up --workspace-folder ${workspaceFolder}`);
+        const response = JSON.parse(res.stdout);
+
+        outcome = response.outcome;
+        containerId = response.containerId;
+        composeProjectName = response.composeProjectName;
+    });
+
+    after(async () => {
+        if (composeProjectName) {
+            await shellExec(`${containerEngine} compose --project-name ${composeProjectName} down -v`, undefined, true, true);
+        }
+        if (containerId) {
+            await shellExec(`${containerEngine} rm -f ${containerId}`, undefined, true, true);
+        }
+    });
+
+    it('should execute successfully for a docker-compose config that builds from Dockerfile.in', () => {
+        assert.equal(outcome, 'success');
+        assert.isDefined(containerId);
+        assert.isDefined(composeProjectName);
+    });
+
+    it('should build the service from preprocessed Dockerfile.in content', async () => {
+        const res = await shellExec(`${containerEngine} exec ${containerId} sh -lc \"command -v python3 && command -v curl && command -v wget\"`);
+
+        assert.include(res.stdout, 'python3');
+        assert.include(res.stdout, 'curl');
+        assert.include(res.stdout, 'wget');
+    });
+});