feat: add build cache support

.env

@@ -11,3 +11,5 @@ OPR_EXECUTOR_DOCKER_HUB_PASSWORD=
 OPR_EXECUTOR_RUNTIME_VERSIONS=v2,v5
 OPR_EXECUTOR_RETRY_ATTEMPTS=5
 OPR_EXECUTOR_RETRY_DELAY_MS=500
+OPR_EXECUTOR_BUILD_CACHE_VOLUME=openruntimes-build-cache
+OPR_EXECUTOR_BUILD_CACHE_HELPER_IMAGE=busybox:1.37

.gitignore

@@ -5,3 +5,4 @@
 /tests/resources/functions/**/code.zip
 /tests/resources/sites/**/code.tar.gz
 /tests/resources/sites/**/code.zip
+.DS_Store

README.md

@@ -61,6 +61,8 @@ services:
       - OPR_EXECUTOR_RETRY_ATTEMPTS
       - OPR_EXECUTOR_RETRY_DELAY_MS
       - OPR_EXECUTOR_IMAGE_PULL
+      - OPR_EXECUTOR_BUILD_CACHE_VOLUME
+      - OPR_EXECUTOR_BUILD_CACHE_HELPER_IMAGE
 
 networks:
   openruntimes-runtimes:
@@ -88,6 +90,8 @@ OPR_EXECUTOR_DOCKER_HUB_PASSWORD=
 OPR_EXECUTOR_RUNTIME_VERSIONS=v5
 OPR_EXECUTOR_RETRY_ATTEMPTS=5
 OPR_EXECUTOR_RETRY_DELAY_MS=500
+OPR_EXECUTOR_BUILD_CACHE_VOLUME=openruntimes-build-cache
+OPR_EXECUTOR_BUILD_CACHE_HELPER_IMAGE=busybox:1.37
 ```
 
 > `OPR_EXECUTOR_CONNECTION_STORAGE` takes a DSN string that represents a connection to your storage device. If you would like to use your local filesystem, you can use `file://localhost`. If using S3 or any other provider for storage, use a DSN of the following format `s3://access_key:access_secret@host:port/bucket_name?region=us-east-1`
@@ -152,6 +156,7 @@ docker compose down
 | `variables` | `json` | Environment variables passed into runtime |  | [ ] |
 | `runtimeEntrypoint` | `string` | Commands to run when creating a container. Maximum of 100 commands are allowed, each 1024 characters long. |  | ' ' |
 | `command` | `string` | Commands to run after container is created. Maximum of 100 commands are allowed, each 1024 characters long. |  | ' ' |
+| `cacheKey` | `string` | Optional key for sharing build caches. Must start with a letter or number. Allowed characters are letters, numbers, dots, underscores, and hyphens. |  | ' ' |
 | `timeout` | `integer` | Commands execution time in seconds |  | 600 |
 | `remove` | `boolean` | Remove a runtime after execution |  | false |
 | `cpus` | `float` | Maximum CPU cores runtime can utilize |  | 1 |
@@ -197,6 +202,8 @@ docker compose down
 | OPR_EXECUTOR_RUNTIME_VERSIONS    | Version tag for runtime environments, ex: `v5`                                                                                                |
 | OPR_EXECUTOR_RETRY_ATTEMPTS      | Number of retry attempts for failed executions, ex: `5`                                                                                       |
 | OPR_EXECUTOR_RETRY_DELAY_MS      | Delay (in milliseconds) between retry attempts, ex: `500`                                                                                    |
+| OPR_EXECUTOR_BUILD_CACHE_VOLUME | Docker volume name used to store build caches by `cacheKey`, ex: `openruntimes-build-cache`                           |
+| OPR_EXECUTOR_BUILD_CACHE_HELPER_IMAGE | Helper image used to create Docker volume subpaths before mounting isolated build caches, ex: `busybox:1.37` |
 
 ## Contributing
 

app/controllers.php

@@ -60,6 +60,7 @@
     ->param('variables', [], new Assoc(), 'Environment variables passed into runtime.', true)
     ->param('runtimeEntrypoint', '', new Text(1024, 0), 'Commands to run when creating a container. Maximum of 100 commands are allowed, each 1024 characters long.', true)
     ->param('command', '', new Text(1024, 0), 'Commands to run after container is created. Maximum of 100 commands are allowed, each 1024 characters long.', true)
+    ->param('cacheKey', '', new Text(128, 0), 'Cache key used for build caches.', true)
     ->param('timeout', 600, new Integer(), 'Commands execution time in seconds.', true)
     ->param('remove', false, new Boolean(), 'Remove a runtime after execution.', true)
     ->param('cpus', 1, new FloatValidator(true), 'Container CPU.', true)
@@ -68,9 +69,13 @@
     ->param('restartPolicy', DockerAPI::RESTART_NO, new WhiteList([DockerAPI::RESTART_NO, DockerAPI::RESTART_ALWAYS, DockerAPI::RESTART_ON_FAILURE, DockerAPI::RESTART_UNLESS_STOPPED], true), 'Define restart policy for the runtime once an exit code is returned. Default value is "no". Possible values are "no", "always", "on-failure", "unless-stopped".', true)
     ->inject('response')
     ->inject('runner')
-    ->action(function (string $runtimeId, string $image, string $entrypoint, string $source, string $destination, string $outputDirectory, array $variables, string $runtimeEntrypoint, string $command, int $timeout, bool $remove, float $cpus, int $memory, string $version, string $restartPolicy, Response $response, Runner $runner): void {
+    ->action(function (string $runtimeId, string $image, string $entrypoint, string $source, string $destination, string $outputDirectory, array $variables, string $runtimeEntrypoint, string $command, string $cacheKey, int $timeout, bool $remove, float $cpus, int $memory, string $version, string $restartPolicy, Response $response, Runner $runner): void {
         $secret = \bin2hex(\random_bytes(16));
 
+        if ($cacheKey !== '' && !\preg_match('/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/', $cacheKey)) {
+            throw new Exception(Exception::EXECUTION_BAD_REQUEST, 'Cache key must start with a letter or number and may only contain letters, numbers, dots, underscores, and hyphens.');
+        }
+
         /**
          * Create container
          */
@@ -102,7 +107,7 @@
 
         $variables = array_map(strval(...), $variables);
 
-        $container = $runner->createRuntime($runtimeId, $secret, $image, $entrypoint, $source, $destination, $variables, $runtimeEntrypoint, $command, $timeout, $remove, $cpus, $memory, $version, $restartPolicy);
+        $container = $runner->createRuntime($runtimeId, $secret, $image, $entrypoint, $source, $destination, $variables, $runtimeEntrypoint, $command, $timeout, $remove, $cpus, $memory, $version, $restartPolicy, $cacheKey);
         $response->setStatusCode(Response::STATUS_CODE_CREATED)->json($container);
     });
 

composer.json

@@ -25,11 +25,11 @@
     "ext-json": "*",
     "ext-swoole": "*",
     "utopia-php/config": "^0.2.2",
-    "utopia-php/console": "^0.1.1",
+    "utopia-php/console": "^0.2.0",
     "utopia-php/di": "0.3.*",
     "utopia-php/dsn": "0.2.*",
     "utopia-php/http": "0.34.*",
-    "utopia-php/orchestration": "^0.19.2",
+    "utopia-php/orchestration": "dev-feature/mount-value-object",
     "utopia-php/storage": "^2.0",
     "utopia-php/system": "0.10.*"
   },

docker-compose.yml

@@ -32,6 +32,8 @@ services:
       - OPR_EXECUTOR_RETRY_ATTEMPTS
       - OPR_EXECUTOR_RETRY_DELAY_MS
       - OPR_EXECUTOR_IMAGE_PULL
+      - OPR_EXECUTOR_BUILD_CACHE_VOLUME
+      - OPR_EXECUTOR_BUILD_CACHE_HELPER_IMAGE
 
 volumes:
   openruntimes-builds:

src/Executor/Runner/Adapter.php

@@ -31,6 +31,7 @@ abstract public function createRuntime(
         int $memory,
         string $version,
         string $restartPolicy,
+        string $cacheKey = '',
         string $region = '',
     ): mixed;
 

src/Executor/Runner/Docker.php

@@ -12,6 +12,7 @@
 use Utopia\Console;
 use Utopia\Http\Response;
 use Utopia\Logger\Log;
+use Utopia\Orchestration\Mount;
 use Utopia\Orchestration\Orchestration;
 use Utopia\Orchestration\Exception\Timeout as TimeoutException;
 use Utopia\Orchestration\Exception\Orchestration as OrchestrationException;
@@ -230,6 +231,7 @@ public function createRuntime(
         int $memory,
         string $version,
         string $restartPolicy,
+        string $cacheKey = '',
         string $region = '',
     ): mixed {
         $runtimeName = System::getHostname() . '-' . $runtimeId;
@@ -318,6 +320,14 @@ public function createRuntime(
                 \dirname($tmpBuild) . ':' . $codeMountPath . ':rw',
             ];
 
+            $cacheEnabled = $cacheKey !== '' && $command !== '' && $command !== '0';
+            if ($cacheEnabled) {
+                $cacheVolume = System::getEnv('OPR_EXECUTOR_BUILD_CACHE_VOLUME', 'openruntimes-build-cache');
+                $this->ensureBuildCacheSubpath($cacheVolume, $cacheKey);
+                $volumes[] = Mount::volume($cacheVolume, '/cache', false, $cacheKey);
+                $variables = \array_merge($variables, $this->getPackageManagerCacheVariables());
+            }
+
             if ($version === 'v5') {
                 $volumes[] = \dirname($tmpLogs . '/logs') . ':/mnt/logs:rw';
                 $volumes[] = \dirname($tmpLogging . '/logging') . ':/tmp/logging:rw';
@@ -344,6 +354,10 @@ public function createRuntime(
                 throw new \Exception('Failed to create runtime');
             }
 
+            if ($cacheEnabled) {
+                $command = $this->withPackageManagerCacheLog($command);
+            }
+
             /**
              * Execute any commands if they were provided
              */
@@ -504,6 +518,44 @@ public function createRuntime(
         return $container;
     }
 
+    private function withPackageManagerCacheLog(string $command): string
+    {
+        return "printf '%s\n' '[build cache] Using package manager cache.'; " . $command;
+    }
+
+    private function ensureBuildCacheSubpath(string $cacheVolume, string $cacheKey): void
+    {
+        $output = '';
+        $stderr = '';
+        $helperImage = System::getEnv('OPR_EXECUTOR_BUILD_CACHE_HELPER_IMAGE', 'busybox:1.37');
+
+        $command = 'docker volume create ' . \escapeshellarg($cacheVolume) . ' >/dev/null && docker run --rm --volume ' . \escapeshellarg($cacheVolume . ':/cache:rw') . ' ' . \escapeshellarg($helperImage) . ' mkdir -p ' . \escapeshellarg('/cache/' . $cacheKey);
+        $status = Console::execute($command, '', $output, $stderr, 30);
+
+        if ($status !== 0) {
+            $error = $stderr !== '' ? $stderr : $output;
+            throw new \Exception('Failed to prepare build cache subpath: ' . $error);
+        }
+    }
+
+    /**
+     * @return array<string, string>
+     */
+    private function getPackageManagerCacheVariables(): array
+    {
+        $cacheRoot = '/cache';
+
+        return [
+            'npm_config_cache' => $cacheRoot . '/npm',
+            'YARN_CACHE_FOLDER' => $cacheRoot . '/yarn',
+            'npm_config_store_dir' => $cacheRoot . '/pnpm',
+            'pnpm_config_store_dir' => $cacheRoot . '/pnpm',
+            'XDG_CACHE_HOME' => $cacheRoot . '/xdg-cache',
+            'XDG_STATE_HOME' => $cacheRoot . '/xdg-state',
+            'BUN_INSTALL_CACHE_DIR' => $cacheRoot . '/bun',
+        ];
+    }
+
     public function deleteRuntime(string $runtimeId): void
     {
         $runtimeName = System::getHostname() . '-' . $runtimeId;