diff --git a/.env b/.env index 4dc03b3f..f6ff2552 100644 --- a/.env +++ b/.env @@ -11,3 +11,5 @@ OPR_EXECUTOR_DOCKER_HUB_PASSWORD= OPR_EXECUTOR_RUNTIME_VERSIONS=v2,v5 OPR_EXECUTOR_RETRY_ATTEMPTS=5 OPR_EXECUTOR_RETRY_DELAY_MS=500 +OPR_EXECUTOR_BUILD_CACHE_VOLUME=openruntimes-build-cache +OPR_EXECUTOR_BUILD_CACHE_HELPER_IMAGE=busybox:1.37 diff --git a/.gitignore b/.gitignore index 68b1d3e9..74188708 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ /tests/resources/functions/**/code.zip /tests/resources/sites/**/code.tar.gz /tests/resources/sites/**/code.zip +.DS_Store diff --git a/README.md b/README.md index cc451d65..a49fe3a2 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,8 @@ services: - OPR_EXECUTOR_RETRY_ATTEMPTS - OPR_EXECUTOR_RETRY_DELAY_MS - OPR_EXECUTOR_IMAGE_PULL + - OPR_EXECUTOR_BUILD_CACHE_VOLUME + - OPR_EXECUTOR_BUILD_CACHE_HELPER_IMAGE networks: openruntimes-runtimes: @@ -88,6 +90,8 @@ OPR_EXECUTOR_DOCKER_HUB_PASSWORD= OPR_EXECUTOR_RUNTIME_VERSIONS=v5 OPR_EXECUTOR_RETRY_ATTEMPTS=5 OPR_EXECUTOR_RETRY_DELAY_MS=500 +OPR_EXECUTOR_BUILD_CACHE_VOLUME=openruntimes-build-cache +OPR_EXECUTOR_BUILD_CACHE_HELPER_IMAGE=busybox:1.37 ``` > `OPR_EXECUTOR_CONNECTION_STORAGE` takes a DSN string that represents a connection to your storage device. If you would like to use your local filesystem, you can use `file://localhost`. If using S3 or any other provider for storage, use a DSN of the following format `s3://access_key:access_secret@host:port/bucket_name?region=us-east-1` @@ -152,6 +156,7 @@ docker compose down | `variables` | `json` | Environment variables passed into runtime | | [ ] | | `runtimeEntrypoint` | `string` | Commands to run when creating a container. Maximum of 100 commands are allowed, each 1024 characters long. | | ' ' | | `command` | `string` | Commands to run after container is created. Maximum of 100 commands are allowed, each 1024 characters long. | | ' ' | +| `cacheKey` | `string` | Optional key for sharing build caches. Must start with a letter or number. Allowed characters are letters, numbers, dots, underscores, and hyphens. | | ' ' | | `timeout` | `integer` | Commands execution time in seconds | | 600 | | `remove` | `boolean` | Remove a runtime after execution | | false | | `cpus` | `float` | Maximum CPU cores runtime can utilize | | 1 | @@ -197,6 +202,8 @@ docker compose down | OPR_EXECUTOR_RUNTIME_VERSIONS | Version tag for runtime environments, ex: `v5` | | OPR_EXECUTOR_RETRY_ATTEMPTS | Number of retry attempts for failed executions, ex: `5` | | OPR_EXECUTOR_RETRY_DELAY_MS | Delay (in milliseconds) between retry attempts, ex: `500` | +| OPR_EXECUTOR_BUILD_CACHE_VOLUME | Docker volume name used to store build caches by `cacheKey`, ex: `openruntimes-build-cache` | +| OPR_EXECUTOR_BUILD_CACHE_HELPER_IMAGE | Helper image used to create Docker volume subpaths before mounting isolated build caches, ex: `busybox:1.37` | ## Contributing diff --git a/app/controllers.php b/app/controllers.php index 5fc619eb..b2f16240 100644 --- a/app/controllers.php +++ b/app/controllers.php @@ -60,6 +60,7 @@ ->param('variables', [], new Assoc(), 'Environment variables passed into runtime.', true) ->param('runtimeEntrypoint', '', new Text(1024, 0), 'Commands to run when creating a container. Maximum of 100 commands are allowed, each 1024 characters long.', true) ->param('command', '', new Text(1024, 0), 'Commands to run after container is created. Maximum of 100 commands are allowed, each 1024 characters long.', true) + ->param('cacheKey', '', new Text(128, 0), 'Cache key used for build caches.', true) ->param('timeout', 600, new Integer(), 'Commands execution time in seconds.', true) ->param('remove', false, new Boolean(), 'Remove a runtime after execution.', true) ->param('cpus', 1, new FloatValidator(true), 'Container CPU.', true) @@ -68,9 +69,13 @@ ->param('restartPolicy', DockerAPI::RESTART_NO, new WhiteList([DockerAPI::RESTART_NO, DockerAPI::RESTART_ALWAYS, DockerAPI::RESTART_ON_FAILURE, DockerAPI::RESTART_UNLESS_STOPPED], true), 'Define restart policy for the runtime once an exit code is returned. Default value is "no". Possible values are "no", "always", "on-failure", "unless-stopped".', true) ->inject('response') ->inject('runner') - ->action(function (string $runtimeId, string $image, string $entrypoint, string $source, string $destination, string $outputDirectory, array $variables, string $runtimeEntrypoint, string $command, int $timeout, bool $remove, float $cpus, int $memory, string $version, string $restartPolicy, Response $response, Runner $runner): void { + ->action(function (string $runtimeId, string $image, string $entrypoint, string $source, string $destination, string $outputDirectory, array $variables, string $runtimeEntrypoint, string $command, string $cacheKey, int $timeout, bool $remove, float $cpus, int $memory, string $version, string $restartPolicy, Response $response, Runner $runner): void { $secret = \bin2hex(\random_bytes(16)); + if ($cacheKey !== '' && !\preg_match('/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/', $cacheKey)) { + throw new Exception(Exception::EXECUTION_BAD_REQUEST, 'Cache key must start with a letter or number and may only contain letters, numbers, dots, underscores, and hyphens.'); + } + /** * Create container */ @@ -102,7 +107,7 @@ $variables = array_map(strval(...), $variables); - $container = $runner->createRuntime($runtimeId, $secret, $image, $entrypoint, $source, $destination, $variables, $runtimeEntrypoint, $command, $timeout, $remove, $cpus, $memory, $version, $restartPolicy); + $container = $runner->createRuntime($runtimeId, $secret, $image, $entrypoint, $source, $destination, $variables, $runtimeEntrypoint, $command, $timeout, $remove, $cpus, $memory, $version, $restartPolicy, $cacheKey); $response->setStatusCode(Response::STATUS_CODE_CREATED)->json($container); }); diff --git a/composer.json b/composer.json index 26fe0c99..2da0fe32 100644 --- a/composer.json +++ b/composer.json @@ -25,11 +25,11 @@ "ext-json": "*", "ext-swoole": "*", "utopia-php/config": "^0.2.2", - "utopia-php/console": "^0.1.1", + "utopia-php/console": "^0.2.0", "utopia-php/di": "0.3.*", "utopia-php/dsn": "0.2.*", "utopia-php/http": "0.34.*", - "utopia-php/orchestration": "^0.19.2", + "utopia-php/orchestration": "dev-feature/mount-value-object", "utopia-php/storage": "^2.0", "utopia-php/system": "0.10.*" }, diff --git a/composer.lock b/composer.lock index 68e044ad..16a8461a 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "fcc19e168530cc9be6ce3a78ceb18bff", + "content-hash": "5388b15866a89c7266460dd828945305", "packages": [ { "name": "brick/math", @@ -1974,20 +1974,21 @@ }, { "name": "utopia-php/console", - "version": "0.1.1", + "version": "0.2.1", "source": { "type": "git", "url": "https://github.com/utopia-php/console.git", - "reference": "d298e43960780e6d76e66de1228c75dc81220e3e" + "reference": "97e3de44424ee9ea207c3129dfcc82f8df37c5b5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/console/zipball/d298e43960780e6d76e66de1228c75dc81220e3e", - "reference": "d298e43960780e6d76e66de1228c75dc81220e3e", + "url": "https://api.github.com/repos/utopia-php/console/zipball/97e3de44424ee9ea207c3129dfcc82f8df37c5b5", + "reference": "97e3de44424ee9ea207c3129dfcc82f8df37c5b5", "shasum": "" }, "require": { - "php": ">=8.0" + "php": ">=8.0", + "utopia-php/validators": "^0.2.0" }, "require-dev": { "laravel/pint": "1.2.*", @@ -2016,9 +2017,9 @@ ], "support": { "issues": "https://github.com/utopia-php/console/issues", - "source": "https://github.com/utopia-php/console/tree/0.1.1" + "source": "https://github.com/utopia-php/console/tree/0.2.1" }, - "time": "2026-02-10T10:20:29+00:00" + "time": "2026-04-20T10:53:53+00:00" }, { "name": "utopia-php/di", @@ -2176,21 +2177,21 @@ }, { "name": "utopia-php/orchestration", - "version": "0.19.2", + "version": "dev-feature/mount-value-object", "source": { "type": "git", "url": "https://github.com/utopia-php/orchestration.git", - "reference": "445895025c4b76707ef21630aa5063e41e0e6d36" + "reference": "34f62718bde0017030cb48e90a1c843e1befa49d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/orchestration/zipball/445895025c4b76707ef21630aa5063e41e0e6d36", - "reference": "445895025c4b76707ef21630aa5063e41e0e6d36", + "url": "https://api.github.com/repos/utopia-php/orchestration/zipball/34f62718bde0017030cb48e90a1c843e1befa49d", + "reference": "34f62718bde0017030cb48e90a1c843e1befa49d", "shasum": "" }, "require": { "php": ">=8.0", - "utopia-php/console": "0.1.*" + "utopia-php/console": "0.2.*" }, "require-dev": { "laravel/pint": "^1.2", @@ -2220,9 +2221,9 @@ ], "support": { "issues": "https://github.com/utopia-php/orchestration/issues", - "source": "https://github.com/utopia-php/orchestration/tree/0.19.2" + "source": "https://github.com/utopia-php/orchestration/tree/feature/mount-value-object" }, - "time": "2026-05-03T04:11:24+00:00" + "time": "2026-06-01T15:18:44+00:00" }, { "name": "utopia-php/servers", @@ -2443,16 +2444,16 @@ }, { "name": "utopia-php/validators", - "version": "0.2.3", + "version": "0.2.4", "source": { "type": "git", "url": "https://github.com/utopia-php/validators.git", - "reference": "9770269c8ed8e6909934965fa8722103c7434c23" + "reference": "b4ee60db4dbae5ffbe53968d01f69b6941251576" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/validators/zipball/9770269c8ed8e6909934965fa8722103c7434c23", - "reference": "9770269c8ed8e6909934965fa8722103c7434c23", + "url": "https://api.github.com/repos/utopia-php/validators/zipball/b4ee60db4dbae5ffbe53968d01f69b6941251576", + "reference": "b4ee60db4dbae5ffbe53968d01f69b6941251576", "shasum": "" }, "require": { @@ -2482,9 +2483,9 @@ ], "support": { "issues": "https://github.com/utopia-php/validators/issues", - "source": "https://github.com/utopia-php/validators/tree/0.2.3" + "source": "https://github.com/utopia-php/validators/tree/0.2.4" }, - "time": "2026-05-14T08:05:44+00:00" + "time": "2026-05-21T12:47:43+00:00" } ], "packages-dev": [ @@ -4594,7 +4595,9 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": {}, + "stability-flags": { + "utopia-php/orchestration": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { diff --git a/docker-compose.yml b/docker-compose.yml index 3c4f2b55..46926a0d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -32,6 +32,8 @@ services: - OPR_EXECUTOR_RETRY_ATTEMPTS - OPR_EXECUTOR_RETRY_DELAY_MS - OPR_EXECUTOR_IMAGE_PULL + - OPR_EXECUTOR_BUILD_CACHE_VOLUME + - OPR_EXECUTOR_BUILD_CACHE_HELPER_IMAGE volumes: openruntimes-builds: diff --git a/src/Executor/Runner/Adapter.php b/src/Executor/Runner/Adapter.php index 974d936b..c7bac752 100644 --- a/src/Executor/Runner/Adapter.php +++ b/src/Executor/Runner/Adapter.php @@ -31,6 +31,7 @@ abstract public function createRuntime( int $memory, string $version, string $restartPolicy, + string $cacheKey = '', string $region = '', ): mixed; diff --git a/src/Executor/Runner/Docker.php b/src/Executor/Runner/Docker.php index 0c7dc9bf..1bb55fd6 100644 --- a/src/Executor/Runner/Docker.php +++ b/src/Executor/Runner/Docker.php @@ -12,6 +12,7 @@ use Utopia\Console; use Utopia\Http\Response; use Utopia\Logger\Log; +use Utopia\Orchestration\Mount; use Utopia\Orchestration\Orchestration; use Utopia\Orchestration\Exception\Timeout as TimeoutException; use Utopia\Orchestration\Exception\Orchestration as OrchestrationException; @@ -230,6 +231,7 @@ public function createRuntime( int $memory, string $version, string $restartPolicy, + string $cacheKey = '', string $region = '', ): mixed { $runtimeName = System::getHostname() . '-' . $runtimeId; @@ -318,6 +320,14 @@ public function createRuntime( \dirname($tmpBuild) . ':' . $codeMountPath . ':rw', ]; + $cacheEnabled = $cacheKey !== '' && $command !== '' && $command !== '0'; + if ($cacheEnabled) { + $cacheVolume = System::getEnv('OPR_EXECUTOR_BUILD_CACHE_VOLUME', 'openruntimes-build-cache'); + $this->ensureBuildCacheSubpath($cacheVolume, $cacheKey); + $volumes[] = Mount::volume($cacheVolume, '/cache', false, $cacheKey); + $variables = \array_merge($variables, $this->getPackageManagerCacheVariables()); + } + if ($version === 'v5') { $volumes[] = \dirname($tmpLogs . '/logs') . ':/mnt/logs:rw'; $volumes[] = \dirname($tmpLogging . '/logging') . ':/tmp/logging:rw'; @@ -344,6 +354,10 @@ public function createRuntime( throw new \Exception('Failed to create runtime'); } + if ($cacheEnabled) { + $command = $this->withPackageManagerCacheLog($command); + } + /** * Execute any commands if they were provided */ @@ -504,6 +518,44 @@ public function createRuntime( return $container; } + private function withPackageManagerCacheLog(string $command): string + { + return "printf '%s\n' '[build cache] Using package manager cache.'; " . $command; + } + + private function ensureBuildCacheSubpath(string $cacheVolume, string $cacheKey): void + { + $output = ''; + $stderr = ''; + $helperImage = System::getEnv('OPR_EXECUTOR_BUILD_CACHE_HELPER_IMAGE', 'busybox:1.37'); + + $command = 'docker volume create ' . \escapeshellarg($cacheVolume) . ' >/dev/null && docker run --rm --volume ' . \escapeshellarg($cacheVolume . ':/cache:rw') . ' ' . \escapeshellarg($helperImage) . ' mkdir -p ' . \escapeshellarg('/cache/' . $cacheKey); + $status = Console::execute($command, '', $output, $stderr, 30); + + if ($status !== 0) { + $error = $stderr !== '' ? $stderr : $output; + throw new \Exception('Failed to prepare build cache subpath: ' . $error); + } + } + + /** + * @return array + */ + private function getPackageManagerCacheVariables(): array + { + $cacheRoot = '/cache'; + + return [ + 'npm_config_cache' => $cacheRoot . '/npm', + 'YARN_CACHE_FOLDER' => $cacheRoot . '/yarn', + 'npm_config_store_dir' => $cacheRoot . '/pnpm', + 'pnpm_config_store_dir' => $cacheRoot . '/pnpm', + 'XDG_CACHE_HOME' => $cacheRoot . '/xdg-cache', + 'XDG_STATE_HOME' => $cacheRoot . '/xdg-state', + 'BUN_INSTALL_CACHE_DIR' => $cacheRoot . '/bun', + ]; + } + public function deleteRuntime(string $runtimeId): void { $runtimeName = System::getHostname() . '-' . $runtimeId; diff --git a/tests/e2e/ExecutorTest.php b/tests/e2e/ExecutorTest.php index 7d748efb..9fab9f00 100644 --- a/tests/e2e/ExecutorTest.php +++ b/tests/e2e/ExecutorTest.php @@ -251,6 +251,21 @@ public function testBuild(): void $response = $this->client->call(Client::METHOD_POST, '/runtimes', [], $params); $this->assertEquals(400, $response['headers']['status-code']); + /** Invalid cache key */ + $params = [ + 'runtimeId' => 'test-build-fail-cache-key-' . $runtimeId, + 'source' => '/storage/functions/php/code.tar.gz', + 'destination' => '/storage/builds/test', + 'entrypoint' => 'index.php', + 'image' => 'openruntimes/php:v5-8.1', + 'command' => 'tar -zxf /tmp/code.tar.gz -C /mnt/code && bash helpers/build.sh "composer install"', + 'cacheKey' => '..', + 'remove' => true + ]; + + $response = $this->client->call(Client::METHOD_POST, '/runtimes', [], $params); + $this->assertEquals(400, $response['headers']['status-code']); + /** Test invalid path */ $params = [ 'runtimeId' => 'test-build-fail-500-' . $runtimeId, @@ -552,6 +567,69 @@ public function testExecute(): void $this->assertEquals(200, $response['headers']['status-code']); } + public function testBuildCache(): void + { + $output = ''; + $stderr = ''; + Console::execute('cd /app/tests/resources/functions/node && tar --exclude code.tar.gz -czf code.tar.gz .', '', $output, $stderr); + + $runtimeId = \bin2hex(\random_bytes(4)); + $cacheKey = 'test-build-cache-' . $runtimeId; + $pnpmCache = '/cache/pnpm'; + + $params = [ + 'runtimeId' => 'test-build-cache-miss-' . $runtimeId, + 'source' => '/storage/functions/node/code.tar.gz', + 'destination' => '/storage/builds/test-cache-miss', + 'entrypoint' => 'index.js', + 'image' => 'openruntimes/node:v5-18.0', + 'command' => 'tar -zxf /tmp/code.tar.gz -C /mnt/code && bash helpers/build.sh "pnpm install && test -d ' . $pnpmCache . ' && touch ' . $pnpmCache . '/.open-runtimes-cache-test"', + 'cacheKey' => $cacheKey, + 'remove' => true, + ]; + + $response = $this->client->call(Client::METHOD_POST, '/runtimes', [], $params); + $this->assertEquals(201, $response['headers']['status-code']); + + $firstBuildOutput = ''; + foreach ($response['body']['output'] as $outputItem) { + $firstBuildOutput .= $outputItem['content']; + } + + $this->assertStringContainsString('[build cache] Using package manager cache.', $firstBuildOutput); + + $params = [ + 'runtimeId' => 'test-build-cache-hit-' . $runtimeId, + 'source' => '/storage/functions/node/code.tar.gz', + 'destination' => '/storage/builds/test-cache-hit', + 'entrypoint' => 'index.js', + 'image' => 'openruntimes/node:v5-18.0', + 'command' => 'tar -zxf /tmp/code.tar.gz -C /mnt/code && bash helpers/build.sh "pnpm install && test -f ' . $pnpmCache . '/.open-runtimes-cache-test"', + 'cacheKey' => $cacheKey, + 'remove' => true, + ]; + + $response = $this->client->call(Client::METHOD_POST, '/runtimes', [], $params); + $this->assertEquals(201, $response['headers']['status-code']); + + $this->assertNotEmpty($response['body']['path']); + + $buildPath = $response['body']['path']; + + $response = $this->client->call(Client::METHOD_POST, '/runtimes/test-exec-cache-' . $runtimeId . '/executions', [], [ + 'source' => $buildPath, + 'entrypoint' => 'index.js', + 'image' => 'openruntimes/node:v5-18.0', + 'runtimeEntrypoint' => 'cp /tmp/code.tar.gz /mnt/code/code.tar.gz && nohup helpers/start.sh "bash helpers/server.sh"', + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals(200, $response['body']['statusCode']); + + $response = $this->client->call(Client::METHOD_DELETE, '/runtimes/test-exec-cache-' . $runtimeId, [], []); + $this->assertEquals(200, $response['headers']['status-code']); + } + // We also test SSR two Set-cookie here public function testSSRLogs(): void {