From 487249c41329257458230146504118a7f48b4351 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Wed, 10 Jun 2026 00:59:01 +0400 Subject: [PATCH 1/6] Add node modules cache key for builds --- docker-compose.yml | 7 +- .../Modules/Functions/Workers/Builds.php | 71 ++++++++++++++++++- src/Executor/Executor.php | 6 +- 3 files changed, 77 insertions(+), 7 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 7d3f6bdc5f..0e61c9b8b5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1218,11 +1218,11 @@ services: - appwrite openruntimes-executor: - container_name: openruntimes-executor + container_name: exc1 hostname: exc1 <<: *x-logging stop_signal: SIGINT - image: openruntimes/executor:0.11.4 + image: openruntimes/executor:0.24.0 restart: unless-stopped networks: - appwrite @@ -1236,6 +1236,8 @@ services: # It's not possible to share mount file between 2 containers without host mount (copying is too slow) - /tmp:/tmp:rw environment: + - OPR_EXECUTOR_CONNECTION_STORAGE=local://localhost + - OPR_EXECUTOR_CONNECTION_BUILD_CACHE_STORAGE= - OPR_EXECUTOR_IMAGE_PULL=enabled - OPR_EXECUTOR_INACTIVE_TRESHOLD=$_APP_COMPUTE_INACTIVE_THRESHOLD - OPR_EXECUTOR_MAINTENANCE_INTERVAL=$_APP_COMPUTE_MAINTENANCE_INTERVAL @@ -1494,3 +1496,4 @@ volumes: appwrite-builds: appwrite-config: appwrite-models: + openruntimes-node-modules-cache: diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index 92e00cd4e6..760e926a21 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -679,6 +679,14 @@ class Builds extends Action deployment: $deployment ); + $cacheKey = $this->getNodeModulesCacheKey($project, $resource, $runtime, $version, $command); + + Span::add('build.node_modules_cache.enabled', $cacheKey !== ''); + Span::add('build.node_modules_cache.key', $cacheKey); + + $cacheDebugLogs = $this->getNodeModulesCacheDebugLog($cacheKey, $runtime, $version, $command, $source); + $deployment = $this->appendBuildLog($dbForProject, $queueForRealtime, $deployment, $cacheDebugLogs); + $response = null; $err = null; @@ -692,7 +700,7 @@ class Builds extends Action $span = Span::current(); Co::join([ - Co\go(function () use ($executor, &$response, $project, $deployment, $source, $resource, $runtime, $vars, $command, $cpus, $memory, $timeout, &$err, $version, $span) { + Co\go(function () use ($executor, &$response, $project, $deployment, $source, $resource, $runtime, $vars, $command, $cacheKey, $cpus, $memory, $timeout, &$err, $version, $span) { try { if ($version === 'v2') { $command = 'tar -zxf /tmp/code.tar.gz -C /usr/code && cd /usr/local/src/ && ./build.sh'; @@ -737,7 +745,8 @@ class Builds extends Action destination: APP_STORAGE_BUILDS . "/app-{$project->getId()}", variables: $vars, command: $command, - outputDirectory: $outputDirectory ?? '' + outputDirectory: $outputDirectory ?? '', + cacheKey: $cacheKey ); } catch (ExecutorTimeout $error) { @@ -888,7 +897,7 @@ class Builds extends Action $logs = $logsBefore . $logsAfter; } - $deployment->setAttribute('buildLogs', $logs); + $deployment->setAttribute('buildLogs', $cacheDebugLogs . $logs); $adapter = null; if ($resource->getCollection() === 'sites' && ! empty($detectionLogs)) { @@ -1342,6 +1351,62 @@ class Builds extends Action return ''; } + protected function getNodeModulesCacheKey(Document $project, Document $resource, array $runtime, string $version, string $command): string + { + if ($version !== 'v5' || $command === '' || $command === '0') { + return ''; + } + + $hashContext = [ + 'version' => 'v1', + 'projectId' => $project->getId(), + 'resourceType' => $resource->getCollection(), + 'resourceId' => $resource->getId(), + 'runtime' => $runtime['image'] ?? '', + ]; + + return \substr(\hash('sha256', \json_encode($hashContext, JSON_THROW_ON_ERROR)), 0, 48); + } + + protected function getNodeModulesCacheDebugLog(string $cacheKey, array $runtime, string $version, string $command, string $source): string + { + $date = \date('H:i:s'); + $runtimeKey = $runtime['key'] ?? ''; + $prefix = "\033[90m[$date] \033[90m[\033[0mappwrite\033[90m]\033[36m"; + + if ($cacheKey !== '') { + return $prefix . " build cache requested. key={$cacheKey} source={$source} \033[0m\n"; + } + + $reason = match (true) { + $version !== 'v5' => 'runtime version is not v5', + $command === '' || $command === '0' => 'build command is empty', + default => 'cache key unavailable', + }; + + return $prefix . " build cache not requested. reason={$reason} runtime={$runtimeKey} version={$version} source={$source} \033[0m\n"; + } + + protected function appendBuildLog(Database $dbForProject, Realtime $queueForRealtime, Document $deployment, string $log): Document + { + if ($log === '') { + return $deployment; + } + + $logs = $deployment->getAttribute('buildLogs', '') . $log; + $deployment->setAttribute('buildLogs', $logs); + + $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document([ + 'buildLogs' => $logs, + ])); + + $queueForRealtime + ->setPayload($deployment->getArrayCopy()) + ->trigger(); + + return $deployment; + } + /** * @throws Structure * @throws \Utopia\Database\Exception diff --git a/src/Executor/Executor.php b/src/Executor/Executor.php index c570970732..53c86f1287 100644 --- a/src/Executor/Executor.php +++ b/src/Executor/Executor.php @@ -71,7 +71,8 @@ class Executor array $variables = [], ?string $command = null, string $outputDirectory = '', - string $runtimeEntrypoint = '' + string $runtimeEntrypoint = '', + string $cacheKey = '' ) { $runtimeId = "$projectId-$deploymentId-build"; $route = "/runtimes"; @@ -95,7 +96,8 @@ class Executor 'version' => $version, 'timeout' => $timeout, 'outputDirectory' => $outputDirectory, - 'runtimeEntrypoint' => $runtimeEntrypoint + 'runtimeEntrypoint' => $runtimeEntrypoint, + 'cacheKey' => $cacheKey ]; From 241602d9e1edad71d89c89337dd66fa7c01a8a84 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Wed, 10 Jun 2026 02:16:10 +0400 Subject: [PATCH 2/6] Update E2E SitesBase to use exc1 container --- tests/e2e/Services/Sites/SitesBase.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/Services/Sites/SitesBase.php b/tests/e2e/Services/Sites/SitesBase.php index 7ce6772c90..21d3a4ff66 100644 --- a/tests/e2e/Services/Sites/SitesBase.php +++ b/tests/e2e/Services/Sites/SitesBase.php @@ -72,7 +72,7 @@ trait SitesBase throw new Critical('Deployment failed: ' . json_encode($deployment['body'], JSON_PRETTY_PRINT)); } - Console::execute("docker inspect openruntimes-executor --format='{{.State.ExitCode}}'", '', $this->stdout, $this->stderr); + Console::execute("docker inspect exc1 --format='{{.State.ExitCode}}'", '', $this->stdout, $this->stderr); if (\trim($this->stdout) !== '0') { $msg = 'Executor has a problem: ' . $this->stderr . ' (' . $this->stdout . '), current status: '; From 1af1da01e203aadb152b76f1b08aaa972ec3bf68 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Wed, 10 Jun 2026 02:18:02 +0400 Subject: [PATCH 3/6] Update E2E SitesBase executor logs lookup --- tests/e2e/Services/Sites/SitesBase.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/Services/Sites/SitesBase.php b/tests/e2e/Services/Sites/SitesBase.php index 21d3a4ff66..f93a534bc4 100644 --- a/tests/e2e/Services/Sites/SitesBase.php +++ b/tests/e2e/Services/Sites/SitesBase.php @@ -76,7 +76,7 @@ trait SitesBase if (\trim($this->stdout) !== '0') { $msg = 'Executor has a problem: ' . $this->stderr . ' (' . $this->stdout . '), current status: '; - Console::execute("docker compose logs openruntimes-executor", '', $this->stdout, $this->stderr); + Console::execute("docker logs exc1", '', $this->stdout, $this->stderr); $msg .= $this->stdout . ' (' . $this->stderr . ')'; throw new Critical($msg . json_encode($deployment['body'], JSON_PRETTY_PRINT)); From 57269867daf25cf49e2836f6111619dff9445ff5 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Wed, 10 Jun 2026 02:36:06 +0400 Subject: [PATCH 4/6] Update install compose executor config --- app/views/install/compose.phtml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index 6ce1fb5cea..d401db9ce0 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -983,12 +983,12 @@ $hostPath = rtrim($this->getParam('hostPath', ''), '/'); - appwrite openruntimes-executor: - container_name: openruntimes-executor + container_name: exc1 hostname: exc1 <<: *x-logging restart: unless-stopped stop_signal: SIGINT - image: openruntimes/executor:0.11.4 + image: openruntimes/executor:0.24.0 networks: - appwrite - runtimes @@ -1001,6 +1001,8 @@ $hostPath = rtrim($this->getParam('hostPath', ''), '/'); # It's not possible to share mount file between 2 containers without host mount (copying is too slow) - /tmp:/tmp:rw environment: + - OPR_EXECUTOR_CONNECTION_STORAGE=local://localhost + - OPR_EXECUTOR_CONNECTION_BUILD_CACHE_STORAGE= - OPR_EXECUTOR_INACTIVE_TRESHOLD=$_APP_COMPUTE_INACTIVE_THRESHOLD - OPR_EXECUTOR_MAINTENANCE_INTERVAL=$_APP_COMPUTE_MAINTENANCE_INTERVAL - OPR_EXECUTOR_NETWORK=$_APP_COMPUTE_RUNTIMES_NETWORK @@ -1209,3 +1211,4 @@ volumes: appwrite-sites: appwrite-builds: appwrite-config: + openruntimes-node-modules-cache: From 8791964e2b83ca234f2c5eeb86efcaaed5ff530b Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Thu, 11 Jun 2026 18:10:29 +0400 Subject: [PATCH 5/6] Remove node modules cache debug build logs --- .../Modules/Functions/Workers/Builds.php | 44 +------------------ 1 file changed, 1 insertion(+), 43 deletions(-) diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index c945fedb6a..69c64e4df9 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -673,9 +673,6 @@ class Builds extends Action Span::add('build.node_modules_cache.enabled', $cacheKey !== ''); Span::add('build.node_modules_cache.key', $cacheKey); - $cacheDebugLogs = $this->getNodeModulesCacheDebugLog($cacheKey, $runtime, $version, $command, $source); - $deployment = $this->appendBuildLog($dbForProject, $queueForRealtime, $deployment, $cacheDebugLogs); - $response = null; $err = null; @@ -866,7 +863,7 @@ class Builds extends Action ['logs' => $logs, 'detectionLogs' => $detectionLogs] = $this->splitSiteDetectionLogs($logs); - $deployment->setAttribute('buildLogs', $cacheDebugLogs . $logs); + $deployment->setAttribute('buildLogs', $logs); $adapter = null; if ($resource->getCollection() === 'sites' && ! empty($detectionLogs)) { @@ -1323,45 +1320,6 @@ class Builds extends Action return \substr(\hash('sha256', \json_encode($hashContext, JSON_THROW_ON_ERROR)), 0, 48); } - protected function getNodeModulesCacheDebugLog(string $cacheKey, array $runtime, string $version, string $command, string $source): string - { - $date = \date('H:i:s'); - $runtimeKey = $runtime['key'] ?? ''; - $prefix = "\033[90m[$date] \033[90m[\033[0mappwrite\033[90m]\033[36m"; - - if ($cacheKey !== '') { - return $prefix . " build cache requested. key={$cacheKey} source={$source} \033[0m\n"; - } - - $reason = match (true) { - $version !== 'v5' => 'runtime version is not v5', - $command === '' || $command === '0' => 'build command is empty', - default => 'cache key unavailable', - }; - - return $prefix . " build cache not requested. reason={$reason} runtime={$runtimeKey} version={$version} source={$source} \033[0m\n"; - } - - protected function appendBuildLog(Database $dbForProject, Realtime $queueForRealtime, Document $deployment, string $log): Document - { - if ($log === '') { - return $deployment; - } - - $logs = $deployment->getAttribute('buildLogs', '') . $log; - $deployment->setAttribute('buildLogs', $logs); - - $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document([ - 'buildLogs' => $logs, - ])); - - $queueForRealtime - ->setPayload($deployment->getArrayCopy()) - ->trigger(); - - return $deployment; - } - protected function prepareSiteBuildCommand(string $command, string $outputDirectory): string { $listFilesCommand = 'echo "{APPWRITE_DETECTION_SEPARATOR_START}" && cd /usr/local/build'; From c6314d52a6d3637f99dfe3e8310ea491bce30b18 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Fri, 12 Jun 2026 14:41:09 +0400 Subject: [PATCH 6/6] Update executor to 0.24.1 --- .github/workflows/ci.yml | 6 +++--- app/views/install/compose.phtml | 2 +- docker-compose.yml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 78e1ea5255..96b579c006 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -429,7 +429,7 @@ jobs: - name: Wait for Open Runtimes timeout-minutes: 3 run: | - while ! docker compose logs openruntimes-executor | grep -q "Executor is ready."; do + while ! docker logs exc1 | grep -q "Executor is ready."; do echo "Waiting for Executor to come online" sleep 1 done @@ -525,7 +525,7 @@ jobs: - name: Wait for Open Runtimes timeout-minutes: 3 run: | - while ! docker compose logs openruntimes-executor | grep -q "Executor is ready."; do + while ! docker logs exc1 | grep -q "Executor is ready."; do echo "Waiting for Executor to come online" sleep 1 done @@ -678,7 +678,7 @@ jobs: - name: Wait for Open Runtimes timeout-minutes: 3 run: | - while ! docker compose logs openruntimes-executor | grep -q "Executor is ready."; do + while ! docker logs exc1 | grep -q "Executor is ready."; do echo "Waiting for Executor to come online" sleep 1 done diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index d401db9ce0..d71ccff0b4 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -988,7 +988,7 @@ $hostPath = rtrim($this->getParam('hostPath', ''), '/'); <<: *x-logging restart: unless-stopped stop_signal: SIGINT - image: openruntimes/executor:0.24.0 + image: openruntimes/executor:0.24.1 networks: - appwrite - runtimes diff --git a/docker-compose.yml b/docker-compose.yml index 3e05c90104..d4b2aacd03 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1249,7 +1249,7 @@ services: hostname: exc1 <<: *x-logging stop_signal: SIGINT - image: openruntimes/executor:0.24.0 + image: openruntimes/executor:0.24.1 restart: unless-stopped networks: - appwrite