diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 78e1ea5255..96b579c006 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -429,7 +429,7 @@ jobs: - name: Wait for Open Runtimes timeout-minutes: 3 run: | - while ! docker compose logs openruntimes-executor | grep -q "Executor is ready."; do + while ! docker logs exc1 | grep -q "Executor is ready."; do echo "Waiting for Executor to come online" sleep 1 done @@ -525,7 +525,7 @@ jobs: - name: Wait for Open Runtimes timeout-minutes: 3 run: | - while ! docker compose logs openruntimes-executor | grep -q "Executor is ready."; do + while ! docker logs exc1 | grep -q "Executor is ready."; do echo "Waiting for Executor to come online" sleep 1 done @@ -678,7 +678,7 @@ jobs: - name: Wait for Open Runtimes timeout-minutes: 3 run: | - while ! docker compose logs openruntimes-executor | grep -q "Executor is ready."; do + while ! docker logs exc1 | grep -q "Executor is ready."; do echo "Waiting for Executor to come online" sleep 1 done diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index 6ce1fb5cea..d71ccff0b4 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -983,12 +983,12 @@ $hostPath = rtrim($this->getParam('hostPath', ''), '/'); - appwrite openruntimes-executor: - container_name: openruntimes-executor + container_name: exc1 hostname: exc1 <<: *x-logging restart: unless-stopped stop_signal: SIGINT - image: openruntimes/executor:0.11.4 + image: openruntimes/executor:0.24.1 networks: - appwrite - runtimes @@ -1001,6 +1001,8 @@ $hostPath = rtrim($this->getParam('hostPath', ''), '/'); # It's not possible to share mount file between 2 containers without host mount (copying is too slow) - /tmp:/tmp:rw environment: + - OPR_EXECUTOR_CONNECTION_STORAGE=local://localhost + - OPR_EXECUTOR_CONNECTION_BUILD_CACHE_STORAGE= - OPR_EXECUTOR_INACTIVE_TRESHOLD=$_APP_COMPUTE_INACTIVE_THRESHOLD - OPR_EXECUTOR_MAINTENANCE_INTERVAL=$_APP_COMPUTE_MAINTENANCE_INTERVAL - OPR_EXECUTOR_NETWORK=$_APP_COMPUTE_RUNTIMES_NETWORK @@ -1209,3 +1211,4 @@ volumes: appwrite-sites: appwrite-builds: appwrite-config: + openruntimes-node-modules-cache: diff --git a/docker-compose.yml b/docker-compose.yml index 212b0f1f47..d4b2aacd03 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1245,11 +1245,11 @@ services: - appwrite openruntimes-executor: - container_name: openruntimes-executor + container_name: exc1 hostname: exc1 <<: *x-logging stop_signal: SIGINT - image: openruntimes/executor:0.11.4 + image: openruntimes/executor:0.24.1 restart: unless-stopped networks: - appwrite @@ -1263,6 +1263,8 @@ services: # It's not possible to share mount file between 2 containers without host mount (copying is too slow) - /tmp:/tmp:rw environment: + - OPR_EXECUTOR_CONNECTION_STORAGE=local://localhost + - OPR_EXECUTOR_CONNECTION_BUILD_CACHE_STORAGE= - OPR_EXECUTOR_IMAGE_PULL=enabled - OPR_EXECUTOR_INACTIVE_TRESHOLD=$_APP_COMPUTE_INACTIVE_THRESHOLD - OPR_EXECUTOR_MAINTENANCE_INTERVAL=$_APP_COMPUTE_MAINTENANCE_INTERVAL @@ -1525,3 +1527,4 @@ volumes: appwrite-builds: appwrite-config: appwrite-models: + openruntimes-node-modules-cache: diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index fae32bf76a..2550db36e2 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -686,6 +686,11 @@ class Builds extends Action deployment: $deployment ); + $cacheKey = $this->getNodeModulesCacheKey($project, $resource, $runtime, $version, $command); + + Span::add('build.node_modules_cache.enabled', $cacheKey !== ''); + Span::add('build.node_modules_cache.key', $cacheKey); + $response = null; $err = null; @@ -699,7 +704,7 @@ class Builds extends Action $span = Span::current(); Co::join([ - Co\go(function () use ($executor, &$response, $project, $deployment, $source, $resource, $runtime, $vars, $command, $cpus, $memory, $timeout, &$err, $version, $span) { + Co\go(function () use ($executor, &$response, $project, $deployment, $source, $resource, $runtime, $vars, $command, $cacheKey, $cpus, $memory, $timeout, &$err, $version, $span) { try { if ($version === 'v2') { $command = 'tar -zxf /tmp/code.tar.gz -C /usr/code && cd /usr/local/src/ && ./build.sh'; @@ -726,7 +731,8 @@ class Builds extends Action destination: APP_STORAGE_BUILDS . "/app-{$project->getId()}", variables: $vars, command: $command, - outputDirectory: $outputDirectory ?? '' + outputDirectory: $outputDirectory ?? '', + cacheKey: $cacheKey ); } catch (ExecutorTimeout $error) { @@ -1315,6 +1321,23 @@ class Builds extends Action return ''; } + protected function getNodeModulesCacheKey(Document $project, Document $resource, array $runtime, string $version, string $command): string + { + if ($version !== 'v5' || $command === '' || $command === '0') { + return ''; + } + + $hashContext = [ + 'version' => 'v1', + 'projectId' => $project->getId(), + 'resourceType' => $resource->getCollection(), + 'resourceId' => $resource->getId(), + 'runtime' => $runtime['image'] ?? '', + ]; + + return \substr(\hash('sha256', \json_encode($hashContext, JSON_THROW_ON_ERROR)), 0, 48); + } + protected function prepareSiteBuildCommand(string $command, string $outputDirectory): string { $listFilesCommand = 'echo "{APPWRITE_DETECTION_SEPARATOR_START}" && cd /usr/local/build'; diff --git a/src/Executor/Executor.php b/src/Executor/Executor.php index c570970732..53c86f1287 100644 --- a/src/Executor/Executor.php +++ b/src/Executor/Executor.php @@ -71,7 +71,8 @@ class Executor array $variables = [], ?string $command = null, string $outputDirectory = '', - string $runtimeEntrypoint = '' + string $runtimeEntrypoint = '', + string $cacheKey = '' ) { $runtimeId = "$projectId-$deploymentId-build"; $route = "/runtimes"; @@ -95,7 +96,8 @@ class Executor 'version' => $version, 'timeout' => $timeout, 'outputDirectory' => $outputDirectory, - 'runtimeEntrypoint' => $runtimeEntrypoint + 'runtimeEntrypoint' => $runtimeEntrypoint, + 'cacheKey' => $cacheKey ]; diff --git a/tests/e2e/Services/Sites/SitesBase.php b/tests/e2e/Services/Sites/SitesBase.php index 1f27ff45de..54344f649a 100644 --- a/tests/e2e/Services/Sites/SitesBase.php +++ b/tests/e2e/Services/Sites/SitesBase.php @@ -72,11 +72,11 @@ trait SitesBase throw new Critical('Deployment failed: ' . json_encode($deployment['body'], JSON_PRETTY_PRINT)); } - $code = Console::execute("docker inspect openruntimes-executor --format='{{.State.ExitCode}}'", '', $this->stdout, $this->stderr); + $code = Console::execute("docker inspect exc1 --format='{{.State.ExitCode}}'", '', $this->stdout, $this->stderr); if ($code === 0 && \trim($this->stdout) !== '' && \trim($this->stdout) !== '0') { $msg = 'Executor has a problem: ' . $this->stderr . ' (' . $this->stdout . '), current status: '; - Console::execute("docker compose logs openruntimes-executor", '', $this->stdout, $this->stderr); + Console::execute("docker logs exc1", '', $this->stdout, $this->stderr); $msg .= $this->stdout . ' (' . $this->stderr . ')'; throw new Critical($msg . json_encode($deployment['body'], JSON_PRETTY_PRINT));