From 1d7a2cfc7173e5054d8bde369b2977d84457086c Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 17 Feb 2022 18:51:36 +0400 Subject: [PATCH] feat: extract build commands to the builds worker --- app/executor.php | 72 ++++------------ app/workers/builds.php | 13 ++- composer.json | 2 +- composer.lock | 176 +++++++++++++++++++------------------- src/Executor/Executor.php | 8 +- 5 files changed, 123 insertions(+), 148 deletions(-) diff --git a/app/executor.php b/app/executor.php index c0d36a11cc..2f5242e507 100644 --- a/app/executor.php +++ b/app/executor.php @@ -18,6 +18,7 @@ use Utopia\Storage\Device\Local; use Utopia\Storage\Storage; use Utopia\Swoole\Request; use Utopia\Swoole\Response; +use Utopia\Validator\ArrayList; use Utopia\Validator\Assoc; use Utopia\Validator\Range as ValidatorRange; use Utopia\Validator\Text; @@ -118,13 +119,14 @@ App::post('/v1/runtimes') ->param('runtimeId', '', new Text(62), 'Unique runtime ID.') ->param('source', '', new Text(0), 'Path to source files.') ->param('destination', '', new Text(0), 'Destination folder to store build files into.') - ->param('vars', '', new Assoc(), 'Environment Variables required for the build') + ->param('vars', [], new Assoc(), 'Environment Variables required for the build') + ->param('commands', [], new ArrayList(new Text(0)), 'Commands required to build the container') ->param('runtime', '', new Text(128), 'Runtime for the cloud function') ->param('baseImage', '', new Text(128), 'Base image name of the runtime') ->inject('orchestrationPool') ->inject('activeRuntimes') ->inject('response') - ->action(function (string $runtimeId, string $source, string $destination, array $vars, string $runtime, string $baseImage, $orchestrationPool, $activeRuntimes, Response $response) { + ->action(function (string $runtimeId, string $source, string $destination, array $vars, array $commands, string $runtime, string $baseImage, $orchestrationPool, $activeRuntimes, Response $response) { $container = 'r-' . $runtimeId; @@ -173,7 +175,7 @@ App::post('/v1/runtimes') $container = 'b-' . $runtimeId; $vars = array_map(fn ($v) => strval($v), $vars); $orchestration - ->setCpus(App::getEnv('_APP_FUNCTIONS_CPUS', 0)) + ->setCpus(App::getEnv('_APP_FUNCTIONS_CPUS', 1)) ->setMemory(App::getEnv('_APP_FUNCTIONS_MEMORY', 256)) ->setSwap(App::getEnv('_APP_FUNCTIONS_MEMORY_SWAP', 256)); @@ -207,61 +209,21 @@ App::post('/v1/runtimes') /** * Extract user code into build container */ - $untarStdout = ''; - $untarStderr = ''; - $untarSuccess = $orchestration->execute( + $status = $orchestration->execute( name: $container, - command: [ - 'sh', - '-c', - 'mkdir -p /usr/code && cp /tmp/code.tar.gz /usr/workspace/code.tar.gz && cd /usr/workspace/ && tar -zxf /usr/workspace/code.tar.gz -C /usr/code && rm /usr/workspace/code.tar.gz' - ], - stdout: $untarStdout, - stderr: $untarStderr, - timeout: 60 - ); - - if (!$untarSuccess) { - throw new Exception('Failed to extract tarfile ' . $untarStderr, 500); - } - - /** - * Build code and install dependenices - */ - $buildSuccess = $orchestration->execute( - name: $container, - command: ['sh', '-c', 'cd /usr/local/src && ./build.sh'], + command: $commands, stdout: $buildStdout, stderr: $buildStderr, - timeout: App::getEnv('_APP_FUNCTIONS_BUILD_TIMEOUT', 900) + timeout: App::getEnv('_APP_FUNCTIONS_TIMEOUT', 900) ); - if (!$buildSuccess) { - throw new Exception('Failed to build dependencies: ' . $buildStderr, 500); - } - - /** - * Repackage code and save - */ - $compressStdout = ''; - $compressStderr = ''; - $compressSuccess = $orchestration->execute( - name: $container, - command: [ - 'tar', '-C', '/usr/code', '-czvf', '/usr/builds/code.tar.gz', './' - ], - stdout: $compressStdout, - stderr: $compressStderr, - timeout: 60 - ); - - if (!$compressSuccess) { - throw new Exception('Failed to compress built code: ' . $compressStderr, 500); + if (!$status) { + throw new Exception('Failed to build dependenices ' . $buildStderr, 500); } // Check if the build was successful by checking if file exists if (!\file_exists($tmpBuild)) { - throw new Exception('Something went wrong during the build process'); + throw new Exception('Something went wrong during the build process', 500); } /** @@ -287,8 +249,8 @@ App::post('/v1/runtimes') $build = [ 'outputPath' => $outputPath, 'status' => 'ready', - 'stdout' => \utf8_encode(\mb_substr($buildStdout, -4096)), - 'stderr' => \utf8_encode(\mb_substr($buildStderr, -4096)), + 'stdout' => \utf8_encode($buildStdout), + 'stderr' => \utf8_encode($buildStderr), 'startTime' => $buildStart, 'endTime' => $buildEnd, 'duration' => $buildEnd - $buildStart, @@ -332,9 +294,9 @@ App::post('/v1/runtimes') $vars = array_map(fn ($v) => strval($v), $vars); $orchestration - ->setCpus(App::getEnv('_APP_FUNCTIONS_CPUS', '1')) - ->setMemory(App::getEnv('_APP_FUNCTIONS_MEMORY', '256')) - ->setSwap(App::getEnv('_APP_FUNCTIONS_MEMORY_SWAP', '256')); + ->setCpus(App::getEnv('_APP_FUNCTIONS_CPUS', 1)) + ->setMemory(App::getEnv('_APP_FUNCTIONS_MEMORY', 256)) + ->setSwap(App::getEnv('_APP_FUNCTIONS_MEMORY_SWAP', 256)); $id = $orchestration->run( image: $baseImage, @@ -463,7 +425,7 @@ App::post('/v1/execution') ->desc('Create an execution') ->param('runtimeId', '', new Text(62), 'The runtimeID to execute') ->param('path', '', new Text(0), 'Path containing the built files.', false) - ->param('vars', '', new Assoc(), 'Environment variables required for the build', false) + ->param('vars', [], new Assoc(), 'Environment variables required for the build', false) ->param('data', '', new Text(8192), 'Data to be forwarded to the function, this is user specified.', true) ->param('runtime', '', new Text(128), 'Runtime for the cloud function', false) ->param('entrypoint', '', new Text(256), 'Entrypoint of the code file') diff --git a/app/workers/builds.php b/app/workers/builds.php index 3e70c1978b..0bdea41126 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -119,7 +119,18 @@ class BuildsV1 extends Worker source: $source, vars: $vars, runtime: $key, - baseImage: $baseImage + baseImage: $baseImage, + commands: [ + 'sh', '-c', + 'mkdir -p /usr/code && \ + cp /tmp/code.tar.gz /usr/workspace/code.tar.gz && \ + cd /usr/workspace/ && \ + tar -zxf /usr/workspace/code.tar.gz -C /usr/code && \ + rm /usr/workspace/code.tar.gz && \ + cd /usr/local/src && \ + ./build.sh && \ + tar -C /usr/code -czf /usr/builds/code.tar.gz ./' + ] ); /** Update the build document */ diff --git a/composer.json b/composer.json index 5a46b5758f..55834b825b 100644 --- a/composer.json +++ b/composer.json @@ -60,7 +60,7 @@ "utopia-php/storage": "0.5.*", "utopia-php/websocket": "0.1.0", "utopia-php/image": "0.5.*", - "utopia-php/orchestration": "0.3.*", + "utopia-php/orchestration": "0.4.*", "resque/php-resque": "1.3.6", "matomo/device-detector": "5.0.1", "dragonmantank/cron-expression": "3.1.0", diff --git a/composer.lock b/composer.lock index a8640d648b..71ff25ebef 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "1a5d84f96eb76e59f7ad0ff7bcd4a8d8", + "content-hash": "e28c5946b102f637cc247b026f42ae0e", "packages": [ { "name": "adhocore/jwt", @@ -119,11 +119,11 @@ "source": { "type": "git", "url": "https://github.com/appwrite/runtimes.git", - "reference": "991a8674c75f78644b557c9723869c370b3290b5" + "reference": "ea44eb19af851a3fe4ecf73e789f374f70d778c5" }, "require": { "php": ">=8.0", - "utopia-php/orchestration": "0.3.x", + "utopia-php/orchestration": "0.4.*", "utopia-php/system": "0.4.*" }, "require-dev": { @@ -160,7 +160,7 @@ "php", "runtimes" ], - "time": "2022-02-01T10:58:43+00:00" + "time": "2022-02-17T12:06:12+00:00" }, { "name": "chillerlan/php-qrcode", @@ -525,12 +525,12 @@ } }, "autoload": { - "psr-4": { - "GuzzleHttp\\": "src/" - }, "files": [ "src/functions_include.php" - ] + ], + "psr-4": { + "GuzzleHttp\\": "src/" + } }, "notification-url": "https://packagist.org/downloads/", "license": [ @@ -1027,12 +1027,12 @@ } }, "autoload": { - "psr-4": { - "MongoDB\\": "src/" - }, "files": [ "src/functions.php" - ] + ], + "psr-4": { + "MongoDB\\": "src/" + } }, "notification-url": "https://packagist.org/downloads/", "license": [ @@ -1760,12 +1760,12 @@ } }, "autoload": { - "psr-4": { - "Symfony\\Polyfill\\Php80\\": "" - }, "files": [ "bootstrap.php" ], + "psr-4": { + "Symfony\\Polyfill\\Php80\\": "" + }, "classmap": [ "Resources/stubs" ] @@ -2252,16 +2252,16 @@ }, { "name": "utopia-php/framework", - "version": "0.19.5", + "version": "0.19.6", "source": { "type": "git", "url": "https://github.com/utopia-php/framework.git", - "reference": "1c28ba9a5b491cf7c90c535fefee5832c7133623" + "reference": "7d9b28365fb794001cb34dd028659452d4e71b7d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/framework/zipball/1c28ba9a5b491cf7c90c535fefee5832c7133623", - "reference": "1c28ba9a5b491cf7c90c535fefee5832c7133623", + "url": "https://api.github.com/repos/utopia-php/framework/zipball/7d9b28365fb794001cb34dd028659452d4e71b7d", + "reference": "7d9b28365fb794001cb34dd028659452d4e71b7d", "shasum": "" }, "require": { @@ -2295,9 +2295,9 @@ ], "support": { "issues": "https://github.com/utopia-php/framework/issues", - "source": "https://github.com/utopia-php/framework/tree/0.19.5" + "source": "https://github.com/utopia-php/framework/tree/0.19.6" }, - "time": "2022-01-04T14:40:23+00:00" + "time": "2022-02-10T17:05:22+00:00" }, { "name": "utopia-php/image", @@ -2470,16 +2470,16 @@ }, { "name": "utopia-php/orchestration", - "version": "0.3.0", + "version": "0.4.0", "source": { "type": "git", "url": "https://github.com/utopia-php/orchestration.git", - "reference": "5398361b39fdb7a15b561ebf243c4cb75a68432a" + "reference": "6f25a47e6d6b14540853b62db43c676aecf80519" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/orchestration/zipball/5398361b39fdb7a15b561ebf243c4cb75a68432a", - "reference": "5398361b39fdb7a15b561ebf243c4cb75a68432a", + "url": "https://api.github.com/repos/utopia-php/orchestration/zipball/6f25a47e6d6b14540853b62db43c676aecf80519", + "reference": "6f25a47e6d6b14540853b62db43c676aecf80519", "shasum": "" }, "require": { @@ -2519,9 +2519,9 @@ ], "support": { "issues": "https://github.com/utopia-php/orchestration/issues", - "source": "https://github.com/utopia-php/orchestration/tree/0.3.0" + "source": "https://github.com/utopia-php/orchestration/tree/0.4.0" }, - "time": "2021-12-20T14:08:08+00:00" + "time": "2022-02-17T11:48:37+00:00" }, { "name": "utopia-php/preloader", @@ -2682,16 +2682,16 @@ }, { "name": "utopia-php/swoole", - "version": "0.3.2", + "version": "0.3.3", "source": { "type": "git", "url": "https://github.com/utopia-php/swoole.git", - "reference": "2b714eddf77cd5eda1889219c9656d7c0a63ce73" + "reference": "8312df69233b5dcd3992de88f131f238002749de" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/swoole/zipball/2b714eddf77cd5eda1889219c9656d7c0a63ce73", - "reference": "2b714eddf77cd5eda1889219c9656d7c0a63ce73", + "url": "https://api.github.com/repos/utopia-php/swoole/zipball/8312df69233b5dcd3992de88f131f238002749de", + "reference": "8312df69233b5dcd3992de88f131f238002749de", "shasum": "" }, "require": { @@ -2732,9 +2732,9 @@ ], "support": { "issues": "https://github.com/utopia-php/swoole/issues", - "source": "https://github.com/utopia-php/swoole/tree/0.3.2" + "source": "https://github.com/utopia-php/swoole/tree/0.3.3" }, - "time": "2021-12-13T15:37:41+00:00" + "time": "2022-01-20T09:58:43+00:00" }, { "name": "utopia-php/system", @@ -3031,12 +3031,12 @@ } }, "autoload": { - "psr-4": { - "Amp\\ByteStream\\": "lib" - }, "files": [ "lib/functions.php" - ] + ], + "psr-4": { + "Amp\\ByteStream\\": "lib" + } }, "notification-url": "https://packagist.org/downloads/", "license": [ @@ -3197,23 +3197,23 @@ }, { "name": "composer/semver", - "version": "3.2.7", + "version": "3.2.9", "source": { "type": "git", "url": "https://github.com/composer/semver.git", - "reference": "deac27056b57e46faf136fae7b449eeaa71661ee" + "reference": "a951f614bd64dcd26137bc9b7b2637ddcfc57649" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/composer/semver/zipball/deac27056b57e46faf136fae7b449eeaa71661ee", - "reference": "deac27056b57e46faf136fae7b449eeaa71661ee", + "url": "https://api.github.com/repos/composer/semver/zipball/a951f614bd64dcd26137bc9b7b2637ddcfc57649", + "reference": "a951f614bd64dcd26137bc9b7b2637ddcfc57649", "shasum": "" }, "require": { "php": "^5.3.2 || ^7.0 || ^8.0" }, "require-dev": { - "phpstan/phpstan": "^0.12.54", + "phpstan/phpstan": "^1.4", "symfony/phpunit-bridge": "^4.2 || ^5" }, "type": "library", @@ -3258,7 +3258,7 @@ "support": { "irc": "irc://irc.freenode.org/composer", "issues": "https://github.com/composer/semver/issues", - "source": "https://github.com/composer/semver/tree/3.2.7" + "source": "https://github.com/composer/semver/tree/3.2.9" }, "funding": [ { @@ -3274,7 +3274,7 @@ "type": "tidelift" } ], - "time": "2022-01-04T09:57:54+00:00" + "time": "2022-02-04T13:58:43+00:00" }, { "name": "composer/xdebug-handler", @@ -3704,12 +3704,12 @@ }, "type": "library", "autoload": { - "psr-4": { - "DeepCopy\\": "src/DeepCopy/" - }, "files": [ "src/DeepCopy/deep_copy.php" - ] + ], + "psr-4": { + "DeepCopy\\": "src/DeepCopy/" + } }, "notification-url": "https://packagist.org/downloads/", "license": [ @@ -3957,16 +3957,16 @@ }, { "name": "phar-io/version", - "version": "3.1.0", + "version": "3.1.1", "source": { "type": "git", "url": "https://github.com/phar-io/version.git", - "reference": "bae7c545bef187884426f042434e561ab1ddb182" + "reference": "15a90844ad40f127afd244c0cad228de2a80052a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phar-io/version/zipball/bae7c545bef187884426f042434e561ab1ddb182", - "reference": "bae7c545bef187884426f042434e561ab1ddb182", + "url": "https://api.github.com/repos/phar-io/version/zipball/15a90844ad40f127afd244c0cad228de2a80052a", + "reference": "15a90844ad40f127afd244c0cad228de2a80052a", "shasum": "" }, "require": { @@ -4002,9 +4002,9 @@ "description": "Library for handling version information and constraints", "support": { "issues": "https://github.com/phar-io/version/issues", - "source": "https://github.com/phar-io/version/tree/3.1.0" + "source": "https://github.com/phar-io/version/tree/3.1.1" }, - "time": "2021-02-23T14:00:09+00:00" + "time": "2022-02-07T21:56:48+00:00" }, { "name": "phpdocumentor/reflection-common", @@ -4613,11 +4613,11 @@ } }, "autoload": { - "classmap": [ - "src/" - ], "files": [ "src/Framework/Assert/Functions.php" + ], + "classmap": [ + "src/" ] }, "notification-url": "https://packagist.org/downloads/", @@ -5213,16 +5213,16 @@ }, { "name": "sebastian/global-state", - "version": "5.0.3", + "version": "5.0.5", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/global-state.git", - "reference": "23bd5951f7ff26f12d4e3242864df3e08dec4e49" + "reference": "0ca8db5a5fc9c8646244e629625ac486fa286bf2" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/global-state/zipball/23bd5951f7ff26f12d4e3242864df3e08dec4e49", - "reference": "23bd5951f7ff26f12d4e3242864df3e08dec4e49", + "url": "https://api.github.com/repos/sebastianbergmann/global-state/zipball/0ca8db5a5fc9c8646244e629625ac486fa286bf2", + "reference": "0ca8db5a5fc9c8646244e629625ac486fa286bf2", "shasum": "" }, "require": { @@ -5265,7 +5265,7 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/global-state/issues", - "source": "https://github.com/sebastianbergmann/global-state/tree/5.0.3" + "source": "https://github.com/sebastianbergmann/global-state/tree/5.0.5" }, "funding": [ { @@ -5273,7 +5273,7 @@ "type": "github" } ], - "time": "2021-06-11T13:31:12+00:00" + "time": "2022-02-14T08:28:10+00:00" }, { "name": "sebastian/lines-of-code", @@ -5839,12 +5839,12 @@ } }, "autoload": { - "psr-4": { - "Symfony\\Polyfill\\Intl\\Grapheme\\": "" - }, "files": [ "bootstrap.php" - ] + ], + "psr-4": { + "Symfony\\Polyfill\\Intl\\Grapheme\\": "" + } }, "notification-url": "https://packagist.org/downloads/", "license": [ @@ -5920,12 +5920,12 @@ } }, "autoload": { - "psr-4": { - "Symfony\\Polyfill\\Intl\\Normalizer\\": "" - }, "files": [ "bootstrap.php" ], + "psr-4": { + "Symfony\\Polyfill\\Intl\\Normalizer\\": "" + }, "classmap": [ "Resources/stubs" ] @@ -6007,12 +6007,12 @@ } }, "autoload": { - "psr-4": { - "Symfony\\Polyfill\\Mbstring\\": "" - }, "files": [ "bootstrap.php" - ] + ], + "psr-4": { + "Symfony\\Polyfill\\Mbstring\\": "" + } }, "notification-url": "https://packagist.org/downloads/", "license": [ @@ -6084,12 +6084,12 @@ } }, "autoload": { - "psr-4": { - "Symfony\\Polyfill\\Php72\\": "" - }, "files": [ "bootstrap.php" - ] + ], + "psr-4": { + "Symfony\\Polyfill\\Php72\\": "" + } }, "notification-url": "https://packagist.org/downloads/", "license": [ @@ -6400,16 +6400,16 @@ }, { "name": "twig/twig", - "version": "v2.14.10", + "version": "v2.14.11", "source": { "type": "git", "url": "https://github.com/twigphp/Twig.git", - "reference": "95fb194cd4dd6ac373a27af2bde2bad5d3f27aba" + "reference": "66baa66f29ee30e487e05f1679903e36eb01d727" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/twigphp/Twig/zipball/95fb194cd4dd6ac373a27af2bde2bad5d3f27aba", - "reference": "95fb194cd4dd6ac373a27af2bde2bad5d3f27aba", + "url": "https://api.github.com/repos/twigphp/Twig/zipball/66baa66f29ee30e487e05f1679903e36eb01d727", + "reference": "66baa66f29ee30e487e05f1679903e36eb01d727", "shasum": "" }, "require": { @@ -6464,7 +6464,7 @@ ], "support": { "issues": "https://github.com/twigphp/Twig/issues", - "source": "https://github.com/twigphp/Twig/tree/v2.14.10" + "source": "https://github.com/twigphp/Twig/tree/v2.14.11" }, "funding": [ { @@ -6476,7 +6476,7 @@ "type": "tidelift" } ], - "time": "2022-01-03T21:13:26+00:00" + "time": "2022-02-04T06:57:25+00:00" }, { "name": "vimeo/psalm", @@ -6555,13 +6555,13 @@ } }, "autoload": { - "psr-4": { - "Psalm\\": "src/Psalm/" - }, "files": [ "src/functions.php", "src/spl_object_id.php" - ] + ], + "psr-4": { + "Psalm\\": "src/Psalm/" + } }, "notification-url": "https://packagist.org/downloads/", "license": [ diff --git a/src/Executor/Executor.php b/src/Executor/Executor.php index 474ad92cbd..26d9773f66 100644 --- a/src/Executor/Executor.php +++ b/src/Executor/Executor.php @@ -37,8 +37,9 @@ class Executor string $source, array $vars, string $runtime, - string $baseImage) - { + string $baseImage, + array $commands + ) { $route = "/runtimes"; $headers = [ 'content-type' => 'application/json', @@ -51,7 +52,8 @@ class Executor 'destination' => APP_STORAGE_BUILDS . "/app-$projectId", 'vars' => $vars, 'runtime' => $runtime, - 'baseImage' => $baseImage + 'baseImage' => $baseImage, + 'commands' => $commands ]; $response = $this->call(self::METHOD_POST, $route, $headers, $params, true, 30);