From f75280779016dcd4115d0d61480b9da587dd2f04 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 18 Feb 2022 04:43:04 +0400 Subject: [PATCH] feat: add 406 status code --- app/executor.php | 161 +++++++++++++++++++++----------------- app/workers/builds.php | 2 + composer.lock | 12 +-- src/Executor/Executor.php | 77 +++++++++++------- 4 files changed, 145 insertions(+), 107 deletions(-) diff --git a/app/executor.php b/app/executor.php index 6bb9f13ce6..af68fb30ad 100644 --- a/app/executor.php +++ b/app/executor.php @@ -20,6 +20,7 @@ use Utopia\Swoole\Request; use Utopia\Swoole\Response; use Utopia\Validator\ArrayList; use Utopia\Validator\Assoc; +use Utopia\Validator\Boolean; use Utopia\Validator\Range as ValidatorRange; use Utopia\Validator\Text; @@ -118,16 +119,18 @@ App::post('/v1/runtimes') ->desc("Create a new runtime server") ->param('runtimeId', '', new Text(62), 'Unique runtime ID.') ->param('source', '', new Text(0), 'Path to source files.') - ->param('destination', '', new Text(0), 'Destination folder to store build files into.') + ->param('destination', '', new Text(0), 'Destination folder to store build files into.', true) ->param('vars', [], new Assoc(), 'Environment Variables required for the build') ->param('commands', [], new ArrayList(new Text(0)), 'Commands required to build the container') ->param('runtime', '', new Text(128), 'Runtime for the cloud function') + ->param('network', '', new Text(128), 'Network to attach the container to') ->param('baseImage', '', new Text(128), 'Base image name of the runtime') - ->param('workdir', '', new Text(256), 'Working directory') + ->param('remove', false, new Boolean(), 'Remove a runtime after execution') + ->param('workdir', '', new Text(256), 'Working directory', true) ->inject('orchestrationPool') ->inject('activeRuntimes') ->inject('response') - ->action(function (string $runtimeId, string $source, string $destination, array $vars, array $commands, string $runtime, string $baseImage, string $workdir, $orchestrationPool, $activeRuntimes, Response $response) { + ->action(function (string $runtimeId, string $source, string $destination, array $vars, array $commands, string $runtime, string $network, string $baseImage, bool $remove, string $workdir, $orchestrationPool, $activeRuntimes, Response $response) { $container = 'r-' . $runtimeId; @@ -172,7 +175,11 @@ App::post('/v1/runtimes') * Create container */ $orchestration = $orchestrationPool->get(); - $container = 'b-' . $runtimeId; + $container = 'r-' . $runtimeId; + $secret = \bin2hex(\random_bytes(16)); + $vars = \array_merge($vars, [ + 'INTERNAL_RUNTIME_KEY' => $secret + ]); $vars = array_map(fn ($v) => strval($v), $vars); $orchestration ->setCpus(App::getEnv('_APP_FUNCTIONS_CPUS', 1)) @@ -190,11 +197,11 @@ App::post('/v1/runtimes') 'openruntimes-created' => strval($buildStart), 'openruntimes-runtime' => $runtime, ], - command: [ - 'tail', - '-f', - '/dev/null' - ], + // command: [ + // 'tail', + // '-f', + // '/dev/null' + // ], mountFolder: \dirname($tmpSource), workdir: $workdir, volumes: [ @@ -206,6 +213,10 @@ App::post('/v1/runtimes') throw new Exception('Failed to create build container', 500); } + if (!empty($network)) { + $orchestration->networkConnect($container, $network); + } + /** * Execute any commands if they were provided */ @@ -247,7 +258,7 @@ App::post('/v1/runtimes') $build['outputPath'] = $outputPath; } - if ($buildStdout === '') { + if (empty($buildStdout)) { $buildStdout = 'Build Successful!'; } @@ -261,86 +272,91 @@ App::post('/v1/runtimes') 'duration' => $buildEnd - $buildStart, ]); + if (!$remove) { + $activeRuntimes->set($container, [ + 'id' => $buildId, + 'name' => $container, + 'created' => $buildStart, + 'updated' => $buildEnd, + 'status' => 'Up ' . \round($buildEnd - $buildStart, 2) . 's', + 'key' => $secret, + ]); + } + + Console::success('Build Stage completed in ' . ($buildEnd - $buildStart) . ' seconds'); } catch (Throwable $th) { Console::error('Build failed: ' . $th->getMessage()); throw new Exception($th->getMessage(), 500); } finally { - if (!empty($buildId)) { + if (!empty($buildId) && $remove) { $orchestration->remove($buildId, true); } $orchestrationPool->put($orchestration); } - /** Create runtime server */ - try { - $orchestration = $orchestrationPool->get(); - /** - * Copy code files from source to a temporary location on the executor - */ - $buffer = $device->read($outputPath); - if(!$device->write($tmpBuild, $buffer)) { - throw new Exception('Failed to copy built code to temporary location.', 500); - }; + // /** Create runtime server */ + // try { + // $orchestration = $orchestrationPool->get(); + // /** + // * Copy code files from source to a temporary location on the executor + // */ + // $buffer = $device->read($source); + // if(!$device->write($tmpSource, $buffer)) { + // throw new Exception('Failed to copy built code to temporary location.', 500); + // }; - /** - * Launch Runtime - */ - $container = 'r-' . $runtimeId; - $secret = \bin2hex(\random_bytes(16)); - $vars = \array_merge($vars, [ - 'INTERNAL_RUNTIME_KEY' => $secret - ]); + // /** + // * Launch Runtime + // */ + // $container = 'r-' . $runtimeId; + // $secret = \bin2hex(\random_bytes(16)); + // $vars = \array_merge($vars, [ + // 'INTERNAL_RUNTIME_KEY' => $secret + // ]); - $executionStart = \microtime(true); - $executionTime = \time(); + // $executionStart = \microtime(true); + // $executionTime = \time(); - $vars = array_map(fn ($v) => strval($v), $vars); + // $vars = array_map(fn ($v) => strval($v), $vars); - $orchestration - ->setCpus(App::getEnv('_APP_FUNCTIONS_CPUS', 1)) - ->setMemory(App::getEnv('_APP_FUNCTIONS_MEMORY', 256)) - ->setSwap(App::getEnv('_APP_FUNCTIONS_MEMORY_SWAP', 256)); + // $orchestration + // ->setCpus(App::getEnv('_APP_FUNCTIONS_CPUS', 1)) + // ->setMemory(App::getEnv('_APP_FUNCTIONS_MEMORY', 256)) + // ->setSwap(App::getEnv('_APP_FUNCTIONS_MEMORY_SWAP', 256)); - $id = $orchestration->run( - image: $baseImage, - name: $container, - vars: $vars, - labels: [ - 'openruntimes-id' => $runtimeId, - 'openruntimes-type' => 'function', - 'openruntimes-created' => strval($executionTime), - 'openruntimes-runtime' => $runtime - ], - hostname: $container, - mountFolder: \dirname($tmpBuild), - ); + // $id = $orchestration->run( + // image: $baseImage, + // name: $container, + // vars: $vars, + // labels: [ + // 'openruntimes-id' => $runtimeId, + // 'openruntimes-type' => 'function', + // 'openruntimes-created' => strval($executionTime), + // 'openruntimes-runtime' => $runtime + // ], + // hostname: $container, + // mountFolder: \dirname($tmpSource), + // ); - if (empty($id)) { - throw new Exception('Failed to create runtime', 500); - } + // if (empty($id)) { + // throw new Exception('Failed to create runtime', 500); + // } - $orchestration->networkConnect($container, App::getEnv('_APP_EXECUTOR_RUNTIME_NETWORK', 'openruntimes')); + // $orchestration->networkConnect($container, App::getEnv('_APP_EXECUTOR_RUNTIME_NETWORK', 'openruntimes')); - $executionEnd = \microtime(true); + // $executionEnd = \microtime(true); - $activeRuntimes->set($container, [ - 'id' => $id, - 'name' => $container, - 'created' => $executionTime, - 'updated' => $executionTime, - 'status' => 'Up ' . \round($executionEnd - $executionStart, 2) . 's', - 'key' => $secret, - ]); + - Console::success('Runtime Server created in ' . ($executionEnd - $executionStart) . ' seconds'); - } catch (\Throwable $th) { - Console::error('Runtime Server Creation Failed: '. $th->getMessage()); - throw new Exception($th->getMessage(), 500); - } finally { - $orchestrationPool->put($orchestration); - } + // Console::success('Runtime Server created in ' . ($executionEnd - $executionStart) . ' seconds'); + // } catch (\Throwable $th) { + // Console::error('Runtime Server Creation Failed: '. $th->getMessage()); + // throw new Exception($th->getMessage(), 500); + // } finally { + // $orchestrationPool->put($orchestration); + // } $response ->setStatusCode(Response::STATUS_CODE_CREATED) @@ -502,7 +518,7 @@ App::post('/v1/execution') // 110 is the Swoole error code for timeout, see: https://www.swoole.co.uk/docs/swoole-error-code if ($errNo !== 0 && $errNo !== CURLE_COULDNT_CONNECT && $errNo !== CURLE_OPERATION_TIMEDOUT && $errNo !== 110) { - throw new Exception('An internal curl error has occurred within the executor! Error Msg: ' . $error, 500); + throw new Exception('An internal curl error has occurred within the executor! Error Msg: ' . $error, 406); } $executionData = []; @@ -567,15 +583,16 @@ App::setResource('activeRuntimes', fn() => $activeRuntimes); App::error(function ($error, $response) { // $route = $utopia->match($request); // logError($error, "httpError", $route); - switch ($error->getCode()) { case 400: // Error allowed publicly case 401: // Error allowed publicly case 402: // Error allowed publicly case 403: // Error allowed publicly case 404: // Error allowed publicly + case 406: // Error allowed publicly case 409: // Error allowed publicly case 412: // Error allowed publicly + case 425: // Error allowed publicly case 429: // Error allowed publicly case 501: // Error allowed publicly case 503: // Error allowed publicly @@ -594,6 +611,8 @@ App::error(function ($error, $response) { 'version' => App::getEnv('OPENRUNTIMES_VERSION', 'UNKNOWN'), ]; + var_dump($output); + $response ->addHeader('Cache-Control', 'no-cache, no-store, must-revalidate') ->addHeader('Expires', '0') diff --git a/app/workers/builds.php b/app/workers/builds.php index 420dba3ad7..bf825d49e5 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -117,10 +117,12 @@ class BuildsV1 extends Worker functionId: $functionId, deploymentId: $deploymentId, source: $source, + destination: APP_STORAGE_BUILDS . "/app-$projectId", vars: $vars, runtime: $key, baseImage: $baseImage, workdir: '/usr/code', + remove: true, commands: [ 'sh', '-c', 'mkdir -p /usr/code && \ diff --git a/composer.lock b/composer.lock index 71ff25ebef..2a68fa1d19 100644 --- a/composer.lock +++ b/composer.lock @@ -2252,16 +2252,16 @@ }, { "name": "utopia-php/framework", - "version": "0.19.6", + "version": "0.19.7", "source": { "type": "git", "url": "https://github.com/utopia-php/framework.git", - "reference": "7d9b28365fb794001cb34dd028659452d4e71b7d" + "reference": "f17afe77a21873b9be18ebc05283813468b4283a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/framework/zipball/7d9b28365fb794001cb34dd028659452d4e71b7d", - "reference": "7d9b28365fb794001cb34dd028659452d4e71b7d", + "url": "https://api.github.com/repos/utopia-php/framework/zipball/f17afe77a21873b9be18ebc05283813468b4283a", + "reference": "f17afe77a21873b9be18ebc05283813468b4283a", "shasum": "" }, "require": { @@ -2295,9 +2295,9 @@ ], "support": { "issues": "https://github.com/utopia-php/framework/issues", - "source": "https://github.com/utopia-php/framework/tree/0.19.6" + "source": "https://github.com/utopia-php/framework/tree/0.19.7" }, - "time": "2022-02-10T17:05:22+00:00" + "time": "2022-02-18T00:04:49+00:00" }, { "name": "utopia-php/image", diff --git a/src/Executor/Executor.php b/src/Executor/Executor.php index 7833b99e02..5ca0f4a706 100644 --- a/src/Executor/Executor.php +++ b/src/Executor/Executor.php @@ -3,7 +3,9 @@ namespace Executor; use Exception; +use phpDocumentor\Reflection\DocBlock\Tags\Var_; use Utopia\App; +use Utopia\CLI\Console; class Executor { @@ -34,10 +36,13 @@ class Executor string $functionId, string $deploymentId, string $projectId, - string $source, + string $source, string $runtime, string $baseImage, - string $workdir, + bool $remove = false, + string $workdir = '', + string $destination = '', + string $network = '', array $vars = [], array $commands = [] ) { @@ -49,14 +54,18 @@ class Executor $params = [ 'runtimeId' => "$projectId-$deploymentId", 'source' => $source, - 'destination' => APP_STORAGE_BUILDS . "/app-$projectId", + 'destination' => $destination, 'runtime' => $runtime, 'baseImage' => $baseImage, 'workdir' => $workdir, + 'network' => empty($network) ? App::getEnv('_APP_EXECUTOR_RUNTIME_NETWORK', 'openruntimes') : $network, 'vars' => $vars, + 'remove' => $remove, 'commands' => $commands ]; + var_dump($params); + $response = $this->call(self::METHOD_POST, $route, $headers, $params, true, 30); $status = $response['headers']['status-code']; @@ -131,35 +140,47 @@ class Executor $response = $this->call(self::METHOD_POST, $route, $headers, $params, true, 30); $status = $response['headers']['status-code']; - + if ($status >= 400) { - switch ($status) { - case 404: - $response = $this->createRuntime($functionId, $deploymentId, $projectId, $path, $runtime, $baseImage, '', $vars); - /** Try to create the execution once more */ - $response = $this->call(self::METHOD_POST, $route, $headers, $params, true, 30); - $status = $response['headers']['status-code']; - if ($status >= 400) { - throw new \Exception($response['body']['message'], $status); - } - break; - case 425: - for ($attempts = 0; $attempts < 3; $attempts++) { - sleep(1); + for ($attempts = 0; $attempts < 3; $attempts++) { + switch ($status) { + case 404: + Console::error("Runtime not found. Creating runtine"); + $response = $this->createRuntime( + functionId: $functionId, + deploymentId: $deploymentId, + projectId: $projectId, + source: $path, + runtime: $runtime, + baseImage: $baseImage, + vars: $vars, + commands: [] + ); + /** Try to create the execution once more */ + Console::error("Creating execution"); $response = $this->call(self::METHOD_POST, $route, $headers, $params, true, 30); $status = $response['headers']['status-code']; + var_dump($status); + break; + case 406: + $response = $this->call(self::METHOD_POST, $route, $headers, $params, true, 30); + $status = $response['headers']['status-code']; + break; + default: + throw new \Exception($response['body']['message'], $status); + } - if ($status < 400) { - break; - } - if ($status !== 425) { - throw new Exception($response['body']['message'], $status); - } - } - throw new Exception($response['body']['message'], 503); - default: + if ($status < 400) { + return $response['body']; + } + + if ($status != 406) { throw new \Exception($response['body']['message'], $status); + } + + sleep(1); } + throw new Exception($response['body']['message'], 503); } return $response['body']; @@ -262,10 +283,6 @@ class Executor $responseHeaders['status-code'] = $responseStatus; - if ($responseStatus === 500) { - echo 'Server error('.$method.': '.$path.'. Params: '.json_encode($params).'): '.json_encode($responseBody)."\n"; - } - return [ 'headers' => $responseHeaders, 'body' => $responseBody