diff --git a/CHANGES.md b/CHANGES.md index cfe2c1a93b..657d0e0ea5 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,15 +6,18 @@ - Refactored E-Mail template (#1422) - Improved locale management (#1440) - Added `$permissions` to execution response (#948) - +- Switch from using Docker CLI to Docker API by intergrating [utopia-php/orchestration](https://github.com/utopia-php/orchestration) +- Added DOCKERHUB_PULL_USERNAME, DOCKERHUB_PULL_PASSWORD and DOCKERHUB_PULL_EMAIL env variables for pulling from private DockerHub repos ## Bugs - Fixed MariaDB timeout after 24 hours (#1510) + # Version 0.9.4 ## Security - Fixed security vulnerability that exposes project ID's from other admin users (#1453) + # Version 0.9.3 ## Bugs diff --git a/app/config/events.php b/app/config/events.php index b27a5eafb9..e0e28c7299 100644 --- a/app/config/events.php +++ b/app/config/events.php @@ -162,6 +162,21 @@ return [ 'model' => Response::MODEL_ANY, 'note' => 'version >= 0.7', ], + 'users.update.email' => [ + 'description' => 'This event triggers when the user email address is updated.', + 'model' => Response::MODEL_USER, + 'note' => 'version >= 0.10', + ], + 'users.update.name' => [ + 'description' => 'This event triggers when the user name is updated.', + 'model' => Response::MODEL_USER, + 'note' => 'version >= 0.10', + ], + 'users.update.password' => [ + 'description' => 'This event triggers when the user password is updated.', + 'model' => Response::MODEL_USER, + 'note' => 'version >= 0.10', + ], 'users.update.status' => [ 'description' => 'This event triggers when a user status is updated from the users API.', 'model' => Response::MODEL_USER, diff --git a/app/config/variables.php b/app/config/variables.php index df7346cd59..bb505fd6be 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -462,6 +462,33 @@ return [ 'question' => '', 'filter' => '' ], + [ + 'name' => 'DOCKERHUB_PULL_USERNAME', + 'description' => 'The username for hub.docker.com. This variable is used to pull images from hub.docker.com.', + 'introduction' => '0.10.0', + 'default' => '', + 'required' => false, + 'question' => '', + 'filter' => '' + ], + [ + 'name' => 'DOCKERHUB_PULL_PASSWORD', + 'description' => 'The password for hub.docker.com. This variable is used to pull images from hub.docker.com.', + 'introduction' => '0.10.0', + 'default' => '', + 'required' => false, + 'question' => '', + 'filter' => '' + ], + [ + 'name' => 'DOCKERHUB_PULL_EMAIL', + 'description' => 'The email for hub.docker.com. This variable is used to pull images from hub.docker.com.', + 'introduction' => '0.10.0', + 'default' => '', + 'required' => false, + 'question' => '', + 'filter' => '' + ], ], [ 'category' => 'Maintenance', diff --git a/app/tasks/migrate.php b/app/tasks/migrate.php index 60aad941d1..be80c6e228 100644 --- a/app/tasks/migrate.php +++ b/app/tasks/migrate.php @@ -72,7 +72,7 @@ $cli $sum = \count($projects); $offset = $offset + $limit; $count = $count + $sum; - + if ($sum > 0) { Console::log('Fetched '.$count.'/'.$consoleDB->getSum().' projects...'); } diff --git a/app/workers/functions.php b/app/workers/functions.php index e3bacaa46d..4e3999534a 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -14,6 +14,11 @@ use Swoole\Runtime; use Utopia\App; use Utopia\CLI\Console; use Utopia\Config\Config; +use Utopia\Orchestration\Orchestration; +use Utopia\Orchestration\Adapter\DockerAPI; +use Utopia\Orchestration\Container; +use Utopia\Orchestration\Exception\Orchestration as OrchestrationException; +use Utopia\Orchestration\Exception\Timeout as TimeoutException; require_once __DIR__.'/../workers.php'; @@ -24,39 +29,27 @@ Console::success(APP_NAME.' functions worker v1 has started'); $runtimes = Config::getParam('runtimes'); +$dockerUser = App::getEnv('DOCKERHUB_PULL_USERNAME', null); +$dockerPass = App::getEnv('DOCKERHUB_PULL_PASSWORD', null); +$dockerEmail = App::getEnv('DOCKERHUB_PULL_EMAIL', null); +$orchestration = new Orchestration(new DockerAPI($dockerUser, $dockerPass, $dockerEmail)); + /** * Warmup Docker Images */ $warmupStart = \microtime(true); -Co\run(function() use ($runtimes) { // Warmup: make sure images are ready to run fast 🚀 - - $dockerUser = App::getEnv('DOCKERHUB_PULL_USERNAME', null); - $dockerPass = App::getEnv('DOCKERHUB_PULL_PASSWORD', null); - - if($dockerUser) { - $stdout = ''; - $stderr = ''; - - Console::execute('docker login --username '.$dockerUser.' --password-stdin', $dockerPass, $stdout, $stderr); - Console::log('Docker Login'. $stdout.$stderr); - } - +Co\run(function() use ($runtimes, $orchestration) { // Warmup: make sure images are ready to run fast 🚀 foreach($runtimes as $runtime) { - go(function() use ($runtime) { - $stdout = ''; - $stderr = ''; - + go(function() use ($runtime, $orchestration) { Console::info('Warming up '.$runtime['name'].' '.$runtime['version'].' environment...'); - - Console::execute('docker pull '.$runtime['image'], '', $stdout, $stderr); - - if(!empty($stdout)) { - Console::log($stdout); - } - - if(!empty($stderr)) { - Console::error($stderr); + + $response = $orchestration->pull($runtime['image']); + + if ($response) { + Console::success("Successfully Warmed up {$runtime['name']} {$runtime['version']}!"); + } else { + Console::error("Failed to Warmup {$runtime['name']} {$runtime['version']}!"); } }); } @@ -75,40 +68,17 @@ $stderr = ''; $executionStart = \microtime(true); -$exitCode = Console::execute('docker ps --all --format "name={{.Names}}&status={{.Status}}&labels={{.Labels}}" --filter label=appwrite-type=function' - , '', $stdout, $stderr, 30); +$response = $orchestration->list(['label' => 'appwrite-type=function']); + +$list = []; + +foreach ($response as $value) { + $list[$value->getName()] = $value; +} $executionEnd = \microtime(true); -$list = []; -$stdout = \explode("\n", $stdout); - -\array_map(function($value) use (&$list) { - $container = []; - - \parse_str($value, $container); - - if(isset($container['name'])) { - $container = [ - 'name' => $container['name'], - 'online' => (\substr($container['status'], 0, 2) === 'Up'), - 'status' => $container['status'], - 'labels' => $container['labels'], - ]; - - \array_map(function($value) use (&$container) { - $value = \explode('=', $value); - - if(isset($value[0]) && isset($value[1])) { - $container[$value[0]] = $value[1]; - } - }, \explode(',', $container['labels'])); - - $list[$container['name']] = $container; - } -}, $stdout); - -Console::info(count($list)." functions listed in " . ($executionEnd - $executionStart) . " seconds with exit code {$exitCode}"); +Console::info(count($list).' functions listed in ' . ($executionEnd - $executionStart) . ' seconds'); /** * 1. Get event args - DONE @@ -298,6 +268,7 @@ class FunctionsV1 extends Worker public function execute(string $trigger, string $projectId, string $executionId, Database $database, Document $function, string $event = '', string $eventData = '', string $data = '', array $webhooks = [], string $userId = '', string $jwt = ''): void { global $list; + global $orchestration; $runtimes = Config::getParam('runtimes'); @@ -356,12 +327,6 @@ class FunctionsV1 extends Worker 'APPWRITE_FUNCTION_PROJECT_ID' => $projectId, ]); - \array_walk($vars, function (&$value, $key) { - $key = $this->filterEnvKey($key); - $value = \escapeshellarg((empty($value)) ? '' : $value); - $value = "--env {$key}={$value}"; - }); - $tagPath = $tag->getAttribute('path', ''); $tagPathTarget = '/tmp/project-'.$projectId.'/'.$tag->getId().'/code.tar.gz'; $tagPathTargetDir = \pathinfo($tagPathTarget, PATHINFO_DIRNAME); @@ -384,12 +349,14 @@ class FunctionsV1 extends Worker } } - if(isset($list[$container]) && !$list[$container]['online']) { // Remove conatiner if not online + if(isset($list[$container]) && !(\substr($list[$container]->getStatus(), 0, 2) === 'Up')) { // Remove conatiner if not online $stdout = ''; $stderr = ''; - - if(Console::execute("docker rm {$container}", '', $stdout, $stderr, 30) !== 0) { - throw new Exception('Failed to remove offline container: '.$stderr); + + try { + $orchestration->remove($container); + } catch (Exception $e) { + Console::warning('Failed to remove container: '.$e->getMessage()); } unset($list[$container]); @@ -410,51 +377,60 @@ class FunctionsV1 extends Worker $executionStart = \microtime(true); $executionTime = \time(); - $cpus = App::getEnv('_APP_FUNCTIONS_CPUS', ''); - $memory = App::getEnv('_APP_FUNCTIONS_MEMORY', ''); - $swap = App::getEnv('_APP_FUNCTIONS_MEMORY_SWAP', ''); - $exitCode = Console::execute("docker run ". - " -d". - " --entrypoint=\"\"". - (empty($cpus) ? "" : (" --cpus=".$cpus)). - (empty($memory) ? "" : (" --memory=".$memory."m")). - (empty($swap) ? "" : (" --memory-swap=".$swap."m")). - " --name={$container}". - " --label appwrite-type=function". - " --label appwrite-created={$executionTime}". - " --volume {$tagPathTargetDir}:/tmp:rw". - " --workdir /usr/local/src". - " ".\implode(" ", $vars). - " {$runtime['image']}". - " tail -f /dev/null" - , '', $stdout, $stderr, 30); - if($exitCode !== 0) { - throw new Exception('Failed to create function environment: '.$stderr); + $orchestration->setCpus(App::getEnv('_APP_FUNCTIONS_CPUS', '1')); + $orchestration->setMemory(App::getEnv('_APP_FUNCTIONS_MEMORY', '256')); + $orchestration->setSwap(App::getEnv('_APP_FUNCTIONS_MEMORY_SWAP', '256')); + + foreach($vars as &$value) { + $value = strval($value); } - $exitCodeUntar = Console::execute("docker exec ". - $container. - " sh -c 'mv /tmp/code.tar.gz /usr/local/src/code.tar.gz && tar -zxf /usr/local/src/code.tar.gz --strip 1 && rm /usr/local/src/code.tar.gz'" - , '', $stdout, $stderr, 60); + $id = $orchestration->run( + image: $runtime['image'], + name: $container, + command: ['tail', + '-f', + '/dev/null' + ], + entrypoint: '', + workdir: '/usr/local/src', + volumes: [], + vars: $vars, + mountFolder: $tagPathTargetDir, + labels: [ + 'appwrite-type' => 'function', + 'appwrite-created' => strval($executionTime) + ]); - if($exitCodeUntar !== 0) { - throw new Exception('Failed to extract tar: '.$stderr); + $untarStdout = ''; + $untarStderr = ''; + + $untarSuccess = $orchestration->execute( + name: $container, + command: [ + 'sh', + '-c', + 'mv /tmp/code.tar.gz /usr/local/src/code.tar.gz && tar -zxf /usr/local/src/code.tar.gz --strip 1 && rm /usr/local/src/code.tar.gz' + ], + stdout: $untarStdout, + stderr: $untarStderr, + vars: $vars, + timeout: 60); + + if (!$untarSuccess) { + throw new Exception('Failed to extract tar: '.$untarStderr); } $executionEnd = \microtime(true); - $list[$container] = [ - 'name' => $container, - 'online' => true, - 'status' => 'Up', - 'labels' => [ + $list[$container] = new Container($container, $id, 'Up', + [ 'appwrite-type' => 'function', - 'appwrite-created' => $executionTime, - ], - ]; + 'appwrite-created' => strval($executionTime), + ]); - Console::info("Function created in " . ($executionEnd - $executionStart) . " seconds with exit code {$exitCode}"); + Console::info('Function created in ' . ($executionEnd - $executionStart) . ' seconds'); } else { Console::info('Container is ready to run'); @@ -465,14 +441,28 @@ class FunctionsV1 extends Worker $executionStart = \microtime(true); - $exitCode = Console::execute("docker exec ".\implode(" ", $vars)." {$container} {$command}" - , '', $stdout, $stderr, $function->getAttribute('timeout', (int) App::getEnv('_APP_FUNCTIONS_TIMEOUT', 900))); + $exitCode = 0; + + try { + $exitCode = (int)!$orchestration->execute( + name: $container, + command: $orchestration->parseCommandString($command), + stdout: $stdout, + stderr: $stderr, + vars: $vars, + timeout: $function->getAttribute('timeout', (int) App::getEnv('_APP_FUNCTIONS_TIMEOUT', 900))); + } catch (TimeoutException $e) { + $exitCode = 124; + } catch (OrchestrationException $e) { + $stderr = $e->getMessage(); + $exitCode = 1; + } $executionEnd = \microtime(true); $executionTime = ($executionEnd - $executionStart); $functionStatus = ($exitCode === 0) ? 'completed' : 'failed'; - Console::info("Function executed in " . ($executionEnd - $executionStart) . " seconds with exit code {$exitCode}"); + Console::info('Function executed in ' . ($executionEnd - $executionStart) . ' seconds, status: ' . $functionStatus); Authorization::disable(); @@ -482,7 +472,7 @@ class FunctionsV1 extends Worker 'exitCode' => $exitCode, 'stdout' => \mb_substr($stdout, -4000), // log last 4000 chars output 'stderr' => \mb_substr($stderr, -4000), // log last 4000 chars output - 'time' => $executionTime, + 'time' => $executionTime ])); Authorization::reset(); @@ -540,6 +530,7 @@ class FunctionsV1 extends Worker public function cleanup(): void { global $list; + global $orchestration; Console::success(count($list).' running containers counted'); @@ -549,19 +540,17 @@ class FunctionsV1 extends Worker Console::info('Starting containers cleanup'); \uasort($list, function ($item1, $item2) { - return (int)($item1['appwrite-created'] ?? 0) <=> (int)($item2['appwrite-created'] ?? 0); + return (int)($item1->getLabels['appwrite-created'] ?? 0) <=> (int)($item2->getLabels['appwrite-created'] ?? 0); }); while(\count($list) > $max) { $first = \array_shift($list); - $stdout = ''; - $stderr = ''; - if(Console::execute("docker rm -f {$first['name']}", '', $stdout, $stderr, 30) !== 0) { - Console::error('Failed to remove container: '.$stderr); - } - else { - Console::info('Removed container: '.$first['name']); + try { + $orchestration->remove($first->getName(), true); + Console::info('Removed container: '.$first->getName()); + } catch (Exception $e) { + Console::error('Failed to remove container: '.$e); } } } diff --git a/composer.json b/composer.json index ed675be1fe..f05f0d9fdb 100644 --- a/composer.json +++ b/composer.json @@ -55,6 +55,7 @@ "utopia-php/image": "0.5.*", "resque/php-resque": "1.3.6", "matomo/device-detector": "4.2.3", + "utopia-php/orchestration": "0.2.0", "dragonmantank/cron-expression": "3.1.0", "influxdb/influxdb-php": "1.15.2", "phpmailer/phpmailer": "6.5.0", diff --git a/composer.lock b/composer.lock index 7531b391e0..ba437ce8e4 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "989bc612da5db17215857941270be27f", + "content-hash": "0a782184d016e458ff18f20a2c49ccfb", "packages": [ { "name": "adhocore/jwt", @@ -1907,6 +1907,61 @@ }, "time": "2021-07-24T11:35:55+00:00" }, + { + "name": "utopia-php/orchestration", + "version": "0.2.0", + "source": { + "type": "git", + "url": "https://github.com/utopia-php/orchestration.git", + "reference": "de10509017768cf2b62363bb39912002ab41dafb" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/orchestration/zipball/de10509017768cf2b62363bb39912002ab41dafb", + "reference": "de10509017768cf2b62363bb39912002ab41dafb", + "shasum": "" + }, + "require": { + "php": ">=8.0", + "utopia-php/cli": "0.11.*" + }, + "require-dev": { + "phpunit/phpunit": "^9.3", + "vimeo/psalm": "4.0.1" + }, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\Orchestration\\": "src/Orchestration" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Eldad Fux", + "email": "eldad@appwrite.io" + } + ], + "description": "Lite & fast micro PHP abstraction library for container orchestration", + "keywords": [ + "docker", + "framework", + "kubernetes", + "orchestration", + "php", + "swarm", + "upf", + "utopia" + ], + "support": { + "issues": "https://github.com/utopia-php/orchestration/issues", + "source": "https://github.com/utopia-php/orchestration/tree/0.2.0" + }, + "time": "2021-08-16T12:52:42+00:00" + }, { "name": "utopia-php/preloader", "version": "0.2.4", @@ -5105,16 +5160,16 @@ }, { "name": "symfony/console", - "version": "v5.3.6", + "version": "v5.3.7", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "51b71afd6d2dc8f5063199357b9880cea8d8bfe2" + "reference": "8b1008344647462ae6ec57559da166c2bfa5e16a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/51b71afd6d2dc8f5063199357b9880cea8d8bfe2", - "reference": "51b71afd6d2dc8f5063199357b9880cea8d8bfe2", + "url": "https://api.github.com/repos/symfony/console/zipball/8b1008344647462ae6ec57559da166c2bfa5e16a", + "reference": "8b1008344647462ae6ec57559da166c2bfa5e16a", "shasum": "" }, "require": { @@ -5184,7 +5239,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v5.3.6" + "source": "https://github.com/symfony/console/tree/v5.3.7" }, "funding": [ { @@ -5200,7 +5255,7 @@ "type": "tidelift" } ], - "time": "2021-07-27T19:10:22+00:00" + "time": "2021-08-25T20:02:16+00:00" }, { "name": "symfony/deprecation-contracts", @@ -5757,16 +5812,16 @@ }, { "name": "symfony/string", - "version": "v5.3.3", + "version": "v5.3.7", "source": { "type": "git", "url": "https://github.com/symfony/string.git", - "reference": "bd53358e3eccec6a670b5f33ab680d8dbe1d4ae1" + "reference": "8d224396e28d30f81969f083a58763b8b9ceb0a5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/string/zipball/bd53358e3eccec6a670b5f33ab680d8dbe1d4ae1", - "reference": "bd53358e3eccec6a670b5f33ab680d8dbe1d4ae1", + "url": "https://api.github.com/repos/symfony/string/zipball/8d224396e28d30f81969f083a58763b8b9ceb0a5", + "reference": "8d224396e28d30f81969f083a58763b8b9ceb0a5", "shasum": "" }, "require": { @@ -5820,7 +5875,7 @@ "utf8" ], "support": { - "source": "https://github.com/symfony/string/tree/v5.3.3" + "source": "https://github.com/symfony/string/tree/v5.3.7" }, "funding": [ { @@ -5836,7 +5891,7 @@ "type": "tidelift" } ], - "time": "2021-06-27T11:44:38+00:00" + "time": "2021-08-26T08:00:08+00:00" }, { "name": "textalk/websocket", diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index 2ed66ba06d..85ddf14ac0 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -43,7 +43,7 @@ abstract class Migration '0.9.2' => 'V08', '0.9.3' => 'V08', '0.9.4' => 'V08', - '0.10.0' => 'V08', + '0.10.0' => 'V09', ]; /** diff --git a/src/Appwrite/Migration/Version/V09.php b/src/Appwrite/Migration/Version/V09.php new file mode 100644 index 0000000000..2d1411c4c3 --- /dev/null +++ b/src/Appwrite/Migration/Version/V09.php @@ -0,0 +1,48 @@ +project; + Console::log('Migrating project: ' . $project->getAttribute('name') . ' (' . $project->getId() . ')'); + + $this->forEachDocument([$this, 'fixDocument']); + } + + protected function fixDocument(Document $document) + { + switch ($document->getAttribute('$collection')) { + /** + * Add version reference to database. + */ + case Database::SYSTEM_COLLECTION_PROJECTS: + $document->setAttribute('version', '0.10.0'); + + break; + } + + foreach ($document as &$attr) { + if ($attr instanceof Document) { + $attr = $this->fixDocument($attr); + } + + if (\is_array($attr)) { + foreach ($attr as &$child) { + if ($child instanceof Document) { + $child = $this->fixDocument($child); + } + } + } + } + + return $document; + } +} diff --git a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php index 454a149c3c..4b52aa26cb 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php @@ -196,7 +196,6 @@ class FunctionsCustomClientTest extends Scope ]); $output = json_decode($executions['body']['stdout'], true); - $this->assertEquals(200, $executions['headers']['status-code']); $this->assertEquals('completed', $executions['body']['status']); $this->assertEquals($functionId, $output['APPWRITE_FUNCTION_ID']); diff --git a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php index 775eca06e2..58740150de 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php @@ -518,7 +518,7 @@ class FunctionsCustomServerTest extends Scope $this->assertEquals($executions['body']['executions'][0]['$id'], $executionId); $this->assertEquals($executions['body']['executions'][0]['trigger'], 'http'); $this->assertEquals($executions['body']['executions'][0]['status'], 'failed'); - $this->assertEquals($executions['body']['executions'][0]['exitCode'], 1); + $this->assertEquals($executions['body']['executions'][0]['exitCode'], 124); $this->assertGreaterThan(2, $executions['body']['executions'][0]['time']); $this->assertLessThan(3, $executions['body']['executions'][0]['time']); $this->assertEquals($executions['body']['executions'][0]['stdout'], ''); diff --git a/tests/unit/Migration/MigrationV09Test.php b/tests/unit/Migration/MigrationV09Test.php new file mode 100644 index 0000000000..d5bee3808f --- /dev/null +++ b/tests/unit/Migration/MigrationV09Test.php @@ -0,0 +1,31 @@ +pdo = new \PDO('sqlite::memory:'); + $this->migration = new V09($this->pdo); + $reflector = new ReflectionClass('Appwrite\Migration\Version\V09'); + $this->method = $reflector->getMethod('fixDocument'); + $this->method->setAccessible(true); + } + + public function testMigration() + { + $document = $this->fixDocument(new Document([ + '$id' => 'project', + '$collection' => Database::SYSTEM_COLLECTION_PROJECTS, + 'version' => '0.9.0' + ])); + + $this->assertEquals($document->getAttribute('version', '0.9.0'), '0.10.0'); + } +}