diff --git a/CHANGES.md b/CHANGES.md index e67b581a9c..42b2faafc7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -3,6 +3,12 @@ - Introduced new execution model for functions - Improved functions execution times - Improved functions execution times +- **[ Breaking ]** **Tags** have been renamed to **Deployments** + - Rename `tagId` to `deplyomentId` in collections + - Rename tags to deployments in the docs + - Updated endpoints to reflect the new terminology + - Updated UI with these changes + - Updated event names from `function.tags.*` to `function.deployments.*` # Version 0.12.1 ## Bugs diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index fe2273d125..47d6c1bd00 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -433,7 +433,7 @@ App::delete('/v1/functions/:functionId') // Request executor to delete deployment containers $ch = \curl_init(); - \curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor:8080/v1/cleanup/function"); + \curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor/v1/cleanup/function"); \curl_setopt($ch, CURLOPT_POST, true); \curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode([ 'functionId' => $functionId @@ -589,7 +589,7 @@ App::post('/v1/functions/:functionId/deployments') $function = $dbForProject->getDocument('functions', $functionId); $ch = \curl_init(); - \curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor:8080/v1/deployment"); + \curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor/v1/deployment"); \curl_setopt($ch, CURLOPT_POST, true); \curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode([ 'functionId' => $function->getId(), @@ -760,7 +760,7 @@ App::delete('/v1/functions/:functionId/deployments/:deploymentId') // Request executor to delete deployment containers $ch = \curl_init(); - \curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor:8080/v1/cleanup/deployment"); + \curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor/v1/cleanup/deployment"); \curl_setopt($ch, CURLOPT_POST, true); \curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode([ 'deploymentId' => $deploymentId @@ -918,7 +918,7 @@ App::post('/v1/functions/:functionId/executions') // Directly execute function. $ch = \curl_init(); - \curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor:8080/v1/execute"); + \curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor/v1/execute"); \curl_setopt($ch, CURLOPT_POST, true); \curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode([ 'trigger' => 'http', @@ -1150,7 +1150,7 @@ App::post('/v1/builds/:buildId') // Retry build $ch = \curl_init(); - \curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor:8080/v1/build/{$buildId}"); + \curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor/v1/build/{$buildId}"); \curl_setopt($ch, CURLOPT_POST, true); \curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); \curl_setopt($ch, CURLOPT_TIMEOUT, 900); diff --git a/app/executor.php b/app/executor.php index 104db66575..26711226fb 100644 --- a/app/executor.php +++ b/app/executor.php @@ -1,40 +1,40 @@ get(); + try { + $orchestration = $orchestrationPool->get(); - Console::info('Warming up ' . $runtime['name'] . ' ' . $runtime['version'] . ' environment...'); + Console::info('Warming up ' . $runtime['name'] . ' ' . $runtime['version'] . ' environment...'); - $response = $orchestration->pull($runtime['image']); + $response = $orchestration->pull($runtime['image']); - if ($response) { - Console::success("Successfully Warmed up {$runtime['name']} {$runtime['version']}!"); - } else { - Console::warning("Failed to Warmup {$runtime['name']} {$runtime['version']}!"); + if ($response) { + Console::success("Successfully Warmed up {$runtime['name']} {$runtime['version']}!"); + } else { + Console::warning("Failed to Warmup {$runtime['name']} {$runtime['version']}!"); + } + } catch (\Throwable $th) { + } finally { + $orchestrationPool->put($orchestration); } - - $orchestrationPool->put($orchestration); }); } }); @@ -123,11 +127,15 @@ try { $activeFunctions->create(); Co\run(function () use ($orchestrationPool, $activeFunctions) { - $orchestration = $orchestrationPool->get(); - $executionStart = \microtime(true); + try { + $orchestration = $orchestrationPool->get(); + $executionStart = \microtime(true); + $residueList = $orchestration->list(['label' => 'appwrite-type=function']); + } catch (\Throwable $th) { + } finally { + $orchestrationPool->put($orchestration); + } - $residueList = $orchestration->list(['label' => 'appwrite-type=function']); - $orchestrationPool->put($orchestration); foreach ($residueList as $value) { go(fn () => $activeFunctions->set($value->getName(), [ @@ -310,9 +318,8 @@ function createRuntimeServer(string $functionId, string $projectId, string $depl } catch (\Throwable $th) { $orchestrationPool->put($orchestration); throw $th; - } finally { - $orchestrationPool->put($orchestration); } + $orchestrationPool->put($orchestration); }; function execute(string $trigger, string $projectId, string $executionId, string $functionId, Database $database, string $event = '', string $eventData = '', string $data = '', array $webhooks = [], string $userId = '', string $jwt = ''): array @@ -402,7 +409,7 @@ function execute(string $trigger, string $projectId, string $executionId, string $database->createDocument('builds', new Document([ '$id' => $buildId, '$read' => ($userId !== '') ? ['user:' . $userId] : [], - '$write' => ['role:all'], + '$write' => [], 'startTime' => time(), 'deploymentId' => $deployment->getId(), 'status' => 'processing', @@ -436,7 +443,7 @@ function execute(string $trigger, string $projectId, string $executionId, string } try { - if (!$activeFunctions->exists($container)) { // Create contianer if not ready + if (!$activeFunctions->exists($container)) { // Create container if not ready createRuntimeServer($functionId, $projectId, $deployment->getId(), $database); } else if ($activeFunctions->get($container)['status'] === 'Down') { sleep(1); @@ -643,311 +650,6 @@ function execute(string $trigger, string $projectId, string $executionId, string ]; }; -App::post('/v1/execute') // Define Route - ->desc('Execute a function') - ->param('trigger', '', new Text(1024)) - ->param('projectId', '', new Text(1024)) - ->param('executionId', '', new Text(1024), '', true) - ->param('functionId', '', new Text(1024)) - ->param('event', '', new Text(1024), '', true) - ->param('eventData', '', new Text(10240), '', true) - ->param('data', '', new Text(1024), '', true) - ->param('webhooks', [], new ArrayList(new JSON()), '', true) - ->param('userId', '', new Text(1024), '', true) - ->param('jwt', '', new Text(1024), '', true) - ->inject('response') - ->inject('dbForProject') - ->action( - function (string $trigger, string $projectId, string $executionId, string $functionId, string $event, string $eventData, string $data, array $webhooks, string $userId, string $jwt, Response $response, Database $dbForProject) { - try { - $data = execute($trigger, $projectId, $executionId, $functionId, $dbForProject, $event, $eventData, $data, $webhooks, $userId, $jwt); - $response->json($data); - } catch (Exception $e) { - logError($e, 'executeEndpoint'); - - $response - ->addHeader('Cache-Control', 'no-cache, no-store, must-revalidate') - ->addHeader('Expires', '0') - ->addHeader('Pragma', 'no-cache') - ->json(['error' => $e->getMessage()]); - } - } - ); - - -// Cleanup Endpoints used internally by appwrite when a function or deployment gets deleted to also clean up their containers -App::post('/v1/cleanup/function') - ->param('functionId', '', new UID()) - ->inject('response') - ->inject('dbForProject') - ->action( - function (string $functionId, Response $response, Database $dbForProject) use ($orchestrationPool) { - try { - /** @var Orchestration $orchestration */ - $orchestration = $orchestrationPool->get(); - - // Get function document - $function = $dbForProject->getDocument('functions', $functionId); - - // Check if function exists - if ($function->isEmpty()) { - throw new Exception('Function not found', 404); - } - - $results = $dbForProject->find('deployments', [new Query('functionId', Query::TYPE_EQUAL, [$functionId])], 999); - - // If amount is 0 then we simply return true - if (count($results) === 0) { - return $response->json(['success' => true]); - } - - // Delete the containers of all deployments - foreach ($results as $deployment) { - try { - // Remove any ongoing builds - if ($deployment->getAttribute('buildId')) { - $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId')); - - if ($build->getAttribute('status') === 'building') { - // Remove the build - $orchestration->remove('build-stage-' . $deployment->getAttribute('buildId'), true); - Console::info('Removed build for deployment ' . $deployment['$id']); - } - } - - $orchestration->remove('appwrite-function-' . $deployment['$id'], true); - Console::info('Removed container for deployment ' . $deployment['$id']); - } catch (Exception $e) { - // Do nothing, we don't care that much if it fails - } - } - - return $response->json(['success' => true]); - } catch (Exception $e) { - logError($e, "cleanupFunction"); - $orchestrationPool->put($orchestration); - - return $response->json(['error' => $e->getMessage()]); - } - $orchestrationPool->put($orchestration); - } - ); - -App::post('/v1/cleanup/deployment') - ->param('deploymentId', '', new UID(), 'Deployment unique ID.') - ->inject('response') - ->inject('dbForProject') - ->action(function (string $deploymentId, Response $response, Database $dbForProject) use ($orchestrationPool) { - try { - /** @var Orchestration $orchestration */ - $orchestration = $orchestrationPool->get(); - - // Get deployment document - $deployment = $dbForProject->getDocument('deployments', $deploymentId); - - // Check if deployment exists - if ($deployment->isEmpty()) { - throw new Exception('Deployment not found', 404); - } - - try { - // Remove any ongoing builds - if ($deployment->getAttribute('buildId')) { - $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId')); - - if ($build->getAttribute('status') == 'building') { - // Remove the build - $orchestration->remove('build-stage-' . $deployment->getAttribute('buildId'), true); - Console::info('Removed build for deployment ' . $deployment['$id']); - } - } - - // Remove the container of the deployment - $orchestration->remove('appwrite-function-' . $deployment['$id'], true); - Console::info('Removed container for deployment ' . $deployment['$id']); - } catch (Exception $e) { - // Do nothing, we don't care that much if it fails - } - } catch (Exception $e) { - logError($e, "cleanupFunction"); - $orchestrationPool->put($orchestration); - - return $response->json(['error' => $e->getMessage()]); - } - $orchestrationPool->put($orchestration); - - return $response->json(['success' => true]); - }); - -App::post('/v1/deployment') - ->param('functionId', '', new UID(), 'Function unique ID.') - ->param('deploymentId', '', new UID(), 'Deployment unique ID.') - ->param('userId', '', new UID(), 'User unique ID.', true) - ->inject('response') - ->inject('dbForProject') - ->inject('projectID') - ->inject('register') - ->action(function (string $functionId, string $deploymentId, string $userId, Response $response, Database $dbForProject, string $projectID, Registry $register) use ($runtimes) { - // Get function document - $function = $dbForProject->getDocument('functions', $functionId); - // Get deployment document - $deployment = $dbForProject->getDocument('deployments', $deploymentId); - - // Check if both documents exist - if ($function->isEmpty()) { - throw new Exception('Function not found', 404); - } - - if ($deployment->isEmpty()) { - throw new Exception('Deployment not found', 404); - } - - $runtime = $runtimes[$function->getAttribute('runtime')] ?? null; - - if (\is_null($runtime)) { - throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported'); - } - - // Create a new build entry - $buildId = $dbForProject->getId(); - - if ($deployment->getAttribute('buildId')) { - $buildId = $deployment->getAttribute('buildId'); - } else { - try { - $dbForProject->createDocument('builds', new Document([ - '$id' => $buildId, - '$read' => (!empty($userId)) ? ['user:' . $userId] : [], - '$write' => ['role:all'], - 'startTime' => time(), - 'deploymentId' => $deploymentId, - 'status' => 'processing', - 'runtime' => $function->getAttribute('runtime'), - 'outputPath' => '', - 'source' => $deployment->getAttribute('path'), - 'sourceType' => Storage::DEVICE_LOCAL, - 'stdout' => '', - 'stderr' => '', - 'endTime' => 0, - 'duration' => 0, - 'search' => implode(' ', [$deployment->getId(), $buildId]) - ])); - - $deployment->setAttribute('buildId', $buildId); - - $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); - } catch (\Throwable $th) { - var_dump($deployment->getArrayCopy()); - throw $th; - } - } - - // Build Code - go(function () use ($projectID, $deploymentId, $buildId, $functionId, $function, $register) { - $db = $register->get('dbPool')->get(); - $redis = $register->get('redisPool')->get(); - $cache = new Cache(new RedisCache($redis)); - - $dbForProject = new Database(new MariaDB($db), $cache); - $dbForProject->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $dbForProject->setNamespace('_project_' . $projectID); - // Build Code - runBuildStage($buildId, $projectID); - - // Update the schedule - $schedule = $function->getAttribute('schedule', ''); - $cron = (empty($function->getAttribute('deployment')) && !empty($schedule)) ? new CronExpression($schedule) : null; - $next = (empty($function->getAttribute('deployment')) && !empty($schedule)) ? $cron->getNextRunDate()->format('U') : 0; - - // Grab deployment - $deployment = $dbForProject->getDocument('deployments', $deploymentId); - - // Grab build - $build = $dbForProject->getDocument('builds', $buildId); - - // If the build failed, it won't be possible to deploy - if ($build->getAttribute('status') !== 'ready') { - return; - } - - if ($deployment->getAttribute('deploy') === true) { - // Update the function document setting the deployment as the active one - $function - ->setAttribute('deployment', $deployment->getId()) - ->setAttribute('scheduleNext', (int)$next); - $function = $dbForProject->updateDocument('functions', $function->getId(), $function); - } - - // Deploy Runtime Server - createRuntimeServer($functionId, $projectID, $deploymentId, $dbForProject); - - $register->get('dbPool')->put($db); - $register->get('redisPool')->put($redis); - }); - - if (false === $function) { - throw new Exception('Failed saving function to DB', 500); - } - - $response->dynamic($function, Response::MODEL_FUNCTION); - }); - -App::get('/v1/') - ->inject('response') - ->action( - function (Response $response) { - $response - ->addHeader('Cache-Control', 'no-cache, no-store, must-revalidate') - ->addHeader('Expires', '0') - ->addHeader('Pragma', 'no-cache') - ->json(['status' => 'online']); - } - ); - -// Build Endpoints -App::post('/v1/build/:buildId') // Start a Build - ->param('buildId', '', new UID(), 'Build unique ID.', false) - ->inject('response') - ->inject('dbForProject') - ->inject('projectID') - ->action(function (string $buildId, Response $response, Database $dbForProject, string $projectID) { - try { - // Get build document - $build = $dbForProject->getDocument('builds', $buildId); - - // Check if build exists - if ($build->isEmpty()) { - throw new Exception('Build not found', 404); - } - - // Check if build is already running - if ($build->getAttribute('status') === 'running') { - throw new Exception('Build is already running', 409); - } - - // Check if build is already finished - if ($build->getAttribute('status') === 'finished') { - throw new Exception('Build is already finished', 409); - } - - go(function () use ($buildId, $dbForProject, $projectID) { - // Build Code - runBuildStage($buildId, $projectID, $dbForProject); - }); - - // return success - return $response->json(['success' => true]); - } catch (Exception $e) { - logError($e, "buildEndpoint"); - - $response - ->addHeader('Cache-Control', 'no-cache, no-store, must-revalidate') - ->addHeader('Expires', '0') - ->addHeader('Pragma', 'no-cache') - ->json(['error' => $e->getMessage()]); - } - }); - function runBuildStage(string $buildId, string $projectID): Document { global $runtimes; @@ -977,7 +679,7 @@ function runBuildStage(string $buildId, string $projectID): Document return $build; } - // Update Deployment Status + // Update Tag Status $build->setAttribute('status', 'building'); $database->updateDocument('builds', $build->getId(), $build); @@ -989,7 +691,7 @@ function runBuildStage(string $buildId, string $projectID): Document throw new Exception('Runtime "' . $build->getAttribute('runtime', '') . '" is not supported'); } - // Grab Deployment Files + // Grab Tag Files $deploymentPath = $build->getAttribute('source', ''); $sourceType = $build->getAttribute('sourceType', ''); @@ -1024,25 +726,7 @@ function runBuildStage(string $buildId, string $projectID): Document throw new Exception('Code is not readable: ' . $build->getAttribute('source', '')); } - $deployment = $database->getDocument('deployments', $build->getAttribute('deploymentId', '')); - $resourceId = $deployment->getAttribute('resourceId', ''); - $resourceType = $deployment->getAttribute('resourceType', ''); - - if (empty($resourceId)) { - throw new Exception('Invalid resource ID on build ' . $build->getId()); - } - - if (empty($resourceType)) { - throw new Exception('Invalid resource type on build' . $build->getId()); - } - - $resource = $database->getDocument($resourceType, $resourceId); - - if ($resource->isEmpty()) { - throw new Exception('Resource not found on build ' . $build->getId()); - } - - $vars = $resource->getAttribute('vars', []); + $vars = $build->getAttribute('vars', []); // Start tracking time $buildStart = \time(); @@ -1061,7 +745,7 @@ function runBuildStage(string $buildId, string $projectID): Document } else { throw new Exception('Can\'t create directory /tmp/project-' . $projectID . '/' . $build->getId() . '/builtCode'); } - }; + } // Launch build container $id = $orchestration->run( @@ -1211,24 +895,335 @@ function runBuildStage(string $buildId, string $projectID): Document $orchestration->remove($id, true); } - $orchestrationPool->put($orchestration); $register->get('dbPool')->put($db); $register->get('redisPool')->put($redis); throw new Exception('Build failed: ' . $e->getMessage()); - } finally { - $orchestrationPool->put($orchestration); - - $register->get('dbPool')->put($db); - $register->get('redisPool')->put($redis); } + $orchestrationPool->put($orchestration); + + $register->get('dbPool')->put($db); + $register->get('redisPool')->put($redis); + return $build; } +App::post('/v1/execute') // Define Route + ->desc('Execute a function') + ->param('trigger', '', new Text(1024)) + ->param('projectId', '', new Text(1024)) + ->param('executionId', '', new Text(1024), '', true) + ->param('functionId', '', new Text(1024)) + ->param('event', '', new Text(1024), '', true) + ->param('eventData', '', new Text(10240), '', true) + ->param('data', '', new Text(1024), '', true) + ->param('webhooks', [], new ArrayList(new JSON()), '', true) + ->param('userId', '', new Text(1024), '', true) + ->param('jwt', '', new Text(1024), '', true) + ->inject('response') + ->inject('dbForProject') + ->action( + function (string $trigger, string $projectId, string $executionId, string $functionId, string $event, string $eventData, string $data, array $webhooks, string $userId, string $jwt, Response $response, Database $dbForProject) { + try { + $data = execute($trigger, $projectId, $executionId, $functionId, $dbForProject, $event, $eventData, $data, $webhooks, $userId, $jwt); + $response->json($data); + } catch (Exception $e) { + logError($e, 'executeEndpoint'); + + $response + ->addHeader('Cache-Control', 'no-cache, no-store, must-revalidate') + ->addHeader('Expires', '0') + ->addHeader('Pragma', 'no-cache') + ->json(['error' => $e->getMessage()]); + } + } + ); + +// Cleanup Endpoints used internally by appwrite when a function or deployment gets deleted to also clean up their containers +App::post('/v1/cleanup/function') + ->param('functionId', '', new UID()) + ->inject('response') + ->inject('dbForProject') + ->action( + function (string $functionId, Response $response, Database $dbForProject) use ($orchestrationPool) { + try { + /** @var Orchestration $orchestration */ + $orchestration = $orchestrationPool->get(); + + // Get function document + $function = $dbForProject->getDocument('functions', $functionId); + + // Check if function exists + if ($function->isEmpty()) { + throw new Exception('Function not found', 404); + } + + $results = $dbForProject->find('deployments', [new Query('functionId', Query::TYPE_EQUAL, [$functionId])], 999); + + // If amount is 0 then we simply return true + if (count($results) === 0) { + return $response->json(['success' => true]); + } + + // Delete the containers of all deployments + foreach ($results as $deployment) { + try { + // Remove any ongoing builds + if ($deployment->getAttribute('buildId')) { + $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId')); + + if ($build->getAttribute('status') === 'building') { + // Remove the build + $orchestration->remove('build-stage-' . $deployment->getAttribute('buildId'), true); + Console::info('Removed build for deployment ' . $deployment['$id']); + } + } + + $orchestration->remove('appwrite-function-' . $deployment['$id'], true); + Console::info('Removed container for deployment ' . $deployment['$id']); + } catch (Exception $e) { + // Do nothing, we don't care that much if it fails + } + } + + return $response->json(['success' => true]); + } catch (Exception $e) { + logError($e, "cleanupFunction"); + $orchestrationPool->put($orchestration); + + return $response->json(['error' => $e->getMessage()]); + } + $orchestrationPool->put($orchestration); + } + ); + +App::post('/v1/cleanup/deployment') + ->param('deploymentId', '', new UID(), 'Deployment unique ID.') + ->inject('response') + ->inject('dbForProject') + ->action(function (string $deploymentId, Response $response, Database $dbForProject) use ($orchestrationPool) { + try { + /** @var Orchestration $orchestration */ + $orchestration = $orchestrationPool->get(); + + // Get deployment document + $deployment = $dbForProject->getDocument('deployments', $deploymentId); + + // Check if deployment exists + if ($deployment->isEmpty()) { + throw new Exception('Deployment not found', 404); + } + + try { + // Remove any ongoing builds + if ($deployment->getAttribute('buildId')) { + $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId')); + + if ($build->getAttribute('status') == 'building') { + // Remove the build + $orchestration->remove('build-stage-' . $deployment->getAttribute('buildId'), true); + Console::info('Removed build for deployment ' . $deployment['$id']); + } + } + + // Remove the container of the deployment + $orchestration->remove('appwrite-function-' . $deployment['$id'], true); + Console::info('Removed container for deployment ' . $deployment['$id']); + } catch (Exception $e) { + // Do nothing, we don't care that much if it fails + } + } catch (Exception $e) { + logError($e, "cleanupFunction"); + $orchestrationPool->put($orchestration); + + return $response->json(['error' => $e->getMessage()]); + } + $orchestrationPool->put($orchestration); + + return $response->json(['success' => true]); + }); + +App::post('/v1/deployment') + ->param('functionId', '', new UID(), 'Function unique ID.') + ->param('deploymentId', '', new UID(), 'Deployment unique ID.') + ->param('userId', '', new UID(), 'User unique ID.', true) + ->inject('response') + ->inject('dbForProject') + ->inject('projectID') + ->inject('register') + ->action(function (string $functionId, string $deploymentId, string $userId, Response $response, Database $dbForProject, string $projectID, Registry $register) use ($runtimes) { + // Get function document + $function = $dbForProject->getDocument('functions', $functionId); + // Get deployment document + $deployment = $dbForProject->getDocument('deployments', $deploymentId); + + // Check if both documents exist + if ($function->isEmpty()) { + throw new Exception('Function not found', 404); + } + + if ($deployment->isEmpty()) { + throw new Exception('Deployment not found', 404); + } + + $runtime = $runtimes[$function->getAttribute('runtime')] ?? null; + + if (\is_null($runtime)) { + throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported'); + } + + // Create a new build entry + $buildId = $dbForProject->getId(); + + if ($deployment->getAttribute('buildId')) { + $buildId = $deployment->getAttribute('buildId'); + } else { + try { + $dbForProject->createDocument('builds', new Document([ + '$id' => $buildId, + '$read' => (!empty($userId)) ? ['user:' . $userId] : [], + '$write' => ['role:all'], + 'dateCreated' => time(), + 'status' => 'processing', + 'runtime' => $function->getAttribute('runtime'), + 'outputPath' => '', + 'source' => $deployment->getAttribute('path'), + 'sourceType' => Storage::DEVICE_LOCAL, + 'stdout' => '', + 'stderr' => '', + 'time' => 0, + 'vars' => [ + 'ENTRYPOINT_NAME' => $deployment->getAttribute('entrypoint'), + 'APPWRITE_FUNCTION_ID' => $function->getId(), + 'APPWRITE_FUNCTION_NAME' => $function->getAttribute('name', ''), + 'APPWRITE_FUNCTION_RUNTIME_NAME' => $runtime['name'], + 'APPWRITE_FUNCTION_RUNTIME_VERSION' => $runtime['version'], + 'APPWRITE_FUNCTION_PROJECT_ID' => $projectID, + ] + ])); + + $deployment->setAttribute('buildId', $buildId); + + $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); + } catch (\Throwable $th) { + var_dump($deployment->getArrayCopy()); + throw $th; + } + } + + // Build Code + go(function () use ($projectID, $deploymentId, $buildId, $functionId, $function, $register) { + try { + $db = $register->get('dbPool')->get(); + $redis = $register->get('redisPool')->get(); + $cache = new Cache(new RedisCache($redis)); + + $dbForProject = new Database(new MariaDB($db), $cache); + $dbForProject->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); + $dbForProject->setNamespace('_project_' . $projectID); + // Build Code + runBuildStage($buildId, $projectID); + + // Update the schedule + $schedule = $function->getAttribute('schedule', ''); + $cron = (empty($function->getAttribute('deployment')) && !empty($schedule)) ? new CronExpression($schedule) : null; + $next = (empty($function->getAttribute('deployment')) && !empty($schedule)) ? $cron->getNextRunDate()->format('U') : 0; + + // Grab deployment + $deployment = $dbForProject->getDocument('deployments', $deploymentId); + + // Grab build + $build = $dbForProject->getDocument('builds', $buildId); + + // If the build failed, it won't be possible to deploy + if ($build->getAttribute('status') !== 'ready') { + return; + } + + if ($deployment->getAttribute('automaticDeploy') === true) { + // Update the function document setting the deployment as the active one + $function + ->setAttribute('deployment', $deployment->getId()) + ->setAttribute('scheduleNext', (int)$next); + $function = $dbForProject->updateDocument('functions', $function->getId(), $function); + } + + // Deploy Runtime Server + createRuntimeServer($functionId, $projectID, $deploymentId, $dbForProject); + } catch (\Throwable $th) { + } finally { + $register->get('dbPool')->put($db); + $register->get('redisPool')->put($redis); + } + }); + + if (false === $function) { + throw new Exception('Failed saving function to DB', 500); + } + + $response->dynamic($function, Response::MODEL_FUNCTION); + }); + +App::get('/v1/') + ->inject('response') + ->action( + function (Response $response) { + $response + ->addHeader('Cache-Control', 'no-cache, no-store, must-revalidate') + ->addHeader('Expires', '0') + ->addHeader('Pragma', 'no-cache') + ->json(['status' => 'online']); + } + ); + +// Build Endpoints +App::post('/v1/build/:buildId') // Start a Build + ->param('buildId', '', new UID(), 'Build unique ID.', false) + ->inject('response') + ->inject('dbForProject') + ->inject('projectID') + ->action(function (string $buildId, Response $response, Database $dbForProject, string $projectID) { + try { + // Get build document + $build = $dbForProject->getDocument('builds', $buildId); + + // Check if build exists + if ($build->isEmpty()) { + throw new Exception('Build not found', 404); + } + + // Check if build is already running + if ($build->getAttribute('status') === 'running') { + throw new Exception('Build is already running', 409); + } + + // Check if build is already finished + if ($build->getAttribute('status') === 'finished') { + throw new Exception('Build is already finished', 409); + } + + go(function () use ($buildId, $dbForProject, $projectID) { + // Build Code + runBuildStage($buildId, $projectID, $dbForProject); + }); + + // return success + return $response->json(['success' => true]); + } catch (Exception $e) { + logError($e, "buildEndpoint"); + + $response + ->addHeader('Cache-Control', 'no-cache, no-store, must-revalidate') + ->addHeader('Expires', '0') + ->addHeader('Pragma', 'no-cache') + ->json(['error' => $e->getMessage()]); + } + }); + App::setMode(App::MODE_TYPE_PRODUCTION); // Define Mode -$http = new Server("0.0.0.0", 8080); +$http = new Server("0.0.0.0", 80); function handleShutdown() { @@ -1408,4 +1403,4 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo } }); -$http->start(); +$http->start(); \ No newline at end of file diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index b4b4b492f4..a4e8aeb0c0 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -194,10 +194,10 @@ services: - _APP_USAGE_STATS - _APP_STATSD_HOST - _APP_STATSD_PORT - - DOCKERHUB_PULL_USERNAME - - DOCKERHUB_PULL_PASSWORD - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG + - DOCKERHUB_PULL_USERNAME + - DOCKERHUB_PULL_PASSWORD appwrite-worker-database: image: /: @@ -324,13 +324,9 @@ services: depends_on: - redis - mariadb - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - appwrite-functions:/storage/functions:rw - - /tmp:/tmp:rw + - appwrite-executor environment: - _APP_ENV - - _APP_OPENSSL_KEY_V1 - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER @@ -341,13 +337,7 @@ services: - _APP_DB_USER - _APP_DB_PASS - _APP_FUNCTIONS_TIMEOUT - - _APP_FUNCTIONS_BUILD_TIMEOUT - - _APP_FUNCTIONS_CONTAINERS - - _APP_FUNCTIONS_CPUS - - _APP_FUNCTIONS_MEMORY - - _APP_FUNCTIONS_MEMORY_SWAP - _APP_EXECUTOR_SECRET - - _APP_FUNCTIONS_RUNTIMES - _APP_USAGE_STATS appwrite-executor: diff --git a/app/workers/functions.php b/app/workers/functions.php index 8dbaef42a4..b4fc22eddf 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -1,10 +1,6 @@ list(['label' => 'appwrite-type=function']); -/** @var Container[] $list */ -$list = []; - -foreach ($response as $value) { - $list[$value->getName()] = $value; -} - -$executionEnd = \microtime(true); - -Console::info(count($list) . ' functions listed in ' . ($executionEnd - $executionStart) . ' seconds'); - -/** - * 1. Get event args - DONE - * 2. Unpackage code in the isolated container - DONE - * 3. Execute in container with timeout - * + messure execution time - DONE - * + pass env vars - DONE - * + pass one-time api key - * 4. Update execution status - DONE - * 5. Update execution stdout & stderr - DONE - * 6. Trigger audit log - DONE - * 7. Trigger usage log - DONE - */ - -// TODO avoid scheduled execution if delay is bigger than X offest - class FunctionsV1 extends Worker { public array $args = []; @@ -266,7 +224,7 @@ class FunctionsV1 extends Worker public function execute(string $trigger, string $projectId, string $executionId, Database $database, Document $function, string $event = '', string $eventData = '', string $data = '', array $webhooks = [], string $userId = '', string $jwt = ''): void { $ch = \curl_init(); - \curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor:8080/v1/execute"); + \curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor/v1/execute"); \curl_setopt($ch, CURLOPT_POST, true); \curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode([ 'trigger' => $trigger, @@ -299,67 +257,6 @@ class FunctionsV1 extends Worker \curl_close($ch); } - /** - * Cleanup any hanging containers above the allowed max containers. - * - * @return void - */ - public function cleanup(): void - { - /** @var Container[] $list */ - global $list; - /** @var Orchestration $orchestration */ - global $orchestration; - - Console::success(count($list) . ' running containers counted'); - - $max = (int) App::getEnv('_APP_FUNCTIONS_CONTAINERS'); - - if (\count($list) > $max) { - Console::info('Starting containers cleanup'); - - \uasort($list, function (Container $item1, Container $item2) { - return (int)($item1->getLabels['appwrite-created'] ?? 0) <=> (int)($item2->getLabels['appwrite-created'] ?? 0); - }); - - while (\count($list) > $max) { - $first = \array_shift($list); - - try { - $orchestration->remove($first->getName(), true); - Console::info('Removed container: ' . $first->getName()); - } catch (Exception $e) { - Console::error('Failed to remove container: ' . $e); - } - } - } - } - - /** - * Filter ENV vars - * - * @param string $string - * - * @return string - */ - public function filterEnvKey(string $string): string - { - if (empty($this->allowed)) { - $this->allowed = array_fill_keys(\str_split('0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_'), true); - } - - $string = \str_split($string); - $output = ''; - - foreach ($string as $char) { - if (\array_key_exists($char, $this->allowed)) { - $output .= $char; - } - } - - return $output; - } - public function shutdown(): void { } diff --git a/docker-compose.yml b/docker-compose.yml index 584c65fe89..4f7afc3d61 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -61,7 +61,6 @@ services: - traefik.http.routers.appwrite_api_https.service=appwrite_api - traefik.http.routers.appwrite_api_https.tls=true volumes: - - /var/run/docker.sock:/var/run/docker.sock - appwrite-uploads:/storage/uploads:rw - appwrite-cache:/storage/cache:rw - appwrite-config:/storage/config:rw @@ -326,17 +325,14 @@ services: networks: - appwrite volumes: - - /var/run/docker.sock:/var/run/docker.sock - - appwrite-functions:/storage/functions:rw - - /tmp:/tmp:rw - ./app:/usr/src/code/app - ./src:/usr/src/code/src depends_on: - redis - mariadb + - appwrite-executor environment: - _APP_ENV - - _APP_OPENSSL_KEY_V1 - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER @@ -347,12 +343,6 @@ services: - _APP_DB_USER - _APP_DB_PASS - _APP_FUNCTIONS_TIMEOUT - - _APP_FUNCTIONS_BUILD_TIMEOUT - - _APP_FUNCTIONS_CONTAINERS - - _APP_FUNCTIONS_RUNTIMES - - _APP_FUNCTIONS_CPUS - - _APP_FUNCTIONS_MEMORY - - _APP_FUNCTIONS_MEMORY_SWAP - _APP_EXECUTOR_SECRET - _APP_USAGE_STATS - DOCKERHUB_PULL_USERNAME