Appwrite/appwrite
1
0
Fork 0
mirror of https://github.com/appwrite/appwrite.git synced 2026-05-26 13:51:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'feat-functions-refactor' into feat-cleanup-collections

Browse Source
This commit is contained in:
Bradley Schofield
2022-01-26 09:59:46 +00:00
parent 4273ad4771 4147faafdc
commit ee2670fcd7
6 changed files with 387 additions and 509 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGES.md
+6
View File
@@ -3,6 +3,12 @@
- Introduced new execution model for functions
- Improved functions execution times
- Improved functions execution times
- **[ Breaking ]** **Tags** have been renamed to **Deployments**
- Rename `tagId` to `deplyomentId` in collections
- Rename tags to deployments in the docs
- Updated endpoints to reflect the new terminology
- Updated UI with these changes
- Updated event names from `function.tags.*` to `function.deployments.*`
# Version 0.12.1
## Bugs
app/controllers/api/functions.php
+5 -5
View File
@@ -433,7 +433,7 @@ App::delete('/v1/functions/:functionId')
// Request executor to delete deployment containers
$ch = \curl_init();
\curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor:8080/v1/cleanup/function");
\curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor/v1/cleanup/function");
\curl_setopt($ch, CURLOPT_POST, true);
\curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode([
'functionId' => $functionId
@@ -589,7 +589,7 @@ App::post('/v1/functions/:functionId/deployments')
$function = $dbForProject->getDocument('functions', $functionId);
$ch = \curl_init();
\curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor:8080/v1/deployment");
\curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor/v1/deployment");
\curl_setopt($ch, CURLOPT_POST, true);
\curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode([
'functionId' => $function->getId(),
@@ -760,7 +760,7 @@ App::delete('/v1/functions/:functionId/deployments/:deploymentId')
// Request executor to delete deployment containers
$ch = \curl_init();
\curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor:8080/v1/cleanup/deployment");
\curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor/v1/cleanup/deployment");
\curl_setopt($ch, CURLOPT_POST, true);
\curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode([
'deploymentId' => $deploymentId
@@ -918,7 +918,7 @@ App::post('/v1/functions/:functionId/executions')
// Directly execute function.
$ch = \curl_init();
\curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor:8080/v1/execute");
\curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor/v1/execute");
\curl_setopt($ch, CURLOPT_POST, true);
\curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode([
'trigger' => 'http',
@@ -1150,7 +1150,7 @@ App::post('/v1/builds/:buildId')
// Retry build
$ch = \curl_init();
\curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor:8080/v1/build/{$buildId}");
\curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor/v1/build/{$buildId}");
\curl_setopt($ch, CURLOPT_POST, true);
\curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
\curl_setopt($ch, CURLOPT_TIMEOUT, 900);
app/executor.php
+370 -375
View File
@@ -1,40 +1,40 @@
<?php
require_once __DIR__ . '/../vendor/autoload.php';
use Utopia\Database\Document;
use Utopia\Database\Database;
use Utopia\Database\Validator\Authorization;
use Utopia\Database\Validator\UID;
use Appwrite\Event\Event;
use Appwrite\Utopia\Response\Model\Execution;
use Appwrite\Messaging\Adapter\Realtime;
use Appwrite\Stats\Stats;
use Utopia\App;
use Utopia\Swoole\Request;
use Appwrite\Utopia\Response;
use Utopia\CLI\Console;
use Swoole\Process;
use Swoole\Http\Server;
use Appwrite\Utopia\Response\Model\Execution;
use Cron\CronExpression;
use Swoole\ConnectionPool;
use Swoole\Coroutine as Co;
use Swoole\Http\Request as SwooleRequest;
use Swoole\Http\Response as SwooleResponse;
use Utopia\Orchestration\Orchestration;
use Utopia\Database\Adapter\MariaDB;
use Swoole\Http\Server;
use Swoole\Process;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Cache\Adapter\Redis as RedisCache;
use Utopia\Cache\Cache;
use Utopia\Config\Config;
use Utopia\Database\Adapter\MariaDB;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Query;
use Utopia\Database\Validator\Authorization;
use Utopia\Database\Validator\UID;
use Utopia\Logger\Log;
use Utopia\Orchestration\Adapter\DockerAPI;
use Utopia\Orchestration\Adapter\DockerCLI;
use Utopia\Orchestration\Orchestration;
use Utopia\Registry\Registry;
use Utopia\Storage\Device\Local;
use Utopia\Storage\Storage;
use Utopia\Swoole\Request;
use Utopia\Validator\ArrayList;
use Utopia\Validator\JSON;
use Utopia\Validator\Text;
use Cron\CronExpression;
use Swoole\ConnectionPool;
use Utopia\Storage\Device\Local;
use Utopia\Storage\Storage;
use Swoole\Coroutine as Co;
use Utopia\Cache\Cache;
use Utopia\Database\Query;
use Utopia\Orchestration\Adapter\DockerCLI;
use Utopia\Logger\Log;
use Utopia\Orchestration\Adapter\DockerAPI;
use Utopia\Registry\Registry;
require_once __DIR__ . '/init.php';
@@ -91,6 +91,7 @@ $orchestrationPool = new ConnectionPool(function () {
return $orchestration;
}, 6);
try {
$runtimes = Config::getParam('runtimes');
@@ -98,19 +99,22 @@ try {
Co\run(function () use ($runtimes, $orchestrationPool) {
foreach ($runtimes as $runtime) {
go(function () use ($runtime, $orchestrationPool) {
$orchestration = $orchestrationPool->get();
try {
$orchestration = $orchestrationPool->get();
Console::info('Warming up ' . $runtime['name'] . ' ' . $runtime['version'] . ' environment...');
Console::info('Warming up ' . $runtime['name'] . ' ' . $runtime['version'] . ' environment...');
$response = $orchestration->pull($runtime['image']);
$response = $orchestration->pull($runtime['image']);
if ($response) {
Console::success("Successfully Warmed up {$runtime['name']} {$runtime['version']}!");
} else {
Console::warning("Failed to Warmup {$runtime['name']} {$runtime['version']}!");
if ($response) {
Console::success("Successfully Warmed up {$runtime['name']} {$runtime['version']}!");
} else {
Console::warning("Failed to Warmup {$runtime['name']} {$runtime['version']}!");
}
} catch (\Throwable $th) {
} finally {
$orchestrationPool->put($orchestration);
}
$orchestrationPool->put($orchestration);
});
}
});
@@ -123,11 +127,15 @@ try {
$activeFunctions->create();
Co\run(function () use ($orchestrationPool, $activeFunctions) {
$orchestration = $orchestrationPool->get();
$executionStart = \microtime(true);
try {
$orchestration = $orchestrationPool->get();
$executionStart = \microtime(true);
$residueList = $orchestration->list(['label' => 'appwrite-type=function']);
} catch (\Throwable $th) {
} finally {
$orchestrationPool->put($orchestration);
}
$residueList = $orchestration->list(['label' => 'appwrite-type=function']);
$orchestrationPool->put($orchestration);
foreach ($residueList as $value) {
go(fn () => $activeFunctions->set($value->getName(), [
@@ -310,9 +318,8 @@ function createRuntimeServer(string $functionId, string $projectId, string $depl
} catch (\Throwable $th) {
$orchestrationPool->put($orchestration);
throw $th;
} finally {
$orchestrationPool->put($orchestration);
}
$orchestrationPool->put($orchestration);
};
function execute(string $trigger, string $projectId, string $executionId, string $functionId, Database $database, string $event = '', string $eventData = '', string $data = '', array $webhooks = [], string $userId = '', string $jwt = ''): array
@@ -402,7 +409,7 @@ function execute(string $trigger, string $projectId, string $executionId, string
$database->createDocument('builds', new Document([
'$id' => $buildId,
'$read' => ($userId !== '') ? ['user:' . $userId] : [],
'$write' => ['role:all'],
'$write' => [],
'startTime' => time(),
'deploymentId' => $deployment->getId(),
'status' => 'processing',
@@ -436,7 +443,7 @@ function execute(string $trigger, string $projectId, string $executionId, string
}
try {
if (!$activeFunctions->exists($container)) { // Create contianer if not ready
if (!$activeFunctions->exists($container)) { // Create container if not ready
createRuntimeServer($functionId, $projectId, $deployment->getId(), $database);
} else if ($activeFunctions->get($container)['status'] === 'Down') {
sleep(1);
@@ -643,311 +650,6 @@ function execute(string $trigger, string $projectId, string $executionId, string
];
};
App::post('/v1/execute') // Define Route
->desc('Execute a function')
->param('trigger', '', new Text(1024))
->param('projectId', '', new Text(1024))
->param('executionId', '', new Text(1024), '', true)
->param('functionId', '', new Text(1024))
->param('event', '', new Text(1024), '', true)
->param('eventData', '', new Text(10240), '', true)
->param('data', '', new Text(1024), '', true)
->param('webhooks', [], new ArrayList(new JSON()), '', true)
->param('userId', '', new Text(1024), '', true)
->param('jwt', '', new Text(1024), '', true)
->inject('response')
->inject('dbForProject')
->action(
function (string $trigger, string $projectId, string $executionId, string $functionId, string $event, string $eventData, string $data, array $webhooks, string $userId, string $jwt, Response $response, Database $dbForProject) {
try {
$data = execute($trigger, $projectId, $executionId, $functionId, $dbForProject, $event, $eventData, $data, $webhooks, $userId, $jwt);
$response->json($data);
} catch (Exception $e) {
logError($e, 'executeEndpoint');
$response
->addHeader('Cache-Control', 'no-cache, no-store, must-revalidate')
->addHeader('Expires', '0')
->addHeader('Pragma', 'no-cache')
->json(['error' => $e->getMessage()]);
}
}
);
// Cleanup Endpoints used internally by appwrite when a function or deployment gets deleted to also clean up their containers
App::post('/v1/cleanup/function')
->param('functionId', '', new UID())
->inject('response')
->inject('dbForProject')
->action(
function (string $functionId, Response $response, Database $dbForProject) use ($orchestrationPool) {
try {
/** @var Orchestration $orchestration */
$orchestration = $orchestrationPool->get();
// Get function document
$function = $dbForProject->getDocument('functions', $functionId);
// Check if function exists
if ($function->isEmpty()) {
throw new Exception('Function not found', 404);
}
$results = $dbForProject->find('deployments', [new Query('functionId', Query::TYPE_EQUAL, [$functionId])], 999);
// If amount is 0 then we simply return true
if (count($results) === 0) {
return $response->json(['success' => true]);
}
// Delete the containers of all deployments
foreach ($results as $deployment) {
try {
// Remove any ongoing builds
if ($deployment->getAttribute('buildId')) {
$build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId'));
if ($build->getAttribute('status') === 'building') {
// Remove the build
$orchestration->remove('build-stage-' . $deployment->getAttribute('buildId'), true);
Console::info('Removed build for deployment ' . $deployment['$id']);
}
}
$orchestration->remove('appwrite-function-' . $deployment['$id'], true);
Console::info('Removed container for deployment ' . $deployment['$id']);
} catch (Exception $e) {
// Do nothing, we don't care that much if it fails
}
}
return $response->json(['success' => true]);
} catch (Exception $e) {
logError($e, "cleanupFunction");
$orchestrationPool->put($orchestration);
return $response->json(['error' => $e->getMessage()]);
}
$orchestrationPool->put($orchestration);
}
);
App::post('/v1/cleanup/deployment')
->param('deploymentId', '', new UID(), 'Deployment unique ID.')
->inject('response')
->inject('dbForProject')
->action(function (string $deploymentId, Response $response, Database $dbForProject) use ($orchestrationPool) {
try {
/** @var Orchestration $orchestration */
$orchestration = $orchestrationPool->get();
// Get deployment document
$deployment = $dbForProject->getDocument('deployments', $deploymentId);
// Check if deployment exists
if ($deployment->isEmpty()) {
throw new Exception('Deployment not found', 404);
}
try {
// Remove any ongoing builds
if ($deployment->getAttribute('buildId')) {
$build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId'));
if ($build->getAttribute('status') == 'building') {
// Remove the build
$orchestration->remove('build-stage-' . $deployment->getAttribute('buildId'), true);
Console::info('Removed build for deployment ' . $deployment['$id']);
}
}
// Remove the container of the deployment
$orchestration->remove('appwrite-function-' . $deployment['$id'], true);
Console::info('Removed container for deployment ' . $deployment['$id']);
} catch (Exception $e) {
// Do nothing, we don't care that much if it fails
}
} catch (Exception $e) {
logError($e, "cleanupFunction");
$orchestrationPool->put($orchestration);
return $response->json(['error' => $e->getMessage()]);
}
$orchestrationPool->put($orchestration);
return $response->json(['success' => true]);
});
App::post('/v1/deployment')
->param('functionId', '', new UID(), 'Function unique ID.')
->param('deploymentId', '', new UID(), 'Deployment unique ID.')
->param('userId', '', new UID(), 'User unique ID.', true)
->inject('response')
->inject('dbForProject')
->inject('projectID')
->inject('register')
->action(function (string $functionId, string $deploymentId, string $userId, Response $response, Database $dbForProject, string $projectID, Registry $register) use ($runtimes) {
// Get function document
$function = $dbForProject->getDocument('functions', $functionId);
// Get deployment document
$deployment = $dbForProject->getDocument('deployments', $deploymentId);
// Check if both documents exist
if ($function->isEmpty()) {
throw new Exception('Function not found', 404);
}
if ($deployment->isEmpty()) {
throw new Exception('Deployment not found', 404);
}
$runtime = $runtimes[$function->getAttribute('runtime')] ?? null;
if (\is_null($runtime)) {
throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported');
}
// Create a new build entry
$buildId = $dbForProject->getId();
if ($deployment->getAttribute('buildId')) {
$buildId = $deployment->getAttribute('buildId');
} else {
try {
$dbForProject->createDocument('builds', new Document([
'$id' => $buildId,
'$read' => (!empty($userId)) ? ['user:' . $userId] : [],
'$write' => ['role:all'],
'startTime' => time(),
'deploymentId' => $deploymentId,
'status' => 'processing',
'runtime' => $function->getAttribute('runtime'),
'outputPath' => '',
'source' => $deployment->getAttribute('path'),
'sourceType' => Storage::DEVICE_LOCAL,
'stdout' => '',
'stderr' => '',
'endTime' => 0,
'duration' => 0,
'search' => implode(' ', [$deployment->getId(), $buildId])
]));
$deployment->setAttribute('buildId', $buildId);
$dbForProject->updateDocument('deployments', $deployment->getId(), $deployment);
} catch (\Throwable $th) {
var_dump($deployment->getArrayCopy());
throw $th;
}
}
// Build Code
go(function () use ($projectID, $deploymentId, $buildId, $functionId, $function, $register) {
$db = $register->get('dbPool')->get();
$redis = $register->get('redisPool')->get();
$cache = new Cache(new RedisCache($redis));
$dbForProject = new Database(new MariaDB($db), $cache);
$dbForProject->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
$dbForProject->setNamespace('_project_' . $projectID);
// Build Code
runBuildStage($buildId, $projectID);
// Update the schedule
$schedule = $function->getAttribute('schedule', '');
$cron = (empty($function->getAttribute('deployment')) && !empty($schedule)) ? new CronExpression($schedule) : null;
$next = (empty($function->getAttribute('deployment')) && !empty($schedule)) ? $cron->getNextRunDate()->format('U') : 0;
// Grab deployment
$deployment = $dbForProject->getDocument('deployments', $deploymentId);
// Grab build
$build = $dbForProject->getDocument('builds', $buildId);
// If the build failed, it won't be possible to deploy
if ($build->getAttribute('status') !== 'ready') {
return;
}
if ($deployment->getAttribute('deploy') === true) {
// Update the function document setting the deployment as the active one
$function
->setAttribute('deployment', $deployment->getId())
->setAttribute('scheduleNext', (int)$next);
$function = $dbForProject->updateDocument('functions', $function->getId(), $function);
}
// Deploy Runtime Server
createRuntimeServer($functionId, $projectID, $deploymentId, $dbForProject);
$register->get('dbPool')->put($db);
$register->get('redisPool')->put($redis);
});
if (false === $function) {
throw new Exception('Failed saving function to DB', 500);
}
$response->dynamic($function, Response::MODEL_FUNCTION);
});
App::get('/v1/')
->inject('response')
->action(
function (Response $response) {
$response
->addHeader('Cache-Control', 'no-cache, no-store, must-revalidate')
->addHeader('Expires', '0')
->addHeader('Pragma', 'no-cache')
->json(['status' => 'online']);
}
);
// Build Endpoints
App::post('/v1/build/:buildId') // Start a Build
->param('buildId', '', new UID(), 'Build unique ID.', false)
->inject('response')
->inject('dbForProject')
->inject('projectID')
->action(function (string $buildId, Response $response, Database $dbForProject, string $projectID) {
try {
// Get build document
$build = $dbForProject->getDocument('builds', $buildId);
// Check if build exists
if ($build->isEmpty()) {
throw new Exception('Build not found', 404);
}
// Check if build is already running
if ($build->getAttribute('status') === 'running') {
throw new Exception('Build is already running', 409);
}
// Check if build is already finished
if ($build->getAttribute('status') === 'finished') {
throw new Exception('Build is already finished', 409);
}
go(function () use ($buildId, $dbForProject, $projectID) {
// Build Code
runBuildStage($buildId, $projectID, $dbForProject);
});
// return success
return $response->json(['success' => true]);
} catch (Exception $e) {
logError($e, "buildEndpoint");
$response
->addHeader('Cache-Control', 'no-cache, no-store, must-revalidate')
->addHeader('Expires', '0')
->addHeader('Pragma', 'no-cache')
->json(['error' => $e->getMessage()]);
}
});
function runBuildStage(string $buildId, string $projectID): Document
{
global $runtimes;
@@ -977,7 +679,7 @@ function runBuildStage(string $buildId, string $projectID): Document
return $build;
}
// Update Deployment Status
// Update Tag Status
$build->setAttribute('status', 'building');
$database->updateDocument('builds', $build->getId(), $build);
@@ -989,7 +691,7 @@ function runBuildStage(string $buildId, string $projectID): Document
throw new Exception('Runtime "' . $build->getAttribute('runtime', '') . '" is not supported');
}
// Grab Deployment Files
// Grab Tag Files
$deploymentPath = $build->getAttribute('source', '');
$sourceType = $build->getAttribute('sourceType', '');
@@ -1024,25 +726,7 @@ function runBuildStage(string $buildId, string $projectID): Document
throw new Exception('Code is not readable: ' . $build->getAttribute('source', ''));
}
$deployment = $database->getDocument('deployments', $build->getAttribute('deploymentId', ''));
$resourceId = $deployment->getAttribute('resourceId', '');
$resourceType = $deployment->getAttribute('resourceType', '');
if (empty($resourceId)) {
throw new Exception('Invalid resource ID on build ' . $build->getId());
}
if (empty($resourceType)) {
throw new Exception('Invalid resource type on build' . $build->getId());
}
$resource = $database->getDocument($resourceType, $resourceId);
if ($resource->isEmpty()) {
throw new Exception('Resource not found on build ' . $build->getId());
}
$vars = $resource->getAttribute('vars', []);
$vars = $build->getAttribute('vars', []);
// Start tracking time
$buildStart = \time();
@@ -1061,7 +745,7 @@ function runBuildStage(string $buildId, string $projectID): Document
} else {
throw new Exception('Can\'t create directory /tmp/project-' . $projectID . '/' . $build->getId() . '/builtCode');
}
};
}
// Launch build container
$id = $orchestration->run(
@@ -1211,24 +895,335 @@ function runBuildStage(string $buildId, string $projectID): Document
$orchestration->remove($id, true);
}
$orchestrationPool->put($orchestration);
$register->get('dbPool')->put($db);
$register->get('redisPool')->put($redis);
throw new Exception('Build failed: ' . $e->getMessage());
} finally {
$orchestrationPool->put($orchestration);
$register->get('dbPool')->put($db);
$register->get('redisPool')->put($redis);
}
$orchestrationPool->put($orchestration);
$register->get('dbPool')->put($db);
$register->get('redisPool')->put($redis);
return $build;
}
App::post('/v1/execute') // Define Route
->desc('Execute a function')
->param('trigger', '', new Text(1024))
->param('projectId', '', new Text(1024))
->param('executionId', '', new Text(1024), '', true)
->param('functionId', '', new Text(1024))
->param('event', '', new Text(1024), '', true)
->param('eventData', '', new Text(10240), '', true)
->param('data', '', new Text(1024), '', true)
->param('webhooks', [], new ArrayList(new JSON()), '', true)
->param('userId', '', new Text(1024), '', true)
->param('jwt', '', new Text(1024), '', true)
->inject('response')
->inject('dbForProject')
->action(
function (string $trigger, string $projectId, string $executionId, string $functionId, string $event, string $eventData, string $data, array $webhooks, string $userId, string $jwt, Response $response, Database $dbForProject) {
try {
$data = execute($trigger, $projectId, $executionId, $functionId, $dbForProject, $event, $eventData, $data, $webhooks, $userId, $jwt);
$response->json($data);
} catch (Exception $e) {
logError($e, 'executeEndpoint');
$response
->addHeader('Cache-Control', 'no-cache, no-store, must-revalidate')
->addHeader('Expires', '0')
->addHeader('Pragma', 'no-cache')
->json(['error' => $e->getMessage()]);
}
}
);
// Cleanup Endpoints used internally by appwrite when a function or deployment gets deleted to also clean up their containers
App::post('/v1/cleanup/function')
->param('functionId', '', new UID())
->inject('response')
->inject('dbForProject')
->action(
function (string $functionId, Response $response, Database $dbForProject) use ($orchestrationPool) {
try {
/** @var Orchestration $orchestration */
$orchestration = $orchestrationPool->get();
// Get function document
$function = $dbForProject->getDocument('functions', $functionId);
// Check if function exists
if ($function->isEmpty()) {
throw new Exception('Function not found', 404);
}
$results = $dbForProject->find('deployments', [new Query('functionId', Query::TYPE_EQUAL, [$functionId])], 999);
// If amount is 0 then we simply return true
if (count($results) === 0) {
return $response->json(['success' => true]);
}
// Delete the containers of all deployments
foreach ($results as $deployment) {
try {
// Remove any ongoing builds
if ($deployment->getAttribute('buildId')) {
$build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId'));
if ($build->getAttribute('status') === 'building') {
// Remove the build
$orchestration->remove('build-stage-' . $deployment->getAttribute('buildId'), true);
Console::info('Removed build for deployment ' . $deployment['$id']);
}
}
$orchestration->remove('appwrite-function-' . $deployment['$id'], true);
Console::info('Removed container for deployment ' . $deployment['$id']);
} catch (Exception $e) {
// Do nothing, we don't care that much if it fails
}
}
return $response->json(['success' => true]);
} catch (Exception $e) {
logError($e, "cleanupFunction");
$orchestrationPool->put($orchestration);
return $response->json(['error' => $e->getMessage()]);
}
$orchestrationPool->put($orchestration);
}
);
App::post('/v1/cleanup/deployment')
->param('deploymentId', '', new UID(), 'Deployment unique ID.')
->inject('response')
->inject('dbForProject')
->action(function (string $deploymentId, Response $response, Database $dbForProject) use ($orchestrationPool) {
try {
/** @var Orchestration $orchestration */
$orchestration = $orchestrationPool->get();
// Get deployment document
$deployment = $dbForProject->getDocument('deployments', $deploymentId);
// Check if deployment exists
if ($deployment->isEmpty()) {
throw new Exception('Deployment not found', 404);
}
try {
// Remove any ongoing builds
if ($deployment->getAttribute('buildId')) {
$build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId'));
if ($build->getAttribute('status') == 'building') {
// Remove the build
$orchestration->remove('build-stage-' . $deployment->getAttribute('buildId'), true);
Console::info('Removed build for deployment ' . $deployment['$id']);
}
}
// Remove the container of the deployment
$orchestration->remove('appwrite-function-' . $deployment['$id'], true);
Console::info('Removed container for deployment ' . $deployment['$id']);
} catch (Exception $e) {
// Do nothing, we don't care that much if it fails
}
} catch (Exception $e) {
logError($e, "cleanupFunction");
$orchestrationPool->put($orchestration);
return $response->json(['error' => $e->getMessage()]);
}
$orchestrationPool->put($orchestration);
return $response->json(['success' => true]);
});
App::post('/v1/deployment')
->param('functionId', '', new UID(), 'Function unique ID.')
->param('deploymentId', '', new UID(), 'Deployment unique ID.')
->param('userId', '', new UID(), 'User unique ID.', true)
->inject('response')
->inject('dbForProject')
->inject('projectID')
->inject('register')
->action(function (string $functionId, string $deploymentId, string $userId, Response $response, Database $dbForProject, string $projectID, Registry $register) use ($runtimes) {
// Get function document
$function = $dbForProject->getDocument('functions', $functionId);
// Get deployment document
$deployment = $dbForProject->getDocument('deployments', $deploymentId);
// Check if both documents exist
if ($function->isEmpty()) {
throw new Exception('Function not found', 404);
}
if ($deployment->isEmpty()) {
throw new Exception('Deployment not found', 404);
}
$runtime = $runtimes[$function->getAttribute('runtime')] ?? null;
if (\is_null($runtime)) {
throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported');
}
// Create a new build entry
$buildId = $dbForProject->getId();
if ($deployment->getAttribute('buildId')) {
$buildId = $deployment->getAttribute('buildId');
} else {
try {
$dbForProject->createDocument('builds', new Document([
'$id' => $buildId,
'$read' => (!empty($userId)) ? ['user:' . $userId] : [],
'$write' => ['role:all'],
'dateCreated' => time(),
'status' => 'processing',
'runtime' => $function->getAttribute('runtime'),
'outputPath' => '',
'source' => $deployment->getAttribute('path'),
'sourceType' => Storage::DEVICE_LOCAL,
'stdout' => '',
'stderr' => '',
'time' => 0,
'vars' => [
'ENTRYPOINT_NAME' => $deployment->getAttribute('entrypoint'),
'APPWRITE_FUNCTION_ID' => $function->getId(),
'APPWRITE_FUNCTION_NAME' => $function->getAttribute('name', ''),
'APPWRITE_FUNCTION_RUNTIME_NAME' => $runtime['name'],
'APPWRITE_FUNCTION_RUNTIME_VERSION' => $runtime['version'],
'APPWRITE_FUNCTION_PROJECT_ID' => $projectID,
]
]));
$deployment->setAttribute('buildId', $buildId);
$dbForProject->updateDocument('deployments', $deployment->getId(), $deployment);
} catch (\Throwable $th) {
var_dump($deployment->getArrayCopy());
throw $th;
}
}
// Build Code
go(function () use ($projectID, $deploymentId, $buildId, $functionId, $function, $register) {
try {
$db = $register->get('dbPool')->get();
$redis = $register->get('redisPool')->get();
$cache = new Cache(new RedisCache($redis));
$dbForProject = new Database(new MariaDB($db), $cache);
$dbForProject->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
$dbForProject->setNamespace('_project_' . $projectID);
// Build Code
runBuildStage($buildId, $projectID);
// Update the schedule
$schedule = $function->getAttribute('schedule', '');
$cron = (empty($function->getAttribute('deployment')) && !empty($schedule)) ? new CronExpression($schedule) : null;
$next = (empty($function->getAttribute('deployment')) && !empty($schedule)) ? $cron->getNextRunDate()->format('U') : 0;
// Grab deployment
$deployment = $dbForProject->getDocument('deployments', $deploymentId);
// Grab build
$build = $dbForProject->getDocument('builds', $buildId);
// If the build failed, it won't be possible to deploy
if ($build->getAttribute('status') !== 'ready') {
return;
}
if ($deployment->getAttribute('automaticDeploy') === true) {
// Update the function document setting the deployment as the active one
$function
->setAttribute('deployment', $deployment->getId())
->setAttribute('scheduleNext', (int)$next);
$function = $dbForProject->updateDocument('functions', $function->getId(), $function);
}
// Deploy Runtime Server
createRuntimeServer($functionId, $projectID, $deploymentId, $dbForProject);
} catch (\Throwable $th) {
} finally {
$register->get('dbPool')->put($db);
$register->get('redisPool')->put($redis);
}
});
if (false === $function) {
throw new Exception('Failed saving function to DB', 500);
}
$response->dynamic($function, Response::MODEL_FUNCTION);
});
App::get('/v1/')
->inject('response')
->action(
function (Response $response) {
$response
->addHeader('Cache-Control', 'no-cache, no-store, must-revalidate')
->addHeader('Expires', '0')
->addHeader('Pragma', 'no-cache')
->json(['status' => 'online']);
}
);
// Build Endpoints
App::post('/v1/build/:buildId') // Start a Build
->param('buildId', '', new UID(), 'Build unique ID.', false)
->inject('response')
->inject('dbForProject')
->inject('projectID')
->action(function (string $buildId, Response $response, Database $dbForProject, string $projectID) {
try {
// Get build document
$build = $dbForProject->getDocument('builds', $buildId);
// Check if build exists
if ($build->isEmpty()) {
throw new Exception('Build not found', 404);
}
// Check if build is already running
if ($build->getAttribute('status') === 'running') {
throw new Exception('Build is already running', 409);
}
// Check if build is already finished
if ($build->getAttribute('status') === 'finished') {
throw new Exception('Build is already finished', 409);
}
go(function () use ($buildId, $dbForProject, $projectID) {
// Build Code
runBuildStage($buildId, $projectID, $dbForProject);
});
// return success
return $response->json(['success' => true]);
} catch (Exception $e) {
logError($e, "buildEndpoint");
$response
->addHeader('Cache-Control', 'no-cache, no-store, must-revalidate')
->addHeader('Expires', '0')
->addHeader('Pragma', 'no-cache')
->json(['error' => $e->getMessage()]);
}
});
App::setMode(App::MODE_TYPE_PRODUCTION); // Define Mode
$http = new Server("0.0.0.0", 8080);
$http = new Server("0.0.0.0", 80);
function handleShutdown()
{
@@ -1408,4 +1403,4 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo
}
});
$http->start();
$http->start();
app/views/install/compose.phtml
+3 -13
View File
@@ -194,10 +194,10 @@ services:
- _APP_USAGE_STATS
- _APP_STATSD_HOST
- _APP_STATSD_PORT
- DOCKERHUB_PULL_USERNAME
- DOCKERHUB_PULL_PASSWORD
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
- DOCKERHUB_PULL_USERNAME
- DOCKERHUB_PULL_PASSWORD
appwrite-worker-database:
image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
@@ -324,13 +324,9 @@ services:
depends_on:
- redis
- mariadb
volumes:
- /var/run/docker.sock:/var/run/docker.sock
- appwrite-functions:/storage/functions:rw
- /tmp:/tmp:rw
- appwrite-executor
environment:
- _APP_ENV
- _APP_OPENSSL_KEY_V1
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
@@ -341,13 +337,7 @@ services:
- _APP_DB_USER
- _APP_DB_PASS
- _APP_FUNCTIONS_TIMEOUT
- _APP_FUNCTIONS_BUILD_TIMEOUT
- _APP_FUNCTIONS_CONTAINERS
- _APP_FUNCTIONS_CPUS
- _APP_FUNCTIONS_MEMORY
- _APP_FUNCTIONS_MEMORY_SWAP
- _APP_EXECUTOR_SECRET
- _APP_FUNCTIONS_RUNTIMES
- _APP_USAGE_STATS
appwrite-executor:
app/workers/functions.php
+2 -105
View File
@@ -1,10 +1,6 @@
<?php
use Appwrite\Event\Event;
use Appwrite\Messaging\Adapter\Realtime;
use Appwrite\Resque\Worker;
use Appwrite\Stats\Stats;
use Appwrite\Utopia\Response\Model\Execution;
use Cron\CronExpression;
use Swoole\Runtime;
use Utopia\App;
@@ -13,11 +9,8 @@ use Utopia\Config\Config;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Validator\Authorization;
use Utopia\Orchestration\Orchestration;
use Utopia\Orchestration\Adapter\DockerAPI;
use Utopia\Orchestration\Container;
use Utopia\Orchestration\Exception\Orchestration as OrchestrationException;
use Utopia\Orchestration\Exception\Timeout as TimeoutException;
use Utopia\Orchestration\Orchestration;
require_once __DIR__.'/../init.php';
@@ -38,41 +31,6 @@ $warmupTime = $warmupEnd - $warmupStart;
Console::success('Finished warmup in ' . $warmupTime . ' seconds');
/**
* List function servers
*/
$stdout = '';
$stderr = '';
$executionStart = \microtime(true);
$response = $orchestration->list(['label' => 'appwrite-type=function']);
/** @var Container[] $list */
$list = [];
foreach ($response as $value) {
$list[$value->getName()] = $value;
}
$executionEnd = \microtime(true);
Console::info(count($list) . ' functions listed in ' . ($executionEnd - $executionStart) . ' seconds');
/**
* 1. Get event args - DONE
* 2. Unpackage code in the isolated container - DONE
* 3. Execute in container with timeout
* + messure execution time - DONE
* + pass env vars - DONE
* + pass one-time api key
* 4. Update execution status - DONE
* 5. Update execution stdout & stderr - DONE
* 6. Trigger audit log - DONE
* 7. Trigger usage log - DONE
*/
// TODO avoid scheduled execution if delay is bigger than X offest
class FunctionsV1 extends Worker
{
public array $args = [];
@@ -266,7 +224,7 @@ class FunctionsV1 extends Worker
public function execute(string $trigger, string $projectId, string $executionId, Database $database, Document $function, string $event = '', string $eventData = '', string $data = '', array $webhooks = [], string $userId = '', string $jwt = ''): void
{
$ch = \curl_init();
\curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor:8080/v1/execute");
\curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor/v1/execute");
\curl_setopt($ch, CURLOPT_POST, true);
\curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode([
'trigger' => $trigger,
@@ -299,67 +257,6 @@ class FunctionsV1 extends Worker
\curl_close($ch);
}
/**
* Cleanup any hanging containers above the allowed max containers.
*
* @return void
*/
public function cleanup(): void
{
/** @var Container[] $list */
global $list;
/** @var Orchestration $orchestration */
global $orchestration;
Console::success(count($list) . ' running containers counted');
$max = (int) App::getEnv('_APP_FUNCTIONS_CONTAINERS');
if (\count($list) > $max) {
Console::info('Starting containers cleanup');
\uasort($list, function (Container $item1, Container $item2) {
return (int)($item1->getLabels['appwrite-created'] ?? 0) <=> (int)($item2->getLabels['appwrite-created'] ?? 0);
});
while (\count($list) > $max) {
$first = \array_shift($list);
try {
$orchestration->remove($first->getName(), true);
Console::info('Removed container: ' . $first->getName());
} catch (Exception $e) {
Console::error('Failed to remove container: ' . $e);
}
}
}
}
/**
* Filter ENV vars
*
* @param string $string
*
* @return string
*/
public function filterEnvKey(string $string): string
{
if (empty($this->allowed)) {
$this->allowed = array_fill_keys(\str_split('0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_'), true);
}
$string = \str_split($string);
$output = '';
foreach ($string as $char) {
if (\array_key_exists($char, $this->allowed)) {
$output .= $char;
}
}
return $output;
}
public function shutdown(): void
{
}
docker-compose.yml
+1 -11
View File
@@ -61,7 +61,6 @@ services:
- traefik.http.routers.appwrite_api_https.service=appwrite_api
- traefik.http.routers.appwrite_api_https.tls=true
volumes:
- /var/run/docker.sock:/var/run/docker.sock
- appwrite-uploads:/storage/uploads:rw
- appwrite-cache:/storage/cache:rw
- appwrite-config:/storage/config:rw
@@ -326,17 +325,14 @@ services:
networks:
- appwrite
volumes:
- /var/run/docker.sock:/var/run/docker.sock
- appwrite-functions:/storage/functions:rw
- /tmp:/tmp:rw
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
depends_on:
- redis
- mariadb
- appwrite-executor
environment:
- _APP_ENV
- _APP_OPENSSL_KEY_V1
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
@@ -347,12 +343,6 @@ services:
- _APP_DB_USER
- _APP_DB_PASS
- _APP_FUNCTIONS_TIMEOUT
- _APP_FUNCTIONS_BUILD_TIMEOUT
- _APP_FUNCTIONS_CONTAINERS
- _APP_FUNCTIONS_RUNTIMES
- _APP_FUNCTIONS_CPUS
- _APP_FUNCTIONS_MEMORY
- _APP_FUNCTIONS_MEMORY_SWAP
- _APP_EXECUTOR_SECRET
- _APP_USAGE_STATS
- DOCKERHUB_PULL_USERNAME