diff --git a/app/cli.php b/app/cli.php index acadd10be1..4106ae35b5 100644 --- a/app/cli.php +++ b/app/cli.php @@ -8,6 +8,10 @@ use Appwrite\Event\Certificate; use Appwrite\Event\Database as EventDatabase; use Appwrite\Event\Func; use Appwrite\Event\Audit; +use Appwrite\Event\Build; +use Appwrite\Event\Event; +use Appwrite\Event\Mail; +use Appwrite\Event\Phone; use Appwrite\Platform\Appwrite; use Utopia\CLI\CLI; use Utopia\Database\Validator\Authorization; @@ -149,27 +153,36 @@ CLI::setResource('influxdb', function (Registry $register) { CLI::setResource('queue', function (Group $pools) { return $pools->get('queue')->pop()->getResource(); }, ['pools']); - -CLI::setResource('deletes', function (Connection $queue) { - return new Delete($queue); +CLI::setResource('messaging', function (Connection $queue) { + return new Phone($queue); }, ['queue']); - -CLI::setResource('queueForFunctions', function (Connection $queue) { - return new Func($queue); +CLI::setResource('mails', function (Connection $queue) { + return new Mail($queue); }, ['queue']); - -CLI::setResource('audits', function (Connection $queue) { - return new Audit($queue); +CLI::setResource('builds', function (Connection $queue) { + return new Build($queue); }, ['queue']); - -CLI::setResource('certificates', function (Connection $queue) { - return new Certificate($queue); -}, ['queue']); - CLI::setResource('database', function (Connection $queue) { return new EventDatabase($queue); }, ['queue']); - +CLI::setResource('deletes', function (Connection $queue) { + return new Delete($queue); +}, ['queue']); +CLI::setResource('events', function (Connection $queue) { + return new Event('', '', $queue); +}, ['queue']); +CLI::setResource('audits', function (Connection $queue) { + return new Audit($queue); +}, ['queue']); +CLI::setResource('events', function (Connection $queue) { + return new Event('', '', $queue); +}, ['queue']); +CLI::setResource('queueForFunctions', function (Connection $queue) { + return new Func($queue); +}, ['queue']); +CLI::setResource('certificates', function (Connection $queue) { + return new Certificate($queue); +}, ['queue']); CLI::setResource('logError', function (Registry $register) { return function (Throwable $error, string $namespace, string $action) use ($register) { $logger = $register->get('logger'); diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 346820d699..3d6e7bf762 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -1291,7 +1291,6 @@ App::get('/v1/account') ->inject('response') ->inject('user') ->action(function (Response $response, Document $user) { - $response->dynamic($user, Response::MODEL_ACCOUNT); }); diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 6fe7c67e69..88b91ab8cd 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -634,7 +634,8 @@ App::post('/v1/functions/:functionId/deployments') ->inject('deviceFunctions') ->inject('deviceLocal') ->inject('dbForConsole') - ->action(function (string $functionId, string $entrypoint, mixed $code, bool $activate, Request $request, Response $response, Database $dbForProject, Event $events, Document $project, Device $deviceFunctions, Device $deviceLocal, Database $dbForConsole) { + ->inject('builds') + ->action(function (string $functionId, string $entrypoint, mixed $code, bool $activate, Request $request, Response $response, Database $dbForProject, Event $events, Document $project, Device $deviceFunctions, Device $deviceLocal, Database $dbForConsole, Build $builds) { $function = $dbForProject->getDocument('functions', $functionId); @@ -754,8 +755,7 @@ App::post('/v1/functions/:functionId/deployments') } // Start the build - $buildEvent = new Build(); - $buildEvent + $builds ->setType(BUILD_TYPE_DEPLOYMENT) ->setResource($function) ->setDeployment($deployment) @@ -1001,7 +1001,8 @@ App::post('/v1/functions/:functionId/deployments/:deploymentId/builds/:buildId') ->inject('dbForProject') ->inject('project') ->inject('events') - ->action(function (string $functionId, string $deploymentId, string $buildId, Response $response, Database $dbForProject, Document $project, Event $events) { + ->inject('builds') + ->action(function (string $functionId, string $deploymentId, string $buildId, Response $response, Database $dbForProject, Document $project, Event $events, Build $builds) { $function = $dbForProject->getDocument('functions', $functionId); $deployment = $dbForProject->getDocument('deployments', $deploymentId); @@ -1029,8 +1030,7 @@ App::post('/v1/functions/:functionId/deployments/:deploymentId/builds/:buildId') ->setParam('deploymentId', $deployment->getId()); // Retry the build - $buildEvent = new Build(); - $buildEvent + $builds ->setType(BUILD_TYPE_RETRY) ->setResource($function) ->setDeployment($deployment) diff --git a/app/init.php b/app/init.php index bb2b6f2a6d..bea032ab75 100644 --- a/app/init.php +++ b/app/init.php @@ -69,6 +69,7 @@ use Utopia\Pools\Group; use Utopia\Pools\Pool; use Ahc\Jwt\JWT; use Ahc\Jwt\JWTException; +use Appwrite\Event\Build; use Appwrite\Event\Certificate; use Appwrite\Event\Func; use MaxMind\Db\Reader; @@ -858,18 +859,21 @@ App::setResource('messaging', function (Connection $queue) { App::setResource('mails', function (Connection $queue) { return new Mail($queue); }, ['queue']); +App::setResource('builds', function (Connection $queue) { + return new Build($queue); +}, ['queue']); App::setResource('database', function (Connection $queue) { return new EventDatabase($queue); -}, ['pools']); +}, ['queue']); App::setResource('deletes', function (Connection $queue) { return new Delete($queue); }, ['queue']); App::setResource('events', function (Connection $queue) { return new Event('', '', $queue); -}, ['pools']); +}, ['queue']); App::setResource('audits', function (Connection $queue) { return new Audit($queue); -}, ['pools']); +}, ['queue']); App::setResource('events', function (Connection $queue) { return new Event('', '', $queue); }, ['queue']); diff --git a/app/worker.php b/app/worker.php index 21785362d2..33fe128a67 100644 --- a/app/worker.php +++ b/app/worker.php @@ -4,9 +4,13 @@ require_once __DIR__ . '/init.php'; use Appwrite\Event\Event; use Appwrite\Event\Audit; +use Appwrite\Event\Build; use Appwrite\Event\Certificate; use Appwrite\Event\Database as EventDatabase; +use Appwrite\Event\Delete; use Appwrite\Event\Func; +use Appwrite\Event\Mail; +use Appwrite\Event\Phone; use Swoole\Runtime; use Utopia\App; use Utopia\Cache\Adapter\Sharding; @@ -15,6 +19,7 @@ use Utopia\CLI\Console; use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\Document; +use Utopia\DSN\DSN; use Utopia\Queue\Adapter\Swoole; use Utopia\Queue\Message; use Utopia\Queue\Server; @@ -22,6 +27,15 @@ use Utopia\Registry\Registry; use Utopia\Logger\Log; use Utopia\Logger\Logger; use Utopia\Pools\Group; +use Utopia\Queue\Connection; +use Utopia\Storage\Device; +use Utopia\Storage\Device\Backblaze; +use Utopia\Storage\Device\DOSpaces; +use Utopia\Storage\Device\Linode; +use Utopia\Storage\Device\Local; +use Utopia\Storage\Device\S3; +use Utopia\Storage\Device\Wasabi; +use Utopia\Storage\Storage; Runtime::enableCoroutine(SWOOLE_HOOK_ALL); @@ -86,16 +100,39 @@ Server::setResource('database', function (Registry $register) { ); }, ['register']); -Server::setResource('queueForFunctions', function (Registry $register) { - $pools = $register->get('pools'); - return new Func( - $pools - ->get('queue') - ->pop() - ->getResource() - ); -}, ['register']); - +Server::setResource('queue', function (Group $pools) { + return $pools->get('queue')->pop()->getResource(); +}, ['pools']); +Server::setResource('messaging', function (Connection $queue) { + return new Phone($queue); +}, ['queue']); +Server::setResource('mails', function (Connection $queue) { + return new Mail($queue); +}, ['queue']); +Server::setResource('builds', function (Connection $queue) { + return new Build($queue); +}, ['queue']); +Server::setResource('database', function (Connection $queue) { + return new EventDatabase($queue); +}, ['queue']); +Server::setResource('deletes', function (Connection $queue) { + return new Delete($queue); +}, ['queue']); +Server::setResource('events', function (Connection $queue) { + return new Event('', '', $queue); +}, ['queue']); +Server::setResource('audits', function (Connection $queue) { + return new Audit($queue); +}, ['queue']); +Server::setResource('events', function (Connection $queue) { + return new Event('', '', $queue); +}, ['queue']); +Server::setResource('queueForFunctions', function (Connection $queue) { + return new Func($queue); +}, ['queue']); +Server::setResource('certificates', function (Connection $queue) { + return new Certificate($queue); +}, ['queue']); Server::setResource('events', function (Registry $register) { $pools = $register->get('pools'); return new Event( @@ -197,3 +234,111 @@ $server Console::error('[Error] File: ' . $error->getFile()); Console::error('[Error] Line: ' . $error->getLine()); }); + +/** + * Get Console DB + * + * @returns Cache + */ +function getCache(): Cache +{ + global $register; + + $pools = $register->get('pools'); + /** @var \Utopia\Pools\Group $pools */ + + $list = Config::getParam('pools-cache', []); + $adapters = []; + + foreach ($list as $value) { + $adapters[] = $pools + ->get($value) + ->pop() + ->getResource(); + } + + return new Cache(new Sharding($adapters)); +} + +/** + * Get Console DB + * + * @returns Database + */ +function getConsoleDB(): Database +{ + global $register; + + /** @var \Utopia\Pools\Group $pools */ + $pools = $register->get('pools'); + + $dbAdapter = $pools + ->get('console') + ->pop() + ->getResource(); + + $database = new Database($dbAdapter, getCache()); + + $database->setNamespace('console'); + + return $database; +} + +/** + * Get Project DB + * + * @param Document $project + * @returns Database + */ +function getProjectDB(Document $project): Database +{ + global $register; + + /** @var \Utopia\Pools\Group $pools */ + $pools = $register->get('pools'); + + if ($project->isEmpty() || $project->getId() === 'console') { + return getConsoleDB(); + } + + $dbAdapter = $pools + ->get($project->getAttribute('database')) + ->pop() + ->getResource(); + + $database = new Database($dbAdapter, getCache()); + $database->setNamespace('_' . $project->getInternalId()); + + return $database; +} + + +/** + * Get Functions Storage Device + * @param string $projectId of the project + * @return Device + */ +function getFunctionsDevice($projectId): Device +{ + return getDevice(APP_STORAGE_FUNCTIONS . '/app-' . $projectId); +} + +/** + * Get Files Storage Device + * @param string $projectId of the project + * @return Device + */ +function getFilesDevice($projectId): Device +{ + return getDevice(APP_STORAGE_UPLOADS . '/app-' . $projectId); +} + +/** + * Get Builds Storage Device + * @param string $projectId of the project + * @return Device + */ +function getBuildsDevice($projectId): Device +{ + return getDevice(APP_STORAGE_BUILDS . '/app-' . $projectId); +} diff --git a/app/workers/builds.php b/app/workers/builds.php index 586663f183..9ca077dfcd 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -1,5 +1,7 @@ getDocument('functions', $function->getId()); + if ($function->isEmpty()) { + throw new Exception('Function not found', 404); } - public function init(): void - { - $this->executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST')); + $deployment = $dbForProject->getDocument('deployments', $deployment->getId()); + if ($deployment->isEmpty()) { + throw new Exception('Deployment not found', 404); } - public function run(): void - { - $type = $this->args['type'] ?? ''; - $project = new Document($this->args['project'] ?? []); - $resource = new Document($this->args['resource'] ?? []); - $deployment = new Document($this->args['deployment'] ?? []); - - switch ($type) { - case BUILD_TYPE_DEPLOYMENT: - case BUILD_TYPE_RETRY: - Console::info('Creating build for deployment: ' . $deployment->getId()); - $this->buildDeployment($project, $resource, $deployment); - break; - - default: - throw new \Exception('Invalid build type'); - break; - } + $runtimes = Config::getParam('runtimes', []); + $key = $function->getAttribute('runtime'); + $runtime = isset($runtimes[$key]) ? $runtimes[$key] : null; + if (\is_null($runtime)) { + throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported'); } - protected function buildDeployment(Document $project, Document $function, Document $deployment) - { - global $register; + $connection = App::getEnv('_APP_CONNECTIONS_STORAGE', ''); /** @TODO : move this to the registry or someplace else */ + $device = Storage::DEVICE_LOCAL; + try { + $dsn = new DSN($connection); + $device = $dsn->getScheme(); + } catch (\Exception $e) { + Console::error($e->getMessage() . 'Invalid DSN. Defaulting to Local device.'); + } - $dbForProject = $this->getProjectDB($project); + $buildId = $deployment->getAttribute('buildId', ''); + $startTime = DateTime::now(); + if (empty($buildId)) { + $buildId = ID::unique(); + $build = $dbForProject->createDocument('builds', new Document([ + '$id' => $buildId, + '$permissions' => [], + 'startTime' => $startTime, + 'deploymentId' => $deployment->getId(), + 'status' => 'processing', + 'outputPath' => '', + 'runtime' => $function->getAttribute('runtime'), + 'source' => $deployment->getAttribute('path'), + 'sourceType' => $device, + 'stdout' => '', + 'stderr' => '', + 'endTime' => null, + 'duration' => 0 + ])); + $deployment->setAttribute('buildId', $buildId); + $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); + } else { + $build = $dbForProject->getDocument('builds', $buildId); + } - $function = $dbForProject->getDocument('functions', $function->getId()); - if ($function->isEmpty()) { - throw new Exception('Function not found', 404); + /** Request the executor to build the code... */ + $build->setAttribute('status', 'building'); + $build = $dbForProject->updateDocument('builds', $buildId, $build); + + /** Trigger Webhook */ + $deploymentModel = new Deployment(); + + $pools = $register->get('pools'); + $connection = $pools->get('queue')->pop()->getResource(); + + $deploymentUpdate = new Event(Event::WEBHOOK_QUEUE_NAME, Event::WEBHOOK_CLASS_NAME, $connection); + $deploymentUpdate + ->setProject($project) + ->setEvent('functions.[functionId].deployments.[deploymentId].update') + ->setParam('functionId', $function->getId()) + ->setParam('deploymentId', $deployment->getId()) + ->setPayload($deployment->getArrayCopy(array_keys($deploymentModel->getRules()))) + ->trigger(); + + /** Trigger Functions */ + $pools = $register->get('pools'); + $connection = $pools->get('queue')->pop(); + + $functions = new Func($connection->getResource()); + $functions + ->from($deploymentUpdate) + ->trigger(); + + $connection->reclaim(); + + /** Trigger Realtime */ + $allEvents = Event::generateEvents('functions.[functionId].deployments.[deploymentId].update', [ + 'functionId' => $function->getId(), + 'deploymentId' => $deployment->getId() + ]); + $target = Realtime::fromPayload( + // Pass first, most verbose event pattern + event: $allEvents[0], + payload: $build, + project: $project + ); + + Realtime::send( + projectId: 'console', + payload: $build->getArrayCopy(), + events: $allEvents, + channels: $target['channels'], + roles: $target['roles'] + ); + + $source = $deployment->getAttribute('path'); + + $vars = array_reduce($function->getAttribute('vars', []), function (array $carry, Document $var) { + $carry[$var->getAttribute('key')] = $var->getAttribute('value'); + return $carry; + }, []); + + try { + $response = $executor->createRuntime( + projectId: $project->getId(), + deploymentId: $deployment->getId(), + source: $source, + image: $runtime['image'], + remove: true, + entrypoint: $deployment->getAttribute('entrypoint'), + workdir: '/usr/code', + destination: APP_STORAGE_BUILDS . "/app-{$project->getId()}", + variables: $vars, + commands: [ + 'sh', '-c', + 'tar -zxf /tmp/code.tar.gz -C /usr/code && \ + cd /usr/local/src/ && ./build.sh' + ] + ); + + $endTime = new \DateTime(); + $endTime->setTimestamp($response['endTimeUnix']); + + /** Update the build document */ + $build->setAttribute('endTime', DateTime::format($endTime)); + $build->setAttribute('duration', \intval($response['duration'])); + $build->setAttribute('status', $response['status']); + $build->setAttribute('outputPath', $response['outputPath']); + $build->setAttribute('stderr', $response['stderr']); + $build->setAttribute('stdout', $response['stdout']); + + /* Also update the deployment buildTime */ + $deployment->setAttribute('buildTime', $response['duration']); + + Console::success("Build id: $buildId created"); + + $function->setAttribute('scheduleUpdatedAt', DateTime::now()); + + /** Set auto deploy */ + if ($deployment->getAttribute('activate') === true) { + $function->setAttribute('deployment', $deployment->getId()); + $function = $dbForProject->updateDocument('functions', $function->getId(), $function); } - $deployment = $dbForProject->getDocument('deployments', $deployment->getId()); - if ($deployment->isEmpty()) { - throw new Exception('Deployment not found', 404); - } + /** Update function schedule */ + $dbForConsole = getConsoleDB(); + $schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId')); + $schedule->setAttribute('resourceUpdatedAt', $function->getAttribute('scheduleUpdatedAt')); - $runtimes = Config::getParam('runtimes', []); - $key = $function->getAttribute('runtime'); - $runtime = isset($runtimes[$key]) ? $runtimes[$key] : null; - if (\is_null($runtime)) { - throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported'); - } + $schedule + ->setAttribute('schedule', $function->getAttribute('schedule')) + ->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment'))); - $connection = App::getEnv('_APP_CONNECTIONS_STORAGE', ''); /** @TODO : move this to the registry or someplace else */ - $device = Storage::DEVICE_LOCAL; - try { - $dsn = new DSN($connection); - $device = $dsn->getScheme(); - } catch (\Exception $e) { - Console::error($e->getMessage() . 'Invalid DSN. Defaulting to Local device.'); - } - $buildId = $deployment->getAttribute('buildId', ''); - $startTime = DateTime::now(); - if (empty($buildId)) { - $buildId = ID::unique(); - $build = $dbForProject->createDocument('builds', new Document([ - '$id' => $buildId, - '$permissions' => [], - 'startTime' => $startTime, - 'deploymentId' => $deployment->getId(), - 'status' => 'processing', - 'outputPath' => '', - 'runtime' => $function->getAttribute('runtime'), - 'source' => $deployment->getAttribute('path'), - 'sourceType' => $device, - 'stdout' => '', - 'stderr' => '', - 'endTime' => null, - 'duration' => 0 - ])); - $deployment->setAttribute('buildId', $buildId); - $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); - } else { - $build = $dbForProject->getDocument('builds', $buildId); - } - - /** Request the executor to build the code... */ - $build->setAttribute('status', 'building'); + Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule)); + } catch (\Throwable $th) { + $endTime = DateTime::now(); + $interval = (new \DateTime($endTime))->diff(new \DateTime($startTime)); + $build->setAttribute('endTime', $endTime); + $build->setAttribute('duration', $interval->format('%s') + 0); + $build->setAttribute('status', 'failed'); + $build->setAttribute('stderr', $th->getMessage()); + Console::error($th->getMessage()); + } finally { $build = $dbForProject->updateDocument('builds', $buildId, $build); - /** Trigger Webhook */ - $deploymentModel = new Deployment(); - - $pools = $register->get('pools'); - $connection = $pools->get('queue')->pop()->getResource(); - - $deploymentUpdate = new Event(Event::WEBHOOK_QUEUE_NAME, Event::WEBHOOK_CLASS_NAME, $connection); - $deploymentUpdate - ->setProject($project) - ->setEvent('functions.[functionId].deployments.[deploymentId].update') - ->setParam('functionId', $function->getId()) - ->setParam('deploymentId', $deployment->getId()) - ->setPayload($deployment->getArrayCopy(array_keys($deploymentModel->getRules()))) - ->trigger(); - - /** Trigger Functions */ - $pools = $register->get('pools'); - $connection = $pools->get('queue')->pop(); - - $functions = new Func($connection->getResource()); - $functions - ->from($deploymentUpdate) - ->trigger(); - - $connection->reclaim(); - - /** Trigger Realtime */ - $allEvents = Event::generateEvents('functions.[functionId].deployments.[deploymentId].update', [ - 'functionId' => $function->getId(), - 'deploymentId' => $deployment->getId() - ]); + /** + * Send realtime Event + */ $target = Realtime::fromPayload( // Pass first, most verbose event pattern event: $allEvents[0], payload: $build, project: $project ); - Realtime::send( projectId: 'console', payload: $build->getArrayCopy(), @@ -164,113 +219,50 @@ class BuildsV1 extends Worker roles: $target['roles'] ); - $source = $deployment->getAttribute('path'); - - $vars = array_reduce($function->getAttribute('vars', []), function (array $carry, Document $var) { - $carry[$var->getAttribute('key')] = $var->getAttribute('value'); - return $carry; - }, []); - - try { - $response = $this->executor->createRuntime( - projectId: $project->getId(), - deploymentId: $deployment->getId(), - source: $source, - image: $runtime['image'], - remove: true, - entrypoint: $deployment->getAttribute('entrypoint'), - workdir: '/usr/code', - destination: APP_STORAGE_BUILDS . "/app-{$project->getId()}", - variables: $vars, - commands: [ - 'sh', '-c', - 'tar -zxf /tmp/code.tar.gz -C /usr/code && \ - cd /usr/local/src/ && ./build.sh' - ] - ); - - $endTime = new \DateTime(); - $endTime->setTimestamp($response['endTimeUnix']); - - /** Update the build document */ - $build->setAttribute('endTime', DateTime::format($endTime)); - $build->setAttribute('duration', \intval($response['duration'])); - $build->setAttribute('status', $response['status']); - $build->setAttribute('outputPath', $response['outputPath']); - $build->setAttribute('stderr', $response['stderr']); - $build->setAttribute('stdout', $response['stdout']); - - /* Also update the deployment buildTime */ - $deployment->setAttribute('buildTime', $response['duration']); - - Console::success("Build id: $buildId created"); - - $function->setAttribute('scheduleUpdatedAt', DateTime::now()); - - /** Set auto deploy */ - if ($deployment->getAttribute('activate') === true) { - $function->setAttribute('deployment', $deployment->getId()); - $function = $dbForProject->updateDocument('functions', $function->getId(), $function); - } - - /** Update function schedule */ - $dbForConsole = $this->getConsoleDB(); - $schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId')); - $schedule->setAttribute('resourceUpdatedAt', $function->getAttribute('scheduleUpdatedAt')); - - $schedule - ->setAttribute('schedule', $function->getAttribute('schedule')) - ->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment'))); - - - Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule)); - } catch (\Throwable $th) { - $endTime = DateTime::now(); - $interval = (new \DateTime($endTime))->diff(new \DateTime($startTime)); - $build->setAttribute('endTime', $endTime); - $build->setAttribute('duration', $interval->format('%s') + 0); - $build->setAttribute('status', 'failed'); - $build->setAttribute('stderr', $th->getMessage()); - Console::error($th->getMessage()); - } finally { - $build = $dbForProject->updateDocument('builds', $buildId, $build); - - /** - * Send realtime Event - */ - $target = Realtime::fromPayload( - // Pass first, most verbose event pattern - event: $allEvents[0], - payload: $build, - project: $project - ); - Realtime::send( - projectId: 'console', - payload: $build->getArrayCopy(), - events: $allEvents, - channels: $target['channels'], - roles: $target['roles'] - ); - - /** Update usage stats */ - if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') { - $statsd = $register->get('statsd'); - $usage = new Stats($statsd); - $usage - ->setParam('projectInternalId', $project->getInternalId()) - ->setParam('projectId', $project->getId()) - ->setParam('functionId', $function->getId()) - ->setParam('builds.{scope}.compute', 1) - ->setParam('buildStatus', $build->getAttribute('status', '')) - ->setParam('buildTime', $build->getAttribute('duration')) - ->setParam('networkRequestSize', 0) - ->setParam('networkResponseSize', 0) - ->submit(); - } + /** Update usage stats */ + if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') { + $statsd = $register->get('statsd'); + $usage = new Stats($statsd); + $usage + ->setParam('projectInternalId', $project->getInternalId()) + ->setParam('projectId', $project->getId()) + ->setParam('functionId', $function->getId()) + ->setParam('builds.{scope}.compute', 1) + ->setParam('buildStatus', $build->getAttribute('status', '')) + ->setParam('buildTime', $build->getAttribute('duration')) + ->setParam('networkRequestSize', 0) + ->setParam('networkResponseSize', 0) + ->submit(); } } - - public function shutdown(): void - { - } } + +$server->job() + ->inject('message') + ->action(function (Message $message) { + $payload = $message->getPayload() ?? []; + + if (empty($payload)) { + throw new Exception('Missing payload'); + } + + $type = $payload['type'] ?? ''; + $project = new Document($payload['project'] ?? []); + $resource = new Document($payload['resource'] ?? []); + $deployment = new Document($payload['deployment'] ?? []); + + switch ($type) { + case BUILD_TYPE_DEPLOYMENT: + case BUILD_TYPE_RETRY: + Console::info('Creating build for deployment: ' . $deployment->getId()); + buildDeployment($project, $resource, $deployment); + break; + + default: + throw new \Exception('Invalid build type'); + break; + } + }); + +$server->workerStart(); +$server->start(); diff --git a/app/workers/databases.php b/app/workers/databases.php index a24111ed7f..b0087e2913 100644 --- a/app/workers/databases.php +++ b/app/workers/databases.php @@ -25,56 +25,6 @@ Authorization::setDefaultStatus(false); const DATABASE_PROJECT = 'project'; const DATABASE_CONSOLE = 'console'; -function getCache(): Cache -{ - global $register; - - $pools = $register->get('pools'); - /** @var \Utopia\Pools\Group $pools */ - - $list = Config::getParam('pools-cache', []); - $adapters = []; - - foreach ($list as $value) { - $adapters[] = $pools - ->get($value) - ->pop() - ->getResource(); - } - - return new Cache(new Sharding($adapters)); -} - -/** - * Get Project DB - * - * @param Document $project - * @returns Database - */ -function getProjectDB(Document $project): Database -{ - global $register; - - /** @var \Utopia\Pools\Group $pools */ - $pools = $register->get('pools'); - - if ($project->isEmpty() || $project->getId() === 'console') { - return getConsoleDB(); - } - - $dbAdapter = $pools - ->get($project->getAttribute('database')) - ->pop() - ->getResource() - ; - - $database = new Database($dbAdapter, getCache()); - $database->setNamespace('_' . $project->getInternalId()); - - return $database; -} - - /** * @param Document $database * @param Document $collection diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 458b341a45..e546c314b9 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -1,5 +1,7 @@ job() + ->inject('message') + ->action(function (Message $message) { + $payload = $message->getPayload() ?? []; -class DeletesV1 extends Worker -{ - public function getName(): string - { - return "deletes"; - } + if (empty($payload)) { + throw new Exception('Missing payload'); + } - public function init(): void - { - } - - public function run(): void - { - $project = new Document($this->args['project'] ?? []); - $type = $this->args['type'] ?? ''; + $project = new Document($payload['project'] ?? []); + $type = $payload['type'] ?? ''; switch (strval($type)) { case DELETE_TYPE_DOCUMENT: - $document = new Document($this->args['document'] ?? []); + $document = new Document($payload['document'] ?? []); switch ($document->getCollection()) { case DELETE_TYPE_DATABASES: - $this->deleteDatabase($document, $project); + deleteDatabase($document, $project); break; case DELETE_TYPE_COLLECTIONS: - $this->deleteCollection($document, $project); + deleteCollection($document, $project); break; case DELETE_TYPE_PROJECTS: - $this->deleteProject($document); + deleteProject($document); break; case DELETE_TYPE_FUNCTIONS: - $this->deleteFunction($document, $project); + deleteFunction($document, $project); break; case DELETE_TYPE_DEPLOYMENTS: - $this->deleteDeployment($document, $project); + deleteDeployment($document, $project); break; case DELETE_TYPE_USERS: - $this->deleteUser($document, $project); + deleteUser($document, $project); break; case DELETE_TYPE_TEAMS: - $this->deleteMemberships($document, $project); + deleteMemberships($document, $project); break; case DELETE_TYPE_BUCKETS: - $this->deleteBucket($document, $project); + deleteBucket($document, $project); break; default: Console::error('No lazy delete operation available for document of type: ' . $document->getCollection()); @@ -72,479 +70,476 @@ class DeletesV1 extends Worker break; case DELETE_TYPE_EXECUTIONS: - $this->deleteExecutionLogs($this->args['datetime']); + deleteExecutionLogs($payload['datetime']); break; case DELETE_TYPE_AUDIT: - $datetime = $this->args['datetime'] ?? null; + $datetime = $payload['datetime'] ?? null; if (!empty($datetime)) { - $this->deleteAuditLogs($datetime); + deleteAuditLogs($datetime); } - $document = new Document($this->args['document'] ?? []); + $document = new Document($payload['document'] ?? []); if (!$document->isEmpty()) { - $this->deleteAuditLogsByResource('document/' . $document->getId(), $project); + deleteAuditLogsByResource('document/' . $document->getId(), $project); } break; case DELETE_TYPE_ABUSE: - $this->deleteAbuseLogs($this->args['datetime']); + deleteAbuseLogs($payload['datetime']); break; case DELETE_TYPE_REALTIME: - $this->deleteRealtimeUsage($this->args['datetime']); + deleteRealtimeUsage($payload['datetime']); break; case DELETE_TYPE_SESSIONS: - $this->deleteExpiredSessions(); + deleteExpiredSessions(); break; case DELETE_TYPE_CERTIFICATES: - $document = new Document($this->args['document']); - $this->deleteCertificates($document); + $document = new Document($payload['document']); + deleteCertificates($document); break; case DELETE_TYPE_USAGE: - $this->deleteUsageStats($this->args['hourlyUsageRetentionDatetime']); + deleteUsageStats($payload['hourlyUsageRetentionDatetime']); break; case DELETE_TYPE_CACHE_BY_RESOURCE: - $this->deleteCacheByResource($this->args['resource']); + deleteCacheByResource($payload['resource']); break; case DELETE_TYPE_CACHE_BY_TIMESTAMP: - $this->deleteCacheByDate(); + deleteCacheByDate($payload); break; case DELETE_TYPE_SCHEDULES: - $this->deleteSchedules($this->args['datetime']); + deleteSchedules($payload['datetime']); break; default: Console::error('No delete operation for type: ' . $type); break; } - } + }); - public function shutdown(): void - { - } /** * @throws Exception */ - protected function deleteSchedules(string $datetime): void - { - $this->listByGroup( - 'schedules', - [ - Query::equal('region', [App::getEnv('_APP_REGION', 'default')]), - Query::equal('resourceType', ['function']), - Query::lessThanEqual('resourceUpdatedAt', $datetime), - Query::equal('active', [false]), - ], - $this->getConsoleDB(), - function (Document $document) { - $project = $this->getConsoleDB()->getDocument('projects', $document->getAttribute('projectId')); +function deleteSchedules(string $datetime): void +{ + listByGroup( + 'schedules', + [ + Query::equal('region', [App::getEnv('_APP_REGION', 'default')]), + Query::equal('resourceType', ['function']), + Query::lessThanEqual('resourceUpdatedAt', $datetime), + Query::equal('active', [false]), + ], + getConsoleDB(), + function (Document $document) { + $project = getConsoleDB()->getDocument('projects', $document->getAttribute('projectId')); - if ($project->isEmpty()) { - Console::warning('Unable to delete schedule for function ' . $document->getAttribute('resourceId')); - return; - } - - $function = $this->getProjectDB($project)->getDocument('functions', $document->getAttribute('resourceId')); - - if ($function->isEmpty()) { - $this->getConsoleDB()->deleteDocument('schedules', $document->getId()); - Console::success('Deleting schedule for function ' . $document->getAttribute('resourceId')); - } + if ($project->isEmpty()) { + Console::warning('Unable to delete schedule for function ' . $document->getAttribute('resourceId')); + return; } - ); - } + + $function = getProjectDB($project)->getDocument('functions', $document->getAttribute('resourceId')); + + if ($function->isEmpty()) { + getConsoleDB()->deleteDocument('schedules', $document->getId()); + Console::success('Deleting schedule for function ' . $document->getAttribute('resourceId')); + } + } + ); +} /** * @param string $resource */ - protected function deleteCacheByResource(string $resource): void - { - $this->deleteCacheFiles([ - Query::equal('resource', [$resource]), - ]); - } +function deleteCacheByResource(string $resource): void +{ + deleteCacheFiles([ + Query::equal('resource', [$resource]), + ]); +} - protected function deleteCacheByDate(): void - { - $this->deleteCacheFiles([ - Query::lessThan('accessedAt', $this->args['datetime']), - ]); - } +function deleteCacheByDate($payload): void +{ + deleteCacheFiles([ + Query::lessThan('accessedAt', $payload['datetime']), + ]); +} - protected function deleteCacheFiles($query): void - { - $this->deleteForProjectIds(function (Document $project) use ($query) { +function deleteCacheFiles($query): void +{ + deleteForProjectIds(function (Document $project) use ($query) { - $projectId = $project->getId(); - $dbForProject = $this->getProjectDB($project); - $cache = new Cache( - new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId) - ); + $projectId = $project->getId(); + $dbForProject = getProjectDB($project); + $cache = new Cache( + new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId) + ); - $this->deleteByGroup( - 'cache', - $query, - $dbForProject, - function (Document $document) use ($cache, $projectId) { - $path = APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId . DIRECTORY_SEPARATOR . $document->getId(); + deleteByGroup( + 'cache', + $query, + $dbForProject, + function (Document $document) use ($cache, $projectId) { + $path = APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId . DIRECTORY_SEPARATOR . $document->getId(); - if ($cache->purge($document->getId())) { - Console::success('Deleting cache file: ' . $path); - } else { - Console::error('Failed to delete cache file: ' . $path); - } + if ($cache->purge($document->getId())) { + Console::success('Deleting cache file: ' . $path); + } else { + Console::error('Failed to delete cache file: ' . $path); } - ); - }); - } + } + ); + }); +} /** * @param Document $document database document * @param Document $projectId */ - protected function deleteDatabase(Document $document, Document $project): void - { - $databaseId = $document->getId(); - $projectId = $project->getId(); +function deleteDatabase(Document $document, Document $project): void +{ + $databaseId = $document->getId(); + $projectId = $project->getId(); - $dbForProject = $this->getProjectDB($project); + $dbForProject = getProjectDB($project); - $this->deleteByGroup('database_' . $document->getInternalId(), [], $dbForProject, function ($document) use ($project) { - $this->deleteCollection($document, $project); - }); + deleteByGroup('database_' . $document->getInternalId(), [], $dbForProject, function ($document) use ($project) { + deleteCollection($document, $project); + }); - $dbForProject->deleteCollection('database_' . $document->getInternalId()); + $dbForProject->deleteCollection('database_' . $document->getInternalId()); - $this->deleteAuditLogsByResource('database/' . $databaseId, $project); - } + deleteAuditLogsByResource('database/' . $databaseId, $project); +} /** * @param Document $document teams document * @param Document $project */ - protected function deleteCollection(Document $document, Document $project): void - { - $collectionId = $document->getId(); - $databaseId = $document->getAttribute('databaseId'); - $databaseInternalId = $document->getAttribute('databaseInternalId'); +function deleteCollection(Document $document, Document $project): void +{ + $collectionId = $document->getId(); + $databaseId = $document->getAttribute('databaseId'); + $databaseInternalId = $document->getAttribute('databaseInternalId'); - $dbForProject = $this->getProjectDB($project); + $dbForProject = getProjectDB($project); - $dbForProject->deleteCollection('database_' . $databaseInternalId . '_collection_' . $document->getInternalId()); + $dbForProject->deleteCollection('database_' . $databaseInternalId . '_collection_' . $document->getInternalId()); - $this->deleteByGroup('attributes', [ - Query::equal('databaseId', [$databaseId]), - Query::equal('collectionId', [$collectionId]) - ], $dbForProject); + deleteByGroup('attributes', [ + Query::equal('databaseId', [$databaseId]), + Query::equal('collectionId', [$collectionId]) + ], $dbForProject); - $this->deleteByGroup('indexes', [ - Query::equal('databaseId', [$databaseId]), - Query::equal('collectionId', [$collectionId]) - ], $dbForProject); + deleteByGroup('indexes', [ + Query::equal('databaseId', [$databaseId]), + Query::equal('collectionId', [$collectionId]) + ], $dbForProject); - $this->deleteAuditLogsByResource('database/' . $databaseId . '/collection/' . $collectionId, $project); - } + deleteAuditLogsByResource('database/' . $databaseId . '/collection/' . $collectionId, $project); +} /** * @param string $hourlyUsageRetentionDatetime */ - protected function deleteUsageStats(string $hourlyUsageRetentionDatetime) - { - $this->deleteForProjectIds(function (Document $project) use ($hourlyUsageRetentionDatetime) { - $dbForProject = $this->getProjectDB($project); - // Delete Usage stats - $this->deleteByGroup('stats', [ - Query::lessThan('time', $hourlyUsageRetentionDatetime), - Query::equal('period', ['1h']), - ], $dbForProject); - }); - } +function deleteUsageStats(string $hourlyUsageRetentionDatetime) +{ + deleteForProjectIds(function (Document $project) use ($hourlyUsageRetentionDatetime) { + $dbForProject = getProjectDB($project); + // Delete Usage stats + deleteByGroup('stats', [ + Query::lessThan('time', $hourlyUsageRetentionDatetime), + Query::equal('period', ['1h']), + ], $dbForProject); + }); +} /** * @param Document $document teams document * @param Document $project */ - protected function deleteMemberships(Document $document, Document $project): void - { - $teamId = $document->getAttribute('teamId', ''); +function deleteMemberships(Document $document, Document $project): void +{ + $teamId = $document->getAttribute('teamId', ''); - // Delete Memberships - $this->deleteByGroup('memberships', [ - Query::equal('teamId', [$teamId]) - ], $this->getProjectDB($project)); - } + // Delete Memberships + deleteByGroup('memberships', [ + Query::equal('teamId', [$teamId]) + ], getProjectDB($project)); +} /** * @param Document $document project document */ - protected function deleteProject(Document $document): void - { - $projectId = $document->getId(); +function deleteProject(Document $document): void +{ + $projectId = $document->getId(); - // Delete all DBs - $this->getProjectDB($document)->delete($projectId); + // Delete all DBs + getProjectDB($document)->delete($projectId); - // Delete all storage directories - $uploads = $this->getFilesDevice($document->getId()); - $cache = new Local(APP_STORAGE_CACHE . '/app-' . $document->getId()); + // Delete all storage directories + $uploads = getFilesDevice($document->getId()); + $cache = new Local(APP_STORAGE_CACHE . '/app-' . $document->getId()); - $uploads->delete($uploads->getRoot(), true); - $cache->delete($cache->getRoot(), true); - } + $uploads->delete($uploads->getRoot(), true); + $cache->delete($cache->getRoot(), true); +} /** * @param Document $document user document * @param Document $project */ - protected function deleteUser(Document $document, Document $project): void - { - $userId = $document->getId(); +function deleteUser(Document $document, Document $project): void +{ + $userId = $document->getId(); - $dbForProject = $this->getProjectDB($project); + $dbForProject = getProjectDB($project); - // Delete all sessions of this user from the sessions table and update the sessions field of the user record - $this->deleteByGroup('sessions', [ - Query::equal('userId', [$userId]) - ], $dbForProject); + // Delete all sessions of this user from the sessions table and update the sessions field of the user record + deleteByGroup('sessions', [ + Query::equal('userId', [$userId]) + ], $dbForProject); - $dbForProject->deleteCachedDocument('users', $userId); + $dbForProject->deleteCachedDocument('users', $userId); - // Delete Memberships and decrement team membership counts - $this->deleteByGroup('memberships', [ - Query::equal('userId', [$userId]) - ], $dbForProject, function (Document $document) use ($dbForProject) { - if ($document->getAttribute('confirm')) { // Count only confirmed members - $teamId = $document->getAttribute('teamId'); - $team = $dbForProject->getDocument('teams', $teamId); - if (!$team->isEmpty()) { - $team = $dbForProject->updateDocument( - 'teams', - $teamId, - // Ensure that total >= 0 - $team->setAttribute('total', \max($team->getAttribute('total', 0) - 1, 0)) - ); - } + // Delete Memberships and decrement team membership counts + deleteByGroup('memberships', [ + Query::equal('userId', [$userId]) + ], $dbForProject, function (Document $document) use ($dbForProject) { + if ($document->getAttribute('confirm')) { // Count only confirmed members + $teamId = $document->getAttribute('teamId'); + $team = $dbForProject->getDocument('teams', $teamId); + if (!$team->isEmpty()) { + $team = $dbForProject->updateDocument( + 'teams', + $teamId, + // Ensure that total >= 0 + $team->setAttribute('total', \max($team->getAttribute('total', 0) - 1, 0)) + ); } - }); + } + }); - // Delete tokens - $this->deleteByGroup('tokens', [ - Query::equal('userId', [$userId]) + // Delete tokens + deleteByGroup('tokens', [ + Query::equal('userId', [$userId]) + ], $dbForProject); +} + + /** + * @param string $datetime + */ +function deleteExecutionLogs(string $datetime): void +{ + deleteForProjectIds(function (Document $project) use ($datetime) { + $dbForProject = getProjectDB($project); + // Delete Executions + deleteByGroup('executions', [ + Query::lessThan('$createdAt', $datetime) ], $dbForProject); - } + }); +} + +function deleteExpiredSessions(): void +{ + $consoleDB = getConsoleDB(); + + deleteForProjectIds(function (Document $project) use ($consoleDB) { + $dbForProject = getProjectDB($project); + + $project = $consoleDB->getDocument('projects', $project->getId()); + $duration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG; + $expired = DateTime::addSeconds(new \DateTime(), -1 * $duration); + + // Delete Sessions + deleteByGroup('sessions', [ + Query::lessThan('$createdAt', $expired) + ], $dbForProject); + }); +} /** * @param string $datetime */ - protected function deleteExecutionLogs(string $datetime): void - { - $this->deleteForProjectIds(function (Document $project) use ($datetime) { - $dbForProject = $this->getProjectDB($project); - // Delete Executions - $this->deleteByGroup('executions', [ - Query::lessThan('$createdAt', $datetime) - ], $dbForProject); - }); - } - - protected function deleteExpiredSessions(): void - { - $consoleDB = $this->getConsoleDB(); - - $this->deleteForProjectIds(function (Document $project) use ($consoleDB) { - $dbForProject = $this->getProjectDB($project); - - $project = $consoleDB->getDocument('projects', $project->getId()); - $duration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG; - $expired = DateTime::addSeconds(new \DateTime(), -1 * $duration); - - // Delete Sessions - $this->deleteByGroup('sessions', [ - Query::lessThan('$createdAt', $expired) - ], $dbForProject); - }); - } - - /** - * @param string $datetime - */ - protected function deleteRealtimeUsage(string $datetime): void - { - $this->deleteForProjectIds(function (Document $project) use ($datetime) { - $dbForProject = $this->getProjectDB($project); - // Delete Dead Realtime Logs - $this->deleteByGroup('realtime', [ - Query::lessThan('timestamp', $datetime) - ], $dbForProject); - }); - } +function deleteRealtimeUsage(string $datetime): void +{ + deleteForProjectIds(function (Document $project) use ($datetime) { + $dbForProject = getProjectDB($project); + // Delete Dead Realtime Logs + deleteByGroup('realtime', [ + Query::lessThan('timestamp', $datetime) + ], $dbForProject); + }); +} /** * @param string $datetime * @throws Exception */ - protected function deleteAbuseLogs(string $datetime): void - { - if (empty($datetime)) { - throw new Exception('Failed to delete audit logs. No datetime provided'); - } - - $this->deleteForProjectIds(function (Document $project) use ($datetime) { - $projectId = $project->getId(); - $dbForProject = $this->getProjectDB($project); - $timeLimit = new TimeLimit("", 0, 1, $dbForProject); - $abuse = new Abuse($timeLimit); - $status = $abuse->cleanup($datetime); - if (!$status) { - throw new Exception('Failed to delete Abuse logs for project ' . $projectId); - } - }); +function deleteAbuseLogs(string $datetime): void +{ + if (empty($datetime)) { + throw new Exception('Failed to delete audit logs. No datetime provided'); } + deleteForProjectIds(function (Document $project) use ($datetime) { + $projectId = $project->getId(); + $dbForProject = getProjectDB($project); + $timeLimit = new TimeLimit("", 0, 1, $dbForProject); + $abuse = new Abuse($timeLimit); + $status = $abuse->cleanup($datetime); + if (!$status) { + throw new Exception('Failed to delete Abuse logs for project ' . $projectId); + } + }); +} + /** * @param string $datetime * @throws Exception */ - protected function deleteAuditLogs(string $datetime): void - { - if (empty($datetime)) { - throw new Exception('Failed to delete audit logs. No datetime provided'); - } - - $this->deleteForProjectIds(function (Document $project) use ($datetime) { - $projectId = $project->getId(); - $dbForProject = $this->getProjectDB($project); - $audit = new Audit($dbForProject); - $status = $audit->cleanup($datetime); - if (!$status) { - throw new Exception('Failed to delete Audit logs for project' . $projectId); - } - }); +function deleteAuditLogs(string $datetime): void +{ + if (empty($datetime)) { + throw new Exception('Failed to delete audit logs. No datetime provided'); } + deleteForProjectIds(function (Document $project) use ($datetime) { + $projectId = $project->getId(); + $dbForProject = getProjectDB($project); + $audit = new Audit($dbForProject); + $status = $audit->cleanup($datetime); + if (!$status) { + throw new Exception('Failed to delete Audit logs for project' . $projectId); + } + }); +} + /** * @param string $resource * @param Document $project */ - protected function deleteAuditLogsByResource(string $resource, Document $project): void - { - $dbForProject = $this->getProjectDB($project); +function deleteAuditLogsByResource(string $resource, Document $project): void +{ + $dbForProject = getProjectDB($project); - $this->deleteByGroup(Audit::COLLECTION, [ - Query::equal('resource', [$resource]) - ], $dbForProject); - } + deleteByGroup(Audit::COLLECTION, [ + Query::equal('resource', [$resource]) + ], $dbForProject); +} /** * @param Document $document function document * @param Document $project */ - protected function deleteFunction(Document $document, Document $project): void - { - $projectId = $project->getId(); - $dbForProject = $this->getProjectDB($project); - $functionId = $document->getId(); - - /** - * Delete Variables - */ - Console::info("Deleting variables for function " . $functionId); - $this->deleteByGroup('variables', [ - Query::equal('functionId', [$functionId]) - ], $dbForProject); - - /** - * Delete Deployments - */ - Console::info("Deleting deployments for function " . $functionId); - $storageFunctions = $this->getFunctionsDevice($projectId); - $deploymentIds = []; - $this->deleteByGroup('deployments', [ - Query::equal('resourceId', [$functionId]) - ], $dbForProject, function (Document $document) use ($storageFunctions, &$deploymentIds) { - $deploymentIds[] = $document->getId(); - if ($storageFunctions->delete($document->getAttribute('path', ''), true)) { - Console::success('Deleted deployment files: ' . $document->getAttribute('path', '')); - } else { - Console::error('Failed to delete deployment files: ' . $document->getAttribute('path', '')); - } - }); - - /** - * Delete builds - */ - Console::info("Deleting builds for function " . $functionId); - $storageBuilds = $this->getBuildsDevice($projectId); - foreach ($deploymentIds as $deploymentId) { - $this->deleteByGroup('builds', [ - Query::equal('deploymentId', [$deploymentId]) - ], $dbForProject, function (Document $document) use ($storageBuilds, $deploymentId) { - if ($storageBuilds->delete($document->getAttribute('outputPath', ''), true)) { - Console::success('Deleted build files: ' . $document->getAttribute('outputPath', '')); - } else { - Console::error('Failed to delete build files: ' . $document->getAttribute('outputPath', '')); - } - }); - } - - /** - * Delete Executions - */ - Console::info("Deleting executions for function " . $functionId); - $this->deleteByGroup('executions', [ - Query::equal('functionId', [$functionId]) - ], $dbForProject); - - // TODO: Request executor to delete runtime - } +function deleteFunction(Document $document, Document $project): void +{ + $projectId = $project->getId(); + $dbForProject = getProjectDB($project); + $functionId = $document->getId(); /** - * @param Document $document deployment document - * @param Document $project + * Delete Variables */ - protected function deleteDeployment(Document $document, Document $project): void - { - $projectId = $project->getId(); - $dbForProject = $this->getProjectDB($project); - $deploymentId = $document->getId(); - $functionId = $document->getAttribute('resourceId'); + Console::info("Deleting variables for function " . $functionId); + deleteByGroup('variables', [ + Query::equal('functionId', [$functionId]) + ], $dbForProject); - /** - * Delete deployment files - */ - Console::info("Deleting deployment files for deployment " . $deploymentId); - $storageFunctions = $this->getFunctionsDevice($projectId); + /** + * Delete Deployments + */ + Console::info("Deleting deployments for function " . $functionId); + $storageFunctions = getFunctionsDevice($projectId); + $deploymentIds = []; + deleteByGroup('deployments', [ + Query::equal('resourceId', [$functionId]) + ], $dbForProject, function (Document $document) use ($storageFunctions, &$deploymentIds) { + $deploymentIds[] = $document->getId(); if ($storageFunctions->delete($document->getAttribute('path', ''), true)) { Console::success('Deleted deployment files: ' . $document->getAttribute('path', '')); } else { Console::error('Failed to delete deployment files: ' . $document->getAttribute('path', '')); } + }); - /** - * Delete builds - */ - Console::info("Deleting builds for deployment " . $deploymentId); - $storageBuilds = $this->getBuildsDevice($projectId); - $this->deleteByGroup('builds', [ + /** + * Delete builds + */ + Console::info("Deleting builds for function " . $functionId); + $storageBuilds = getBuildsDevice($projectId); + foreach ($deploymentIds as $deploymentId) { + deleteByGroup('builds', [ Query::equal('deploymentId', [$deploymentId]) - ], $dbForProject, function (Document $document) use ($storageBuilds) { + ], $dbForProject, function (Document $document) use ($storageBuilds, $deploymentId) { if ($storageBuilds->delete($document->getAttribute('outputPath', ''), true)) { Console::success('Deleted build files: ' . $document->getAttribute('outputPath', '')); } else { Console::error('Failed to delete build files: ' . $document->getAttribute('outputPath', '')); } }); - - // TODO: Request executor to delete runtime } + /** + * Delete Executions + */ + Console::info("Deleting executions for function " . $functionId); + deleteByGroup('executions', [ + Query::equal('functionId', [$functionId]) + ], $dbForProject); + + // TODO: Request executor to delete runtime +} + + /** + * @param Document $document deployment document + * @param Document $project + */ +function deleteDeployment(Document $document, Document $project): void +{ + $projectId = $project->getId(); + $dbForProject = getProjectDB($project); + $deploymentId = $document->getId(); + $functionId = $document->getAttribute('resourceId'); + + /** + * Delete deployment files + */ + Console::info("Deleting deployment files for deployment " . $deploymentId); + $storageFunctions = getFunctionsDevice($projectId); + if ($storageFunctions->delete($document->getAttribute('path', ''), true)) { + Console::success('Deleted deployment files: ' . $document->getAttribute('path', '')); + } else { + Console::error('Failed to delete deployment files: ' . $document->getAttribute('path', '')); + } + + /** + * Delete builds + */ + Console::info("Deleting builds for deployment " . $deploymentId); + $storageBuilds = getBuildsDevice($projectId); + deleteByGroup('builds', [ + Query::equal('deploymentId', [$deploymentId]) + ], $dbForProject, function (Document $document) use ($storageBuilds) { + if ($storageBuilds->delete($document->getAttribute('outputPath', ''), true)) { + Console::success('Deleted build files: ' . $document->getAttribute('outputPath', '')); + } else { + Console::error('Failed to delete build files: ' . $document->getAttribute('outputPath', '')); + } + }); + + // TODO: Request executor to delete runtime +} + /** * @param Document $document to be deleted @@ -553,183 +548,185 @@ class DeletesV1 extends Worker * * @return bool */ - protected function deleteById(Document $document, Database $database, callable $callback = null): bool - { - if ($database->deleteDocument($document->getCollection(), $document->getId())) { - Console::success('Deleted document "' . $document->getId() . '" successfully'); +function deleteById(Document $document, Database $database, callable $callback = null): bool +{ + if ($database->deleteDocument($document->getCollection(), $document->getId())) { + Console::success('Deleted document "' . $document->getId() . '" successfully'); + if (is_callable($callback)) { + $callback($document); + } + + return true; + } else { + Console::error('Failed to delete document: ' . $document->getId()); + return false; + } +} + + /** + * @param callable $callback + */ +function deleteForProjectIds(callable $callback): void +{ + // TODO: @Meldiron name of this method no longer matches. It does not delete, and it gives whole document + $count = 0; + $chunk = 0; + $limit = 50; + $projects = []; + $sum = $limit; + + $executionStart = \microtime(true); + + while ($sum === $limit) { + $projects = getConsoleDB()->find('projects', [Query::limit($limit), Query::offset($chunk * $limit)]); + + $chunk++; + + /** @var string[] $projectIds */ + $sum = count($projects); + + Console::info('Executing delete function for chunk #' . $chunk . '. Found ' . $sum . ' projects'); + foreach ($projects as $project) { + $callback($project); + $count++; + } + } + + $executionEnd = \microtime(true); + Console::info("Found {$count} projects " . ($executionEnd - $executionStart) . " seconds"); +} + + /** + * @param string $collection collectionID + * @param Query[] $queries + * @param Database $database + * @param callable $callback + */ +function deleteByGroup(string $collection, array $queries, Database $database, callable $callback = null): void +{ + $count = 0; + $chunk = 0; + $limit = 50; + $results = []; + $sum = $limit; + + $executionStart = \microtime(true); + + while ($sum === $limit) { + $chunk++; + + $results = $database->find($collection, \array_merge([Query::limit($limit)], $queries)); + + $sum = count($results); + + Console::info('Deleting chunk #' . $chunk . '. Found ' . $sum . ' documents'); + + foreach ($results as $document) { + deleteById($document, $database, $callback); + $count++; + } + } + + $executionEnd = \microtime(true); + + Console::info("Deleted {$count} document by group in " . ($executionEnd - $executionStart) . " seconds"); +} + + /** + * @param string $collection collectionID + * @param Query[] $queries + * @param Database $database + * @param callable $callback + */ +function listByGroup(string $collection, array $queries, Database $database, callable $callback = null): void +{ + $count = 0; + $chunk = 0; + $limit = 50; + $results = []; + $sum = $limit; + + $executionStart = \microtime(true); + + while ($sum === $limit) { + $chunk++; + + $results = $database->find($collection, \array_merge([Query::limit($limit)], $queries)); + + $sum = count($results); + + foreach ($results as $document) { if (is_callable($callback)) { $callback($document); } - return true; - } else { - Console::error('Failed to delete document: ' . $document->getId()); - return false; + $count++; } } - /** - * @param callable $callback - */ - protected function deleteForProjectIds(callable $callback): void - { - // TODO: @Meldiron name of this method no longer matches. It does not delete, and it gives whole document - $count = 0; - $chunk = 0; - $limit = 50; - $projects = []; - $sum = $limit; + $executionEnd = \microtime(true); - $executionStart = \microtime(true); - - while ($sum === $limit) { - $projects = $this->getConsoleDB()->find('projects', [Query::limit($limit), Query::offset($chunk * $limit)]); - - $chunk++; - - /** @var string[] $projectIds */ - $sum = count($projects); - - Console::info('Executing delete function for chunk #' . $chunk . '. Found ' . $sum . ' projects'); - foreach ($projects as $project) { - $callback($project); - $count++; - } - } - - $executionEnd = \microtime(true); - Console::info("Found {$count} projects " . ($executionEnd - $executionStart) . " seconds"); - } - - /** - * @param string $collection collectionID - * @param Query[] $queries - * @param Database $database - * @param callable $callback - */ - protected function deleteByGroup(string $collection, array $queries, Database $database, callable $callback = null): void - { - $count = 0; - $chunk = 0; - $limit = 50; - $results = []; - $sum = $limit; - - $executionStart = \microtime(true); - - while ($sum === $limit) { - $chunk++; - - $results = $database->find($collection, \array_merge([Query::limit($limit)], $queries)); - - $sum = count($results); - - Console::info('Deleting chunk #' . $chunk . '. Found ' . $sum . ' documents'); - - foreach ($results as $document) { - $this->deleteById($document, $database, $callback); - $count++; - } - } - - $executionEnd = \microtime(true); - - Console::info("Deleted {$count} document by group in " . ($executionEnd - $executionStart) . " seconds"); - } - - /** - * @param string $collection collectionID - * @param Query[] $queries - * @param Database $database - * @param callable $callback - */ - protected function listByGroup(string $collection, array $queries, Database $database, callable $callback = null): void - { - $count = 0; - $chunk = 0; - $limit = 50; - $results = []; - $sum = $limit; - - $executionStart = \microtime(true); - - while ($sum === $limit) { - $chunk++; - - $results = $database->find($collection, \array_merge([Query::limit($limit)], $queries)); - - $sum = count($results); - - foreach ($results as $document) { - if (is_callable($callback)) { - $callback($document); - } - - $count++; - } - } - - $executionEnd = \microtime(true); - - Console::info("Listed {$count} document by group in " . ($executionEnd - $executionStart) . " seconds"); - } + Console::info("Listed {$count} document by group in " . ($executionEnd - $executionStart) . " seconds"); +} /** * @param Document $document certificates document */ - protected function deleteCertificates(Document $document): void - { - $consoleDB = $this->getConsoleDB(); +function deleteCertificates(Document $document): void +{ + $consoleDB = getConsoleDB(); - // If domain has certificate generated - if (isset($document['certificateId'])) { - $domainUsingCertificate = $consoleDB->findOne('domains', [ - Query::equal('certificateId', [$document['certificateId']]) - ]); + // If domain has certificate generated + if (isset($document['certificateId'])) { + $domainUsingCertificate = $consoleDB->findOne('domains', [ + Query::equal('certificateId', [$document['certificateId']]) + ]); - if (!$domainUsingCertificate) { - $mainDomain = App::getEnv('_APP_DOMAIN_TARGET', ''); - if ($mainDomain === $document->getAttribute('domain')) { - $domainUsingCertificate = $mainDomain; - } - } - - // If certificate is still used by some domain, mark we can't delete. - // Current domain should not be found, because we only have copy. Original domain is already deleted from database. - if ($domainUsingCertificate) { - Console::warning("Skipping certificate deletion, because a domain is still using it."); - return; + if (!$domainUsingCertificate) { + $mainDomain = App::getEnv('_APP_DOMAIN_TARGET', ''); + if ($mainDomain === $document->getAttribute('domain')) { + $domainUsingCertificate = $mainDomain; } } - $domain = $document->getAttribute('domain'); - $directory = APP_STORAGE_CERTIFICATES . '/' . $domain; - $checkTraversal = realpath($directory) === $directory; - - if ($domain && $checkTraversal && is_dir($directory)) { - // Delete certificate document, so Appwrite is aware of change - if (isset($document['certificateId'])) { - $consoleDB->deleteDocument('certificates', $document['certificateId']); - } - - // Delete files, so Traefik is aware of change - array_map('unlink', glob($directory . '/*.*')); - rmdir($directory); - Console::info("Deleted certificate files for {$domain}"); - } else { - Console::info("No certificate files found for {$domain}"); + // If certificate is still used by some domain, mark we can't delete. + // Current domain should not be found, because we only have copy. Original domain is already deleted from database. + if ($domainUsingCertificate) { + Console::warning("Skipping certificate deletion, because a domain is still using it."); + return; } } - protected function deleteBucket(Document $document, Document $project) - { - $projectId = $project->getId(); - $dbForProject = $this->getProjectDB($project); - $dbForProject->deleteCollection('bucket_' . $document->getInternalId()); + $domain = $document->getAttribute('domain'); + $directory = APP_STORAGE_CERTIFICATES . '/' . $domain; + $checkTraversal = realpath($directory) === $directory; - $device = $this->getFilesDevice($projectId); + if ($domain && $checkTraversal && is_dir($directory)) { + // Delete certificate document, so Appwrite is aware of change + if (isset($document['certificateId'])) { + $consoleDB->deleteDocument('certificates', $document['certificateId']); + } - $device->deletePath($document->getId()); + // Delete files, so Traefik is aware of change + array_map('unlink', glob($directory . '/*.*')); + rmdir($directory); + Console::info("Deleted certificate files for {$domain}"); + } else { + Console::info("No certificate files found for {$domain}"); } } + +function deleteBucket(Document $document, Document $project) +{ + $projectId = $project->getId(); + $dbForProject = getProjectDB($project); + $dbForProject->deleteCollection('bucket_' . $document->getInternalId()); + + $device = getFilesDevice($projectId); + + $device->deletePath($document->getId()); +} + +$server->workerStart(); +$server->start(); diff --git a/src/Appwrite/Event/Audit.php b/src/Appwrite/Event/Audit.php index 9be6e27b33..08fd3cad3f 100644 --- a/src/Appwrite/Event/Audit.php +++ b/src/Appwrite/Event/Audit.php @@ -2,7 +2,6 @@ namespace Appwrite\Event; -use Resque; use Utopia\Queue\Client; use Utopia\Queue\Connection; @@ -15,7 +14,7 @@ class Audit extends Event public function __construct(protected Connection $connection) { - parent::__construct(Event::AUDITS_QUEUE_NAME, Event::AUDITS_CLASS_NAME); + parent::__construct(Event::AUDITS_QUEUE_NAME, Event::AUDITS_CLASS_NAME, $connection); } /** diff --git a/src/Appwrite/Event/Build.php b/src/Appwrite/Event/Build.php index 4d4b338118..1009dcedf4 100644 --- a/src/Appwrite/Event/Build.php +++ b/src/Appwrite/Event/Build.php @@ -2,8 +2,9 @@ namespace Appwrite\Event; -use Resque; use Utopia\Database\Document; +use Utopia\Queue\Client; +use Utopia\Queue\Connection; class Build extends Event { @@ -11,9 +12,9 @@ class Build extends Event protected ?Document $resource = null; protected ?Document $deployment = null; - public function __construct() + public function __construct(protected Connection $connection) { - parent::__construct(Event::BUILDS_QUEUE_NAME, Event::BUILDS_CLASS_NAME); + parent::__construct(Event::BUILDS_QUEUE_NAME, Event::BUILDS_CLASS_NAME, $connection); } /** @@ -93,7 +94,11 @@ class Build extends Event */ public function trigger(): string|bool { - return Resque::enqueue($this->queue, $this->class, [ + $client = new Client($this->queue, $this->connection); + + $events = $this->getEvent() ? Event::generateEvents($this->getEvent(), $this->getParams()) : null; + + return $client->enqueue([ 'project' => $this->project, 'resource' => $this->resource, 'deployment' => $this->deployment, diff --git a/src/Appwrite/Event/Certificate.php b/src/Appwrite/Event/Certificate.php index 0f0cbaa73d..1a755b7332 100644 --- a/src/Appwrite/Event/Certificate.php +++ b/src/Appwrite/Event/Certificate.php @@ -2,7 +2,6 @@ namespace Appwrite\Event; -use Resque; use Utopia\Database\Document; use Utopia\Queue\Client; use Utopia\Queue\Connection; @@ -14,7 +13,7 @@ class Certificate extends Event public function __construct(protected Connection $connection) { - parent::__construct(Event::CERTIFICATES_QUEUE_NAME, Event::CERTIFICATES_CLASS_NAME); + parent::__construct(Event::CERTIFICATES_QUEUE_NAME, Event::CERTIFICATES_CLASS_NAME, $connection); } /** diff --git a/src/Appwrite/Event/Database.php b/src/Appwrite/Event/Database.php index 2c6aaea926..7d95b347cd 100644 --- a/src/Appwrite/Event/Database.php +++ b/src/Appwrite/Event/Database.php @@ -2,7 +2,6 @@ namespace Appwrite\Event; -use Resque; use Utopia\Database\Document; use Utopia\Queue\Client; use Utopia\Queue\Connection; @@ -16,7 +15,7 @@ class Database extends Event public function __construct(protected Connection $connection) { - parent::__construct(Event::DATABASE_QUEUE_NAME, Event::DATABASE_CLASS_NAME); + parent::__construct(Event::DATABASE_QUEUE_NAME, Event::DATABASE_CLASS_NAME, $connection); } /** diff --git a/src/Appwrite/Event/Delete.php b/src/Appwrite/Event/Delete.php index ad69172062..32c84ee90c 100644 --- a/src/Appwrite/Event/Delete.php +++ b/src/Appwrite/Event/Delete.php @@ -2,8 +2,8 @@ namespace Appwrite\Event; -use Resque; use Utopia\Database\Document; +use Utopia\Queue\Client; use Utopia\Queue\Connection; class Delete extends Event @@ -122,7 +122,11 @@ class Delete extends Event */ public function trigger(): string|bool { - return Resque::enqueue($this->queue, $this->class, [ + $client = new Client($this->queue, $this->connection); + + $events = $this->getEvent() ? Event::generateEvents($this->getEvent(), $this->getParams()) : null; + + return $client->enqueue([ 'project' => $this->project, 'type' => $this->type, 'document' => $this->document, diff --git a/src/Appwrite/Event/Mail.php b/src/Appwrite/Event/Mail.php index cffee488f7..7c1a7b53b4 100644 --- a/src/Appwrite/Event/Mail.php +++ b/src/Appwrite/Event/Mail.php @@ -2,7 +2,6 @@ namespace Appwrite\Event; -use Resque; use Utopia\Database\Document; use Utopia\Queue\Client; use Utopia\Queue\Connection; @@ -18,7 +17,7 @@ class Mail extends Event public function __construct(protected Connection $connection) { - parent::__construct(Event::MAILS_QUEUE_NAME, Event::MAILS_CLASS_NAME); + parent::__construct(Event::MAILS_QUEUE_NAME, Event::MAILS_CLASS_NAME, $connection); } /** diff --git a/src/Appwrite/Event/Phone.php b/src/Appwrite/Event/Phone.php index 0399e27e45..c18b861e24 100644 --- a/src/Appwrite/Event/Phone.php +++ b/src/Appwrite/Event/Phone.php @@ -12,7 +12,7 @@ class Phone extends Event public function __construct(protected Connection $connection) { - parent::__construct(Event::MESSAGING_QUEUE_NAME, Event::MESSAGING_CLASS_NAME); + parent::__construct(Event::MESSAGING_QUEUE_NAME, Event::MESSAGING_CLASS_NAME, $connection); } /**