diff --git a/app/config/errors.php b/app/config/errors.php index c1174509ab..461521f5e0 100644 --- a/app/config/errors.php +++ b/app/config/errors.php @@ -349,11 +349,6 @@ return [ 'description' => 'Team with the requested ID could not be found.', 'code' => 404, ], - Exception::TEAM_INVITE_ALREADY_EXISTS => [ - 'name' => Exception::TEAM_INVITE_ALREADY_EXISTS, - 'description' => 'User has already been invited or is already a member of this team', - 'code' => 409, - ], Exception::TEAM_INVITE_NOT_FOUND => [ 'name' => Exception::TEAM_INVITE_NOT_FOUND, 'description' => 'The requested team invitation could not be found.', diff --git a/app/config/variables.php b/app/config/variables.php index 113fbae335..57810525c7 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -268,6 +268,24 @@ return [ 'question' => '', 'filter' => '' ], + [ + 'name' => '_APP_COMPRESSION_ENABLED', + 'description' => 'This option allows you to enable or disable the response compression for the Appwrite API. It\'s enabled by default with value "enabled", and to disable it, pass value "disabled".', + 'introduction' => '1.6.0', + 'default' => 'enabled', + 'required' => false, + 'question' => '', + 'filter' => '' + ], + [ + 'name' => '_APP_COMPRESSION_MIN_SIZE_BYTES', + 'description' => 'This option allows you to set the minimum size in bytes for the response compression to be applied. The default value is 1024 bytes.', + 'introduction' => '1.6.0', + 'default' => 1024, + 'required' => false, + 'question' => '', + 'filter' => '' + ] ], ], [ diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index d4d5fc64cf..efd6967408 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -400,6 +400,7 @@ App::post('/v1/functions') $allEvents = Event::generateEvents('rules.[ruleId].create', [ 'ruleId' => $rule->getId(), ]); + $target = Realtime::fromPayload( // Pass first, most verbose event pattern event: $allEvents[0], diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index 461e90da58..d98cfe6f0b 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -546,47 +546,58 @@ App::post('/v1/teams/:teamId/memberships') throw new Exception(Exception::USER_UNAUTHORIZED, 'User is not allowed to send invitations for this team'); } - $secret = Auth::tokenGenerator(); - - $membershipId = ID::unique(); - $membership = new Document([ - '$id' => $membershipId, - '$permissions' => [ - Permission::read(Role::any()), - Permission::update(Role::user($invitee->getId())), - Permission::update(Role::team($team->getId(), 'owner')), - Permission::delete(Role::user($invitee->getId())), - Permission::delete(Role::team($team->getId(), 'owner')), - ], - 'userId' => $invitee->getId(), - 'userInternalId' => $invitee->getInternalId(), - 'teamId' => $team->getId(), - 'teamInternalId' => $team->getInternalId(), - 'roles' => $roles, - 'invited' => DateTime::now(), - 'joined' => ($isPrivilegedUser || $isAppUser) ? DateTime::now() : null, - 'confirm' => ($isPrivilegedUser || $isAppUser), - 'secret' => Auth::hash($secret), - 'search' => implode(' ', [$membershipId, $invitee->getId()]) + $membership = $dbForProject->findOne('memberships', [ + Query::equal('userInternalId', [$invitee->getInternalId()]), + Query::equal('teamInternalId', [$team->getInternalId()]), ]); - if ($isPrivilegedUser || $isAppUser) { // Allow admin to create membership - try { - $membership = Authorization::skip(fn () => $dbForProject->createDocument('memberships', $membership)); - } catch (Duplicate $th) { - throw new Exception(Exception::TEAM_INVITE_ALREADY_EXISTS); - } + if ($membership->isEmpty()) { + $secret = Auth::tokenGenerator(); + $membershipId = ID::unique(); + $membership = new Document([ + '$id' => $membershipId, + '$permissions' => [ + Permission::read(Role::any()), + Permission::update(Role::user($invitee->getId())), + Permission::update(Role::team($team->getId(), 'owner')), + Permission::delete(Role::user($invitee->getId())), + Permission::delete(Role::team($team->getId(), 'owner')), + ], + 'userId' => $invitee->getId(), + 'userInternalId' => $invitee->getInternalId(), + 'teamId' => $team->getId(), + 'teamInternalId' => $team->getInternalId(), + 'roles' => $roles, + 'invited' => DateTime::now(), + 'joined' => ($isPrivilegedUser || $isAppUser) ? DateTime::now() : null, + 'confirm' => ($isPrivilegedUser || $isAppUser), + 'secret' => Auth::hash($secret), + 'search' => implode(' ', [$membershipId, $invitee->getId()]) + ]); + + $membership = ($isPrivilegedUser || $isAppUser) ? + Authorization::skip(fn () => $dbForProject->createDocument('memberships', $membership)) : + $dbForProject->createDocument('memberships', $membership); Authorization::skip(fn () => $dbForProject->increaseDocumentAttribute('teams', $team->getId(), 'total', 1)); - $dbForProject->purgeCachedDocument('users', $invitee->getId()); } else { - try { - $membership = $dbForProject->createDocument('memberships', $membership); - } catch (Duplicate $th) { - throw new Exception(Exception::TEAM_INVITE_ALREADY_EXISTS); + $membership->setAttribute('invited', DateTime::now()); + + if ($isPrivilegedUser || $isAppUser) { + $membership->setAttribute('joined', DateTime::now()); + $membership->setAttribute('confirm', true); } + $membership = ($isPrivilegedUser || $isAppUser) ? + Authorization::skip(fn () => $dbForProject->updateDocument('memberships', $membership->getId(), $membership)) : + $dbForProject->updateDocument('memberships', $membership->getId(), $membership); + } + + + if ($isPrivilegedUser || $isAppUser) { + $dbForProject->purgeCachedDocument('users', $invitee->getId()); + } else { $url = Template::parseURL($url); $url['query'] = Template::mergeQuery(((isset($url['query'])) ? $url['query'] : ''), ['membershipId' => $membership->getId(), 'userId' => $invitee->getId(), 'secret' => $secret, 'teamId' => $teamId]); $url = Template::unParseURL($url); @@ -656,7 +667,7 @@ App::post('/v1/teams/:teamId/memberships') 'owner' => $user->getAttribute('name'), 'direction' => $locale->getText('settings.direction'), /* {{user}}, {{team}}, {{redirect}} and {{project}} are required in default and custom templates */ - 'user' => $user->getAttribute('name'), + 'user' => $name, 'team' => $team->getAttribute('name'), 'redirect' => $url, 'project' => $projectName @@ -668,8 +679,8 @@ App::post('/v1/teams/:teamId/memberships') ->setRecipient($invitee->getAttribute('email')) ->setName($invitee->getAttribute('name')) ->setVariables($emailVariables) - ->trigger() - ; + ->trigger(); + } elseif (!empty($phone)) { if (empty(System::getEnv('_APP_SMS_PROVIDER'))) { throw new Exception(Exception::GENERAL_PHONE_DISABLED, 'Phone provider not configured'); diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 182151a6c3..43f9cef5a3 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -58,7 +58,7 @@ $parseLabel = function (string $label, array $responsePayload, array $requestPar return $label; }; -$eventDatabaseListener = function (Document $document, Response $response, Event $queueForEvents, Func $queueForFunctions, Webhook $queueForWebhooks, Realtime $queueForRealtime) { +$eventDatabaseListener = function (Document $project, Document $document, Response $response, Event $queueForEvents, Func $queueForFunctions, Webhook $queueForWebhooks, Realtime $queueForRealtime) { // Only trigger events for user creation with the database listener. if ($document->getCollection() !== 'users') { return; @@ -74,17 +74,20 @@ $eventDatabaseListener = function (Document $document, Response $response, Event ->from($queueForEvents) ->trigger(); - $queueForWebhooks - ->from($queueForEvents) - ->trigger(); - if ($queueForEvents->getProject()->getId() === 'console') { - return; + /** Trigger webhooks events only if a project has them enabled */ + if (!empty($project->getAttribute('webhooks'))) { + $queueForWebhooks + ->from($queueForEvents) + ->trigger(); } - $queueForRealtime - ->from($queueForEvents) - ->trigger(); + /** Trigger realtime events only for non console events */ + if ($queueForEvents->getProject()->getId() !== 'console') { + $queueForRealtime + ->from($queueForEvents) + ->trigger(); + } }; $usageDatabaseListener = function (string $event, Document $document, Usage $queueForUsage) { @@ -527,6 +530,7 @@ App::init() ->on(Database::EVENT_DOCUMENT_CREATE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $queueForUsage)) ->on(Database::EVENT_DOCUMENT_DELETE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $queueForUsage)) ->on(Database::EVENT_DOCUMENT_CREATE, 'create-trigger-events', fn ($event, $document) => $eventDatabaseListener( + $project, $document, $response, $queueForEventsClone->from($queueForEvents), @@ -679,10 +683,6 @@ App::shutdown() $queueForEvents->setPayload($responsePayload); } - $queueForWebhooks - ->from($queueForEvents) - ->trigger(); - $queueForFunctions ->from($queueForEvents) ->trigger(); @@ -692,6 +692,17 @@ App::shutdown() ->from($queueForEvents) ->trigger(); } + + /** Trigger webhooks events only if a project has them enabled + * A future optimisation is to only trigger webhooks if the webhook is "enabled" + * But it might have performance implications on the API due to the number of webhooks etc. + * Some profiling is needed to see if this is a problem. + */ + if (!empty($project->getAttribute('webhooks'))) { + $queueForWebhooks + ->from($queueForEvents) + ->trigger(); + } } $route = $utopia->getRoute(); diff --git a/app/http.php b/app/http.php index 3a7562ffd1..dac941d25b 100644 --- a/app/http.php +++ b/app/http.php @@ -334,7 +334,7 @@ $http->on(Constant::EVENT_REQUEST, function (SwooleRequest $swooleRequest, Swool } $app = new App('UTC'); - $app->setCompression(true); + $app->setCompression(System::getEnv('_APP_COMPRESSION_ENABLED', 'enabled') === 'enabled'); $app->setCompressionMinSize(intval(System::getEnv('_APP_COMPRESSION_MIN_SIZE_BYTES', '1024'))); // 1KB $pools = $register->get('pools'); diff --git a/app/init.php b/app/init.php index ba775956bd..f812ef094c 100644 --- a/app/init.php +++ b/app/init.php @@ -202,6 +202,7 @@ const DELETE_TYPE_TOPIC = 'topic'; const DELETE_TYPE_TARGET = 'target'; const DELETE_TYPE_EXPIRED_TARGETS = 'invalid_targets'; const DELETE_TYPE_SESSION_TARGETS = 'session_targets'; +const DELETE_TYPE_MAINTENANCE = 'maintenance'; // Message types const MESSAGE_SEND_TYPE_INTERNAL = 'internal'; diff --git a/src/Appwrite/Event/Audit.php b/src/Appwrite/Event/Audit.php index 406f64b370..4b9aa9f5c5 100644 --- a/src/Appwrite/Event/Audit.php +++ b/src/Appwrite/Event/Audit.php @@ -2,7 +2,6 @@ namespace Appwrite\Event; -use Utopia\Queue\Client; use Utopia\Queue\Connection; class Audit extends Event @@ -139,20 +138,13 @@ class Audit extends Event } /** - * Executes the event and sends it to the audit worker. + * Prepare payload for queue. * - * @return string|bool - * @throws \InvalidArgumentException + * @return array */ - public function trigger(): string|bool + protected function preparePayload(): array { - if ($this->paused) { - return false; - } - - $client = new Client($this->queue, $this->connection); - - return $client->enqueue([ + return [ 'project' => $this->project, 'user' => $this->user, 'payload' => $this->payload, @@ -162,6 +154,6 @@ class Audit extends Event 'userAgent' => $this->userAgent, 'event' => $this->event, 'hostname' => $this->hostname - ]); + ]; } } diff --git a/src/Appwrite/Event/Build.php b/src/Appwrite/Event/Build.php index 1fbf20a9f9..831adf8e41 100644 --- a/src/Appwrite/Event/Build.php +++ b/src/Appwrite/Event/Build.php @@ -3,7 +3,6 @@ namespace Appwrite\Event; use Utopia\Database\Document; -use Utopia\Queue\Client; use Utopia\Queue\Connection; class Build extends Event @@ -105,26 +104,19 @@ class Build extends Event } /** - * Executes the function event and sends it to the functions worker. + * Prepare payload for queue. * - * @return string|bool - * @throws \InvalidArgumentException + * @return array */ - public function trigger(): string|bool + protected function preparePayload(): array { - if ($this->paused) { - return false; - } - - $client = new Client($this->queue, $this->connection); - - return $client->enqueue([ + return [ 'project' => $this->project, 'resource' => $this->resource, 'deployment' => $this->deployment, 'type' => $this->type, 'template' => $this->template - ]); + ]; } /** diff --git a/src/Appwrite/Event/Certificate.php b/src/Appwrite/Event/Certificate.php index 5d30c3d5ac..6a395417ed 100644 --- a/src/Appwrite/Event/Certificate.php +++ b/src/Appwrite/Event/Certificate.php @@ -3,7 +3,6 @@ namespace Appwrite\Event; use Utopia\Database\Document; -use Utopia\Queue\Client; use Utopia\Queue\Connection; class Certificate extends Event @@ -67,23 +66,16 @@ class Certificate extends Event } /** - * Executes the event and sends it to the certificates worker. + * Prepare the payload for the event * - * @return string|bool - * @throws \InvalidArgumentException + * @return array */ - public function trigger(): string|bool + protected function preparePayload(): array { - if ($this->paused) { - return false; - } - - $client = new Client($this->queue, $this->connection); - - return $client->enqueue([ + return [ 'project' => $this->project, 'domain' => $this->domain, 'skipRenewCheck' => $this->skipRenewCheck - ]); + ]; } } diff --git a/src/Appwrite/Event/Database.php b/src/Appwrite/Event/Database.php index 1b0ea6851c..24123de6c1 100644 --- a/src/Appwrite/Event/Database.php +++ b/src/Appwrite/Event/Database.php @@ -4,7 +4,6 @@ namespace Appwrite\Event; use Utopia\Database\Document; use Utopia\DSN\DSN; -use Utopia\Queue\Client; use Utopia\Queue\Connection; class Database extends Event @@ -100,18 +99,8 @@ class Database extends Event return $this->document; } - /** - * Executes the event and send it to the database worker. - * - * @return string|bool - * @throws \InvalidArgumentException - */ - public function trigger(): string|bool + public function getQueue(): string { - if ($this->paused) { - return false; - } - try { $dsn = new DSN($this->getProject()->getAttribute('database')); } catch (\InvalidArgumentException) { @@ -119,23 +108,25 @@ class Database extends Event $dsn = new DSN('mysql://' . $this->getProject()->getAttribute('database')); } - $this->setQueue($dsn->getHost()); + $this->queue = $dsn->getHost(); + return $this->queue; + } - $client = new Client($this->queue, $this->connection); - - try { - $result = $client->enqueue([ - 'project' => $this->project, - 'user' => $this->user, - 'type' => $this->type, - 'collection' => $this->collection, - 'document' => $this->document, - 'database' => $this->database, - 'events' => Event::generateEvents($this->getEvent(), $this->getParams()) - ]); - return $result; - } catch (\Throwable $th) { - return false; - } + /** + * Prepare the payload for the event + * + * @return array + */ + protected function preparePayload(): array + { + return [ + 'project' => $this->project, + 'user' => $this->user, + 'type' => $this->type, + 'collection' => $this->collection, + 'document' => $this->document, + 'database' => $this->database, + 'events' => Event::generateEvents($this->getEvent(), $this->getParams()) + ]; } } diff --git a/src/Appwrite/Event/Delete.php b/src/Appwrite/Event/Delete.php index 1a4c9318e3..f0af20f21b 100644 --- a/src/Appwrite/Event/Delete.php +++ b/src/Appwrite/Event/Delete.php @@ -3,7 +3,6 @@ namespace Appwrite\Event; use Utopia\Database\Document; -use Utopia\Queue\Client; use Utopia\Queue\Connection; class Delete extends Event @@ -131,22 +130,14 @@ class Delete extends Event return $this->document; } - /** - * Executes this event and sends it to the deletes worker. + * Prepare the payload for the event * - * @return string|bool - * @throws \InvalidArgumentException + * @return array */ - public function trigger(): string|bool + protected function preparePayload(): array { - if ($this->paused) { - return false; - } - - $client = new Client($this->queue, $this->connection); - - return $client->enqueue([ + return [ 'project' => $this->project, 'type' => $this->type, 'document' => $this->document, @@ -154,6 +145,6 @@ class Delete extends Event 'resourceType' => $this->resourceType, 'datetime' => $this->datetime, 'hourlyUsageRetentionDatetime' => $this->hourlyUsageRetentionDatetime - ]); + ]; } } diff --git a/src/Appwrite/Event/Event.php b/src/Appwrite/Event/Event.php index e3a2e394cf..5cd5f8e7d6 100644 --- a/src/Appwrite/Event/Event.php +++ b/src/Appwrite/Event/Event.php @@ -137,7 +137,6 @@ class Event public function setProject(Document $project): self { $this->project = $project; - return $this; } @@ -312,6 +311,27 @@ class Event return $this->params; } + /** + * Get trimmed values for sensitive/large payload fields. + * Override this method in child classes to add more fields to trim. + * + * @return array + */ + protected function trimPayload(): array + { + $trimmed = []; + + if ($this->project) { + $trimmed['project'] = new Document([ + '$id' => $this->project->getId(), + '$internalId' => $this->project->getInternalId(), + 'database' => $this->project->getAttribute('database') + ]); + } + + return $trimmed; + } + /** * Execute Event. * @@ -324,16 +344,30 @@ class Event return false; } - $client = new Client($this->queue, $this->connection); + /** The getter is required since events like Databases need to override the queue name depending on the project */ + $client = new Client($this->getQueue(), $this->connection); - return $client->enqueue([ + // Merge the base payload with any trimmed values + $payload = array_merge($this->preparePayload(), $this->trimPayload()); + + return $client->enqueue($payload); + } + + /** + * Prepare payload for queue. Can be overridden by child classes to customize payload. + * + * @return array + */ + protected function preparePayload(): array + { + return [ 'project' => $this->project, 'user' => $this->user, 'userId' => $this->userId, 'payload' => $this->payload, 'context' => $this->context, 'events' => Event::generateEvents($this->getEvent(), $this->getParams()) - ]); + ]; } /** diff --git a/src/Appwrite/Event/Func.php b/src/Appwrite/Event/Func.php index 0ad639a9f5..b3945fccb8 100644 --- a/src/Appwrite/Event/Func.php +++ b/src/Appwrite/Event/Func.php @@ -3,7 +3,6 @@ namespace Appwrite\Event; use Utopia\Database\Document; -use Utopia\Queue\Client; use Utopia\Queue\Connection; class Func extends Event @@ -173,13 +172,13 @@ class Func extends Event } /** - * Returns set custom data for the function event. + * Returns set JWT for the function event. * * @return string */ - public function getData(): string + public function getJWT(): string { - return $this->data; + return $this->jwt; } /** @@ -191,37 +190,19 @@ class Func extends Event public function setJWT(string $jwt): self { $this->jwt = $jwt; - return $this; } /** - * Returns set JWT for the function event. + * Prepare payload for the function event. * - * @return string + * @return array */ - public function getJWT(): string + protected function preparePayload(): array { - return $this->jwt; - } - - /** - * Executes the function event and sends it to the functions worker. - * - * @return string|bool - * @throws \InvalidArgumentException - */ - public function trigger(): string|bool - { - if ($this->paused) { - return false; - } - - $client = new Client($this->queue, $this->connection); - $events = $this->getEvent() ? Event::generateEvents($this->getEvent(), $this->getParams()) : null; - return $client->enqueue([ + return [ 'project' => $this->project, 'user' => $this->user, 'userId' => $this->userId, @@ -236,6 +217,6 @@ class Func extends Event 'path' => $this->path, 'headers' => $this->headers, 'method' => $this->method, - ]); + ]; } } diff --git a/src/Appwrite/Event/Mail.php b/src/Appwrite/Event/Mail.php index a0fca75688..1c9e539cdb 100644 --- a/src/Appwrite/Event/Mail.php +++ b/src/Appwrite/Event/Mail.php @@ -2,7 +2,6 @@ namespace Appwrite\Event; -use Utopia\Queue\Client; use Utopia\Queue\Connection; class Mail extends Event @@ -397,20 +396,13 @@ class Mail extends Event } /** - * Executes the event and sends it to the mails worker. + * Prepare the payload for the event * - * @return string|bool - * @throws \InvalidArgumentException + * @return array */ - public function trigger(): string|bool + protected function preparePayload(): array { - if ($this->paused) { - return false; - } - - $client = new Client($this->queue, $this->connection); - - return $client->enqueue([ + return [ 'project' => $this->project, 'recipient' => $this->recipient, 'name' => $this->name, @@ -421,6 +413,6 @@ class Mail extends Event 'variables' => $this->variables, 'attachment' => $this->attachment, 'events' => Event::generateEvents($this->getEvent(), $this->getParams()) - ]); + ]; } } diff --git a/src/Appwrite/Event/Messaging.php b/src/Appwrite/Event/Messaging.php index 755b8c9158..61dbe9c427 100644 --- a/src/Appwrite/Event/Messaging.php +++ b/src/Appwrite/Event/Messaging.php @@ -3,7 +3,6 @@ namespace Appwrite\Event; use Utopia\Database\Document; -use Utopia\Queue\Client; use Utopia\Queue\Connection; class Messaging extends Event @@ -176,19 +175,13 @@ class Messaging extends Event } /** - * Executes the event and sends it to the messaging worker. - * @return string|bool - * @throws \InvalidArgumentException + * Prepare the payload for the event + * + * @return array */ - public function trigger(): string | bool + protected function preparePayload(): array { - if ($this->paused) { - return false; - } - - $client = new Client($this->queue, $this->connection); - - return $client->enqueue([ + return [ 'type' => $this->type, 'project' => $this->project, 'user' => $this->user, @@ -196,6 +189,6 @@ class Messaging extends Event 'message' => $this->message, 'recipients' => $this->recipients, 'providerType' => $this->providerType, - ]); + ]; } } diff --git a/src/Appwrite/Event/Migration.php b/src/Appwrite/Event/Migration.php index 789b8e2160..5fb2d5a106 100644 --- a/src/Appwrite/Event/Migration.php +++ b/src/Appwrite/Event/Migration.php @@ -3,7 +3,6 @@ namespace Appwrite\Event; use Utopia\Database\Document; -use Utopia\Queue\Client; use Utopia\Queue\Connection; class Migration extends Event @@ -68,23 +67,16 @@ class Migration extends Event } /** - * Executes the migration event and sends it to the migrations worker. + * Prepare the payload for the migration event. * - * @return string|bool - * @throws \InvalidArgumentException + * @return array */ - public function trigger(): string|bool + protected function preparePayload(): array { - if ($this->paused) { - return false; - } - - $client = new Client($this->queue, $this->connection); - - return $client->enqueue([ + return [ 'project' => $this->project, 'user' => $this->user, 'migration' => $this->migration, - ]); + ]; } } diff --git a/src/Appwrite/Event/Usage.php b/src/Appwrite/Event/Usage.php index 89e900d2ab..5609859f37 100644 --- a/src/Appwrite/Event/Usage.php +++ b/src/Appwrite/Event/Usage.php @@ -3,7 +3,6 @@ namespace Appwrite\Event; use Utopia\Database\Document; -use Utopia\Queue\Client; use Utopia\Queue\Connection; class Usage extends Event @@ -51,6 +50,20 @@ class Usage extends Event return $this; } + /** + * Prepare the payload for the usage event. + * + * @return array + */ + protected function preparePayload(): array + { + return [ + 'project' => $this->project, + 'reduce' => $this->reduce, + 'metrics' => $this->metrics, + ]; + } + /** * Sends metrics to the usage worker. * @@ -58,20 +71,8 @@ class Usage extends Event */ public function trigger(): string|bool { - if ($this->paused) { - return false; - } - - $client = new Client($this->queue, $this->connection); - - $result = $client->enqueue([ - 'project' => $this->getProject(), - 'reduce' => $this->reduce, - 'metrics' => $this->metrics, - ]); - + parent::trigger(); $this->metrics = []; - - return $result; + return true; } } diff --git a/src/Appwrite/Event/UsageDump.php b/src/Appwrite/Event/UsageDump.php index 2998e4e104..6f44de4eda 100644 --- a/src/Appwrite/Event/UsageDump.php +++ b/src/Appwrite/Event/UsageDump.php @@ -2,7 +2,6 @@ namespace Appwrite\Event; -use Utopia\Queue\Client; use Utopia\Queue\Connection; class UsageDump extends Event @@ -32,20 +31,14 @@ class UsageDump extends Event } /** - * Sends metrics to the usage worker. + * Prepare the payload for the usage dump event. * - * @return string|bool + * @return array */ - public function trigger(): string|bool + protected function preparePayload(): array { - if ($this->paused) { - return false; - } - - $client = new Client($this->queue, $this->connection); - - return $client->enqueue([ + return [ 'stats' => $this->stats, - ]); + ]; } } diff --git a/src/Appwrite/Event/Webhook.php b/src/Appwrite/Event/Webhook.php index 36c6923cae..3e0dbe446f 100644 --- a/src/Appwrite/Event/Webhook.php +++ b/src/Appwrite/Event/Webhook.php @@ -2,7 +2,6 @@ namespace Appwrite\Event; -use Utopia\Database\Document; use Utopia\Queue\Connection; class Webhook extends Event @@ -16,15 +15,17 @@ class Webhook extends Event ->setClass(Event::WEBHOOK_CLASS_NAME); } - public function trigger(): string|bool + /** + * Trim the payload for the webhook event. + * + * @return array + */ + public function trimPayload(): array { - /** Filter out context and trim project to keep the payload small */ - $this->context = []; - $this->project = new Document([ - '$id' => $this->project->getId(), - '$internalId' => $this->project->getInternalId(), - ]); - - return parent::trigger(); + $trimmed = parent::trimPayload(); + if (isset($this->context)) { + $trimmed['context'] = []; + } + return $trimmed; } } diff --git a/src/Appwrite/Extend/Exception.php b/src/Appwrite/Extend/Exception.php index 432a5abb1b..d4f47ca177 100644 --- a/src/Appwrite/Extend/Exception.php +++ b/src/Appwrite/Extend/Exception.php @@ -112,7 +112,6 @@ class Exception extends \Exception /** Teams */ public const TEAM_NOT_FOUND = 'team_not_found'; - public const TEAM_INVITE_ALREADY_EXISTS = 'team_invite_already_exists'; public const TEAM_INVITE_NOT_FOUND = 'team_invite_not_found'; public const TEAM_INVALID_SECRET = 'team_invalid_secret'; public const TEAM_MEMBERSHIP_MISMATCH = 'team_membership_mismatch'; diff --git a/src/Appwrite/Platform/Tasks/Maintenance.php b/src/Appwrite/Platform/Tasks/Maintenance.php index c789cbdaac..2d37bdbf70 100644 --- a/src/Appwrite/Platform/Tasks/Maintenance.php +++ b/src/Appwrite/Platform/Tasks/Maintenance.php @@ -47,9 +47,12 @@ class Maintenance extends Action Console::info("[{$time}] Notifying workers with maintenance tasks every {$interval} seconds"); $this->foreachProject($dbForPlatform, function (Document $project) use ($queueForDeletes, $usageStatsRetentionHourly) { - $queueForDeletes->setProject($project); + $queueForDeletes + ->setType(DELETE_TYPE_MAINTENANCE) + ->setProject($project) + ->setUsageRetentionHourlyDateTime(DateTime::addSeconds(new \DateTime(), -1 * $usageStatsRetentionHourly)) + ->trigger(); - $this->notifyProjects($queueForDeletes, $usageStatsRetentionHourly); }); $this->notifyDeleteConnections($queueForDeletes); @@ -59,18 +62,6 @@ class Maintenance extends Action }, $interval, $delay); } - /** - * Hook to allow sub-classes to extend project-level maintenance functionality. - */ - protected function notifyProjects(Delete $queueForDeletes, int $usageStatsRetentionHourly): void - { - $this->notifyDeleteTargets($queueForDeletes); - $this->notifyDeleteExecutionLogs($queueForDeletes); - $this->notifyDeleteAuditLogs($queueForDeletes); - $this->notifyDeleteUsageStats($usageStatsRetentionHourly, $queueForDeletes); - $this->notifyDeleteExpiredSessions($queueForDeletes); - } - protected function foreachProject(Database $dbForPlatform, callable $callback): void { // TODO: @Meldiron name of this method no longer matches. It does not delete, and it gives whole document @@ -98,28 +89,6 @@ class Maintenance extends Action Console::info("Found {$count} projects " . ($executionEnd - $executionStart) . " seconds"); } - private function notifyDeleteExecutionLogs(Delete $queueForDeletes): void - { - $queueForDeletes - ->setType(DELETE_TYPE_EXECUTIONS) - ->trigger(); - } - - private function notifyDeleteAuditLogs(Delete $queueForDeletes): void - { - $queueForDeletes - ->setType(DELETE_TYPE_AUDIT) - ->trigger(); - } - - private function notifyDeleteUsageStats(int $usageStatsRetentionHourly, Delete $queueForDeletes): void - { - $queueForDeletes - ->setType(DELETE_TYPE_USAGE) - ->setUsageRetentionHourlyDateTime(DateTime::addSeconds(new \DateTime(), -1 * $usageStatsRetentionHourly)) - ->trigger(); - } - private function notifyDeleteConnections(Delete $queueForDeletes): void { $queueForDeletes @@ -128,13 +97,6 @@ class Maintenance extends Action ->trigger(); } - private function notifyDeleteExpiredSessions(Delete $queueForDeletes): void - { - $queueForDeletes - ->setType(DELETE_TYPE_SESSIONS) - ->trigger(); - } - private function renewCertificates(Database $dbForPlatform, Certificate $queueForCertificate): void { $time = DateTime::now(); @@ -177,11 +139,4 @@ class Maintenance extends Action ->setDatetime(DateTime::addSeconds(new \DateTime(), -1 * $interval)) ->trigger(); } - - private function notifyDeleteTargets(Delete $queueForDeletes): void - { - $queueForDeletes - ->setType(DELETE_TYPE_EXPIRED_TARGETS) - ->trigger(); - } } diff --git a/src/Appwrite/Platform/Workers/Builds.php b/src/Appwrite/Platform/Workers/Builds.php index bef78a7514..4f5d6eb694 100644 --- a/src/Appwrite/Platform/Workers/Builds.php +++ b/src/Appwrite/Platform/Workers/Builds.php @@ -46,6 +46,7 @@ class Builds extends Action $this ->desc('Builds worker') ->inject('message') + ->inject('project') ->inject('dbForPlatform') ->inject('queueForEvents') ->inject('queueForFunctions') @@ -54,11 +55,12 @@ class Builds extends Action ->inject('dbForProject') ->inject('deviceForFunctions') ->inject('log') - ->callback(fn ($message, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, Usage $usage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, Log $log) => $this->action($message, $dbForPlatform, $queueForEvents, $queueForFunctions, $usage, $cache, $dbForProject, $deviceForFunctions, $log)); + ->callback(fn ($message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, Usage $usage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, Log $log) => $this->action($message, $project, $dbForPlatform, $queueForEvents, $queueForFunctions, $usage, $cache, $dbForProject, $deviceForFunctions, $log)); } /** * @param Message $message + * @param Document $project * @param Database $dbForPlatform * @param Event $queueForEvents * @param Func $queueForFunctions @@ -70,7 +72,7 @@ class Builds extends Action * @return void * @throws \Utopia\Database\Exception */ - public function action(Message $message, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, Usage $queueForUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, Log $log): void + public function action(Message $message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, Usage $queueForUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, Log $log): void { $payload = $message->getPayload() ?? []; @@ -79,7 +81,6 @@ class Builds extends Action } $type = $payload['type'] ?? ''; - $project = new Document($payload['project'] ?? []); $resource = new Document($payload['resource'] ?? []); $deployment = new Document($payload['deployment'] ?? []); $template = new Document($payload['template'] ?? []); diff --git a/src/Appwrite/Platform/Workers/Databases.php b/src/Appwrite/Platform/Workers/Databases.php index 5b73e6a75c..9345f31165 100644 --- a/src/Appwrite/Platform/Workers/Databases.php +++ b/src/Appwrite/Platform/Workers/Databases.php @@ -34,21 +34,23 @@ class Databases extends Action $this ->desc('Databases worker') ->inject('message') + ->inject('project') ->inject('dbForPlatform') ->inject('dbForProject') ->inject('log') - ->callback(fn (Message $message, Database $dbForPlatform, Database $dbForProject, Log $log) => $this->action($message, $dbForPlatform, $dbForProject, $log)); + ->callback(fn (Message $message, Document $project, Database $dbForPlatform, Database $dbForProject, Log $log) => $this->action($message, $project, $dbForPlatform, $dbForProject, $log)); } /** * @param Message $message + * @param Document $project * @param Database $dbForPlatform * @param Database $dbForProject * @param Log $log * @return void * @throws \Exception */ - public function action(Message $message, Database $dbForPlatform, Database $dbForProject, Log $log): void + public function action(Message $message, Document $project, Database $dbForPlatform, Database $dbForProject, Log $log): void { $payload = $message->getPayload() ?? []; @@ -57,7 +59,6 @@ class Databases extends Action } $type = $payload['type']; - $project = new Document($payload['project']); $collection = new Document($payload['collection'] ?? []); $document = new Document($payload['document'] ?? []); $database = new Document($payload['database'] ?? []); diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 9aaf19f412..539bbd61f9 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -43,6 +43,7 @@ class Deletes extends Action $this ->desc('Deletes worker') ->inject('message') + ->inject('project') ->inject('dbForPlatform') ->inject('getProjectDB') ->inject('timelimit') @@ -55,8 +56,8 @@ class Deletes extends Action ->inject('auditRetention') ->inject('log') ->callback( - fn ($message, $dbForPlatform, callable $getProjectDB, callable $timelimit, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, string $auditRetention, Log $log) => - $this->action($message, $dbForPlatform, $getProjectDB, $timelimit, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $executionRetention, $auditRetention, $log) + fn ($message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $timelimit, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, string $auditRetention, Log $log) => + $this->action($message, $project, $dbForPlatform, $getProjectDB, $timelimit, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $executionRetention, $auditRetention, $log) ); } @@ -64,7 +65,7 @@ class Deletes extends Action * @throws Exception * @throws Throwable */ - public function action(Message $message, Database $dbForPlatform, callable $getProjectDB, callable $timelimit, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, string $auditRetention, Log $log): void + public function action(Message $message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $timelimit, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, string $auditRetention, Log $log): void { $payload = $message->getPayload() ?? []; @@ -78,7 +79,6 @@ class Deletes extends Action $resource = $payload['resource'] ?? null; $resourceType = $payload['resourceType'] ?? null; $document = new Document($payload['document'] ?? []); - $project = new Document($payload['project'] ?? []); $log->addTag('projectId', $project->getId()); $log->addTag('type', $type); @@ -153,6 +153,13 @@ class Deletes extends Action case DELETE_TYPE_SESSION_TARGETS: $this->deleteSessionTargets($project, $getProjectDB, $document); break; + case DELETE_TYPE_MAINTENANCE: + $this->deleteExpiredTargets($project, $getProjectDB); + $this->deleteExecutionLogs($project, $getProjectDB, $executionRetention); + $this->deleteAuditLogs($project, $getProjectDB, $auditRetention); + $this->deleteUsageStats($project, $getProjectDB, $hourlyUsageRetentionDatetime); + $this->deleteExpiredSessions($project, $getProjectDB); + break; default: throw new \Exception('No delete operation for type: ' . \strval($type)); } diff --git a/src/Appwrite/Platform/Workers/Messaging.php b/src/Appwrite/Platform/Workers/Messaging.php index 6e27f16c2a..7837957c5d 100644 --- a/src/Appwrite/Platform/Workers/Messaging.php +++ b/src/Appwrite/Platform/Workers/Messaging.php @@ -59,15 +59,17 @@ class Messaging extends Action $this ->desc('Messaging worker') ->inject('message') + ->inject('project') ->inject('log') ->inject('dbForProject') ->inject('deviceForFiles') ->inject('queueForUsage') - ->callback(fn (Message $message, Log $log, Database $dbForProject, Device $deviceForFiles, Usage $queueForUsage) => $this->action($message, $log, $dbForProject, $deviceForFiles, $queueForUsage)); + ->callback(fn (Message $message, Document $project, Log $log, Database $dbForProject, Device $deviceForFiles, Usage $queueForUsage) => $this->action($message, $project, $log, $dbForProject, $deviceForFiles, $queueForUsage)); } /** * @param Message $message + * @param Document $project * @param Log $log * @param Database $dbForProject * @param Device $deviceForFiles @@ -77,6 +79,7 @@ class Messaging extends Action */ public function action( Message $message, + Document $project, Log $log, Database $dbForProject, Device $deviceForFiles, @@ -90,7 +93,6 @@ class Messaging extends Action } $type = $payload['type'] ?? ''; - $project = new Document($payload['project'] ?? []); switch ($type) { case MESSAGE_SEND_TYPE_INTERNAL: diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index f6af0eb5f2..078c9fa0ff 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -51,16 +51,17 @@ class Migrations extends Action $this ->desc('Migrations worker') ->inject('message') + ->inject('project') ->inject('dbForProject') ->inject('dbForPlatform') ->inject('logError') - ->callback(fn (Message $message, Database $dbForProject, Database $dbForPlatform, callable $logError) => $this->action($message, $dbForProject, $dbForPlatform, $logError)); + ->callback(fn (Message $message, Document $project, Database $dbForProject, Database $dbForPlatform, callable $logError) => $this->action($message, $project, $dbForProject, $dbForPlatform, $logError)); } /** * @throws Exception */ - public function action(Message $message, Database $dbForProject, Database $dbForPlatform, callable $logError): void + public function action(Message $message, Document $project, Database $dbForProject, Database $dbForPlatform, callable $logError): void { $payload = $message->getPayload() ?? []; @@ -69,7 +70,6 @@ class Migrations extends Action } $events = $payload['events'] ?? []; - $project = new Document($payload['project'] ?? []); $migration = new Document($payload['migration'] ?? []); if ($project->getId() === 'console') { diff --git a/src/Appwrite/Platform/Workers/Usage.php b/src/Appwrite/Platform/Workers/Usage.php index 8199fe73a7..3687eeab67 100644 --- a/src/Appwrite/Platform/Workers/Usage.php +++ b/src/Appwrite/Platform/Workers/Usage.php @@ -34,10 +34,11 @@ class Usage extends Action $this ->desc('Usage worker') ->inject('message') + ->inject('project') ->inject('getProjectDB') ->inject('queueForUsageDump') - ->callback(function (Message $message, callable $getProjectDB, UsageDump $queueForUsageDump) { - $this->action($message, $getProjectDB, $queueForUsageDump); + ->callback(function (Message $message, Document $project, callable $getProjectDB, UsageDump $queueForUsageDump) { + $this->action($message, $project, $getProjectDB, $queueForUsageDump); }); $this->aggregationInterval = (int) System::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '20'); @@ -46,21 +47,20 @@ class Usage extends Action /** * @param Message $message + * @param Document $project * @param callable $getProjectDB * @param UsageDump $queueForUsageDump * @return void * @throws \Utopia\Database\Exception * @throws Exception */ - public function action(Message $message, callable $getProjectDB, UsageDump $queueForUsageDump): void + public function action(Message $message, Document $project, callable $getProjectDB, UsageDump $queueForUsageDump): void { $payload = $message->getPayload() ?? []; if (empty($payload)) { throw new Exception('Missing payload'); } - $document = $payload['project'] ?? []; - $project = new Document($document); if (empty($project->getAttribute('database'))) { var_dump($payload); diff --git a/src/Appwrite/Platform/Workers/UsageDump.php b/src/Appwrite/Platform/Workers/UsageDump.php index 3e50ba0363..2f1d13f29a 100644 --- a/src/Appwrite/Platform/Workers/UsageDump.php +++ b/src/Appwrite/Platform/Workers/UsageDump.php @@ -59,26 +59,11 @@ class UsageDump extends Action foreach ($payload['stats'] ?? [] as $stats) { - //$project = new Document($stats['project'] ?? []); - - /** - * Start temp bug fallback - */ - $document = $stats['project'] ?? []; - if (!empty($document['$uid'])) { - $document['$id'] = $document['$uid']; - } - - $project = new Document($document); - - if (empty($project->getAttribute('database'))) { - continue; - } + $project = new Document($stats['project'] ?? []); /** * End temp bug fallback */ - $numberOfKeys = !empty($stats['keys']) ? count($stats['keys']) : 0; $receivedAt = $stats['receivedAt'] ?? 'NONE'; if ($numberOfKeys === 0) { diff --git a/src/Appwrite/Platform/Workers/Webhooks.php b/src/Appwrite/Platform/Workers/Webhooks.php index 50f64ea02d..a76e4f17b0 100644 --- a/src/Appwrite/Platform/Workers/Webhooks.php +++ b/src/Appwrite/Platform/Workers/Webhooks.php @@ -32,22 +32,24 @@ class Webhooks extends Action $this ->desc('Webhooks worker') ->inject('message') + ->inject('project') ->inject('dbForPlatform') ->inject('queueForMails') ->inject('queueForUsage') ->inject('log') - ->callback(fn (Message $message, Database $dbForPlatform, Mail $queueForMails, Usage $queueForUsage, Log $log) => $this->action($message, $dbForPlatform, $queueForMails, $queueForUsage, $log)); + ->callback(fn (Message $message, Document $project, Database $dbForPlatform, Mail $queueForMails, Usage $queueForUsage, Log $log) => $this->action($message, $project, $dbForPlatform, $queueForMails, $queueForUsage, $log)); } /** * @param Message $message + * @param Document $project * @param Database $dbForPlatform * @param Mail $queueForMails * @param Log $log * @return void * @throws Exception */ - public function action(Message $message, Database $dbForPlatform, Mail $queueForMails, Usage $queueForUsage, Log $log): void + public function action(Message $message, Document $project, Database $dbForPlatform, Mail $queueForMails, Usage $queueForUsage, Log $log): void { $this->errors = []; $payload = $message->getPayload() ?? []; @@ -60,8 +62,6 @@ class Webhooks extends Action $webhookPayload = json_encode($payload['payload']); $user = new Document($payload['user'] ?? []); - $project = new Document($payload['project']); - $project = $dbForPlatform->getDocument('projects', $project->getId()); $log->addTag('projectId', $project->getId()); foreach ($project->getAttribute('webhooks', []) as $webhook) { diff --git a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php index 9b9f03a100..d8d1eb8eb5 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php @@ -375,7 +375,7 @@ class FunctionsCustomServerTest extends Scope $this->assertEquals(200, $deployment['headers']['status-code']); $this->assertEquals('ready', $deployment['body']['status']); - }, 500000, 1000); + }, 50000, 1000); $function = $this->getFunction($functionId); diff --git a/tests/e2e/Services/Teams/TeamsBaseClient.php b/tests/e2e/Services/Teams/TeamsBaseClient.php index ca6082f6d0..3381b80120 100644 --- a/tests/e2e/Services/Teams/TeamsBaseClient.php +++ b/tests/e2e/Services/Teams/TeamsBaseClient.php @@ -291,10 +291,7 @@ trait TeamsBaseClient $this->assertEquals($secondName, $lastEmail['to'][0]['name']); $this->assertEquals('Invitation to ' . $teamName . ' Team at ' . $this->getProject()['name'], $lastEmail['subject']); - /** - * Test for FAILURE - */ - + // test for resending invitation $response = $this->client->call(Client::METHOD_POST, '/teams/' . $teamUid . '/memberships', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], @@ -305,7 +302,11 @@ trait TeamsBaseClient 'url' => 'http://localhost:5000/join-us#title' ]); - $this->assertEquals(409, $response['headers']['status-code']); + $this->assertEquals(201, $response['headers']['status-code']); + + /** + * Test for FAILURE + */ $response = $this->client->call(Client::METHOD_POST, '/teams/' . $teamUid . '/memberships', array_merge([ 'content-type' => 'application/json', diff --git a/tests/e2e/Services/Teams/TeamsBaseServer.php b/tests/e2e/Services/Teams/TeamsBaseServer.php index 6a1d05e9d4..bade16cf2f 100644 --- a/tests/e2e/Services/Teams/TeamsBaseServer.php +++ b/tests/e2e/Services/Teams/TeamsBaseServer.php @@ -185,10 +185,7 @@ trait TeamsBaseServer // $this->assertContains('team:'.$teamUid.'/admin', $response['body']['roles']); // $this->assertContains('team:'.$teamUid.'/editor', $response['body']['roles']); - /** - * Test for FAILURE - */ - + // test for resending invitation $response = $this->client->call(Client::METHOD_POST, '/teams/' . $teamUid . '/memberships', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], @@ -199,7 +196,11 @@ trait TeamsBaseServer 'url' => 'http://localhost:5000/join-us#title' ]); - $this->assertEquals(409, $response['headers']['status-code']); + $this->assertEquals(201, $response['headers']['status-code']); + + /** + * Test for FAILURE + */ $response = $this->client->call(Client::METHOD_POST, '/teams/' . $teamUid . '/memberships', array_merge([ 'content-type' => 'application/json',