From a804cba99957beb491d57fa4ddcf0d4690dbaebc Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 14:01:26 +0000 Subject: [PATCH] Refactor usage metrics to stateless publisher pattern Co-Authored-By: Claude Opus 4.6 --- CONTRIBUTING.md | 45 ++-- app/cli.php | 42 +-- app/controllers/api/account.php | 26 +- app/controllers/shared/api.php | 196 +++++++------- app/init/resources.php | 231 +++++++++-------- app/worker.php | 58 +++-- src/Appwrite/Bus/Listeners/Usage.php | 43 ++-- src/Appwrite/Event/Message/Base.php | 21 ++ src/Appwrite/Event/Message/Usage.php | 49 ++++ src/Appwrite/Event/Publisher/Base.php | 33 +++ src/Appwrite/Event/Publisher/Usage.php | 39 +++ src/Appwrite/Event/StatsUsage.php | 96 ------- .../Http/Account/MFA/Challenges/Create.php | 14 +- .../Modules/Avatars/Http/Screenshots/Get.php | 8 +- .../Documents/Attribute/Decrement.php | 8 +- .../Documents/Attribute/Increment.php | 8 +- .../Collections/Documents/Bulk/Delete.php | 10 +- .../Collections/Documents/Bulk/Update.php | 10 +- .../Collections/Documents/Bulk/Upsert.php | 12 +- .../Collections/Documents/Create.php | 10 +- .../Collections/Documents/Delete.php | 8 +- .../Databases/Collections/Documents/Get.php | 8 +- .../Collections/Documents/Update.php | 8 +- .../Collections/Documents/Upsert.php | 8 +- .../Databases/Collections/Documents/XList.php | 8 +- .../Databases/Http/Databases/Delete.php | 1 - .../Http/Databases/Transactions/Update.php | 15 +- .../Databases/Http/TablesDB/Delete.php | 1 - .../Http/TablesDB/Tables/Rows/Bulk/Delete.php | 2 +- .../Http/TablesDB/Tables/Rows/Bulk/Update.php | 2 +- .../Http/TablesDB/Tables/Rows/Bulk/Upsert.php | 2 +- .../TablesDB/Tables/Rows/Column/Decrement.php | 2 +- .../TablesDB/Tables/Rows/Column/Increment.php | 2 +- .../Http/TablesDB/Tables/Rows/Create.php | 2 +- .../Http/TablesDB/Tables/Rows/Delete.php | 2 +- .../Http/TablesDB/Tables/Rows/Get.php | 2 +- .../Http/TablesDB/Tables/Rows/Update.php | 2 +- .../Http/TablesDB/Tables/Rows/Upsert.php | 2 +- .../Http/TablesDB/Tables/Rows/XList.php | 2 +- .../Http/TablesDB/Transactions/Update.php | 2 +- .../Functions/Http/Executions/Create.php | 8 +- .../Modules/Functions/Workers/Builds.php | 241 ++++++++---------- .../Health/Http/Health/Queue/Failed/Get.php | 8 +- .../Http/Health/Queue/StatsUsage/Get.php | 8 +- .../Modules/Teams/Http/Memberships/Create.php | 87 +++---- src/Appwrite/Platform/Workers/Messaging.php | 34 ++- src/Appwrite/Platform/Workers/Migrations.php | 42 +-- src/Appwrite/Platform/Workers/Webhooks.php | 33 ++- src/Appwrite/Usage/Context.php | 74 ++++++ .../Services/Migrations/MigrationsBase.php | 6 +- 50 files changed, 852 insertions(+), 729 deletions(-) create mode 100644 src/Appwrite/Event/Message/Base.php create mode 100644 src/Appwrite/Event/Message/Usage.php create mode 100644 src/Appwrite/Event/Publisher/Base.php create mode 100644 src/Appwrite/Event/Publisher/Usage.php delete mode 100644 src/Appwrite/Event/StatsUsage.php create mode 100644 src/Appwrite/Usage/Context.php diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0ccc8e8372..5d7ab96a4f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -409,14 +409,16 @@ Next follow the appropriate steps below depending on whether you're adding the m **API** -In file `app/controllers/shared/api.php` On the database listener, add to an existing or create a new switch case. Add a call to the usage worker with your new metric const like so: +In file `app/controllers/shared/api.php` On the database listener, add to an existing or create a new switch case. Accumulate metrics in the usage context like so: ```php case $document->getCollection() === 'teams': - $queueForStatsUsage - ->addMetric(METRIC_TEAMS, $value); // per project + $usage->addMetric(METRIC_TEAMS, $value); // per project break; ``` + +The metrics will be automatically published by the shutdown hook at the end of the request. There is no need to manually trigger or publish. + There are cases when you need to handle metric that has a parent entity, like buckets. Files are linked to a parent bucket, you should verify you remove the files stats when you delete a bucket. @@ -425,14 +427,13 @@ In that case you need also to handle children removal using addReduce() method c ```php case $document->getCollection() === 'buckets': //buckets - $queueForStatsUsage - ->addMetric(METRIC_BUCKETS, $value); // per project + $usage->addMetric(METRIC_BUCKETS, $value); // per project if ($event === Database::EVENT_DOCUMENT_DELETE) { - $queueForStatsUsage + $usage ->addReduce($document); } break; - + ``` In addition, you will also need to add some logic to the `reduce()` method of the Usage worker located in `/src/Appwrite/Platform/Workers/Usage.php`, like so: @@ -460,8 +461,12 @@ case $document->getCollection() === 'buckets': **Background worker** -You need to inject the usage queue in the desired worker on the constructor method +You need to inject the usage context and publisher in the desired worker on the constructor method ```php +use Appwrite\Usage\Context; +use Appwrite\Event\Publisher\Usage as UsagePublisher; +use Appwrite\Event\Message\Usage as UsageMessage; + /** * @throws Exception */ @@ -474,24 +479,32 @@ public function __construct() ->inject('dbForProject') ->inject('queueForFunctions') ->inject('queueForEvents') - ->inject('queueForStatsUsage') + ->inject('usage') + ->inject('publisherForUsage') ->inject('log') - ->callback(fn (Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, StatsUsage $queueForStatsUsage, Log $log) => $this->action($message, $dbForProject, $queueForFunctions, $queueForEvents, $queueForStatsUsage, $log)); + ->callback(fn (Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, Context $usage, UsagePublisher $publisherForUsage, Log $log) => $this->action($message, $dbForProject, $queueForFunctions, $queueForEvents, $usage, $publisherForUsage, $log)); } ``` -and then trigger the queue with the new metric like so: +and then accumulate metrics, create a message, and publish like so: ```php -$queueForStatsUsage +$usage ->addMetric(METRIC_BUILDS, 1) ->addMetric(METRIC_BUILDS_STORAGE, $build->getAttribute('size', 0)) ->addMetric(METRIC_BUILDS_COMPUTE, (int)$build->getAttribute('duration', 0) * 1000) - ->addMetric(str_replace('{functionInternalId}', $function->getSequence(), METRIC_FUNCTION_ID_BUILDS), 1) + ->addMetric(str_replace('{functionInternalId}', $function->getSequence(), METRIC_FUNCTION_ID_BUILDS), 1) ->addMetric(str_replace('{functionInternalId}', $function->getSequence(), METRIC_FUNCTION_ID_BUILDS_STORAGE), $build->getAttribute('size', 0)) - ->addMetric(str_replace('{functionInternalId}', $function->getSequence(), METRIC_FUNCTION_ID_BUILDS_COMPUTE), (int)$build->getAttribute('duration', 0) * 1000) - ->setProject($project) - ->trigger(); + ->addMetric(str_replace('{functionInternalId}', $function->getSequence(), METRIC_FUNCTION_ID_BUILDS_COMPUTE), (int)$build->getAttribute('duration', 0) * 1000); + +// Publish the accumulated metrics (workers don't have shutdown hooks) +$message = new UsageMessage( + project: $project, + metrics: $usage->getMetrics(), + reduce: $usage->getReduce() +); +$publisherForUsage->enqueue($message); +$usage->reset(); ``` diff --git a/app/cli.php b/app/cli.php index 052643f004..ee134b9487 100644 --- a/app/cli.php +++ b/app/cli.php @@ -4,11 +4,13 @@ require_once __DIR__ . '/init.php'; use Appwrite\Event\Certificate; use Appwrite\Event\Delete; +use Appwrite\Event\Event; use Appwrite\Event\Func; +use Appwrite\Event\Publisher\Usage as UsagePublisher; use Appwrite\Event\StatsResources; -use Appwrite\Event\StatsUsage; use Appwrite\Platform\Appwrite; use Appwrite\Runtimes\Runtimes; +use Appwrite\Usage\Context as UsageContext; use Appwrite\Utopia\Database\Documents\User; use Executor\Executor; use Swoole\Runtime; @@ -29,6 +31,7 @@ use Utopia\Platform\Service; use Utopia\Pools\Group; use Utopia\Queue\Broker\Pool as BrokerPool; use Utopia\Queue\Publisher; +use Utopia\Queue\Queue; use Utopia\Registry\Registry; use Utopia\System\System; use Utopia\Telemetry\Adapter\None as NoTelemetry; @@ -47,7 +50,7 @@ $platform = new Appwrite(); $args = $platform->getEnv('argv'); \array_shift($args); -if (!isset($args[0])) { +if (! isset($args[0])) { Console::error('Missing task name'); Console::exit(1); } @@ -85,6 +88,7 @@ $setResource('pools', function (Registry $register) { $setResource('authorization', function () { $authorization = new Authorization(); $authorization->disable(); + return $authorization; }, []); @@ -113,7 +117,7 @@ $setResource('dbForPlatform', function ($pools, $cache, $authorization) { $collections = Config::getParam('collections', [])['console']; $last = \array_key_last($collections); - if (!($dbForPlatform->exists($dbForPlatform->getDatabase(), $last))) { /** TODO cache ready variable using registry */ + if (! ($dbForPlatform->exists($dbForPlatform->getDatabase(), $last))) { /** TODO cache ready variable using registry */ throw new Exception('Tables not ready yet.'); } @@ -122,10 +126,10 @@ $setResource('dbForPlatform', function ($pools, $cache, $authorization) { Console::warning($err->getMessage()); sleep($sleep); } - } while ($attempts < $maxAttempts && !$ready); + } while ($attempts < $maxAttempts && ! $ready); - if (!$ready) { - throw new Exception("Console is not ready yet. Please try again later."); + if (! $ready) { + throw new Exception('Console is not ready yet. Please try again later.'); } return $dbForPlatform; @@ -163,7 +167,7 @@ $setResource('getProjectDB', function (Group $pools, Database $dbForPlatform, $c if (\in_array($dsn->getHost(), $sharedTables)) { $database ->setSharedTables(true) - ->setTenant((int)$project->getSequence()) + ->setTenant((int) $project->getSequence()) ->setNamespace($dsn->getParam('namespace')); } else { $database @@ -184,7 +188,7 @@ $setResource('getProjectDB', function (Group $pools, Database $dbForPlatform, $c if (\in_array($dsn->getHost(), $sharedTables)) { $database ->setSharedTables(true) - ->setTenant((int)$project->getSequence()) + ->setTenant((int) $project->getSequence()) ->setNamespace($dsn->getParam('namespace')); } else { $database @@ -207,8 +211,9 @@ $setResource('getLogsDB', function (Group $pools, Cache $cache, Authorization $a $database = null; return function (?Document $project = null) use ($pools, $cache, $database, $authorization) { - if ($database !== null && $project !== null && !$project->isEmpty() && $project->getId() !== 'console') { - $database->setTenant((int)$project->getSequence()); + if ($database !== null && $project !== null && ! $project->isEmpty() && $project->getId() !== 'console') { + $database->setTenant((int) $project->getSequence()); + return $database; } @@ -224,8 +229,8 @@ $setResource('getLogsDB', function (Group $pools, Cache $cache, Authorization $a ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES); // set tenant - if ($project !== null && !$project->isEmpty() && $project->getId() !== 'console') { - $database->setTenant((int)$project->getSequence()); + if ($project !== null && ! $project->isEmpty() && $project->getId() !== 'console') { + $database->setTenant((int) $project->getSequence()); } return $database; @@ -243,15 +248,16 @@ $setResource('publisherFunctions', function (BrokerPool $publisher) { $setResource('publisherMigrations', function (BrokerPool $publisher) { return $publisher; }, ['publisher']); -$setResource('publisherStatsUsage', function (BrokerPool $publisher) { - return $publisher; -}, ['publisher']); $setResource('publisherMessaging', function (BrokerPool $publisher) { return $publisher; }, ['publisher']); -$setResource('queueForStatsUsage', function (Publisher $publisher) { - return new StatsUsage($publisher); -}, ['publisher']); +$setResource('usage', function () { + return new UsageContext(); +}, []); +$setResource('publisherForUsage', fn (Publisher $publisher) => new UsagePublisher( + $publisher, + new Queue(System::getEnv('_APP_STATS_USAGE_QUEUE_NAME', Event::STATS_USAGE_QUEUE_NAME)) +), ['publisher']); $setResource('queueForStatsResources', function (Publisher $publisher) { return new StatsResources($publisher); }, ['publisher']); diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index b58a9b4185..a780bfdac3 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -14,7 +14,6 @@ use Appwrite\Event\Delete; use Appwrite\Event\Event; use Appwrite\Event\Mail; use Appwrite\Event\Messaging; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\Hooks\Hooks; use Appwrite\Network\Validator\Email as EmailValidator; @@ -28,6 +27,7 @@ use Appwrite\SDK\MethodType; use Appwrite\SDK\Response as SDKResponse; use Appwrite\Template\Template; use Appwrite\URL\URL as URLParser; +use Appwrite\Usage\Context; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Database\Validator\CustomId; use Appwrite\Utopia\Database\Validator\Queries\Identities; @@ -2801,12 +2801,12 @@ Http::post('/v1/account/tokens/phone') ->inject('queueForMessaging') ->inject('locale') ->inject('timelimit') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('plan') ->inject('store') ->inject('proofForCode') ->inject('authorization') - ->action(function (string $userId, string $phone, Request $request, Response $response, User $user, Document $project, array $platform, Database $dbForProject, Event $queueForEvents, Messaging $queueForMessaging, Locale $locale, callable $timelimit, StatsUsage $queueForStatsUsage, array $plan, Store $store, ProofsCode $proofForCode, Authorization $authorization) { + ->action(function (string $userId, string $phone, Request $request, Response $response, User $user, Document $project, array $platform, Database $dbForProject, Event $queueForEvents, Messaging $queueForMessaging, Locale $locale, callable $timelimit, Context $usage, array $plan, Store $store, ProofsCode $proofForCode, Authorization $authorization) { if (empty(System::getEnv('_APP_SMS_PROVIDER'))) { throw new Exception(Exception::GENERAL_PHONE_DISABLED, 'Phone provider not configured'); } @@ -2955,16 +2955,12 @@ Http::post('/v1/account/tokens/phone') $countryCode = $helper->parse($phone)->getCountryCode(); if (!empty($countryCode)) { - $queueForStatsUsage - ->addMetric(str_replace('{countryCode}', $countryCode, METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE), 1); + $usage->addMetric(str_replace('{countryCode}', $countryCode, METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE), 1); } } catch (NumberParseException $e) { // Ignore invalid phone number for country code stats } - $queueForStatsUsage - ->addMetric(METRIC_AUTH_METHOD_PHONE, 1) - ->setProject($project) - ->trigger(); + $usage->addMetric(METRIC_AUTH_METHOD_PHONE, 1); } $token->setAttribute('secret', $secret); @@ -4199,11 +4195,11 @@ Http::post('/v1/account/verifications/phone') ->inject('project') ->inject('locale') ->inject('timelimit') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('plan') ->inject('proofForCode') ->inject('authorization') - ->action(function (Request $request, Response $response, User $user, Database $dbForProject, Event $queueForEvents, Messaging $queueForMessaging, Document $project, Locale $locale, callable $timelimit, StatsUsage $queueForStatsUsage, array $plan, ProofsCode $proofForCode, Authorization $authorization) { + ->action(function (Request $request, Response $response, User $user, Database $dbForProject, Event $queueForEvents, Messaging $queueForMessaging, Document $project, Locale $locale, callable $timelimit, Context $usage, array $plan, ProofsCode $proofForCode, Authorization $authorization) { if (empty(System::getEnv('_APP_SMS_PROVIDER'))) { throw new Exception(Exception::GENERAL_PHONE_DISABLED, 'Phone provider not configured'); } @@ -4288,16 +4284,12 @@ Http::post('/v1/account/verifications/phone') $countryCode = $helper->parse($phone)->getCountryCode(); if (!empty($countryCode)) { - $queueForStatsUsage - ->addMetric(str_replace('{countryCode}', $countryCode, METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE), 1); + $usage->addMetric(str_replace('{countryCode}', $countryCode, METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE), 1); } } catch (NumberParseException $e) { // Ignore invalid phone number for country code stats } - $queueForStatsUsage - ->addMetric(METRIC_AUTH_METHOD_PHONE, 1) - ->setProject($project) - ->trigger(); + $usage->addMetric(METRIC_AUTH_METHOD_PHONE, 1); } $verification->setAttribute('secret', $secret); diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 29ccc90179..c8824d3708 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -10,14 +10,16 @@ use Appwrite\Event\Delete; use Appwrite\Event\Event; use Appwrite\Event\Func; use Appwrite\Event\Mail; +use Appwrite\Event\Message\Usage as UsageMessage; use Appwrite\Event\Messaging; +use Appwrite\Event\Publisher\Usage as UsagePublisher; use Appwrite\Event\Realtime; -use Appwrite\Event\StatsUsage; use Appwrite\Event\Webhook; use Appwrite\Extend\Exception; use Appwrite\Extend\Exception as AppwriteException; use Appwrite\Functions\EventProcessor; use Appwrite\SDK\Method; +use Appwrite\Usage\Context; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Request; use Appwrite\Utopia\Response; @@ -53,7 +55,7 @@ $parseLabel = function (string $label, array $responsePayload, array $requestPar $replace = $parts[1] ?? ''; $params = match ($namespace) { - 'user' => (array)$user, + 'user' => (array) $user, 'request' => $requestParams, default => $responsePayload, }; @@ -61,13 +63,13 @@ $parseLabel = function (string $label, array $responsePayload, array $requestPar if (array_key_exists($replace, $params)) { $replacement = $params[$replace]; // Convert to string if it's not already a string - if (!is_string($replacement)) { + if (! is_string($replacement)) { if (is_array($replacement)) { $replacement = json_encode($replacement); } elseif (is_object($replacement) && method_exists($replacement, '__toString')) { - $replacement = (string)$replacement; + $replacement = (string) $replacement; } elseif (is_scalar($replacement)) { - $replacement = (string)$replacement; + $replacement = (string) $replacement; } else { throw new Exception(Exception::GENERAL_SERVER_ERROR, "The server encountered an error while parsing the label: $label. Please create an issue on GitHub to allow us to investigate further https://github.com/appwrite/appwrite/issues/new/choose"); } @@ -75,6 +77,7 @@ $parseLabel = function (string $label, array $responsePayload, array $requestPar $label = \str_replace($find, $replacement, $label); } } + return $label; }; @@ -160,7 +163,7 @@ Http::init() $scopes = $roles[$role]['scopes']; // Step 5: API Key Authentication - if (!empty($apiKey)) { + if (! empty($apiKey)) { // Check if key is expired if ($apiKey->isExpired()) { throw new Exception(Exception::PROJECT_KEY_EXPIRED); @@ -170,7 +173,6 @@ Http::init() $role = $apiKey->getRole(); $scopes = $apiKey->getScopes(); - // Handle special app role case if ($apiKey->getRole() === User::ROLE_APPS) { // Disable authorization checks for project API keys @@ -193,19 +195,19 @@ Http::init() // For standard keys, update last accessed time if (\in_array($apiKey->getType(), [API_KEY_STANDARD, API_KEY_ORGANIZATION, API_KEY_ACCOUNT])) { $dbKey = null; - if (!empty($apiKey->getProjectId())) { + if (! empty($apiKey->getProjectId())) { $dbKey = $project->find( key: 'secret', find: $request->getHeader('x-appwrite-key', ''), subject: 'keys' ); - } elseif (!empty($apiKey->getUserId())) { + } elseif (! empty($apiKey->getUserId())) { $dbKey = $user->find( key: 'secret', find: $request->getHeader('x-appwrite-key', ''), subject: 'keys' ); - } elseif (!empty($apiKey->getTeamId())) { + } elseif (! empty($apiKey->getTeamId())) { $dbKey = $team->find( key: 'secret', find: $request->getHeader('x-appwrite-key', ''), @@ -213,9 +215,7 @@ Http::init() ); } - if (!$dbKey) { - \var_dump($apiKey); - \var_dump($request->getHeader('x-appwrite-key', '')); + if (! $dbKey) { throw new Exception(Exception::USER_UNAUTHORIZED); } @@ -233,7 +233,7 @@ Http::init() if ($sdk !== 'UNKNOWN' && $sdkValidator->isValid($sdk)) { $sdks = $dbKey->getAttribute('sdks', []); - if (!in_array($sdk, $sdks)) { + if (! in_array($sdk, $sdks)) { $sdks[] = $sdk; $updates->setAttribute('sdks', $sdks); @@ -241,14 +241,14 @@ Http::init() } } - if (!$updates->isEmpty()) { + if (! $updates->isEmpty()) { $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->updateDocument('keys', $dbKey->getId(), $updates)); - if (!empty($apiKey->getProjectId())) { + if (! empty($apiKey->getProjectId())) { $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->purgeCachedDocument('projects', $project->getId())); - } elseif (!empty($apiKey->getUserId())) { + } elseif (! empty($apiKey->getUserId())) { $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->purgeCachedDocument('users', $user->getId())); - } elseif (!empty($apiKey->getTeamId())) { + } elseif (! empty($apiKey->getTeamId())) { $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->purgeCachedDocument('teams', $team->getId())); } } @@ -285,7 +285,7 @@ Http::init() } } } // Admin User Authentication - elseif (($project->getId() === 'console' && !$team->isEmpty() && !$user->isEmpty()) || ($project->getId() !== 'console' && !$user->isEmpty() && $mode === APP_MODE_ADMIN)) { + elseif (($project->getId() === 'console' && ! $team->isEmpty() && ! $user->isEmpty()) || ($project->getId() !== 'console' && ! $user->isEmpty() && $mode === APP_MODE_ADMIN)) { $teamId = $team->getId(); $adminRoles = []; $memberships = $user->getAttribute('memberships', []); @@ -310,7 +310,7 @@ Http::init() // Useful for those who have project-specific roles but don't have team-wide role. $scopes = ['teams.read', 'projects.read']; foreach ($adminRoles as $adminRole) { - $isTeamWideRole = !str_starts_with($adminRole, 'project-'); + $isTeamWideRole = ! str_starts_with($adminRole, 'project-'); $isProjectSpecificRole = $projectId !== 'console' && str_starts_with($adminRole, 'project-' . $projectId); if ($isTeamWideRole || $isProjectSpecificRole) { @@ -348,18 +348,18 @@ Http::init() * But, for actions on resources (sites, functions, etc.) in a non-console project, we explicitly check * whether the admin user has necessary permission on the project (sites, functions, etc. don't have permissions associated to them). */ - if (empty($apiKey) && !$user->isEmpty() && $project->getId() !== 'console' && $mode === APP_MODE_ADMIN) { + if (empty($apiKey) && ! $user->isEmpty() && $project->getId() !== 'console' && $mode === APP_MODE_ADMIN) { $input = new Input(Database::PERMISSION_READ, $project->getPermissionsByType(Database::PERMISSION_READ)); $initialStatus = $authorization->getStatus(); $authorization->enable(); - if (!$authorization->isValid($input)) { + if (! $authorization->isValid($input)) { throw new Exception(Exception::PROJECT_NOT_FOUND); } $authorization->setStatus($initialStatus); } // Step 6: Update project and user last activity - if (!$project->isEmpty() && $project->getId() !== 'console') { + if (! $project->isEmpty() && $project->getId() !== 'console') { $accessedAt = $project->getAttribute('accessedAt', 0); if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { $authorization->skip(fn () => $dbForPlatform->updateDocument('projects', $project->getId(), new Document([ @@ -368,12 +368,12 @@ Http::init() } } - if (!empty($user->getId())) { + if (! empty($user->getId())) { $accessedAt = $user->getAttribute('accessedAt', 0); if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_USER_ACCESS)) > $accessedAt) { $user->setAttribute('accessedAt', DateTime::now()); - if ($project->getId() !== 'console' && APP_MODE_ADMIN !== $mode) { + if ($project->getId() !== 'console' && $mode !== APP_MODE_ADMIN) { $dbForProject->updateDocument('users', $user->getId(), new Document([ 'accessedAt' => $user->getAttribute('accessedAt') ])); @@ -397,26 +397,26 @@ Http::init() $method = $method[0]; } - if (!empty($method)) { + if (! empty($method)) { $namespace = $method->getNamespace(); if ( array_key_exists($namespace, $project->getAttribute('services', [])) - && !$project->getAttribute('services', [])[$namespace] - && !(User::isPrivileged($authorization->getRoles()) || User::isApp($authorization->getRoles())) + && ! $project->getAttribute('services', [])[$namespace] + && ! (User::isPrivileged($authorization->getRoles()) || User::isApp($authorization->getRoles())) ) { throw new Exception(Exception::GENERAL_SERVICE_DISABLED); } } // Step 9: Validate scope permissions - $allowed = (array)$route->getLabel('scope', 'none'); + $allowed = (array) $route->getLabel('scope', 'none'); if (empty(\array_intersect($allowed, $scopes))) { throw new Exception(Exception::GENERAL_UNAUTHORIZED_SCOPE, $user->getAttribute('email', 'User') . ' (role: ' . \strtolower($roles[$role]['label']) . ') missing scopes (' . \json_encode($allowed) . ')'); } // Step 10: Check if user is blocked - if (false === $user->getAttribute('status')) { // Account is blocked + if ($user->getAttribute('status') === false) { // Account is blocked throw new Exception(Exception::USER_BLOCKED); } @@ -434,7 +434,7 @@ Http::init() $minimumFactors = ($mfaEnabled && $hasMoreFactors) ? 2 : 1; // Step 13: Handle Multi-Factor Authentication - if (!in_array('mfa', $route->getGroups())) { + if (! in_array('mfa', $route->getGroups())) { if ($session && \count($session->getAttribute('factors', [])) < $minimumFactors) { throw new Exception(Exception::USER_MORE_FACTORS_REQUIRED); } @@ -454,7 +454,7 @@ Http::init() ->inject('queueForDeletes') ->inject('queueForDatabase') ->inject('queueForBuilds') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('queueForFunctions') ->inject('queueForMails') ->inject('dbForProject') @@ -467,14 +467,14 @@ Http::init() ->inject('telemetry') ->inject('platform') ->inject('authorization') - ->action(function (Http $utopia, Request $request, Response $response, Document $project, Document $user, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Mail $queueForMails, Database $dbForProject, callable $timelimit, Document $resourceToken, string $mode, ?Key $apiKey, array $plan, Document $devKey, Telemetry $telemetry, array $platform, Authorization $authorization) { + ->action(function (Http $utopia, Request $request, Response $response, Document $project, Document $user, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Context $usage, Func $queueForFunctions, Mail $queueForMails, Database $dbForProject, callable $timelimit, Document $resourceToken, string $mode, ?Key $apiKey, array $plan, Document $devKey, Telemetry $telemetry, array $platform, Authorization $authorization) { $route = $utopia->getRoute(); if ( array_key_exists('rest', $project->getAttribute('apis', [])) - && !$project->getAttribute('apis', [])['rest'] - && !(User::isPrivileged($authorization->getRoles()) || User::isApp($authorization->getRoles())) + && ! $project->getAttribute('apis', [])['rest'] + && ! (User::isPrivileged($authorization->getRoles()) || User::isApp($authorization->getRoles())) ) { throw new AppwriteException(AppwriteException::GENERAL_API_DISABLED); } @@ -486,7 +486,7 @@ Http::init() $abuseKeyLabel = $route->getLabel('abuse-key', 'url:{url},ip:{ip}'); $timeLimitArray = []; - $abuseKeyLabel = (!is_array($abuseKeyLabel)) ? [$abuseKeyLabel] : $abuseKeyLabel; + $abuseKeyLabel = (! is_array($abuseKeyLabel)) ? [$abuseKeyLabel] : $abuseKeyLabel; foreach ($abuseKeyLabel as $abuseKey) { $start = $request->getContentRangeStart(); @@ -499,7 +499,7 @@ Http::init() ->setParam('{ip}', $request->getIP()) ->setParam('{url}', $request->getHostname() . $route->getPath()) ->setParam('{method}', $request->getMethod()) - ->setParam('{chunkId}', (int)($start / ($end + 1 - $start))); + ->setParam('{chunkId}', (int) ($start / ($end + 1 - $start))); $timeLimitArray[] = $timeLimit; } @@ -511,7 +511,7 @@ Http::init() foreach ($timeLimitArray as $timeLimit) { foreach ($request->getParams() as $key => $value) { // Set request params as potential abuse keys - if (!empty($value)) { + if (! empty($value)) { $timeLimit->setParam('{param-' . $key . '}', (\is_array($value)) ? \json_encode($value) : $value); } } @@ -534,8 +534,8 @@ Http::init() if ( $enabled // Abuse is enabled - && !$isAppUser // User is not API key - && !$isPrivilegedUser // User is not an admin + && ! $isAppUser // User is not API key + && ! $isPrivilegedUser // User is not an admin && $devKey->isEmpty() // request doesn't not contain development key && $abuse->check() // Route is rate-limited ) { @@ -564,19 +564,13 @@ Http::init() ->setProject($project); /* If a session exists, use the user associated with the session */ - if (!$user->isEmpty()) { + if (! $user->isEmpty()) { $userClone = clone $user; // $user doesn't support `type` and can cause unintended effects. $userClone->setAttribute('type', ACTIVITY_TYPE_USER); $queueForAudits->setUser($userClone); } - if (!empty($apiKey) && !empty($apiKey->getDisabledMetrics())) { - foreach ($apiKey->getDisabledMetrics() as $key) { - $queueForStatsUsage->disableMetric($key); - } - } - /* Auto-set projects */ $queueForDeletes->setProject($project); $queueForDatabase->setProject($project); @@ -590,69 +584,64 @@ Http::init() $queueForBuilds->setPlatform($platform); $queueForMails->setPlatform($platform); - $useCache = $route->getLabel('cache', false); $storageCacheOperationsCounter = $telemetry->createCounter('storage.cache.operations.load'); if ($useCache) { $route = $utopia->match($request); $isImageTransformation = $route->getPath() === '/v1/storage/buckets/:bucketId/files/:fileId/preview'; - $isDisabled = isset($plan['imageTransformations']) && $plan['imageTransformations'] === -1 && !User::isPrivileged($authorization->getRoles()); + $isDisabled = isset($plan['imageTransformations']) && $plan['imageTransformations'] === -1 && ! User::isPrivileged($authorization->getRoles()); $key = $request->cacheIdentifier(); - $cacheLog = $authorization->skip(fn () => $dbForProject->getDocument('cache', $key)); + $cacheLog = $authorization->skip(fn () => $dbForProject->getDocument('cache', $key)); $cache = new Cache( new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $project->getId()) ); $timestamp = 60 * 60 * 24 * 180; // Temporarily increase the TTL to 180 day to ensure files in the cache are still fetched. $data = $cache->load($key, $timestamp); - if (!empty($data) && !$cacheLog->isEmpty()) { - $usageMetric = $route->getLabel('usage.metric', null); - if ($usageMetric === METRIC_AVATARS_SCREENSHOTS_GENERATED) { - $queueForStatsUsage->disableMetric(METRIC_AVATARS_SCREENSHOTS_GENERATED); - } + if (! empty($data) && ! $cacheLog->isEmpty()) { $parts = explode('/', $cacheLog->getAttribute('resourceType', '')); $type = $parts[0] ?? null; - if ($type === 'bucket' && (!$isImageTransformation || !$isDisabled)) { + if ($type === 'bucket' && (! $isImageTransformation || ! $isDisabled)) { $bucketId = $parts[1] ?? null; $bucket = $authorization->skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); - $isToken = !$resourceToken->isEmpty() && $resourceToken->getAttribute('bucketInternalId') === $bucket->getSequence(); + $isToken = ! $resourceToken->isEmpty() && $resourceToken->getAttribute('bucketInternalId') === $bucket->getSequence(); $isPrivilegedUser = User::isPrivileged($authorization->getRoles()); - if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAppUser && !$isPrivilegedUser)) { + if ($bucket->isEmpty() || (! $bucket->getAttribute('enabled') && ! $isAppUser && ! $isPrivilegedUser)) { throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); } - if (!$bucket->getAttribute('transformations', true) && !$isAppUser && !$isPrivilegedUser) { + if (! $bucket->getAttribute('transformations', true) && ! $isAppUser && ! $isPrivilegedUser) { throw new Exception(Exception::STORAGE_BUCKET_TRANSFORMATIONS_DISABLED); } $fileSecurity = $bucket->getAttribute('fileSecurity', false); $valid = $authorization->isValid(new Input(Database::PERMISSION_READ, $bucket->getRead())); - if (!$fileSecurity && !$valid && !$isToken) { + if (! $fileSecurity && ! $valid && ! $isToken) { throw new Exception(Exception::USER_UNAUTHORIZED); } $parts = explode('/', $cacheLog->getAttribute('resource')); $fileId = $parts[1] ?? null; - if ($fileSecurity && !$valid && !$isToken) { + if ($fileSecurity && ! $valid && ! $isToken) { $file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); } else { $file = $authorization->skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); } - if (!$resourceToken->isEmpty() && $resourceToken->getAttribute('fileInternalId') !== $file->getSequence()) { + if (! $resourceToken->isEmpty() && $resourceToken->getAttribute('fileInternalId') !== $file->getSequence()) { throw new Exception(Exception::USER_UNAUTHORIZED); } if ($file->isEmpty()) { throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); } - //Do not update transformedAt if it's a console user - if (!User::isPrivileged($authorization->getRoles())) { + // Do not update transformedAt if it's a console user + if (! User::isPrivileged($authorization->getRoles())) { $transformedAt = $file->getAttribute('transformedAt', ''); if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $transformedAt) { $file->setAttribute('transformedAt', DateTime::now()); @@ -668,7 +657,7 @@ Http::init() ->addHeader('X-Appwrite-Cache', 'hit') ->setContentType($cacheLog->getAttribute('mimeType')); $storageCacheOperationsCounter->add(1, ['result' => 'hit']); - if (!$isImageTransformation || !$isDisabled) { + if (! $isImageTransformation || ! $isDisabled) { $response->send($data); } } else { @@ -691,7 +680,7 @@ Http::init() return; } - if (!$user->isEmpty()) { + if (! $user->isEmpty()) { throw new Exception(Exception::USER_SESSION_ALREADY_EXISTS); } }); @@ -745,7 +734,8 @@ Http::shutdown() ->inject('user') ->inject('queueForEvents') ->inject('queueForAudits') - ->inject('queueForStatsUsage') + ->inject('usage') + ->inject('publisherForUsage') ->inject('queueForDeletes') ->inject('queueForDatabase') ->inject('queueForBuilds') @@ -758,11 +748,12 @@ Http::shutdown() ->inject('timelimit') ->inject('eventProcessor') ->inject('bus') - ->action(function (Http $utopia, Request $request, Response $response, Document $project, User $user, Event $queueForEvents, Audit $queueForAudits, StatsUsage $queueForStatsUsage, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Messaging $queueForMessaging, Func $queueForFunctions, Event $queueForWebhooks, Realtime $queueForRealtime, Database $dbForProject, Authorization $authorization, callable $timelimit, EventProcessor $eventProcessor, Bus $bus) use ($parseLabel) { + ->inject('apiKey') + ->action(function (Http $utopia, Request $request, Response $response, Document $project, User $user, Event $queueForEvents, Audit $queueForAudits, Context $usage, UsagePublisher $publisherForUsage, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Messaging $queueForMessaging, Func $queueForFunctions, Event $queueForWebhooks, Realtime $queueForRealtime, Database $dbForProject, Authorization $authorization, callable $timelimit, EventProcessor $eventProcessor, Bus $bus, ?Key $apiKey) use ($parseLabel) { $responsePayload = $response->getPayload(); - if (!empty($queueForEvents->getEvent())) { + if (! empty($queueForEvents->getEvent())) { if (empty($queueForEvents->getPayload())) { $queueForEvents->setPayload($responsePayload); } @@ -784,7 +775,7 @@ Http::shutdown() } // Only trigger functions if there are matching function events - if (!empty($functionsEvents)) { + if (! empty($functionsEvents)) { foreach ($generatedEvents as $event) { if (isset($functionsEvents[$event])) { $queueForFunctions @@ -796,7 +787,7 @@ Http::shutdown() } // Only trigger webhooks if there are matching webhook events - if (!empty($webhooksEvents)) { + if (! empty($webhooksEvents)) { foreach ($generatedEvents as $event) { if (isset($webhooksEvents[$event])) { $queueForWebhooks @@ -820,7 +811,7 @@ Http::shutdown() if ($abuseEnabled && \count($abuseResetCode) > 0 && \in_array($response->getStatusCode(), $abuseResetCode)) { $abuseKeyLabel = $route->getLabel('abuse-key', 'url:{url},ip:{ip}'); - $abuseKeyLabel = (!is_array($abuseKeyLabel)) ? [$abuseKeyLabel] : $abuseKeyLabel; + $abuseKeyLabel = (! is_array($abuseKeyLabel)) ? [$abuseKeyLabel] : $abuseKeyLabel; foreach ($abuseKeyLabel as $abuseKey) { $start = $request->getContentRangeStart(); @@ -833,10 +824,10 @@ Http::shutdown() ->setParam('{ip}', $request->getIP()) ->setParam('{url}', $request->getHostname() . $route->getPath()) ->setParam('{method}', $request->getMethod()) - ->setParam('{chunkId}', (int)($start / ($end + 1 - $start))); + ->setParam('{chunkId}', (int) ($start / ($end + 1 - $start))); foreach ($request->getParams() as $key => $value) { // Set request params as potential abuse keys - if (!empty($value)) { + if (! empty($value)) { $timeLimit->setParam('{param-' . $key . '}', (\is_array($value)) ? \json_encode($value) : $value); } } @@ -850,14 +841,14 @@ Http::shutdown() * Audit labels */ $pattern = $route->getLabel('audits.resource', null); - if (!empty($pattern)) { + if (! empty($pattern)) { $resource = $parseLabel($pattern, $responsePayload, $requestParams, $user); - if (!empty($resource) && $resource !== $pattern) { + if (! empty($resource) && $resource !== $pattern) { $queueForAudits->setResource($resource); } } - if (!$user->isEmpty()) { + if (! $user->isEmpty()) { $userClone = clone $user; // $user doesn't support `type` and can cause unintended effects. $userClone->setAttribute('type', ACTIVITY_TYPE_USER); @@ -883,13 +874,13 @@ Http::shutdown() $queueForAudits->setUser($user); } - if (!empty($queueForAudits->getResource()) && !$queueForAudits->getUser()->isEmpty()) { + if (! empty($queueForAudits->getResource()) && ! $queueForAudits->getUser()->isEmpty()) { /** * audits.payload is switched to default true * in order to auto audit payload for all endpoints */ $pattern = $route->getLabel('audits.payload', true); - if (!empty($pattern)) { + if (! empty($pattern)) { $queueForAudits->setPayload($responsePayload); } @@ -900,19 +891,19 @@ Http::shutdown() $queueForAudits->trigger(); } - if (!empty($queueForDeletes->getType())) { + if (! empty($queueForDeletes->getType())) { $queueForDeletes->trigger(); } - if (!empty($queueForDatabase->getType())) { + if (! empty($queueForDatabase->getType())) { $queueForDatabase->trigger(); } - if (!empty($queueForBuilds->getType())) { + if (! empty($queueForBuilds->getType())) { $queueForBuilds->trigger(); } - if (!empty($queueForMessaging->getType())) { + if (! empty($queueForMessaging->getType())) { $queueForMessaging->trigger(); } @@ -921,14 +912,14 @@ Http::shutdown() if ($useCache) { $resource = $resourceType = null; $data = $response->getPayload(); - if (!empty($data['payload'])) { + if (! empty($data['payload'])) { $pattern = $route->getLabel('cache.resource', null); - if (!empty($pattern)) { + if (! empty($pattern)) { $resource = $parseLabel($pattern, $responsePayload, $requestParams, $user); } $pattern = $route->getLabel('cache.resourceType', null); - if (!empty($pattern)) { + if (! empty($pattern)) { $resourceType = $parseLabel($pattern, $responsePayload, $requestParams, $user); } @@ -938,7 +929,7 @@ Http::shutdown() $key = $request->cacheIdentifier(); $signature = md5($data['payload']); - $cacheLog = $authorization->skip(fn () => $dbForProject->getDocument('cache', $key)); + $cacheLog = $authorization->skip(fn () => $dbForProject->getDocument('cache', $key)); $accessedAt = $cacheLog->getAttribute('accessedAt', 0); $now = DateTime::now(); if ($cacheLog->isEmpty()) { @@ -971,7 +962,7 @@ Http::shutdown() } if ($project->getId() !== 'console') { - if (!User::isPrivileged($authorization->getRoles())) { + if (! User::isPrivileged($authorization->getRoles())) { $bus->dispatch(new RequestCompleted( project: $project->getArrayCopy(), request: $request, @@ -979,9 +970,32 @@ Http::shutdown() )); } - $queueForStatsUsage - ->setProject($project) - ->trigger(); + // Publish usage metrics if context has data + if (! $usage->isEmpty()) { + $metrics = $usage->getMetrics(); + + // Filter out API key disabled metrics using suffix pattern matching + $disabledMetrics = $apiKey?->getDisabledMetrics() ?? []; + if (! empty($disabledMetrics)) { + $metrics = array_values(array_filter($metrics, function ($metric) use ($disabledMetrics) { + foreach ($disabledMetrics as $pattern) { + if (str_ends_with($metric['key'], $pattern)) { + return false; + } + } + + return true; + })); + } + + $message = new UsageMessage( + project: $project, + metrics: $metrics, + reduce: $usage->getReduce() + ); + + $publisherForUsage->enqueue($message); + } } }); diff --git a/app/init/resources.php b/app/init/resources.php index d5486c2a49..1bab4491a4 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -14,10 +14,10 @@ use Appwrite\Event\Func; use Appwrite\Event\Mail; use Appwrite\Event\Messaging; use Appwrite\Event\Migration; +use Appwrite\Event\Publisher\Usage as UsagePublisher; use Appwrite\Event\Realtime; use Appwrite\Event\Screenshot; use Appwrite\Event\StatsResources; -use Appwrite\Event\StatsUsage; use Appwrite\Event\Webhook; use Appwrite\Extend\Exception; use Appwrite\Functions\EventProcessor; @@ -26,6 +26,7 @@ use Appwrite\Network\Cors; use Appwrite\Network\Platform; use Appwrite\Network\Validator\Origin; use Appwrite\Network\Validator\Redirect; +use Appwrite\Usage\Context as UsageContext; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Request; use Appwrite\Utopia\Response; @@ -57,6 +58,7 @@ use Utopia\Logger\Log; use Utopia\Pools\Group; use Utopia\Queue\Broker\Pool as BrokerPool; use Utopia\Queue\Publisher; +use Utopia\Queue\Queue; use Utopia\Storage\Device; use Utopia\Storage\Device\AWS; use Utopia\Storage\Device\Backblaze; @@ -88,6 +90,7 @@ Http::setResource('register', fn () => $register); Http::setResource('locale', function () { $locale = new Locale(System::getEnv('_APP_LOCALE', 'en')); $locale->setFallback(System::getEnv('_APP_LOCALE', 'en')); + return $locale; }); @@ -108,9 +111,6 @@ Http::setResource('publisherFunctions', function (Publisher $publisher) { Http::setResource('publisherMigrations', function (Publisher $publisher) { return $publisher; }, ['publisher']); -Http::setResource('publisherStatsUsage', function (Publisher $publisher) { - return $publisher; -}, ['publisher']); Http::setResource('publisherMails', function (Publisher $publisher) { return $publisher; }, ['publisher']); @@ -150,9 +150,13 @@ Http::setResource('queueForWebhooks', function (Publisher $publisher) { Http::setResource('queueForRealtime', function () { return new Realtime(); }, []); -Http::setResource('queueForStatsUsage', function (Publisher $publisher) { - return new StatsUsage($publisher); -}, ['publisher']); +Http::setResource('usage', function () { + return new UsageContext(); +}, []); +Http::setResource('publisherForUsage', fn (Publisher $publisher) => new UsagePublisher( + $publisher, + new Queue(System::getEnv('_APP_STATS_USAGE_QUEUE_NAME', Event::STATS_USAGE_QUEUE_NAME)) +), ['publisher']); Http::setResource('queueForAudits', function (Publisher $publisher) { return new AuditEvent($publisher); }, ['publisher']); @@ -186,14 +190,14 @@ Http::setResource('allowedHostnames', function (array $platform, Document $proje $allowed = [...($platform['hostnames'] ?? [])]; /* Add platform configured hostnames */ - if (!$project->isEmpty() && $project->getId() !== 'console') { + if (! $project->isEmpty() && $project->getId() !== 'console') { $platforms = $project->getAttribute('platforms', []); $hostnames = Platform::getHostnames($platforms); $allowed = [...$allowed, ...$hostnames]; } /* Add the request hostname if a dev key is found */ - if (!$devKey->isEmpty()) { + if (! $devKey->isEmpty()) { $allowed[] = $request->getHostname(); } @@ -211,12 +215,12 @@ Http::setResource('allowedHostnames', function (array $platform, Document $proje } /* Allow the request origin of rule */ - if (!$rule->isEmpty() && !empty($rule->getAttribute('domain', ''))) { + if (! $rule->isEmpty() && ! empty($rule->getAttribute('domain', ''))) { $allowed[] = $rule->getAttribute('domain', ''); } /* Allow the request origin if a dev key is found */ - if (!$devKey->isEmpty() && !empty($hostname)) { + if (! $devKey->isEmpty() && ! empty($hostname)) { $allowed[] = $hostname; } @@ -229,7 +233,7 @@ Http::setResource('allowedHostnames', function (array $platform, Document $proje Http::setResource('allowedSchemes', function (array $platform, Document $project) { $allowed = [...($platform['schemas'] ?? [])]; - if (!$project->isEmpty() && $project->getId() !== 'console') { + if (! $project->isEmpty() && $project->getId() !== 'console') { /* Add hardcoded schemes */ $allowed[] = 'exp'; $allowed[] = 'appwrite-callback-' . $project->getId(); @@ -273,7 +277,7 @@ Http::setResource('rule', function (Request $request, Database $dbForPlatform, D // Temporary implementation until custom wildcard domains are an official feature // Allow trusted projects; Used for Console (website) previews - if (!$permitsCurrentProject && !$rule->isEmpty() && !empty($rule->getAttribute('projectId', ''))) { + if (! $permitsCurrentProject && ! $rule->isEmpty() && ! empty($rule->getAttribute('projectId', ''))) { $trustedProjects = []; foreach (\explode(',', System::getEnv('_APP_CONSOLE_TRUSTED_PROJECTS', '')) as $trustedProject) { if (empty($trustedProject)) { @@ -286,7 +290,7 @@ Http::setResource('rule', function (Request $request, Database $dbForPlatform, D } } - if (!$permitsCurrentProject) { + if (! $permitsCurrentProject) { return new Document(); } @@ -309,16 +313,18 @@ Http::setResource('cors', function (array $allowedHostnames) { }, ['allowedHostnames']); Http::setResource('originValidator', function (Document $devKey, array $allowedHostnames, array $allowedSchemes) { - if (!$devKey->isEmpty()) { + if (! $devKey->isEmpty()) { return new URL(); } + return new Origin($allowedHostnames, $allowedSchemes); }, ['devKey', 'allowedHostnames', 'allowedSchemes']); Http::setResource('redirectValidator', function (Document $devKey, array $allowedHostnames, array $allowedSchemes) { - if (!$devKey->isEmpty()) { + if (! $devKey->isEmpty()) { return new URL(); } + return new Redirect($allowedHostnames, $allowedSchemes); }, ['devKey', 'allowedHostnames', 'allowedSchemes']); @@ -342,12 +348,11 @@ Http::setResource('user', function (string $mode, Document $project, Document $c * overwriting the previous value. * 7. If account API key is passed, use user of the account API key as long as user ID header matches too */ - $authorization->setDefaultStatus(true); $store->setKey('a_session_' . $project->getId()); - if (APP_MODE_ADMIN === $mode) { + if ($mode === APP_MODE_ADMIN) { $store->setKey('a_session_' . $console->getId()); } @@ -362,7 +367,7 @@ Http::setResource('user', function (string $mode, Document $project, Document $c if (empty($store->getProperty('id', '')) && empty($store->getProperty('secret', ''))) { $sessionHeader = $request->getHeader('x-appwrite-session', ''); - if (!empty($sessionHeader)) { + if (! empty($sessionHeader)) { $store->decode($sessionHeader); } } @@ -382,14 +387,14 @@ Http::setResource('user', function (string $mode, Document $project, Document $c } $user = null; - if (APP_MODE_ADMIN === $mode) { + if ($mode === APP_MODE_ADMIN) { /** @var User $user */ $user = $dbForPlatform->getDocument('users', $store->getProperty('id', '')); } else { if ($project->isEmpty()) { $user = new User([]); } else { - if (!empty($store->getProperty('id', ''))) { + if (! empty($store->getProperty('id', ''))) { if ($project->getId() === 'console') { /** @var User $user */ $user = $dbForPlatform->getDocument('users', $store->getProperty('id', '')); @@ -402,16 +407,16 @@ Http::setResource('user', function (string $mode, Document $project, Document $c } if ( - !$user || + ! $user || $user->isEmpty() // Check a document has been found in the DB - || !$user->sessionVerify($store->getProperty('secret', ''), $proofForToken) + || ! $user->sessionVerify($store->getProperty('secret', ''), $proofForToken) ) { // Validate user has valid login token $user = new User([]); } $authJWT = $request->getHeader('x-appwrite-jwt', ''); - if (!empty($authJWT) && !$project->isEmpty()) { // JWT authentication - if (!$user->isEmpty()) { + if (! empty($authJWT) && ! $project->isEmpty()) { // JWT authentication + if (! $user->isEmpty()) { throw new Exception(Exception::USER_JWT_AND_COOKIE_SET); } @@ -423,7 +428,7 @@ Http::setResource('user', function (string $mode, Document $project, Document $c } $jwtUserId = $payload['userId'] ?? ''; - if (!empty($jwtUserId)) { + if (! empty($jwtUserId)) { if ($mode === APP_MODE_ADMIN) { $user = $dbForPlatform->getDocument('users', $jwtUserId); } else { @@ -431,7 +436,7 @@ Http::setResource('user', function (string $mode, Document $project, Document $c } } $jwtSessionId = $payload['sessionId'] ?? ''; - if (!empty($jwtSessionId)) { + if (! empty($jwtSessionId)) { if (empty($user->find('$id', $jwtSessionId, 'sessions'))) { // Match JWT to active token $user = new User([]); } @@ -441,22 +446,22 @@ Http::setResource('user', function (string $mode, Document $project, Document $c // Account based on account API key $accountKey = $request->getHeader('x-appwrite-key', ''); $accountKeyUserId = $request->getHeader('x-appwrite-user', ''); - if (!empty($accountKeyUserId) && !empty($accountKey)) { - if (!$user->isEmpty()) { + if (! empty($accountKeyUserId) && ! empty($accountKey)) { + if (! $user->isEmpty()) { throw new Exception(Exception::USER_API_KEY_AND_SESSION_SET); } $accountKeyUser = $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->getDocument('users', $accountKeyUserId)); - if (!$accountKeyUser->isEmpty()) { + if (! $accountKeyUser->isEmpty()) { $key = $accountKeyUser->find( key: 'secret', find: $accountKey, subject: 'keys' ); - if (!empty($key)) { + if (! empty($key)) { $expire = $key->getAttribute('expire'); - if (!empty($expire) && $expire < DatabaseDateTime::formatTz(DatabaseDateTime::now())) { + if (! empty($expire) && $expire < DatabaseDateTime::formatTz(DatabaseDateTime::now())) { throw new Exception(Exception::ACCOUNT_KEY_EXPIRED); } @@ -475,10 +480,9 @@ Http::setResource('project', function ($dbForPlatform, $request, $console, $auth /** @var Appwrite\Utopia\Request $request */ /** @var Utopia\Database\Database $dbForPlatform */ /** @var Utopia\Database\Document $console */ - $projectId = $request->getParam('project', $request->getHeader('x-appwrite-project', '')); // Realtime channel "project" can send project=Query array - if (!\is_string($projectId)) { + if (! \is_string($projectId)) { $projectId = $request->getHeader('x-appwrite-project', ''); } @@ -499,7 +503,7 @@ Http::setResource('session', function (User $user, Store $store, Token $proofFor $sessions = $user->getAttribute('sessions', []); $sessionId = $user->sessionVerify($store->getProperty('secret', ''), $proofForToken); - if (!$sessionId) { + if (! $sessionId) { return; } foreach ($sessions as $session) { @@ -509,7 +513,6 @@ Http::setResource('session', function (User $user, Store $store, Token $proofFor } } - return; }, ['user', 'store', 'proofForToken']); Http::setResource('store', function (): Store { @@ -533,12 +536,14 @@ Http::setResource('proofForPassword', function (): Password { Http::setResource('proofForToken', function (): Token { $token = new Token(); $token->setHash(new Sha()); + return $token; }); Http::setResource('proofForCode', function (): Code { $code = new Code(); $code->setHash(new Sha()); + return $code; }); @@ -550,7 +555,7 @@ Http::setResource('authorization', function () { return new Authorization(); }, []); -Http::setResource('dbForProject', function (Group $pools, Database $dbForPlatform, Cache $cache, Document $project, Response $response, Publisher $publisher, Publisher $publisherFunctions, Publisher $publisherWebhooks, Event $queueForEvents, Func $queueForFunctions, Webhook $queueForWebhooks, Realtime $queueForRealtime, StatsUsage $queueForStatsUsage, Authorization $authorization) { +Http::setResource('dbForProject', function (Group $pools, Database $dbForPlatform, Cache $cache, Document $project, Response $response, Publisher $publisher, Publisher $publisherFunctions, Publisher $publisherWebhooks, Event $queueForEvents, Func $queueForFunctions, Webhook $queueForWebhooks, Realtime $queueForRealtime, UsageContext $usage, Authorization $authorization) { if ($project->isEmpty() || $project->getId() === 'console') { return $dbForPlatform; } @@ -615,9 +620,8 @@ Http::setResource('dbForProject', function (Group $pools, Database $dbForPlatfor ->from($queueForEvents) ->trigger(); - /** Trigger webhooks events only if a project has them enabled */ - if (!empty($project->getAttribute('webhooks'))) { + if (! empty($project->getAttribute('webhooks'))) { $queueForWebhooks ->from($queueForEvents) ->trigger(); @@ -636,7 +640,6 @@ Http::setResource('dbForProject', function (Group $pools, Database $dbForPlatfor */ $functionsEventsCacheListener = function (string $event, Document $document, Document $project, Database $dbForProject) { - if ($document->getCollection() !== 'functions') { return; } @@ -658,7 +661,7 @@ Http::setResource('dbForProject', function (Group $pools, Database $dbForPlatfor $dbForProject->getCache()->purge($cacheKey); }; - $usageDatabaseListener = function (string $event, Document $document, StatsUsage $queueForStatsUsage) { + $usageDatabaseListener = function (string $event, Document $document, UsageContext $usage) { $value = 1; switch ($event) { @@ -678,81 +681,78 @@ Http::setResource('dbForProject', function (Group $pools, Database $dbForPlatfor switch (true) { case $document->getCollection() === 'teams': - $queueForStatsUsage->addMetric(METRIC_TEAMS, $value); // per project + $usage->addMetric(METRIC_TEAMS, $value); // per project break; case $document->getCollection() === 'users': - $queueForStatsUsage->addMetric(METRIC_USERS, $value); // per project + $usage->addMetric(METRIC_USERS, $value); // per project if ($event === Database::EVENT_DOCUMENT_DELETE) { - $queueForStatsUsage->addReduce($document); + $usage->addReduce($document); } break; case $document->getCollection() === 'sessions': // sessions - $queueForStatsUsage->addMetric(METRIC_SESSIONS, $value); //per project + $usage->addMetric(METRIC_SESSIONS, $value); // per project break; case $document->getCollection() === 'databases': // databases - $queueForStatsUsage->addMetric(METRIC_DATABASES, $value); // per project + $usage->addMetric(METRIC_DATABASES, $value); // per project if ($event === Database::EVENT_DOCUMENT_DELETE) { - $queueForStatsUsage->addReduce($document); + $usage->addReduce($document); } break; - case str_starts_with($document->getCollection(), 'database_') && !str_contains($document->getCollection(), 'collection'): //collections + case str_starts_with($document->getCollection(), 'database_') && ! str_contains($document->getCollection(), 'collection'): // collections $parts = explode('_', $document->getCollection()); $databaseInternalId = $parts[1] ?? 0; - $queueForStatsUsage + $usage ->addMetric(METRIC_COLLECTIONS, $value) // per project ->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_COLLECTIONS), $value); if ($event === Database::EVENT_DOCUMENT_DELETE) { - $queueForStatsUsage->addReduce($document); + $usage->addReduce($document); } break; - case str_starts_with($document->getCollection(), 'database_') && str_contains($document->getCollection(), '_collection_'): //documents + case str_starts_with($document->getCollection(), 'database_') && str_contains($document->getCollection(), '_collection_'): // documents $parts = explode('_', $document->getCollection()); - $databaseInternalId = $parts[1] ?? 0; + $databaseInternalId = $parts[1] ?? 0; $collectionInternalId = $parts[3] ?? 0; - $queueForStatsUsage + $usage ->addMetric(METRIC_DOCUMENTS, $value) // per project ->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_DOCUMENTS), $value) // per database ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $collectionInternalId], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS), $value); // per collection break; - case $document->getCollection() === 'buckets': //buckets - $queueForStatsUsage - ->addMetric(METRIC_BUCKETS, $value); // per project + case $document->getCollection() === 'buckets': // buckets + $usage->addMetric(METRIC_BUCKETS, $value); // per project if ($event === Database::EVENT_DOCUMENT_DELETE) { - $queueForStatsUsage + $usage ->addReduce($document); } break; case str_starts_with($document->getCollection(), 'bucket_'): // files $parts = explode('_', $document->getCollection()); - $bucketInternalId = $parts[1]; - $queueForStatsUsage + $bucketInternalId = $parts[1]; + $usage ->addMetric(METRIC_FILES, $value) // per project ->addMetric(METRIC_FILES_STORAGE, $document->getAttribute('sizeOriginal') * $value) // per project ->addMetric(str_replace('{bucketInternalId}', $bucketInternalId, METRIC_BUCKET_ID_FILES), $value) // per bucket ->addMetric(str_replace('{bucketInternalId}', $bucketInternalId, METRIC_BUCKET_ID_FILES_STORAGE), $document->getAttribute('sizeOriginal') * $value); // per bucket break; case $document->getCollection() === 'functions': - $queueForStatsUsage - ->addMetric(METRIC_FUNCTIONS, $value); // per project + $usage->addMetric(METRIC_FUNCTIONS, $value); // per project if ($event === Database::EVENT_DOCUMENT_DELETE) { - $queueForStatsUsage + $usage ->addReduce($document); } break; case $document->getCollection() === 'sites': - $queueForStatsUsage - ->addMetric(METRIC_SITES, $value); // per project + $usage->addMetric(METRIC_SITES, $value); // per project if ($event === Database::EVENT_DOCUMENT_DELETE) { - $queueForStatsUsage + $usage ->addReduce($document); } break; case $document->getCollection() === 'deployments': - $queueForStatsUsage + $usage ->addMetric(METRIC_DEPLOYMENTS, $value) // per project ->addMetric(METRIC_DEPLOYMENTS_STORAGE, $document->getAttribute('size') * $value) // per project ->addMetric(str_replace(['{resourceType}'], [$document->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_DEPLOYMENTS), $value) // per function @@ -772,30 +772,27 @@ Http::setResource('dbForProject', function (Group $pools, Database $dbForPlatfor $queueForWebhooks = new Webhook($publisherWebhooks); $queueForRealtime = new Realtime(); - $database - ->on(Database::EVENT_DOCUMENT_CREATE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $queueForStatsUsage)) - ->on(Database::EVENT_DOCUMENT_DELETE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $queueForStatsUsage)) - ->on(Database::EVENT_DOCUMENTS_CREATE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $queueForStatsUsage)) - ->on(Database::EVENT_DOCUMENTS_DELETE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $queueForStatsUsage)) - ->on(Database::EVENT_DOCUMENTS_UPSERT, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $queueForStatsUsage)) - ->on(Database::EVENT_DOCUMENT_CREATE, 'create-trigger-events', fn ($event, $document) => $eventDatabaseListener( - $project, - $document, - $response, - $queueForEventsClone->from($queueForEvents), - $queueForFunctions->from($queueForEvents), - $queueForWebhooks->from($queueForEvents), - $queueForRealtime->from($queueForEvents) - )) - ->on(Database::EVENT_DOCUMENT_CREATE, 'purge-function-events-cache', fn ($event, $document) => $functionsEventsCacheListener($event, $document, $project, $database)) - ->on(Database::EVENT_DOCUMENT_UPDATE, 'purge-function-events-cache', fn ($event, $document) => $functionsEventsCacheListener($event, $document, $project, $database)) - ->on(Database::EVENT_DOCUMENT_DELETE, 'purge-function-events-cache', fn ($event, $document) => $functionsEventsCacheListener($event, $document, $project, $database)) - ; - + ->on(Database::EVENT_DOCUMENT_CREATE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $usage)) + ->on(Database::EVENT_DOCUMENT_DELETE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $usage)) + ->on(Database::EVENT_DOCUMENTS_CREATE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $usage)) + ->on(Database::EVENT_DOCUMENTS_DELETE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $usage)) + ->on(Database::EVENT_DOCUMENTS_UPSERT, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $usage)) + ->on(Database::EVENT_DOCUMENT_CREATE, 'create-trigger-events', fn ($event, $document) => $eventDatabaseListener( + $project, + $document, + $response, + $queueForEventsClone->from($queueForEvents), + $queueForFunctions->from($queueForEvents), + $queueForWebhooks->from($queueForEvents), + $queueForRealtime->from($queueForEvents) + )) + ->on(Database::EVENT_DOCUMENT_CREATE, 'purge-function-events-cache', fn ($event, $document) => $functionsEventsCacheListener($event, $document, $project, $database)) + ->on(Database::EVENT_DOCUMENT_UPDATE, 'purge-function-events-cache', fn ($event, $document) => $functionsEventsCacheListener($event, $document, $project, $database)) + ->on(Database::EVENT_DOCUMENT_DELETE, 'purge-function-events-cache', fn ($event, $document) => $functionsEventsCacheListener($event, $document, $project, $database)); return $database; -}, ['pools', 'dbForPlatform', 'cache', 'project', 'response', 'publisher', 'publisherFunctions', 'publisherWebhooks', 'queueForEvents', 'queueForFunctions', 'queueForWebhooks', 'queueForRealtime', 'queueForStatsUsage', 'authorization']); +}, ['pools', 'dbForPlatform', 'cache', 'project', 'response', 'publisher', 'publisherFunctions', 'publisherWebhooks', 'queueForEvents', 'queueForFunctions', 'queueForWebhooks', 'queueForRealtime', 'usage', 'authorization']); Http::setResource('dbForPlatform', function (Group $pools, Cache $cache, Authorization $authorization) { @@ -844,8 +841,7 @@ Http::setResource('getProjectDB', function (Group $pools, Database $dbForPlatfor ->setMetadata('project', $project->getId()) ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_API) ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES) - ->setDocumentType('users', User::class) - ; + ->setDocumentType('users', User::class); $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); @@ -865,6 +861,7 @@ Http::setResource('getProjectDB', function (Group $pools, Database $dbForPlatfor if (isset($databases[$dsn->getHost()])) { $database = $databases[$dsn->getHost()]; $configure($database); + return $database; } @@ -881,8 +878,9 @@ Http::setResource('getLogsDB', function (Group $pools, Cache $cache, Authorizati $database = null; return function (?Document $project = null) use ($pools, $cache, $authorization, &$database) { - if ($database !== null && $project !== null && !$project->isEmpty() && $project->getId() !== 'console') { + if ($database !== null && $project !== null && ! $project->isEmpty() && $project->getId() !== 'console') { $database->setTenant((int) $project->getSequence()); + return $database; } @@ -898,7 +896,7 @@ Http::setResource('getLogsDB', function (Group $pools, Cache $cache, Authorizati ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES); // set tenant - if ($project !== null && !$project->isEmpty() && $project->getId() !== 'console') { + if ($project !== null && ! $project->isEmpty() && $project->getId() !== 'console') { $database->setTenant((int) $project->getSequence()); } @@ -908,6 +906,7 @@ Http::setResource('getLogsDB', function (Group $pools, Cache $cache, Authorizati Http::setResource('audit', function ($dbForProject) { $adapter = new AdapterDatabase($dbForProject); + return new Audit($adapter); }, ['dbForProject']); @@ -923,6 +922,7 @@ Http::setResource('cache', function (Group $pools, Telemetry $telemetry) { $cache = new Cache(new Sharding($adapters)); $cache->setTelemetry($telemetry); + return $cache; }, ['pools', 'telemetry']); @@ -968,9 +968,9 @@ Http::setResource('deviceForBuilds', function ($project, Telemetry $telemetry) { function getDevice(string $root, string $connection = ''): Device { - $connection = !empty($connection) ? $connection : System::getEnv('_APP_CONNECTIONS_STORAGE', ''); + $connection = ! empty($connection) ? $connection : System::getEnv('_APP_CONNECTIONS_STORAGE', ''); - if (!empty($connection)) { + if (! empty($connection)) { $acl = 'private'; $device = Storage::DEVICE_LOCAL; $accessKey = ''; @@ -992,8 +992,9 @@ function getDevice(string $root, string $connection = ''): Device switch ($device) { case Storage::DEVICE_S3: - if (!empty($url)) { - $bucketRoot = (!empty($bucket) ? $bucket . '/' : '') . \ltrim($root, '/'); + if (! empty($url)) { + $bucketRoot = (! empty($bucket) ? $bucket . '/' : '') . \ltrim($root, '/'); + return new S3($bucketRoot, $accessKey, $accessSecret, $url, $region, $acl); } else { return new AWS($root, $accessKey, $accessSecret, $bucket, $region, $acl); @@ -1002,6 +1003,7 @@ function getDevice(string $root, string $connection = ''): Device case STORAGE::DEVICE_DO_SPACES: $device = new DOSpaces($root, $accessKey, $accessSecret, $bucket, $region, $acl); $device->setHttpVersion(S3::HTTP_VERSION_1_1); + return $device; case Storage::DEVICE_BACKBLAZE: return new Backblaze($root, $accessKey, $accessSecret, $bucket, $region, $acl); @@ -1025,8 +1027,9 @@ function getDevice(string $root, string $connection = ''): Device $s3Bucket = System::getEnv('_APP_STORAGE_S3_BUCKET', ''); $s3Acl = 'private'; $s3EndpointUrl = System::getEnv('_APP_STORAGE_S3_ENDPOINT', ''); - if (!empty($s3EndpointUrl)) { - $bucketRoot = (!empty($s3Bucket) ? $s3Bucket . '/' : '') . \ltrim($root, '/'); + if (! empty($s3EndpointUrl)) { + $bucketRoot = (! empty($s3Bucket) ? $s3Bucket . '/' : '') . \ltrim($root, '/'); + return new S3($bucketRoot, $s3AccessKey, $s3SecretKey, $s3EndpointUrl, $s3Region, $s3Acl); } else { return new AWS($root, $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl); @@ -1040,6 +1043,7 @@ function getDevice(string $root, string $connection = ''): Device $doSpacesAcl = 'private'; $device = new DOSpaces($root, $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl); $device->setHttpVersion(S3::HTTP_VERSION_1_1); + return $device; case Storage::DEVICE_BACKBLAZE: $backblazeAccessKey = System::getEnv('_APP_STORAGE_BACKBLAZE_ACCESS_KEY', ''); @@ -1047,6 +1051,7 @@ function getDevice(string $root, string $connection = ''): Device $backblazeRegion = System::getEnv('_APP_STORAGE_BACKBLAZE_REGION', ''); $backblazeBucket = System::getEnv('_APP_STORAGE_BACKBLAZE_BUCKET', ''); $backblazeAcl = 'private'; + return new Backblaze($root, $backblazeAccessKey, $backblazeSecretKey, $backblazeBucket, $backblazeRegion, $backblazeAcl); case Storage::DEVICE_LINODE: $linodeAccessKey = System::getEnv('_APP_STORAGE_LINODE_ACCESS_KEY', ''); @@ -1054,6 +1059,7 @@ function getDevice(string $root, string $connection = ''): Device $linodeRegion = System::getEnv('_APP_STORAGE_LINODE_REGION', ''); $linodeBucket = System::getEnv('_APP_STORAGE_LINODE_BUCKET', ''); $linodeAcl = 'private'; + return new Linode($root, $linodeAccessKey, $linodeSecretKey, $linodeBucket, $linodeRegion, $linodeAcl); case Storage::DEVICE_WASABI: $wasabiAccessKey = System::getEnv('_APP_STORAGE_WASABI_ACCESS_KEY', ''); @@ -1061,6 +1067,7 @@ function getDevice(string $root, string $connection = ''): Device $wasabiRegion = System::getEnv('_APP_STORAGE_WASABI_REGION', ''); $wasabiBucket = System::getEnv('_APP_STORAGE_WASABI_BUCKET', ''); $wasabiAcl = 'private'; + return new Wasabi($root, $wasabiAccessKey, $wasabiSecretKey, $wasabiBucket, $wasabiRegion, $wasabiAcl); } } @@ -1087,7 +1094,6 @@ Http::setResource('passwordsDictionary', function ($register) { return $register->get('passwordsDictionary'); }, ['register']); - Http::setResource('servers', function () { $platforms = Config::getParam('sdks'); $server = $platforms[APP_SDK_PLATFORM_SERVER]; @@ -1195,16 +1201,17 @@ Http::setResource('gitHub', function (Cache $cache) { }, ['cache']); Http::setResource('requestTimestamp', function ($request) { - //TODO: Move this to the Request class itself + // TODO: Move this to the Request class itself $timestampHeader = $request->getHeader('x-appwrite-timestamp'); $requestTimestamp = null; - if (!empty($timestampHeader)) { + if (! empty($timestampHeader)) { try { $requestTimestamp = new \DateTime($timestampHeader); } catch (\Throwable $e) { throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'Invalid X-Appwrite-Timestamp header value'); } } + return $requestTimestamp; }, ['request']); @@ -1221,13 +1228,13 @@ Http::setResource('devKey', function (Request $request, Document $project, array // Check if given key match project's development keys $key = $project->find('secret', $devKey, 'devKeys'); - if (!$key) { + if (! $key) { return new Document([]); } // check expiration $expire = $key->getAttribute('expire'); - if (!empty($expire) && $expire < DatabaseDateTime::formatTz(DatabaseDateTime::now())) { + if (! empty($expire) && $expire < DatabaseDateTime::formatTz(DatabaseDateTime::now())) { return new Document([]); } @@ -1248,7 +1255,7 @@ Http::setResource('devKey', function (Request $request, Document $project, array if ($sdk !== 'UNKNOWN' && $sdkValidator->isValid($sdk)) { $sdks = $key->getAttribute('sdks', []); - if (!in_array($sdk, $sdks)) { + if (! in_array($sdk, $sdks)) { $sdks[] = $sdk; $key->setAttribute('sdks', $sdks); @@ -1271,7 +1278,7 @@ Http::setResource('team', function (Document $project, Database $dbForPlatform, $teamInternalId = $project->getAttribute('teamInternalId', ''); } else { $route = $utopia->match($request); - $path = !empty($route) ? $route->getPath() : $request->getURI(); + $path = ! empty($route) ? $route->getPath() : $request->getURI(); $orgHeader = $request->getHeader('x-appwrite-organization', ''); if (str_starts_with($path, '/v1/projects/:projectId')) { $uri = $request->getURI(); @@ -1286,8 +1293,9 @@ Http::setResource('team', function (Document $project, Database $dbForPlatform, } $team = $authorization->skip(fn () => $dbForPlatform->getDocument('teams', $teamId)); + return $team; - } elseif (!empty($orgHeader)) { + } elseif (! empty($orgHeader)) { return $authorization->skip(fn () => $dbForPlatform->getDocument('teams', $orgHeader)); } } @@ -1317,13 +1325,13 @@ Http::setResource('previewHostname', function (Request $request, ?Key $apiKey) { if (Http::isDevelopment()) { $allowed = true; - } elseif (!\is_null($apiKey) && $apiKey->getHostnameOverride() === true) { + } elseif (! \is_null($apiKey) && $apiKey->getHostnameOverride() === true) { $allowed = true; } if ($allowed) { $host = $request->getQuery('appwrite-hostname', $request->getHeader('x-appwrite-hostname', '')) ?? ''; - if (!empty($host)) { + if (! empty($host)) { return $host; } } @@ -1344,19 +1352,19 @@ Http::setResource('apiKey', function (Request $request, Document $project, Docum $organizationHeader = $request->getHeader('x-appwrite-organization'); $projectHeader = $request->getHeader('x-appwrite-project'); - if (!empty($key->getProjectId())) { + if (! empty($key->getProjectId())) { if (empty($projectHeader) || $projectHeader !== $key->getProjectId()) { throw new Exception(Exception::PROJECT_ID_MISSING); } } - if (!empty($key->getUserId())) { + if (! empty($key->getUserId())) { if (empty($userHeader) || $userHeader !== $key->getUserId()) { throw new Exception(Exception::USER_ID_MISSING); } } - if (!empty($key->getTeamId())) { + if (! empty($key->getTeamId())) { if (empty($organizationHeader) || $organizationHeader !== $key->getTeamId()) { throw new Exception(Exception::ORGANIZATION_ID_MISSING); } @@ -1370,7 +1378,7 @@ Http::setResource('executor', fn () => new Executor()); Http::setResource('resourceToken', function ($project, $dbForProject, $request, Authorization $authorization) { $tokenJWT = $request->getParam('token'); - if (!empty($tokenJWT) && !$project->isEmpty()) { // JWT authentication + if (! empty($tokenJWT) && ! $project->isEmpty()) { // JWT authentication // Use a large but reasonable maxAge to avoid auto-exp when token has no expiry $jwt = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), RESOURCE_TOKEN_ALGORITHM, RESOURCE_TOKEN_MAX_AGE, RESOURCE_TOKEN_LEEWAY); // Instantiate with key, algo, maxAge and leeway. @@ -1430,6 +1438,7 @@ Http::setResource('resourceToken', function ($project, $dbForProject, $request, default => throw new Exception(Exception::TOKEN_RESOURCE_TYPE_INVALID), }; } + return new Document([]); }, ['project', 'dbForProject', 'request', 'authorization']); diff --git a/app/worker.php b/app/worker.php index 2ee1803ddc..db036b6a99 100644 --- a/app/worker.php +++ b/app/worker.php @@ -13,11 +13,12 @@ use Appwrite\Event\Func; use Appwrite\Event\Mail; use Appwrite\Event\Messaging; use Appwrite\Event\Migration; +use Appwrite\Event\Publisher\Usage as UsagePublisher; use Appwrite\Event\Realtime; use Appwrite\Event\Screenshot; -use Appwrite\Event\StatsUsage; use Appwrite\Event\Webhook; use Appwrite\Platform\Appwrite; +use Appwrite\Usage\Context; use Appwrite\Utopia\Database\Documents\User; use Executor\Executor; use Swoole\Runtime; @@ -42,6 +43,7 @@ use Utopia\Pools\Group; use Utopia\Queue\Broker\Pool as BrokerPool; use Utopia\Queue\Message; use Utopia\Queue\Publisher; +use Utopia\Queue\Queue; use Utopia\Queue\Server; use Utopia\Registry\Registry; use Utopia\Storage\Device\Telemetry as TelemetryDevice; @@ -58,7 +60,8 @@ Server::setResource('register', fn () => $register); Server::setResource('authorization', function () { $authorization = new Authorization(); $authorization->disable(); - return $authorization; + + return $authorization; }, []); Server::setResource('dbForPlatform', function (Cache $cache, Registry $register, Authorization $authorization) { @@ -70,9 +73,7 @@ Server::setResource('dbForPlatform', function (Cache $cache, Registry $register, ->setDatabase(APP_DATABASE) ->setAuthorization($authorization) ->setNamespace('_console') - ->setDocumentType('users', User::class) - ; - + ->setDocumentType('users', User::class); return $dbForPlatform; }, ['cache', 'register', 'authorization']); @@ -111,7 +112,7 @@ Server::setResource('dbForProject', function (Cache $cache, Registry $register, if (\in_array($dsn->getHost(), $sharedTables)) { $database ->setSharedTables(true) - ->setTenant((int)$project->getSequence()) + ->setTenant((int) $project->getSequence()) ->setNamespace($dsn->getParam('namespace')); } else { $database @@ -151,7 +152,7 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForPlatf if (\in_array($dsn->getHost(), $sharedTables)) { $database ->setSharedTables(true) - ->setTenant((int)$project->getSequence()) + ->setTenant((int) $project->getSequence()) ->setNamespace($dsn->getParam('namespace')); } else { $database @@ -173,7 +174,7 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForPlatf if (\in_array($dsn->getHost(), $sharedTables)) { $database ->setSharedTables(true) - ->setTenant((int)$project->getSequence()) + ->setTenant((int) $project->getSequence()) ->setNamespace($dsn->getParam('namespace')); } else { $database @@ -193,9 +194,11 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForPlatf Server::setResource('getLogsDB', function (Group $pools, Cache $cache, Authorization $authorization) { $database = null; + return function (?Document $project = null) use ($pools, $cache, $database, $authorization) { - if ($database !== null && $project !== null && !$project->isEmpty() && $project->getId() !== 'console') { - $database->setTenant((int)$project->getSequence()); + if ($database !== null && $project !== null && ! $project->isEmpty() && $project->getId() !== 'console') { + $database->setTenant((int) $project->getSequence()); + return $database; } @@ -211,8 +214,8 @@ Server::setResource('getLogsDB', function (Group $pools, Cache $cache, Authoriza ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES_WORKER); // set tenant - if ($project !== null && !$project->isEmpty() && $project->getId() !== 'console') { - $database->setTenant((int)$project->getSequence()); + if ($project !== null && ! $project->isEmpty() && $project->getId() !== 'console') { + $database->setTenant((int) $project->getSequence()); } return $database; @@ -227,6 +230,7 @@ Server::setResource('auditRetention', function (Document $project) { if ($project->getId() === 'console') { return DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE', 15778800)); // 6 months } + return DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 1209600)); // 14 days }, ['project']); @@ -252,7 +256,7 @@ Server::setResource('redis', function () { $pass = System::getEnv('_APP_REDIS_PASS', ''); $redis = new \Redis(); - @$redis->pconnect($host, (int)$port); + @$redis->pconnect($host, (int) $port); if ($pass) { $redis->auth($pass); } @@ -269,7 +273,6 @@ Server::setResource('timelimit', function (\Redis $redis) { Server::setResource('log', fn () => new Log()); - Server::setResource('publisher', function (Group $pools) { return new BrokerPool(publisher: $pools->get('publisher')); }, ['pools']); @@ -286,10 +289,6 @@ Server::setResource('publisherMigrations', function (BrokerPool $publisher) { return $publisher; }, ['publisher']); -Server::setResource('publisherStatsUsage', function (BrokerPool $publisher) { - return $publisher; -}, ['publisher']); - Server::setResource('publisherMessaging', function (BrokerPool $publisher) { return $publisher; }, ['publisher']); @@ -310,9 +309,13 @@ Server::setResource('consumerStatsUsage', function (BrokerPool $consumer) { return $consumer; }, ['consumer']); -Server::setResource('queueForStatsUsage', function (Publisher $publisher) { - return new StatsUsage($publisher); -}, ['publisher']); +Server::setResource('usage', function () { + return new Context(); +}, []); +Server::setResource('publisherForUsage', fn (Publisher $publisher) => new UsagePublisher( + $publisher, + new Queue(System::getEnv('_APP_STATS_USAGE_QUEUE_NAME', Event::STATS_USAGE_QUEUE_NAME)) +), ['publisher']); Server::setResource('queueForDatabase', function (Publisher $publisher) { return new EventDatabase($publisher); @@ -354,7 +357,6 @@ Server::setResource('queueForFunctions', function (Publisher $publisher) { return new Func($publisher); }, ['publisher']); - Server::setResource('queueForRealtime', function () { return new Realtime(); }, []); @@ -484,11 +486,13 @@ Server::setResource('getAudit', function (Database $dbForPlatform, callable $get return function (Document $project) use ($dbForPlatform, $getProjectDB) { if ($project->isEmpty() || $project->getId() === 'console') { $adapter = new AdapterDatabase($dbForPlatform); + return new UtopiaAudit($adapter); } $dbForProject = $getProjectDB($project); $adapter = new AdapterDatabase($dbForProject); + return new UtopiaAudit($adapter); }; }, ['dbForPlatform', 'getProjectDB']); @@ -505,7 +509,7 @@ $pools = $register->get('pools'); $platform = new Appwrite(); $args = $platform->getEnv('argv'); -if (!isset($args[1])) { +if (! isset($args[1])) { Console::error('Missing worker name'); Console::exit(1); } @@ -530,10 +534,10 @@ try { 'workersNum' => System::getEnv('_APP_WORKERS_NUM', 1), 'connection' => $pools->get('consumer')->pop()->getResource(), 'workerName' => strtolower($workerName) ?? null, - 'queueName' => $queueName + 'queueName' => $queueName, ]); } catch (\Throwable $e) { - Console::error($e->getMessage() . ', File: ' . $e->getFile() . ', Line: ' . $e->getLine()); + Console::error($e->getMessage() . ', File: ' . $e->getFile() . ', Line: ' . $e->getLine()); } $worker = $platform->getWorker(); @@ -550,11 +554,11 @@ $worker ->inject('pools') ->inject('project') ->inject('authorization') - ->action(function (Throwable $error, ?Logger $logger, Log $log, Group $pools, Document $project, Authorization $authorization) use ($worker, $queueName) { + ->action(function (Throwable $error, ?Logger $logger, Log $log, Group $pools, Document $project, Authorization $authorization) use ($queueName) { $version = System::getEnv('_APP_VERSION', 'UNKNOWN'); if ($logger) { - $log->setNamespace("appwrite-worker"); + $log->setNamespace('appwrite-worker'); $log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname())); $log->setVersion($version); $log->setType(Log::TYPE_ERROR); diff --git a/src/Appwrite/Bus/Listeners/Usage.php b/src/Appwrite/Bus/Listeners/Usage.php index 219287033d..48178f1dee 100644 --- a/src/Appwrite/Bus/Listeners/Usage.php +++ b/src/Appwrite/Bus/Listeners/Usage.php @@ -4,11 +4,12 @@ namespace Appwrite\Bus\Listeners; use Appwrite\Bus\Events\ExecutionCompleted; use Appwrite\Bus\Events\RequestCompleted; -use Appwrite\Event\StatsUsage; +use Appwrite\Event\Message\Usage as UsageMessage; +use Appwrite\Event\Publisher\Usage as Publisher; +use Appwrite\Usage\Context; use Utopia\Bus\Event; use Utopia\Bus\Listener; use Utopia\Database\Document; -use Utopia\Queue\Publisher; class Usage extends Listener { @@ -29,20 +30,21 @@ class Usage extends Listener { $this ->desc('Records usage metrics') - ->inject('publisherStatsUsage') + ->inject('publisherForUsage') + ->inject('usage') ->callback($this->handle(...)); } - public function handle(Event $event, Publisher $publisher): void + public function handle(Event $event, Publisher $publisherForUsage, Context $usage): void { match (true) { - $event instanceof ExecutionCompleted => $this->handleExecutionCompleted($event, $publisher), - $event instanceof RequestCompleted => $this->handleRequestCompleted($event, $publisher), + $event instanceof ExecutionCompleted => $this->handleExecutionCompleted($event, $publisherForUsage), + $event instanceof RequestCompleted => $this->handleRequestCompleted($event, $usage), default => null, }; } - private function handleExecutionCompleted(ExecutionCompleted $event, Publisher $publisher): void + private function handleExecutionCompleted(ExecutionCompleted $event, Publisher $publisherForUsage): void { $execution = new Document($event->execution); $resource = new Document($event->resource); @@ -61,9 +63,7 @@ class Usage extends Listener $compute = (int)($duration * 1000); $mbSeconds = (int)(($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT) * $duration * ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT)); - $queueForStatsUsage = new StatsUsage($publisher); - $queueForStatsUsage - ->setProject($project) + $context = (new Context()) ->addMetric(METRIC_EXECUTIONS, 1) ->addMetric(str_replace(['{resourceType}'], [$resourceType], METRIC_RESOURCE_TYPE_EXECUTIONS), 1) ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$resourceType, $resourceInternalId], METRIC_RESOURCE_TYPE_ID_EXECUTIONS), 1) @@ -72,11 +72,18 @@ class Usage extends Listener ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$resourceType, $resourceInternalId], METRIC_RESOURCE_TYPE_ID_EXECUTIONS_COMPUTE), $compute) ->addMetric(METRIC_EXECUTIONS_MB_SECONDS, $mbSeconds) ->addMetric(str_replace(['{resourceType}'], [$resourceType], METRIC_RESOURCE_TYPE_EXECUTIONS_MB_SECONDS), $mbSeconds) - ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$resourceType, $resourceInternalId], METRIC_RESOURCE_TYPE_ID_EXECUTIONS_MB_SECONDS), $mbSeconds) - ->trigger(); + ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$resourceType, $resourceInternalId], METRIC_RESOURCE_TYPE_ID_EXECUTIONS_MB_SECONDS), $mbSeconds); + + $message = new UsageMessage( + project: $project, + metrics: $context->getMetrics(), + reduce: $context->getReduce() + ); + + $publisherForUsage->enqueue($message); } - private function handleRequestCompleted(RequestCompleted $event, Publisher $publisher): void + private function handleRequestCompleted(RequestCompleted $event, Context $usage): void { $fileSize = 0; $file = $event->request->getFiles('file'); @@ -84,18 +91,14 @@ class Usage extends Listener $fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size']; } - $project = new Document($event->project); $deployment = new Document($event->deployment); - $queueForStatsUsage = new StatsUsage($publisher); $inbound = $event->request->getSize() + $fileSize; $outbound = $event->response->getSize(); - $queueForStatsUsage->setProject($project); - if ($deployment->getAttribute('resourceType') === 'sites') { $siteInternalId = $deployment->getAttribute('resourceInternalId', ''); - $queueForStatsUsage + $usage ->addMetric(METRIC_SITES_REQUESTS, 1) ->addMetric(METRIC_SITES_INBOUND, $inbound) ->addMetric(METRIC_SITES_OUTBOUND, $outbound) @@ -103,12 +106,10 @@ class Usage extends Listener ->addMetric(str_replace('{siteInternalId}', $siteInternalId, METRIC_SITES_ID_INBOUND), $inbound) ->addMetric(str_replace('{siteInternalId}', $siteInternalId, METRIC_SITES_ID_OUTBOUND), $outbound); } else { - $queueForStatsUsage + $usage ->addMetric(METRIC_NETWORK_REQUESTS, 1) ->addMetric(METRIC_NETWORK_INBOUND, $inbound) ->addMetric(METRIC_NETWORK_OUTBOUND, $outbound); } - - $queueForStatsUsage->trigger(); } } diff --git a/src/Appwrite/Event/Message/Base.php b/src/Appwrite/Event/Message/Base.php new file mode 100644 index 0000000000..38b6d5edee --- /dev/null +++ b/src/Appwrite/Event/Message/Base.php @@ -0,0 +1,21 @@ + $metrics + * @param array $reduce + */ + public function __construct( + public readonly Document $project, + public readonly array $metrics, + public readonly array $reduce = [], + ) { + } + + /** + * @return array + */ + public function toArray(): array + { + return [ + 'project' => [ + '$id' => $this->project->getId(), + '$sequence' => $this->project->getSequence(), + 'database' => $this->project->getAttribute('database', ''), + ], + 'metrics' => $this->metrics, + 'reduce' => array_map(fn (Document $doc) => $doc->getArrayCopy(), $this->reduce), + ]; + } + + /** + * @param array $data + * @return static + */ + public static function fromArray(array $data): static + { + return new self( + project: new Document($data['project'] ?? []), + metrics: $data['metrics'] ?? [], + reduce: array_map(fn (array $doc) => new Document($doc), $data['reduce'] ?? []), + ); + } +} diff --git a/src/Appwrite/Event/Publisher/Base.php b/src/Appwrite/Event/Publisher/Base.php new file mode 100644 index 0000000000..2063864723 --- /dev/null +++ b/src/Appwrite/Event/Publisher/Base.php @@ -0,0 +1,33 @@ +toArray(); + + return $this->publisher->enqueue($queue, $payload); + } + + /** + * Get the size of a queue + */ + public function getQueueSize(Queue $queue, bool $failed = false): int + { + return $this->publisher->getQueueSize($queue, $failed); + } +} diff --git a/src/Appwrite/Event/Publisher/Usage.php b/src/Appwrite/Event/Publisher/Usage.php new file mode 100644 index 0000000000..104690671b --- /dev/null +++ b/src/Appwrite/Event/Publisher/Usage.php @@ -0,0 +1,39 @@ +publish($this->queue, $message); + } catch (\Throwable $th) { + Console::error('[Usage] Failed to publish usage message: ' . $th->getMessage()); + return false; + } + } + + /** + * Get the size of the usage queue + */ + public function getSize(bool $failed = false): int + { + return $this->getQueueSize($this->queue, $failed); + } +} diff --git a/src/Appwrite/Event/StatsUsage.php b/src/Appwrite/Event/StatsUsage.php deleted file mode 100644 index a944d70c94..0000000000 --- a/src/Appwrite/Event/StatsUsage.php +++ /dev/null @@ -1,96 +0,0 @@ -setQueue(System::getEnv('_APP_STATS_USAGE_QUEUE_NAME', Event::STATS_USAGE_QUEUE_NAME)) - ->setClass(System::getEnv('_APP_STATS_USAGE_CLASS_NAME', Event::STATS_USAGE_CLASS_NAME)); - } - - /** - * Add reduce. - * - * @param Document $document - * @return self - */ - public function addReduce(Document $document): self - { - $this->reduce[] = $document; - - return $this; - } - - /** - * Add metric. - * - * @param string $key - * @param int $value - * @return self - */ - public function addMetric(string $key, int $value): self - { - $this->metrics[] = [ - 'key' => $key, - 'value' => $value, - ]; - - return $this; - } - - /** - * Set disabled metrics. - * - * @param string $key - * @return self - */ - public function disableMetric(string $key): self - { - $this->disabled[] = $key; - - return $this; - } - - /** - * Prepare the payload for the event - * - * @return array - */ - protected function preparePayload(): array - { - return [ - 'project' => $this->getProject(), - 'reduce' => $this->reduce, - 'metrics' => \array_filter($this->metrics, function ($metric) { - foreach ($this->disabled as $disabledMetric) { - if (\str_ends_with($metric['key'], $disabledMetric)) { - return false; - } - } - return true; - }), - ]; - } - - public function reset(): Event - { - $this->metrics = []; - parent::reset(); - return $this; - } -} diff --git a/src/Appwrite/Platform/Modules/Account/Http/Account/MFA/Challenges/Create.php b/src/Appwrite/Platform/Modules/Account/Http/Account/MFA/Challenges/Create.php index cde6b90fd6..20a6afed2e 100644 --- a/src/Appwrite/Platform/Modules/Account/Http/Account/MFA/Challenges/Create.php +++ b/src/Appwrite/Platform/Modules/Account/Http/Account/MFA/Challenges/Create.php @@ -7,7 +7,6 @@ use Appwrite\Detector\Detector; use Appwrite\Event\Event; use Appwrite\Event\Mail; use Appwrite\Event\Messaging; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; @@ -15,6 +14,7 @@ use Appwrite\SDK\Deprecated; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; use Appwrite\Template\Template; +use Appwrite\Usage\Context; use Appwrite\Utopia\Request; use Appwrite\Utopia\Response; use libphonenumber\NumberParseException; @@ -104,7 +104,7 @@ class Create extends Action ->inject('queueForMessaging') ->inject('queueForMails') ->inject('timelimit') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('plan') ->inject('proofForToken') ->inject('proofForCode') @@ -124,7 +124,7 @@ class Create extends Action Messaging $queueForMessaging, Mail $queueForMails, callable $timelimit, - StatsUsage $queueForStatsUsage, + Context $usage, array $plan, ProofsToken $proofForToken, ProofsCode $proofForCode @@ -201,16 +201,12 @@ class Create extends Action $countryCode = $helper->parse($phone)->getCountryCode(); if (!empty($countryCode)) { - $queueForStatsUsage - ->addMetric(str_replace('{countryCode}', $countryCode, METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE), 1); + $usage->addMetric(str_replace('{countryCode}', $countryCode, METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE), 1); } } catch (NumberParseException $e) { // Ignore invalid phone number for country code stats } - $queueForStatsUsage - ->addMetric(METRIC_AUTH_METHOD_PHONE, 1) - ->setProject($project) - ->trigger(); + $usage->addMetric(METRIC_AUTH_METHOD_PHONE, 1); break; case Type::EMAIL: if (empty(System::getEnv('_APP_SMTP_HOST'))) { diff --git a/src/Appwrite/Platform/Modules/Avatars/Http/Screenshots/Get.php b/src/Appwrite/Platform/Modules/Avatars/Http/Screenshots/Get.php index b6fd354ee3..2df12b17d1 100644 --- a/src/Appwrite/Platform/Modules/Avatars/Http/Screenshots/Get.php +++ b/src/Appwrite/Platform/Modules/Avatars/Http/Screenshots/Get.php @@ -2,7 +2,6 @@ namespace Appwrite\Platform\Modules\Avatars\Http\Screenshots; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\Platform\Modules\Avatars\Http\Action; use Appwrite\SDK\AuthType; @@ -10,6 +9,7 @@ use Appwrite\SDK\ContentType; use Appwrite\SDK\Method; use Appwrite\SDK\MethodType; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Usage\Context; use Appwrite\Utopia\Response; use Utopia\Config\Config; use Utopia\Domains\Domain; @@ -84,11 +84,11 @@ class Get extends Action ->param('quality', -1, new Range(-1, 100), 'Screenshot quality. Pass an integer between 0 to 100. Defaults to keep existing image quality.', true, example: '85') ->param('output', '', new WhiteList(\array_keys(Config::getParam('storage-outputs')), true), 'Output format type (jpeg, jpg, png, gif and webp).', true, example: 'jpeg') ->inject('response') - ->inject('queueForStatsUsage') + ->inject('usage') ->callback($this->action(...)); } - public function action(string $url, array $headers, int $viewportWidth, int $viewportHeight, float $scale, string $theme, string $userAgent, bool $fullpage, string $locale, string $timezone, float $latitude, float $longitude, float $accuracy, bool $touch, array $permissions, int $sleep, int $width, int $height, int $quality, string $output, Response $response, StatsUsage $queueForStatsUsage) + public function action(string $url, array $headers, int $viewportWidth, int $viewportHeight, float $scale, string $theme, string $userAgent, bool $fullpage, string $locale, string $timezone, float $latitude, float $longitude, float $accuracy, bool $touch, array $permissions, int $sleep, int $width, int $height, int $quality, string $output, Response $response, Context $usage) { if (!\extension_loaded('imagick')) { throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Imagick extension is missing'); @@ -210,7 +210,7 @@ class Get extends Action $outputs = Config::getParam('storage-outputs'); $contentType = $outputs[$output] ?? $outputs['png']; - $queueForStatsUsage->addMetric(METRIC_AVATARS_SCREENSHOTS_GENERATED, 1); + $usage->addMetric(METRIC_AVATARS_SCREENSHOTS_GENERATED, 1); $response ->addHeader('Cache-Control', 'private, max-age=2592000') // 30 days diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Attribute/Decrement.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Attribute/Decrement.php index 85813b2354..54557eaac0 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Attribute/Decrement.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Attribute/Decrement.php @@ -3,7 +3,6 @@ namespace Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documents\Attribute; use Appwrite\Event\Event; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documents\Action; use Appwrite\SDK\AuthType; @@ -11,6 +10,7 @@ use Appwrite\SDK\ContentType; use Appwrite\SDK\Deprecated; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Usage\Context; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Response as UtopiaResponse; use InvalidArgumentException; @@ -83,13 +83,13 @@ class Decrement extends Action ->inject('response') ->inject('dbForProject') ->inject('queueForEvents') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('plan') ->inject('authorization') ->callback($this->action(...)); } - public function action(string $databaseId, string $collectionId, string $documentId, string $attribute, int|float $value, int|float|null $min, ?string $transactionId, UtopiaResponse $response, Database $dbForProject, Event $queueForEvents, StatsUsage $queueForStatsUsage, array $plan, Authorization $authorization): void + public function action(string $databaseId, string $collectionId, string $documentId, string $attribute, int|float $value, int|float|null $min, ?string $transactionId, UtopiaResponse $response, Database $dbForProject, Event $queueForEvents, Context $usage, array $plan, Authorization $authorization): void { $isAPIKey = User::isApp($authorization->getRoles()); $isPrivilegedUser = User::isPrivileged($authorization->getRoles()); @@ -200,7 +200,7 @@ class Decrement extends Action ) ); - $queueForStatsUsage + $usage ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, 1) ->addMetric(str_replace('{databaseInternalId}', $database->getSequence(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1); diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Attribute/Increment.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Attribute/Increment.php index 031b5abcc6..b9c19b2d06 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Attribute/Increment.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Attribute/Increment.php @@ -3,7 +3,6 @@ namespace Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documents\Attribute; use Appwrite\Event\Event; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documents\Action; use Appwrite\SDK\AuthType; @@ -11,6 +10,7 @@ use Appwrite\SDK\ContentType; use Appwrite\SDK\Deprecated; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Usage\Context; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Response as UtopiaResponse; use InvalidArgumentException; @@ -83,13 +83,13 @@ class Increment extends Action ->inject('response') ->inject('dbForProject') ->inject('queueForEvents') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('plan') ->inject('authorization') ->callback($this->action(...)); } - public function action(string $databaseId, string $collectionId, string $documentId, string $attribute, int|float $value, int|float|null $max, ?string $transactionId, UtopiaResponse $response, Database $dbForProject, Event $queueForEvents, StatsUsage $queueForStatsUsage, array $plan, Authorization $authorization): void + public function action(string $databaseId, string $collectionId, string $documentId, string $attribute, int|float $value, int|float|null $max, ?string $transactionId, UtopiaResponse $response, Database $dbForProject, Event $queueForEvents, Context $usage, array $plan, Authorization $authorization): void { $isAPIKey = User::isApp($authorization->getRoles()); $isPrivilegedUser = User::isPrivileged($authorization->getRoles()); @@ -200,7 +200,7 @@ class Increment extends Action ) ); - $queueForStatsUsage + $usage ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, 1) ->addMetric(str_replace('{databaseInternalId}', $database->getSequence(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1); diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Delete.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Delete.php index 6ab67318c7..f45b126f16 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Delete.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Delete.php @@ -3,7 +3,6 @@ namespace Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documents\Bulk; use Appwrite\Event\Event; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\Functions\EventProcessor; use Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documents\Action; @@ -12,6 +11,7 @@ use Appwrite\SDK\ContentType; use Appwrite\SDK\Deprecated; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Usage\Context; use Appwrite\Utopia\Response as UtopiaResponse; use Utopia\Database\Database; use Utopia\Database\Document; @@ -76,7 +76,7 @@ class Delete extends Action ->param('transactionId', null, fn (Database $dbForProject) => new Nullable(new UID($dbForProject->getAdapter()->getMaxUIDLength())), 'Transaction ID for staging the operation.', true, ['dbForProject']) ->inject('response') ->inject('dbForProject') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('queueForEvents') ->inject('queueForRealtime') ->inject('queueForFunctions') @@ -86,7 +86,7 @@ class Delete extends Action ->callback($this->action(...)); } - public function action(string $databaseId, string $collectionId, array $queries, ?string $transactionId, UtopiaResponse $response, Database $dbForProject, StatsUsage $queueForStatsUsage, Event $queueForEvents, Event $queueForRealtime, Event $queueForFunctions, Event $queueForWebhooks, array $plan, EventProcessor $eventProcessor): void + public function action(string $databaseId, string $collectionId, array $queries, ?string $transactionId, UtopiaResponse $response, Database $dbForProject, Context $usage, Event $queueForEvents, Event $queueForRealtime, Event $queueForFunctions, Event $queueForWebhooks, array $plan, EventProcessor $eventProcessor): void { $database = $dbForProject->getDocument('databases', $databaseId); if ($database->isEmpty()) { @@ -185,10 +185,10 @@ class Delete extends Action foreach ($documents as $document) { $document->setAttribute('$databaseId', $database->getId()); - $document->setAttribute('$'.$this->getCollectionsEventsContext().'Id', $collection->getId()); + $document->setAttribute('$' . $this->getCollectionsEventsContext() . 'Id', $collection->getId()); } - $queueForStatsUsage + $usage ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, \max(1, $modified)) ->addMetric(str_replace('{databaseInternalId}', $database->getSequence(), METRIC_DATABASE_ID_OPERATIONS_WRITES), \max(1, $modified)); diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Update.php index d306414a89..000b59ff07 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Update.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Update.php @@ -3,7 +3,6 @@ namespace Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documents\Bulk; use Appwrite\Event\Event; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\Functions\EventProcessor; use Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documents\Action; @@ -12,6 +11,7 @@ use Appwrite\SDK\ContentType; use Appwrite\SDK\Deprecated; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Usage\Context; use Appwrite\Utopia\Response as UtopiaResponse; use Utopia\Database\Database; use Utopia\Database\Document; @@ -80,7 +80,7 @@ class Update extends Action ->param('transactionId', null, fn (Database $dbForProject) => new Nullable(new UID($dbForProject->getAdapter()->getMaxUIDLength())), 'Transaction ID for staging the operation.', true, ['dbForProject']) ->inject('response') ->inject('dbForProject') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('queueForEvents') ->inject('queueForRealtime') ->inject('queueForFunctions') @@ -90,7 +90,7 @@ class Update extends Action ->callback($this->action(...)); } - public function action(string $databaseId, string $collectionId, string|array $data, array $queries, ?string $transactionId, UtopiaResponse $response, Database $dbForProject, StatsUsage $queueForStatsUsage, Event $queueForEvents, Event $queueForRealtime, Event $queueForFunctions, Event $queueForWebhooks, array $plan, EventProcessor $eventProcessor): void + public function action(string $databaseId, string $collectionId, string|array $data, array $queries, ?string $transactionId, UtopiaResponse $response, Database $dbForProject, Context $usage, Event $queueForEvents, Event $queueForRealtime, Event $queueForFunctions, Event $queueForWebhooks, array $plan, EventProcessor $eventProcessor): void { $data = \is_string($data) ? \json_decode($data, true) @@ -216,10 +216,10 @@ class Update extends Action foreach ($documents as $document) { $document->setAttribute('$databaseId', $database->getId()); - $document->setAttribute('$'.$this->getCollectionsEventsContext().'Id', $collection->getId()); + $document->setAttribute('$' . $this->getCollectionsEventsContext() . 'Id', $collection->getId()); } - $queueForStatsUsage + $usage ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, \max(1, $modified)) ->addMetric(str_replace('{databaseInternalId}', $database->getSequence(), METRIC_DATABASE_ID_OPERATIONS_WRITES), \max(1, $modified)); diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Upsert.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Upsert.php index cf30fee173..564b5ee7b6 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Upsert.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Upsert.php @@ -3,7 +3,6 @@ namespace Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documents\Bulk; use Appwrite\Event\Event; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\Functions\EventProcessor; use Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documents\Action; @@ -12,6 +11,7 @@ use Appwrite\SDK\ContentType; use Appwrite\SDK\Deprecated; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Usage\Context; use Appwrite\Utopia\Response as UtopiaResponse; use Utopia\Database\Database; use Utopia\Database\Document; @@ -78,7 +78,7 @@ class Upsert extends Action ->param('transactionId', null, fn (Database $dbForProject) => new Nullable(new UID($dbForProject->getAdapter()->getMaxUIDLength())), 'Transaction ID for staging the operation.', true, ['dbForProject']) ->inject('response') ->inject('dbForProject') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('queueForEvents') ->inject('queueForRealtime') ->inject('queueForFunctions') @@ -88,7 +88,7 @@ class Upsert extends Action ->callback($this->action(...)); } - public function action(string $databaseId, string $collectionId, array $documents, ?string $transactionId, UtopiaResponse $response, Database $dbForProject, StatsUsage $queueForStatsUsage, Event $queueForEvents, Event $queueForRealtime, Event $queueForFunctions, Event $queueForWebhooks, array $plan, EventProcessor $eventProcessor): void + public function action(string $databaseId, string $collectionId, array $documents, ?string $transactionId, UtopiaResponse $response, Database $dbForProject, Context $usage, Event $queueForEvents, Event $queueForRealtime, Event $queueForFunctions, Event $queueForWebhooks, array $plan, EventProcessor $eventProcessor): void { $database = $dbForProject->getDocument('databases', $databaseId); if ($database->isEmpty()) { @@ -106,7 +106,7 @@ class Upsert extends Action ); if ($hasRelationships) { - throw new Exception(Exception::GENERAL_BAD_REQUEST, 'Bulk upsert is not supported for ' . $this->getSDKNamespace() . ' with relationship attributes'); + throw new Exception(Exception::GENERAL_BAD_REQUEST, 'Bulk upsert is not supported for ' . $this->getSDKNamespace() . ' with relationship attributes'); } foreach ($documents as $key => $document) { @@ -191,10 +191,10 @@ class Upsert extends Action foreach ($upserted as $document) { $document->setAttribute('$databaseId', $database->getId()); - $document->setAttribute('$'.$this->getCollectionsEventsContext().'Id', $collection->getId()); + $document->setAttribute('$' . $this->getCollectionsEventsContext() . 'Id', $collection->getId()); } - $queueForStatsUsage + $usage ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, \max(1, $modified)) ->addMetric(str_replace('{databaseInternalId}', $database->getSequence(), METRIC_DATABASE_ID_OPERATIONS_WRITES), \max(1, $modified)); diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Create.php index 9b14122abf..0bbe7c75cf 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Create.php @@ -3,7 +3,6 @@ namespace Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documents; use Appwrite\Event\Event; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\Functions\EventProcessor; use Appwrite\SDK\AuthType; @@ -12,6 +11,7 @@ use Appwrite\SDK\Deprecated; use Appwrite\SDK\Method; use Appwrite\SDK\Parameter; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Usage\Context; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Database\Validator\CustomId; use Appwrite\Utopia\Response as UtopiaResponse; @@ -129,7 +129,7 @@ class Create extends Action ->inject('dbForProject') ->inject('user') ->inject('queueForEvents') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('queueForRealtime') ->inject('queueForFunctions') ->inject('queueForWebhooks') @@ -138,7 +138,7 @@ class Create extends Action ->inject('eventProcessor') ->callback($this->action(...)); } - public function action(string $databaseId, string $documentId, string $collectionId, string|array $data, ?array $permissions, ?array $documents, ?string $transactionId, UtopiaResponse $response, Database $dbForProject, Document $user, Event $queueForEvents, StatsUsage $queueForStatsUsage, Event $queueForRealtime, Event $queueForFunctions, Event $queueForWebhooks, array $plan, Authorization $authorization, EventProcessor $eventProcessor): void + public function action(string $databaseId, string $documentId, string $collectionId, string|array $data, ?array $permissions, ?array $documents, ?string $transactionId, UtopiaResponse $response, Database $dbForProject, Document $user, Event $queueForEvents, Context $usage, Event $queueForRealtime, Event $queueForFunctions, Event $queueForWebhooks, array $plan, Authorization $authorization, EventProcessor $eventProcessor): void { $data = \is_string($data) ? \json_decode($data, true) @@ -205,7 +205,7 @@ class Create extends Action ); if ($isBulk && $hasRelationships) { - throw new Exception(Exception::GENERAL_BAD_REQUEST, 'Bulk create is not supported for ' . $this->getSDKNamespace() .' with relationship ' . $this->getStructureContext()); + throw new Exception(Exception::GENERAL_BAD_REQUEST, 'Bulk create is not supported for ' . $this->getSDKNamespace() . ' with relationship ' . $this->getStructureContext()); } $setPermissions = function (Document $document, ?array $permissions) use ($user, $isAPIKey, $isPrivilegedUser, $isBulk, $dbForProject, $authorization) { @@ -489,7 +489,7 @@ class Create extends Action ); } - $queueForStatsUsage + $usage ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, \max(1, $operations)) ->addMetric(str_replace('{databaseInternalId}', $database->getSequence(), METRIC_DATABASE_ID_OPERATIONS_WRITES), \max(1, $operations)); // per collection diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Delete.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Delete.php index 3171fe7aaf..0996fa24ab 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Delete.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Delete.php @@ -4,13 +4,13 @@ namespace Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documen use Appwrite\Databases\TransactionState; use Appwrite\Event\Event; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; use Appwrite\SDK\Deprecated; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Usage\Context; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Response as UtopiaResponse; use Utopia\Database\Database; @@ -80,7 +80,7 @@ class Delete extends Action ->inject('response') ->inject('dbForProject') ->inject('queueForEvents') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('transactionState') ->inject('plan') ->inject('authorization') @@ -96,7 +96,7 @@ class Delete extends Action UtopiaResponse $response, Database $dbForProject, Event $queueForEvents, - StatsUsage $queueForStatsUsage, + Context $usage, TransactionState $transactionState, array $plan, Authorization $authorization @@ -210,7 +210,7 @@ class Delete extends Action authorization: $authorization ); - $queueForStatsUsage + $usage ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, 1) ->addMetric(str_replace('{databaseInternalId}', $database->getSequence(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1); // per collection diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Get.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Get.php index 515b7029e6..8784d30667 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Get.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Get.php @@ -3,13 +3,13 @@ namespace Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documents; use Appwrite\Databases\TransactionState; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; use Appwrite\SDK\Deprecated; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Usage\Context; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Response as UtopiaResponse; use Utopia\Database\Database; @@ -68,13 +68,13 @@ class Get extends Action ->param('transactionId', null, fn (Database $dbForProject) => new Nullable(new UID($dbForProject->getAdapter()->getMaxUIDLength())), 'Transaction ID to read uncommitted changes within the transaction.', true, ['dbForProject']) ->inject('response') ->inject('dbForProject') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('transactionState') ->inject('authorization') ->callback($this->action(...)); } - public function action(string $databaseId, string $collectionId, string $documentId, array $queries, ?string $transactionId, UtopiaResponse $response, Database $dbForProject, StatsUsage $queueForStatsUsage, TransactionState $transactionState, Authorization $authorization): void + public function action(string $databaseId, string $collectionId, string $documentId, array $queries, ?string $transactionId, UtopiaResponse $response, Database $dbForProject, Context $usage, TransactionState $transactionState, Authorization $authorization): void { $isAPIKey = User::isApp($authorization->getRoles()); $isPrivilegedUser = User::isPrivileged($authorization->getRoles()); @@ -130,7 +130,7 @@ class Get extends Action operations: $operations ); - $queueForStatsUsage + $usage ->addMetric(METRIC_DATABASES_OPERATIONS_READS, max($operations, 1)) ->addMetric(str_replace('{databaseInternalId}', $database->getSequence(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations); diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Update.php index 06eca79dad..ca7935dfbd 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Update.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Update.php @@ -4,13 +4,13 @@ namespace Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documen use Appwrite\Databases\TransactionState; use Appwrite\Event\Event; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; use Appwrite\SDK\Deprecated; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Usage\Context; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Response as UtopiaResponse; use Utopia\Database\Database; @@ -84,14 +84,14 @@ class Update extends Action ->inject('response') ->inject('dbForProject') ->inject('queueForEvents') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('transactionState') ->inject('plan') ->inject('authorization') ->callback($this->action(...)); } - public function action(string $databaseId, string $collectionId, string $documentId, string|array $data, ?array $permissions, ?string $transactionId, ?\DateTime $requestTimestamp, UtopiaResponse $response, Database $dbForProject, Event $queueForEvents, StatsUsage $queueForStatsUsage, TransactionState $transactionState, array $plan, Authorization $authorization): void + public function action(string $databaseId, string $collectionId, string $documentId, string|array $data, ?array $permissions, ?string $transactionId, ?\DateTime $requestTimestamp, UtopiaResponse $response, Database $dbForProject, Event $queueForEvents, Context $usage, TransactionState $transactionState, array $plan, Authorization $authorization): void { $data = (\is_string($data)) ? \json_decode($data, true) : $data; // Cast to JSON array @@ -246,7 +246,7 @@ class Update extends Action $setCollection($collection, $newDocument); - $queueForStatsUsage + $usage ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, max($operations, 1)) ->addMetric(str_replace('{databaseInternalId}', $database->getSequence(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations); diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Upsert.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Upsert.php index ef76ebe7cd..dc6655dfd3 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Upsert.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Upsert.php @@ -4,13 +4,13 @@ namespace Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documen use Appwrite\Databases\TransactionState; use Appwrite\Event\Event; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; use Appwrite\SDK\Deprecated; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Usage\Context; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Database\Validator\CustomId; use Appwrite\Utopia\Response as UtopiaResponse; @@ -88,14 +88,14 @@ class Upsert extends Action ->inject('user') ->inject('dbForProject') ->inject('queueForEvents') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('transactionState') ->inject('plan') ->inject('authorization') ->callback($this->action(...)); } - public function action(string $databaseId, string $collectionId, string $documentId, string|array $data, ?array $permissions, ?string $transactionId, ?\DateTime $requestTimestamp, UtopiaResponse $response, Document $user, Database $dbForProject, Event $queueForEvents, StatsUsage $queueForStatsUsage, TransactionState $transactionState, array $plan, Authorization $authorization): void + public function action(string $databaseId, string $collectionId, string $documentId, string|array $data, ?array $permissions, ?string $transactionId, ?\DateTime $requestTimestamp, UtopiaResponse $response, Document $user, Database $dbForProject, Event $queueForEvents, Context $usage, TransactionState $transactionState, array $plan, Authorization $authorization): void { $data = (\is_string($data)) ? \json_decode($data, true) : $data; // Cast to JSON array @@ -256,7 +256,7 @@ class Upsert extends Action $setCollection($collection, $newDocument); - $queueForStatsUsage + $usage ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, \max(1, $operations)) ->addMetric(str_replace('{databaseInternalId}', $database->getSequence(), METRIC_DATABASE_ID_OPERATIONS_WRITES), \max(1, $operations)); diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/XList.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/XList.php index 0c93eaf105..3384153971 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/XList.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/XList.php @@ -3,13 +3,13 @@ namespace Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documents; use Appwrite\Databases\TransactionState; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; use Appwrite\SDK\Deprecated; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Usage\Context; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Response as UtopiaResponse; use Utopia\Database\Database; @@ -75,13 +75,13 @@ class XList extends Action ->inject('response') ->inject('dbForProject') ->inject('user') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('transactionState') ->inject('authorization') ->callback($this->action(...)); } - public function action(string $databaseId, string $collectionId, array $queries, ?string $transactionId, bool $includeTotal, int $ttl, UtopiaResponse $response, Database $dbForProject, Document $user, StatsUsage $queueForStatsUsage, TransactionState $transactionState, Authorization $authorization): void + public function action(string $databaseId, string $collectionId, array $queries, ?string $transactionId, bool $includeTotal, int $ttl, UtopiaResponse $response, Database $dbForProject, Document $user, Context $usage, TransactionState $transactionState, Authorization $authorization): void { $isAPIKey = User::isApp($authorization->getRoles()); $isPrivilegedUser = User::isPrivileged($authorization->getRoles()); @@ -228,7 +228,7 @@ class XList extends Action ); } - $queueForStatsUsage + $usage ->addMetric(METRIC_DATABASES_OPERATIONS_READS, max($operations, 1)) ->addMetric(str_replace('{databaseInternalId}', $database->getSequence(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations); diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Delete.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Delete.php index f849de94c1..1046d7e566 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Delete.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Delete.php @@ -60,7 +60,6 @@ class Delete extends Action ->inject('dbForProject') ->inject('queueForDatabase') ->inject('queueForEvents') - ->inject('queueForStatsUsage') ->callback($this->action(...)); } diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Update.php index a73ad70786..9a5a63ea91 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Update.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Update.php @@ -5,13 +5,13 @@ namespace Appwrite\Platform\Modules\Databases\Http\Databases\Transactions; use Appwrite\Databases\TransactionState; use Appwrite\Event\Delete; use Appwrite\Event\Event; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\Functions\EventProcessor; use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Usage\Context; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Response as UtopiaResponse; use Utopia\Database\Database; @@ -73,7 +73,7 @@ class Update extends Action ->inject('transactionState') ->inject('queueForDeletes') ->inject('queueForEvents') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('queueForRealtime') ->inject('queueForFunctions') ->inject('queueForWebhooks') @@ -92,7 +92,7 @@ class Update extends Action * @param TransactionState $transactionState * @param Delete $queueForDeletes * @param Event $queueForEvents - * @param StatsUsage $queueForStatsUsage + * @param Context $usage * @param Event $queueForRealtime * @param Event $queueForFunctions * @param Event $queueForWebhooks @@ -106,7 +106,7 @@ class Update extends Action * @throws Structure * @throws \Utopia\Http\Exception */ - public function action(string $transactionId, bool $commit, bool $rollback, UtopiaResponse $response, Database $dbForProject, Document $user, TransactionState $transactionState, Delete $queueForDeletes, Event $queueForEvents, StatsUsage $queueForStatsUsage, Event $queueForRealtime, Event $queueForFunctions, Event $queueForWebhooks, Authorization $authorization, EventProcessor $eventProcessor): void + public function action(string $transactionId, bool $commit, bool $rollback, UtopiaResponse $response, Database $dbForProject, Document $user, TransactionState $transactionState, Delete $queueForDeletes, Event $queueForEvents, Context $usage, Event $queueForRealtime, Event $queueForFunctions, Event $queueForWebhooks, Authorization $authorization, EventProcessor $eventProcessor): void { if (!$commit && !$rollback) { throw new Exception(Exception::GENERAL_BAD_REQUEST, 'Either commit or rollback must be true'); @@ -142,7 +142,7 @@ class Update extends Action $currentDocumentId = null; try { - $dbForProject->withTransaction(function () use ($dbForProject, $transactionState, $queueForDeletes, $transactionId, &$transaction, &$operations, &$totalOperations, &$databaseOperations, &$currentDocumentId, $queueForEvents, $queueForStatsUsage, $queueForRealtime, $queueForFunctions, $queueForWebhooks, $authorization) { + $dbForProject->withTransaction(function () use ($dbForProject, $transactionState, $queueForDeletes, $transactionId, &$transaction, &$operations, &$totalOperations, &$databaseOperations, &$currentDocumentId, $queueForEvents, $usage, $queueForRealtime, $queueForFunctions, $queueForWebhooks, $authorization) { $authorization->skip(fn () => $dbForProject->updateDocument('transactions', $transactionId, new Document([ 'status' => 'committing', ]))); @@ -279,11 +279,10 @@ class Update extends Action throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); } - $queueForStatsUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $totalOperations); + $usage->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $totalOperations); foreach ($databaseOperations as $sequence => $count) { - $queueForStatsUsage->addMetric( + $usage->addMetric( str_replace('{databaseInternalId}', $sequence, METRIC_DATABASE_ID_OPERATIONS_WRITES), $count ); diff --git a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Delete.php b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Delete.php index c6cd0c6999..7873d369e6 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Delete.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Delete.php @@ -50,7 +50,6 @@ class Delete extends DatabaseDelete ->inject('dbForProject') ->inject('queueForDatabase') ->inject('queueForEvents') - ->inject('queueForStatsUsage') ->callback($this->action(...)); } } diff --git a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Bulk/Delete.php b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Bulk/Delete.php index a1695bdbc6..adaf83ccf1 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Bulk/Delete.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Bulk/Delete.php @@ -61,7 +61,7 @@ class Delete extends DocumentsDelete ->param('transactionId', null, fn (Database $dbForProject) => new Nullable(new UID($dbForProject->getAdapter()->getMaxUIDLength())), 'Transaction ID for staging the operation.', true, ['dbForProject']) ->inject('response') ->inject('dbForProject') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('queueForEvents') ->inject('queueForRealtime') ->inject('queueForFunctions') diff --git a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Bulk/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Bulk/Update.php index a6bc78b3e9..d706d1f28b 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Bulk/Update.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Bulk/Update.php @@ -63,7 +63,7 @@ class Update extends DocumentsUpdate ->param('transactionId', null, fn (Database $dbForProject) => new Nullable(new UID($dbForProject->getAdapter()->getMaxUIDLength())), 'Transaction ID for staging the operation.', true, ['dbForProject']) ->inject('response') ->inject('dbForProject') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('queueForEvents') ->inject('queueForRealtime') ->inject('queueForFunctions') diff --git a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Bulk/Upsert.php b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Bulk/Upsert.php index 6c0815312d..58da5064f9 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Bulk/Upsert.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Bulk/Upsert.php @@ -63,7 +63,7 @@ class Upsert extends DocumentsUpsert ->param('transactionId', null, fn (Database $dbForProject) => new Nullable(new UID($dbForProject->getAdapter()->getMaxUIDLength())), 'Transaction ID for staging the operation.', true, ['dbForProject']) ->inject('response') ->inject('dbForProject') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('queueForEvents') ->inject('queueForRealtime') ->inject('queueForFunctions') diff --git a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Column/Decrement.php b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Column/Decrement.php index 63d70b40e2..e1e717e9b1 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Column/Decrement.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Column/Decrement.php @@ -66,7 +66,7 @@ class Decrement extends DecrementDocumentAttribute ->inject('response') ->inject('dbForProject') ->inject('queueForEvents') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('plan') ->inject('authorization') ->callback($this->action(...)); diff --git a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Column/Increment.php b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Column/Increment.php index 5beb8468d9..0b20450254 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Column/Increment.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Column/Increment.php @@ -66,7 +66,7 @@ class Increment extends IncrementDocumentAttribute ->inject('response') ->inject('dbForProject') ->inject('queueForEvents') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('plan') ->inject('authorization') ->callback($this->action(...)); diff --git a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Create.php index 4385303ffa..fde8005d2b 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Create.php @@ -106,7 +106,7 @@ class Create extends DocumentCreate ->inject('dbForProject') ->inject('user') ->inject('queueForEvents') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('queueForRealtime') ->inject('queueForFunctions') ->inject('queueForWebhooks') diff --git a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Delete.php b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Delete.php index bee4dc1093..1845edc307 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Delete.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Delete.php @@ -68,7 +68,7 @@ class Delete extends DocumentDelete ->inject('response') ->inject('dbForProject') ->inject('queueForEvents') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('transactionState') ->inject('plan') ->inject('authorization') diff --git a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Get.php b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Get.php index f0a7fcbbc2..43b799e5b1 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Get.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Get.php @@ -57,7 +57,7 @@ class Get extends DocumentGet ->param('transactionId', null, fn (Database $dbForProject) => new Nullable(new UID($dbForProject->getAdapter()->getMaxUIDLength())), 'Transaction ID to read uncommitted changes within the transaction.', true, ['dbForProject']) ->inject('response') ->inject('dbForProject') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('transactionState') ->inject('authorization') ->callback($this->action(...)); diff --git a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Update.php index 71abb5d167..c0d90f9531 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Update.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Update.php @@ -66,7 +66,7 @@ class Update extends DocumentUpdate ->inject('response') ->inject('dbForProject') ->inject('queueForEvents') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('transactionState') ->inject('plan') ->inject('authorization') diff --git a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Upsert.php b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Upsert.php index 0bcf9f9a63..7f0aa0ad7d 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Upsert.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Upsert.php @@ -69,7 +69,7 @@ class Upsert extends DocumentUpsert ->inject('user') ->inject('dbForProject') ->inject('queueForEvents') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('transactionState') ->inject('plan') ->inject('authorization') diff --git a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/XList.php b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/XList.php index 3b8ac0a70e..6e5dcd9370 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/XList.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/XList.php @@ -61,7 +61,7 @@ class XList extends DocumentXList ->inject('response') ->inject('dbForProject') ->inject('user') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('transactionState') ->inject('authorization') ->callback($this->action(...)); diff --git a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Transactions/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Transactions/Update.php index 7da389b265..68ea2b8901 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Transactions/Update.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Transactions/Update.php @@ -57,7 +57,7 @@ class Update extends TransactionsUpdate ->inject('transactionState') ->inject('queueForDeletes') ->inject('queueForEvents') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('queueForRealtime') ->inject('queueForFunctions') ->inject('queueForWebhooks') diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php index bc506c654a..ee33abe9e1 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php @@ -6,7 +6,6 @@ use Ahc\Jwt\JWT; use Appwrite\Event\Delete as DeleteEvent; use Appwrite\Event\Event; use Appwrite\Event\Func; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\Extend\Exception as AppwriteException; use Appwrite\Functions\Validator\Headers; @@ -15,6 +14,7 @@ use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Usage\Context; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Response; use Executor\Executor; @@ -93,7 +93,7 @@ class Create extends Base ->inject('dbForPlatform') ->inject('user') ->inject('queueForEvents') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('queueForFunctions') ->inject('geodb') ->inject('store') @@ -121,7 +121,7 @@ class Create extends Base Database $dbForPlatform, Document $user, Event $queueForEvents, - StatsUsage $queueForStatsUsage, + Context $usage, Func $queueForFunctions, Reader $geodb, Store $store, @@ -499,7 +499,7 @@ class Create extends Base throw $th; } } finally { - $queueForStatsUsage + $usage ->addMetric(METRIC_EXECUTIONS, 1) ->addMetric(str_replace(['{resourceType}'], [RESOURCE_TYPE_FUNCTIONS], METRIC_RESOURCE_TYPE_EXECUTIONS), 1) ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS, $function->getSequence()], METRIC_RESOURCE_TYPE_ID_EXECUTIONS), 1) diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index c080f5d3dd..f8d8b464aa 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -5,11 +5,13 @@ namespace Appwrite\Platform\Modules\Functions\Workers; use Ahc\Jwt\JWT; use Appwrite\Event\Event; use Appwrite\Event\Func; +use Appwrite\Event\Message\Usage as UsageMessage; +use Appwrite\Event\Publisher\Usage as UsagePublisher; use Appwrite\Event\Realtime; use Appwrite\Event\Screenshot; -use Appwrite\Event\StatsUsage; use Appwrite\Event\Webhook; use Appwrite\Filter\BranchDomain as BranchDomainFilter; +use Appwrite\Usage\Context; use Appwrite\Utopia\Response\Model\Deployment; use Appwrite\Vcs\Comment; use Exception; @@ -60,7 +62,8 @@ class Builds extends Action ->inject('queueForWebhooks') ->inject('queueForFunctions') ->inject('queueForRealtime') - ->inject('queueForStatsUsage') + ->inject('usage') + ->inject('publisherForUsage') ->inject('cache') ->inject('dbForProject') ->inject('deviceForFunctions') @@ -74,24 +77,6 @@ class Builds extends Action } /** - * @param Message $message - * @param Document $project - * @param Database $dbForPlatform - * @param Event $queueForEvents - * @param Screenshot $queueForScreenshots - * @param Webhook $queueForWebhooks - * @param Func $queueForFunctions - * @param Realtime $queueForRealtime - * @param StatsUsage $queueForStatsUsage - * @param Cache $cache - * @param Database $dbForProject - * @param Device $deviceForFunctions - * @param Device $deviceForSites - * @param Device $deviceForFiles - * @param Log $log - * @param Executor $executor - * @param array $plan - * @return void * @throws \Utopia\Database\Exception */ public function action( @@ -103,7 +88,8 @@ class Builds extends Action Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, - StatsUsage $queueForStatsUsage, + Context $usage, + UsagePublisher $publisherForUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, @@ -145,7 +131,8 @@ class Builds extends Action $queueForFunctions, $queueForRealtime, $queueForEvents, - $queueForStatsUsage, + $usage, + $publisherForUsage, $dbForPlatform, $dbForProject, $github, @@ -167,28 +154,7 @@ class Builds extends Action } /** - * @param Device $deviceForFunctions - * @param Device $deviceForSites - * @param Device $deviceForFiles - * @param Screenshot $queueForScreenshots - * @param Webhook $queueForWebhooks - * @param Func $queueForFunctions - * @param Realtime $queueForRealtime - * @param Event $queueForEvents - * @param StatsUsage $queueForStatsUsage - * @param Database $dbForPlatform - * @param Database $dbForProject - * @param GitHub $github - * @param Document $project - * @param Document $resource - * @param Document $deployment - * @param Document $template - * @param Log $log - * @param Executor $executor - * @param array $plan - * @return void * @throws \Utopia\Database\Exception - * * @throws Exception */ protected function buildDeployment( @@ -200,7 +166,8 @@ class Builds extends Action Func $queueForFunctions, Realtime $queueForRealtime, Event $queueForEvents, - StatsUsage $queueForStatsUsage, + Context $usage, + UsagePublisher $publisherForUsage, Database $dbForPlatform, Database $dbForProject, GitHub $github, @@ -272,6 +239,7 @@ class Builds extends Action if ($deployment->getAttribute('status') === 'canceled') { $this->cancelDeployment($deployment->getId(), $dbForProject, $queueForRealtime); + return; } @@ -298,7 +266,7 @@ class Builds extends Action $installationId = $deployment->getAttribute('installationId', ''); $providerRepositoryId = $deployment->getAttribute('providerRepositoryId', ''); $providerCommitHash = $deployment->getAttribute('providerCommitHash', ''); - $isVcsEnabled = !empty($providerRepositoryId); + $isVcsEnabled = ! empty($providerRepositoryId); $owner = ''; $repositoryName = ''; @@ -312,7 +280,7 @@ class Builds extends Action } try { - if (!$isVcsEnabled) { + if (! $isVcsEnabled) { // Non-VCS + Template $templateRepositoryName = $template->getAttribute('repositoryName', ''); $templateOwnerName = $template->getAttribute('ownerName', ''); @@ -324,7 +292,7 @@ class Builds extends Action $templateRootDirectory = \ltrim($templateRootDirectory, '.'); $templateRootDirectory = \ltrim($templateRootDirectory, '/'); - if (!empty($templateRepositoryName) && !empty($templateOwnerName) && !empty($templateReferenceType) && !empty($templateReferenceValue)) { + if (! empty($templateRepositoryName) && ! empty($templateOwnerName) && ! empty($templateReferenceType) && ! empty($templateReferenceValue)) { $stdout = ''; $stderr = ''; @@ -358,8 +326,8 @@ class Builds extends Action $source = $device->getPath($deployment->getId() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION)); $result = $localDevice->transfer($tmpPathFile, $source, $device); - if (!$result) { - throw new \Exception("Unable to move file"); + if (! $result) { + throw new \Exception('Unable to move file'); } Console::execute('rm -rf ' . \escapeshellarg($tmpTemplateDirectory), '', $stdout, $stderr); @@ -400,7 +368,7 @@ class Builds extends Action $cloneVersion = $branchName; $cloneType = GitHub::CLONE_TYPE_BRANCH; - if (!empty($commitHash)) { + if (! empty($commitHash)) { $cloneVersion = $commitHash; $cloneType = GitHub::CLONE_TYPE_COMMIT; } @@ -413,6 +381,7 @@ class Builds extends Action if ($dbForProject->getDocument('deployments', $deploymentId)->getAttribute('status') === 'canceled') { $this->cancelDeployment($deployment->getId(), $dbForProject, $queueForRealtime); + return; } @@ -437,7 +406,6 @@ class Builds extends Action $rootDirectory = $rootDirectoryWithoutSpaces; } - // Build from template $templateRepositoryName = $template->getAttribute('repositoryName', ''); $templateOwnerName = $template->getAttribute('ownerName', ''); @@ -449,7 +417,7 @@ class Builds extends Action $templateRootDirectory = \ltrim($templateRootDirectory, '.'); $templateRootDirectory = \ltrim($templateRootDirectory, '/'); - if (!empty($templateRepositoryName) && !empty($templateOwnerName) && !empty($templateReferenceType) && !empty($templateReferenceValue)) { + if (! empty($templateRepositoryName) && ! empty($templateOwnerName) && ! empty($templateReferenceType) && ! empty($templateReferenceValue)) { // Clone template repo $tmpTemplateDirectory = '/tmp/builds/' . $deploymentId . '/template'; @@ -468,7 +436,7 @@ class Builds extends Action Console::execute('rsync -av --exclude \'.git\' ' . \escapeshellarg($tmpTemplateDirectory . '/' . $templateRootDirectory . '/') . ' ' . \escapeshellarg($tmpDirectory . '/' . $rootDirectory), '', $stdout, $stderr); // Commit and push - $exit = Console::execute('git config --global user.email '. \escapeshellarg(APP_VCS_GITHUB_EMAIL) .' && git config --global user.name '. \escapeshellarg(APP_VCS_GITHUB_USERNAME) .' && cd ' . \escapeshellarg($tmpDirectory) . ' && git checkout -b ' . \escapeshellarg($branchName) . ' && git add . && git commit -m "Create ' . \escapeshellarg($resource->getAttribute('name', '')) . ' function" && git push origin ' . \escapeshellarg($branchName), '', $stdout, $stderr); + $exit = Console::execute('git config --global user.email ' . \escapeshellarg(APP_VCS_GITHUB_EMAIL) . ' && git config --global user.name ' . \escapeshellarg(APP_VCS_GITHUB_USERNAME) . ' && cd ' . \escapeshellarg($tmpDirectory) . ' && git checkout -b ' . \escapeshellarg($branchName) . ' && git add . && git commit -m "Create ' . \escapeshellarg($resource->getAttribute('name', '')) . ' function" && git push origin ' . \escapeshellarg($branchName), '', $stdout, $stderr); if ($exit !== 0) { throw new \Exception('Unable to push code repository: ' . $stderr); @@ -511,7 +479,7 @@ class Builds extends Action } $directorySize = $localDevice->getDirectorySize($tmpDirectory); - $sizeLimit = (int)System::getEnv('_APP_COMPUTE_SIZE_LIMIT', '30000000'); + $sizeLimit = (int) System::getEnv('_APP_COMPUTE_SIZE_LIMIT', '30000000'); if (isset($plan['deploymentSize'])) { $sizeLimit = (int) $plan['deploymentSize'] * 1000 * 1000; @@ -529,8 +497,8 @@ class Builds extends Action $source = $device->getPath($deployment->getId() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION)); $result = $localDevice->transfer($tmpPathFile, $source, $device); - if (!$result) { - throw new \Exception("Unable to move file"); + if (! $result) { + throw new \Exception('Unable to move file'); } Console::execute('rm -rf ' . \escapeshellarg($tmpPath), '', $stdout, $stderr); @@ -623,16 +591,15 @@ class Builds extends Action } $cpus = $spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT; - $memory = max($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT, $minMemory); + $memory = max($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT, $minMemory); $timeout = (int) System::getEnv('_APP_COMPUTE_BUILD_TIMEOUT', 900); - - $jwtExpiry = (int)System::getEnv('_APP_COMPUTE_BUILD_TIMEOUT', 900); + $jwtExpiry = (int) System::getEnv('_APP_COMPUTE_BUILD_TIMEOUT', 900); $jwtObj = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', $jwtExpiry, 0); $apiKey = $jwtObj->encode([ 'projectId' => $project->getId(), - 'scopes' => $resource->getAttribute('scopes', []) + 'scopes' => $resource->getAttribute('scopes', []), ]); // Appwrite vars @@ -700,6 +667,7 @@ class Builds extends Action if ($dbForProject->getDocument('deployments', $deploymentId)->getAttribute('status') === 'canceled') { $this->cancelDeployment($deployment->getId(), $dbForProject, $queueForRealtime); + return; } @@ -721,7 +689,7 @@ class Builds extends Action $listFilesCommand .= 'echo "{APPWRITE_DETECTION_SEPARATOR_START}" && cd /usr/local/build'; // Enter output directory, if set - if (!empty($outputDirectory)) { + if (! empty($outputDirectory)) { $listFilesCommand .= ' && cd ' . \escapeshellarg($outputDirectory); } @@ -748,7 +716,7 @@ class Builds extends Action cpus: $cpus, memory: $memory, timeout: $timeout, - remove: true, + remove: true, entrypoint: $deployment->getAttribute('entrypoint', ''), destination: APP_STORAGE_BUILDS . "/app-{$project->getId()}", variables: $vars, @@ -782,6 +750,7 @@ class Builds extends Action if ($deployment->getAttribute('status') === 'canceled') { $isCanceled = true; Console::info('Ignoring realtime logs because build has been canceled'); + return; } @@ -789,7 +758,7 @@ class Builds extends Action $logs = \mb_substr($logs, 0, null, 'UTF-8'); // Do not stream logs added for SSR detection - if (!$insideSeparation) { + if (! $insideSeparation) { $separator = \strpos($logs, '{APPWRITE_DETECTION_SEPARATOR_START}'); if ($separator !== false) { $logs = \substr($logs, 0, $separator); @@ -819,19 +788,19 @@ class Builds extends Action $currentLogs = $deployment->getAttribute('buildLogs', ''); $affected = false; - $streamLogs = \str_replace("\\n", "{APPWRITE_LINEBREAK_PLACEHOLDER}", $logs); + $streamLogs = \str_replace('\\n', '{APPWRITE_LINEBREAK_PLACEHOLDER}', $logs); foreach (\explode("\n", $streamLogs) as $streamLog) { if (empty($streamLog)) { continue; } - $streamLog = \str_replace("{APPWRITE_LINEBREAK_PLACEHOLDER}", "\n", $streamLog); - $streamParts = \explode(" ", $streamLog, 2); + $streamLog = \str_replace('{APPWRITE_LINEBREAK_PLACEHOLDER}', "\n", $streamLog); + $streamParts = \explode(' ', $streamLog, 2); // TODO: use part[0] as timestamp when switching to dbForLogs for build logs $currentLogs .= $streamParts[1]; - if (!empty($streamParts[1])) { + if (! empty($streamParts[1])) { $affected = true; } } @@ -863,6 +832,7 @@ class Builds extends Action if ($dbForProject->getDocument('deployments', $deploymentId)->getAttribute('status') === 'canceled') { $this->cancelDeployment($deployment->getId(), $dbForProject, $queueForRealtime); + return; } @@ -870,7 +840,7 @@ class Builds extends Action throw $err; } - $buildSizeLimit = (int)System::getEnv('_APP_COMPUTE_BUILD_SIZE_LIMIT', '2000000000'); + $buildSizeLimit = (int) System::getEnv('_APP_COMPUTE_BUILD_SIZE_LIMIT', '2000000000'); if (isset($plan['buildSize'])) { $buildSizeLimit = $plan['buildSize'] * 1000 * 1000; } @@ -898,7 +868,7 @@ class Builds extends Action $deployment->setAttribute('buildLogs', $logs); $adapter = null; - if ($resource->getCollection() === 'sites' && !empty($detectionLogs)) { + if ($resource->getCollection() === 'sites' && ! empty($detectionLogs)) { $files = \explode("\n", $detectionLogs); // Parse output $files = \array_filter($files); // Remove empty $files = \array_map(fn ($file) => \trim($file), $files); // Remove whitepsaces @@ -970,9 +940,9 @@ class Builds extends Action // Check if current active deployment started later than this deployment $resource = $dbForProject->getDocument($resource->getCollection(), $resource->getId()); $currentActiveDeploymentId = $resource->getAttribute('deploymentId', ''); - if (!empty($currentActiveDeploymentId)) { + if (! empty($currentActiveDeploymentId)) { $currentActiveDeployment = $dbForProject->getDocument('deployments', $currentActiveDeploymentId); - if (!$currentActiveDeployment->isEmpty()) { + if (! $currentActiveDeployment->isEmpty()) { $currentActiveStartTime = $currentActiveDeployment->getCreatedAt(); $deploymentStartTime = $deployment->getCreatedAt(); @@ -1058,7 +1028,7 @@ class Builds extends Action if ($resource->getCollection() === 'sites') { // VCS branch $branchName = $deployment->getAttribute('providerBranch'); - if (!empty($branchName)) { + if (! empty($branchName)) { $domain = (new BranchDomainFilter())->apply([ 'branch' => $branchName, 'resourceId' => $resource->getId(), @@ -1085,7 +1055,7 @@ class Builds extends Action 'certificateId' => '', 'search' => implode(' ', [$ruleId, $domain]), 'owner' => 'Appwrite', - 'region' => $project->getAttribute('region') + 'region' => $project->getAttribute('region'), ])); } catch (Duplicate $err) { $rule = $dbForPlatform->updateDocument('rules', $ruleId, new Document([ @@ -1126,6 +1096,7 @@ class Builds extends Action if ($dbForProject->getDocument('deployments', $deploymentId)->getAttribute('status') === 'canceled') { $this->cancelDeployment($deployment->getId(), $dbForProject, $queueForRealtime); + return; } @@ -1139,7 +1110,7 @@ class Builds extends Action $schedule ->setAttribute('resourceUpdatedAt', DateTime::now()) ->setAttribute('schedule', $resource->getAttribute('schedule')) - ->setAttribute('active', !empty($resource->getAttribute('schedule')) && !empty($resource->getAttribute('deploymentId'))); + ->setAttribute('active', ! empty($resource->getAttribute('schedule')) && ! empty($resource->getAttribute('deploymentId'))); $dbForPlatform->updateDocument('schedules', $schedule->getId(), new Document([ 'resourceUpdatedAt' => $schedule->getAttribute('resourceUpdatedAt'), 'schedule' => $schedule->getAttribute('schedule'), @@ -1167,13 +1138,14 @@ class Builds extends Action if ($dbForProject->getDocument('deployments', $deploymentId)->getAttribute('status') === 'canceled') { $this->cancelDeployment($deployment->getId(), $dbForProject, $queueForRealtime); + return; } // Color message red $message = $th->getMessage(); - if (!\str_contains($message, '')) { - $message = "" . $message; + if (! \str_contains($message, '')) { + $message = '' . $message; } $message = \str_replace('{APPWRITE_DETECTION_SEPARATOR_START}', '', $message); @@ -1181,9 +1153,9 @@ class Builds extends Action // Combine with previous logs if deployment got past build process $previousLogs = ''; - if (!is_null($deployment->getAttribute('buildSize', null))) { + if (! is_null($deployment->getAttribute('buildSize', null))) { $previousLogs = $deployment->getAttribute('buildLogs', ''); - if (!empty($previousLogs)) { + if (! empty($previousLogs)) { $message = $previousLogs . "\n" . $message; } } @@ -1219,102 +1191,102 @@ class Builds extends Action ->trigger(); $this->sendUsage( - resource:$resource, + resource: $resource, deployment: $deployment, project: $project, - queue: $queueForStatsUsage + usage: $usage, + publisherForUsage: $publisherForUsage ); } } - protected function sendUsage(Document $resource, Document $deployment, Document $project, StatsUsage $queue): void + protected function sendUsage(Document $resource, Document $deployment, Document $project, Context $usage, UsagePublisher $publisherForUsage): void { $spec = Config::getParam('specifications')[$resource->getAttribute('buildSpecification', APP_COMPUTE_SPECIFICATION_DEFAULT)]; switch ($deployment->getAttribute('status')) { case 'ready': - $queue + $usage ->addMetric(METRIC_BUILDS_SUCCESS, 1) // per project - ->addMetric(METRIC_BUILDS_COMPUTE_SUCCESS, (int)$deployment->getAttribute('buildDuration', 0) * 1000) + ->addMetric(METRIC_BUILDS_COMPUTE_SUCCESS, (int) $deployment->getAttribute('buildDuration', 0) * 1000) ->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_SUCCESS), 1) // per function - ->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_COMPUTE_SUCCESS), (int)$deployment->getAttribute('buildDuration', 0) * 1000) + ->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_COMPUTE_SUCCESS), (int) $deployment->getAttribute('buildDuration', 0) * 1000) ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_SUCCESS), 1) // per function - ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_COMPUTE_SUCCESS), (int)$deployment->getAttribute('buildDuration', 0) * 1000); + ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_COMPUTE_SUCCESS), (int) $deployment->getAttribute('buildDuration', 0) * 1000); break; case 'failed': - $queue + $usage ->addMetric(METRIC_BUILDS_FAILED, 1) // per project - ->addMetric(METRIC_BUILDS_COMPUTE_FAILED, (int)$deployment->getAttribute('buildDuration', 0) * 1000) + ->addMetric(METRIC_BUILDS_COMPUTE_FAILED, (int) $deployment->getAttribute('buildDuration', 0) * 1000) ->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_FAILED), 1) // per function - ->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_COMPUTE_FAILED), (int)$deployment->getAttribute('buildDuration', 0) * 1000) + ->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_COMPUTE_FAILED), (int) $deployment->getAttribute('buildDuration', 0) * 1000) ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_FAILED), 1) // per function - ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_COMPUTE_FAILED), (int)$deployment->getAttribute('buildDuration', 0) * 1000); + ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_COMPUTE_FAILED), (int) $deployment->getAttribute('buildDuration', 0) * 1000); break; } - $queue + $usage ->addMetric(METRIC_BUILDS, 1) // per project ->addMetric(METRIC_BUILDS_STORAGE, $deployment->getAttribute('buildSize', 0)) - ->addMetric(METRIC_BUILDS_COMPUTE, (int)$deployment->getAttribute('buildDuration', 0) * 1000) - ->addMetric(METRIC_BUILDS_MB_SECONDS, (int)(($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT) * $deployment->getAttribute('buildDuration', 0) * ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT))) + ->addMetric(METRIC_BUILDS_COMPUTE, (int) $deployment->getAttribute('buildDuration', 0) * 1000) + ->addMetric(METRIC_BUILDS_MB_SECONDS, (int) (($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT) * $deployment->getAttribute('buildDuration', 0) * ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT))) ->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS), 1) // per function ->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_STORAGE), $deployment->getAttribute('buildSize', 0)) - ->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_COMPUTE), (int)$deployment->getAttribute('buildDuration', 0) * 1000) - ->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_MB_SECONDS), (int)(($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT) * $deployment->getAttribute('buildDuration', 0) * ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT))) + ->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_COMPUTE), (int) $deployment->getAttribute('buildDuration', 0) * 1000) + ->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_MB_SECONDS), (int) (($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT) * $deployment->getAttribute('buildDuration', 0) * ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT))) ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS), 1) // per function ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_STORAGE), $deployment->getAttribute('buildSize', 0)) - ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_COMPUTE), (int)$deployment->getAttribute('buildDuration', 0) * 1000) - ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_MB_SECONDS), (int)(($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT) * $deployment->getAttribute('buildDuration', 0) * ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT))) - ->setProject($project) - ->trigger(); + ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_COMPUTE), (int) $deployment->getAttribute('buildDuration', 0) * 1000) + ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_MB_SECONDS), (int) (($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT) * $deployment->getAttribute('buildDuration', 0) * ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT))); + + // Publish usage metrics + if (! $usage->isEmpty()) { + $message = new UsageMessage( + project: $project, + metrics: $usage->getMetrics(), + reduce: $usage->getReduce() + ); + $publisherForUsage->enqueue($message); + $usage->reset(); + } } /** * Hook to run after build success * - * @param Realtime $queueForRealtime - * @param Database $dbForProject - * @param Document $deployment - * @param array $runtime - * @param string|null $adapter - * @return void * @throws Exception */ protected function afterBuildSuccess(Realtime $queueForRealtime, Database $dbForProject, Document &$deployment, array $runtime, ?string $adapter): void { - if (!($queueForRealtime instanceof Realtime)) { + if (! ($queueForRealtime instanceof Realtime)) { throw new Exception('queueForRealtime must be an instance of Realtime'); } - if (!($dbForProject instanceof Database)) { + if (! ($dbForProject instanceof Database)) { throw new Exception('dbForProject must be an instance of Database'); } - if (!($deployment instanceof Document)) { + if (! ($deployment instanceof Document)) { throw new Exception('deployment must be an instance of Document'); } - if (!is_array($runtime)) { + if (! is_array($runtime)) { throw new Exception('runtime must be an array'); } - if (!is_string($adapter) && !is_null($adapter)) { + if (! is_string($adapter) && ! is_null($adapter)) { throw new Exception('adapter must be a string or null'); } } /** * Hook to run after deployment is activated - * - * @param Document $project - * @param Document $deployment - * @return void */ protected function afterDeploymentSuccess( Document $project, Document $deployment, ): void { - if (!($project instanceof Document)) { + if (! ($project instanceof Document)) { throw new Exception('project must be an instance of Document'); } - if (!($deployment instanceof Document)) { + if (! ($deployment instanceof Document)) { throw new Exception('deployment must be an instance of Document'); } } @@ -1322,7 +1294,7 @@ class Builds extends Action protected function getRuntime(Document $resource, string $version): array { $runtimes = Config::getParam($version === 'v2' ? 'runtimes-v2' : 'runtimes', []); - $key = $resource->getAttribute('runtime'); + $key = $resource->getAttribute('runtime'); $runtime = match ($resource->getCollection()) { 'functions' => $runtimes[$resource->getAttribute('runtime')] ?? null, 'sites' => $runtimes[$resource->getAttribute('buildRuntime')] ?? null, @@ -1355,7 +1327,7 @@ class Builds extends Action $envCommand = ''; $bundleCommand = ''; - if (!is_null($framework)) { + if (! is_null($framework)) { $envCommand = $framework['envCommand'] ?? ''; $bundleCommand = $framework['bundleCommand'] ?? ''; } @@ -1364,7 +1336,7 @@ class Builds extends Action $commands[] = $deployment->getAttribute('buildCommands', ''); $commands[] = $bundleCommand; - $commands = array_filter($commands, fn ($command) => !empty($command)); + $commands = array_filter($commands, fn ($command) => ! empty($command)); return implode(' && ', $commands); } @@ -1373,19 +1345,6 @@ class Builds extends Action } /** - * @param string $status - * @param GitHub $github - * @param string $providerCommitHash - * @param string $owner - * @param string $repositoryName - * @param Document $project - * @param Document $resource - * @param string $deploymentId - * @param Database $dbForProject - * @param Database $dbForPlatform - * @param Realtime $queueForRealtime - * @param array $platform - * @return void * @throws Structure * @throws \Utopia\Database\Exception * @throws Conflict @@ -1413,7 +1372,7 @@ class Builds extends Action $deployment = $dbForProject->getDocument('deployments', $deploymentId); $commentId = $deployment->getAttribute('providerCommentId', ''); - if (!empty($providerCommitHash)) { + if (! empty($providerCommitHash)) { $message = match ($status) { 'ready' => 'Build succeeded.', 'failed' => 'Build failed.', @@ -1448,7 +1407,7 @@ class Builds extends Action $github->updateCommitStatus($repositoryName, $providerCommitHash, $owner, $state, $message, $providerTargetUrl, $name); } - if (!empty($commentId)) { + if (! empty($commentId)) { $retries = 0; while (true) { @@ -1456,7 +1415,7 @@ class Builds extends Action try { $dbForPlatform->createDocument('vcsCommentLocks', new Document([ - '$id' => $commentId + '$id' => $commentId, ])); break; } catch (\Throwable $err) { @@ -1470,22 +1429,22 @@ class Builds extends Action // Wrap in try/finally to ensure lock file gets deleted try { - $resourceType = match($resource->getCollection()) { + $resourceType = match ($resource->getCollection()) { 'functions' => 'function', 'sites' => 'site', default => throw new \Exception('Invalid resource type') }; $rule = $dbForPlatform->findOne('rules', [ - Query::equal("projectInternalId", [$project->getSequence()]), - Query::equal("type", ["deployment"]), - Query::equal("deploymentInternalId", [$deployment->getSequence()]), + Query::equal('projectInternalId', [$project->getSequence()]), + Query::equal('type', ['deployment']), + Query::equal('deploymentInternalId', [$deployment->getSequence()]), ]); $protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS') == 'disabled' ? 'http' : 'https'; - $previewUrl = match($resource->getCollection()) { + $previewUrl = match ($resource->getCollection()) { 'functions' => '', - 'sites' => !empty($rule) ? ("{$protocol}://" . $rule->getAttribute('domain', '')) : '', + 'sites' => ! empty($rule) ? ("{$protocol}://" . $rule->getAttribute('domain', '')) : '', default => throw new \Exception('Invalid resource type') }; @@ -1498,7 +1457,7 @@ class Builds extends Action } } } catch (\Throwable $th) { - Console::warning("Git action failed:"); + Console::warning('Git action failed:'); Console::warning($th->getMessage()); Console::warning($th->getTraceAsString()); diff --git a/src/Appwrite/Platform/Modules/Health/Http/Health/Queue/Failed/Get.php b/src/Appwrite/Platform/Modules/Health/Http/Health/Queue/Failed/Get.php index dc84d0ee37..cb3640746f 100644 --- a/src/Appwrite/Platform/Modules/Health/Http/Health/Queue/Failed/Get.php +++ b/src/Appwrite/Platform/Modules/Health/Http/Health/Queue/Failed/Get.php @@ -12,9 +12,9 @@ use Appwrite\Event\Func; use Appwrite\Event\Mail; use Appwrite\Event\Messaging; use Appwrite\Event\Migration; +use Appwrite\Event\Publisher\Usage as UsagePublisher; use Appwrite\Event\Screenshot; use Appwrite\Event\StatsResources; -use Appwrite\Event\StatsUsage; use Appwrite\Event\Webhook; use Appwrite\Platform\Modules\Health\Http\Health\Queue\Base; use Appwrite\SDK\AuthType; @@ -79,7 +79,7 @@ class Get extends Base ->inject('queueForMails') ->inject('queueForFunctions') ->inject('queueForStatsResources') - ->inject('queueForStatsUsage') + ->inject('publisherForUsage') ->inject('queueForWebhooks') ->inject('queueForCertificates') ->inject('queueForBuilds') @@ -99,7 +99,7 @@ class Get extends Base Mail $queueForMails, Func $queueForFunctions, StatsResources $queueForStatsResources, - StatsUsage $queueForStatsUsage, + UsagePublisher $publisherForUsage, Webhook $queueForWebhooks, Certificate $queueForCertificates, Build $queueForBuilds, @@ -116,7 +116,7 @@ class Get extends Base System::getEnv('_APP_MAILS_QUEUE_NAME', Event::MAILS_QUEUE_NAME) => $queueForMails, System::getEnv('_APP_FUNCTIONS_QUEUE_NAME', Event::FUNCTIONS_QUEUE_NAME) => $queueForFunctions, System::getEnv('_APP_STATS_RESOURCES_QUEUE_NAME', Event::STATS_RESOURCES_QUEUE_NAME) => $queueForStatsResources, - System::getEnv('_APP_STATS_USAGE_QUEUE_NAME', Event::STATS_USAGE_QUEUE_NAME) => $queueForStatsUsage, + System::getEnv('_APP_STATS_USAGE_QUEUE_NAME', Event::STATS_USAGE_QUEUE_NAME) => $publisherForUsage, System::getEnv('_APP_WEBHOOK_QUEUE_NAME', Event::WEBHOOK_QUEUE_NAME) => $queueForWebhooks, System::getEnv('_APP_CERTIFICATES_QUEUE_NAME', Event::CERTIFICATES_QUEUE_NAME) => $queueForCertificates, System::getEnv('_APP_BUILDS_QUEUE_NAME', Event::BUILDS_QUEUE_NAME) => $queueForBuilds, diff --git a/src/Appwrite/Platform/Modules/Health/Http/Health/Queue/StatsUsage/Get.php b/src/Appwrite/Platform/Modules/Health/Http/Health/Queue/StatsUsage/Get.php index 10678efbc3..65b3d228a6 100644 --- a/src/Appwrite/Platform/Modules/Health/Http/Health/Queue/StatsUsage/Get.php +++ b/src/Appwrite/Platform/Modules/Health/Http/Health/Queue/StatsUsage/Get.php @@ -2,7 +2,7 @@ namespace Appwrite\Platform\Modules\Health\Http\Health\Queue\StatsUsage; -use Appwrite\Event\StatsUsage; +use Appwrite\Event\Publisher\Usage as UsagePublisher; use Appwrite\Platform\Modules\Health\Http\Health\Queue\Base; use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; @@ -42,16 +42,16 @@ class Get extends Base contentType: ContentType::JSON )) ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) - ->inject('queueForStatsUsage') + ->inject('publisherForUsage') ->inject('response') ->callback($this->action(...)); } - public function action(int|string $threshold, StatsUsage $queueForStatsUsage, Response $response): void + public function action(int|string $threshold, UsagePublisher $publisherForUsage, Response $response): void { $threshold = (int) $threshold; - $size = $queueForStatsUsage->getSize(); + $size = $publisherForUsage->getSize(); $this->assertQueueThreshold($size, $threshold); diff --git a/src/Appwrite/Platform/Modules/Teams/Http/Memberships/Create.php b/src/Appwrite/Platform/Modules/Teams/Http/Memberships/Create.php index 3bf597eaca..221c3aa521 100644 --- a/src/Appwrite/Platform/Modules/Teams/Http/Memberships/Create.php +++ b/src/Appwrite/Platform/Modules/Teams/Http/Memberships/Create.php @@ -6,7 +6,6 @@ use Appwrite\Auth\Validator\Phone; use Appwrite\Event\Event; use Appwrite\Event\Mail; use Appwrite\Event\Messaging; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\Network\Validator\Email as EmailValidator; use Appwrite\Platform\Action; @@ -14,6 +13,7 @@ use Appwrite\SDK\AuthType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; use Appwrite\Template\Template; +use Appwrite\Usage\Context; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Response; use libphonenumber\NumberParseException; @@ -70,7 +70,7 @@ class Create extends Action new SDKResponse( code: Response::STATUS_CODE_CREATED, model: Response::MODEL_MEMBERSHIP, - ) + ), ] )) ->label('abuse-limit', 10) @@ -91,20 +91,20 @@ class Create extends Action ->inject('queueForMessaging') ->inject('queueForEvents') ->inject('timelimit') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('plan') ->inject('proofForPassword') ->inject('proofForToken') ->callback($this->action(...)); } - public function action(string $teamId, string $email, string $userId, string $phone, array $roles, string $url, string $name, Response $response, Document $project, Document $user, Database $dbForProject, Authorization $authorization, Locale $locale, Mail $queueForMails, Messaging $queueForMessaging, Event $queueForEvents, callable $timelimit, StatsUsage $queueForStatsUsage, array $plan, Password $proofForPassword, Token $proofForToken) + public function action(string $teamId, string $email, string $userId, string $phone, array $roles, string $url, string $name, Response $response, Document $project, Document $user, Database $dbForProject, Authorization $authorization, Locale $locale, Mail $queueForMails, Messaging $queueForMessaging, Event $queueForEvents, callable $timelimit, Context $usage, array $plan, Password $proofForPassword, Token $proofForToken) { $isAppUser = User::isApp($authorization->getRoles()); $isPrivilegedUser = User::isPrivileged($authorization->getRoles()); if (empty($url)) { - if (!$isAppUser && !$isPrivilegedUser) { + if (! $isAppUser && ! $isPrivilegedUser) { throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'URL is required'); } } @@ -113,7 +113,7 @@ class Create extends Action throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'At least one of userId, email, or phone is required'); } - if (!$isPrivilegedUser && !$isAppUser && empty(System::getEnv('_APP_SMTP_HOST'))) { + if (! $isPrivilegedUser && ! $isAppUser && empty(System::getEnv('_APP_SMTP_HOST'))) { throw new Exception(Exception::GENERAL_SMTP_DISABLED); } @@ -124,28 +124,28 @@ class Create extends Action if ($team->isEmpty()) { throw new Exception(Exception::TEAM_NOT_FOUND); } - if (!empty($userId)) { + if (! empty($userId)) { $invitee = $dbForProject->getDocument('users', $userId); if ($invitee->isEmpty()) { throw new Exception(Exception::USER_NOT_FOUND, 'User with given userId doesn\'t exist.', 404); } - if (!empty($email) && $invitee->getAttribute('email', '') !== $email) { + if (! empty($email) && $invitee->getAttribute('email', '') !== $email) { throw new Exception(Exception::USER_ALREADY_EXISTS, 'Given userId and email doesn\'t match', 409); } - if (!empty($phone) && $invitee->getAttribute('phone', '') !== $phone) { + if (! empty($phone) && $invitee->getAttribute('phone', '') !== $phone) { throw new Exception(Exception::USER_ALREADY_EXISTS, 'Given userId and phone doesn\'t match', 409); } $email = $invitee->getAttribute('email', ''); $phone = $invitee->getAttribute('phone', ''); $name = $invitee->getAttribute('name', '') ?: $name; - } elseif (!empty($email)) { + } elseif (! empty($email)) { $invitee = $dbForProject->findOne('users', [Query::equal('email', [$email])]); // Get user by email address - if (!$invitee->isEmpty() && !empty($phone) && $invitee->getAttribute('phone', '') !== $phone) { + if (! $invitee->isEmpty() && ! empty($phone) && $invitee->getAttribute('phone', '') !== $phone) { throw new Exception(Exception::USER_ALREADY_EXISTS, 'Given email and phone doesn\'t match', 409); } - } elseif (!empty($phone)) { + } elseif (! empty($phone)) { $invitee = $dbForProject->findOne('users', [Query::equal('phone', [$phone])]); - if (!$invitee->isEmpty() && !empty($email) && $invitee->getAttribute('email', '') !== $email) { + if (! $invitee->isEmpty() && ! empty($email) && $invitee->getAttribute('email', '') !== $email) { throw new Exception(Exception::USER_ALREADY_EXISTS, 'Given phone and email doesn\'t match', 409); } } @@ -153,7 +153,7 @@ class Create extends Action if ($invitee->isEmpty()) { // Create new user if no user with same email found $limit = $project->getAttribute('auths', [])['limit'] ?? 0; - if (!$isPrivilegedUser && !$isAppUser && $limit !== 0 && $project->getId() !== 'console') { // check users limit, console invites are allways allowed. + if (! $isPrivilegedUser && ! $isAppUser && $limit !== 0 && $project->getId() !== 'console') { // check users limit, console invites are allways allowed. $total = $dbForProject->count('users', [], APP_LIMIT_USERS); if ($total >= $limit) { @@ -165,7 +165,7 @@ class Create extends Action $identityWithMatchingEmail = $dbForProject->findOne('identities', [ Query::equal('providerEmail', [$email]), ]); - if (!$identityWithMatchingEmail->isEmpty()) { + if (! $identityWithMatchingEmail->isEmpty()) { throw new Exception(Exception::USER_EMAIL_ALREADY_EXISTS); } @@ -225,7 +225,7 @@ class Create extends Action $isOwner = $authorization->hasRole('team:' . $team->getId() . '/owner'); - if (!$isOwner && !$isPrivilegedUser && !$isAppUser) { // Not owner, not admin, not app (server) + if (! $isOwner && ! $isPrivilegedUser && ! $isAppUser) { // Not owner, not admin, not app (server) throw new Exception(Exception::USER_UNAUTHORIZED, 'User is not allowed to send invitations for this team'); } @@ -255,7 +255,7 @@ class Create extends Action 'joined' => ($isPrivilegedUser || $isAppUser) ? DateTime::now() : null, 'confirm' => ($isPrivilegedUser || $isAppUser), 'secret' => $proofForToken->hash($secret), - 'search' => implode(' ', [$membershipId, $invitee->getId()]) + 'search' => implode(' ', [$membershipId, $invitee->getId()]), ]); $membership = ($isPrivilegedUser || $isAppUser) ? @@ -292,22 +292,22 @@ class Create extends Action $url = Template::parseURL($url); $url['query'] = Template::mergeQuery(((isset($url['query'])) ? $url['query'] : ''), ['membershipId' => $membership->getId(), 'userId' => $invitee->getId(), 'secret' => $secret, 'teamId' => $teamId, 'teamName' => $team->getAttribute('name')]); $url = Template::unParseURL($url); - if (!empty($email)) { + if (! empty($email)) { $projectName = $project->isEmpty() ? 'Console' : $project->getAttribute('name', '[APP-NAME]'); - $body = $locale->getText("emails.invitation.body"); - $preview = $locale->getText("emails.invitation.preview"); - $subject = $locale->getText("emails.invitation.subject"); + $body = $locale->getText('emails.invitation.body'); + $preview = $locale->getText('emails.invitation.preview'); + $subject = $locale->getText('emails.invitation.subject'); $customTemplate = $project->getAttribute('templates', [])['email.invitation-' . $locale->default] ?? []; $message = Template::fromFile(APP_CE_CONFIG_DIR . '/locale/templates/email-inner-base.tpl'); $message ->setParam('{{body}}', $body, escapeHtml: false) - ->setParam('{{hello}}', $locale->getText("emails.invitation.hello")) - ->setParam('{{footer}}', $locale->getText("emails.invitation.footer")) - ->setParam('{{thanks}}', $locale->getText("emails.invitation.thanks")) - ->setParam('{{buttonText}}', $locale->getText("emails.invitation.buttonText")) - ->setParam('{{signature}}', $locale->getText("emails.invitation.signature")); + ->setParam('{{hello}}', $locale->getText('emails.invitation.hello')) + ->setParam('{{footer}}', $locale->getText('emails.invitation.footer')) + ->setParam('{{thanks}}', $locale->getText('emails.invitation.thanks')) + ->setParam('{{buttonText}}', $locale->getText('emails.invitation.buttonText')) + ->setParam('{{signature}}', $locale->getText('emails.invitation.signature')); $body = $message->render(); $smtp = $project->getAttribute('smtp', []); @@ -315,16 +315,16 @@ class Create extends Action $senderEmail = System::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM); $senderName = System::getEnv('_APP_SYSTEM_EMAIL_NAME', APP_NAME . ' Server'); - $replyTo = ""; + $replyTo = ''; if ($smtpEnabled) { - if (!empty($smtp['senderEmail'])) { + if (! empty($smtp['senderEmail'])) { $senderEmail = $smtp['senderEmail']; } - if (!empty($smtp['senderName'])) { + if (! empty($smtp['senderName'])) { $senderName = $smtp['senderName']; } - if (!empty($smtp['replyTo'])) { + if (! empty($smtp['replyTo'])) { $replyTo = $smtp['replyTo']; } @@ -335,14 +335,14 @@ class Create extends Action ->setSmtpPassword($smtp['password'] ?? '') ->setSmtpSecure($smtp['secure'] ?? ''); - if (!empty($customTemplate)) { - if (!empty($customTemplate['senderEmail'])) { + if (! empty($customTemplate)) { + if (! empty($customTemplate['senderEmail'])) { $senderEmail = $customTemplate['senderEmail']; } - if (!empty($customTemplate['senderName'])) { + if (! empty($customTemplate['senderName'])) { $senderName = $customTemplate['senderName']; } - if (!empty($customTemplate['replyTo'])) { + if (! empty($customTemplate['replyTo'])) { $replyTo = $customTemplate['replyTo']; } @@ -363,7 +363,7 @@ class Create extends Action 'user' => $name, 'team' => $team->getAttribute('name'), 'redirect' => $url, - 'project' => $projectName + 'project' => $projectName, ]; $queueForMails @@ -374,7 +374,7 @@ class Create extends Action ->setName($invitee->getAttribute('name', '')) ->appendVariables($emailVariables) ->trigger(); - } elseif (!empty($phone)) { + } elseif (! empty($phone)) { if (empty(System::getEnv('_APP_SMS_PROVIDER'))) { throw new Exception(Exception::GENERAL_PHONE_DISABLED, 'Phone provider not configured'); } @@ -382,7 +382,7 @@ class Create extends Action $message = Template::fromFile(APP_CE_CONFIG_DIR . '/locale/templates/sms-base.tpl'); $customTemplate = $project->getAttribute('templates', [])['sms.invitation-' . $locale->default] ?? []; - if (!empty($customTemplate)) { + if (! empty($customTemplate)) { $message = $customTemplate['message']; } @@ -406,25 +406,20 @@ class Create extends Action try { $countryCode = $helper->parse($phone)->getCountryCode(); - if (!empty($countryCode)) { - $queueForStatsUsage - ->addMetric(str_replace('{countryCode}', $countryCode, METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE), 1); + if (! empty($countryCode)) { + $usage->addMetric(str_replace('{countryCode}', $countryCode, METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE), 1); } } catch (NumberParseException $e) { // Ignore invalid phone number for country code stats } - $queueForStatsUsage - ->addMetric(METRIC_AUTH_METHOD_PHONE, 1) - ->setProject($project) - ->trigger(); + $usage->addMetric(METRIC_AUTH_METHOD_PHONE, 1); } } $queueForEvents ->setParam('userId', $invitee->getId()) ->setParam('teamId', $team->getId()) - ->setParam('membershipId', $membership->getId()) - ; + ->setParam('membershipId', $membership->getId()); $response ->setStatusCode(Response::STATUS_CODE_CREATED) diff --git a/src/Appwrite/Platform/Workers/Messaging.php b/src/Appwrite/Platform/Workers/Messaging.php index d866cc2bd0..af7d2027e3 100644 --- a/src/Appwrite/Platform/Workers/Messaging.php +++ b/src/Appwrite/Platform/Workers/Messaging.php @@ -2,8 +2,10 @@ namespace Appwrite\Platform\Workers; -use Appwrite\Event\StatsUsage; +use Appwrite\Event\Message\Usage; +use Appwrite\Event\Publisher\Usage as UsagePublisher; use Appwrite\Messaging\Status as MessageStatus; +use Appwrite\Usage\Context as UsageContext; use libphonenumber\NumberParseException; use libphonenumber\PhoneNumberUtil; use Swoole\Runtime; @@ -71,7 +73,7 @@ class Messaging extends Action ->inject('log') ->inject('dbForProject') ->inject('deviceForFiles') - ->inject('queueForStatsUsage') + ->inject('publisherForUsage') ->callback($this->action(...)); } @@ -81,7 +83,7 @@ class Messaging extends Action * @param Log $log * @param Database $dbForProject * @param Device $deviceForFiles - * @param StatsUsage $queueForStatsUsage + * @param UsagePublisher $publisherForUsage * @return void * @throws \Exception */ @@ -91,7 +93,7 @@ class Messaging extends Action Log $log, Database $dbForProject, Device $deviceForFiles, - StatsUsage $queueForStatsUsage + UsagePublisher $publisherForUsage ): void { Runtime::setHookFlags(SWOOLE_HOOK_ALL ^ SWOOLE_HOOK_TCP); $payload = $message->getPayload() ?? []; @@ -115,7 +117,7 @@ class Messaging extends Action case MESSAGE_SEND_TYPE_EXTERNAL: $message = $dbForProject->getDocument('messages', $payload['messageId']); - $this->sendExternalMessage($dbForProject, $message, $deviceForFiles, $project, $queueForStatsUsage); + $this->sendExternalMessage($dbForProject, $message, $deviceForFiles, $project, $publisherForUsage); break; default: throw new \Exception('Unknown message type: ' . $type); @@ -133,7 +135,7 @@ class Messaging extends Action Document $message, Device $deviceForFiles, Document $project, - StatsUsage $queueForStatsUsage + UsagePublisher $publisherForUsage ): void { $topicIds = $message->getAttribute('topics', []); $targetIds = $message->getAttribute('targets', []); @@ -239,8 +241,8 @@ class Messaging extends Action /** * @var array $results */ - $results = batch(\array_map(function ($providerId) use ($identifiers, &$providers, $default, $message, $dbForProject, $deviceForFiles, $project, $queueForStatsUsage) { - return function () use ($providerId, $identifiers, &$providers, $default, $message, $dbForProject, $deviceForFiles, $project, $queueForStatsUsage) { + $results = batch(\array_map(function ($providerId) use ($identifiers, &$providers, $default, $message, $dbForProject, $deviceForFiles, $project, $publisherForUsage) { + return function () use ($providerId, $identifiers, &$providers, $default, $message, $dbForProject, $deviceForFiles, $project, $publisherForUsage) { if (\array_key_exists($providerId, $providers)) { $provider = $providers[$providerId]; } else { @@ -267,8 +269,8 @@ class Messaging extends Action $adapter->getMaxMessagesPerRequest() ); - return batch(\array_map(function ($batch) use ($message, $provider, $adapter, $dbForProject, $deviceForFiles, $project, $queueForStatsUsage) { - return function () use ($batch, $message, $provider, $adapter, $dbForProject, $deviceForFiles, $project, $queueForStatsUsage) { + return batch(\array_map(function ($batch) use ($message, $provider, $adapter, $dbForProject, $deviceForFiles, $project, $publisherForUsage) { + return function () use ($batch, $message, $provider, $adapter, $dbForProject, $deviceForFiles, $project, $publisherForUsage) { $deliveredTotal = 0; $deliveryErrors = []; $messageData = clone $message; @@ -308,8 +310,8 @@ class Messaging extends Action $deliveryErrors[] = 'Failed sending to targets with error: ' . $e->getMessage(); } finally { $errorTotal = \count($deliveryErrors); - $queueForStatsUsage - ->setProject($project) + $usage = new UsageContext(); + $usage ->addMetric(METRIC_MESSAGES, ($deliveredTotal + $errorTotal)) ->addMetric(METRIC_MESSAGES_SENT, $deliveredTotal) ->addMetric(METRIC_MESSAGES_FAILED, $errorTotal) @@ -318,8 +320,12 @@ class Messaging extends Action ->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_FAILED), $errorTotal) ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER), ($deliveredTotal + $errorTotal)) ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_SENT), $deliveredTotal) - ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_FAILED), $errorTotal) - ->trigger(); + ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_FAILED), $errorTotal); + + $publisherForUsage->enqueue(new Usage( + project: $project, + metrics: $usage->getMetrics(), + )); return [ 'deliveredTotal' => $deliveredTotal, diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index c4cb9ce415..ce20358626 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -4,10 +4,12 @@ namespace Appwrite\Platform\Workers; use Ahc\Jwt\JWT; use Appwrite\Event\Mail; +use Appwrite\Event\Message\Usage as UsageMessage; +use Appwrite\Event\Publisher\Usage as UsagePublisher; use Appwrite\Event\Realtime; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\Template\Template; +use Appwrite\Usage\Context; use Utopia\Compression\Compression; use Utopia\Config\Config; use Utopia\Console; @@ -84,7 +86,8 @@ class Migrations extends Action ->inject('deviceForMigrations') ->inject('deviceForFiles') ->inject('queueForMails') - ->inject('queueForStatsUsage') + ->inject('usage') + ->inject('publisherForUsage') ->inject('plan') ->inject('authorization') ->callback($this->action(...)); @@ -103,7 +106,8 @@ class Migrations extends Action Device $deviceForMigrations, Device $deviceForFiles, Mail $queueForMails, - StatsUsage $queueForStatsUsage, + Context $usage, + UsagePublisher $publisherForUsage, array $plan, Authorization $authorization, ): void { @@ -147,7 +151,8 @@ class Migrations extends Action $migration, $queueForRealtime, $queueForMails, - $queueForStatsUsage, + $usage, + $publisherForUsage, $platform, $authorization ); @@ -345,7 +350,8 @@ class Migrations extends Action Document $migration, Realtime $queueForRealtime, Mail $queueForMails, - StatsUsage $queueForStatsUsage, + Context $usage, + UsagePublisher $publisherForUsage, array $platform, Authorization $authorization, ): void { @@ -360,7 +366,7 @@ class Migrations extends Action throw new \Exception('_APP_MIGRATION_HOST is not set'); } - $endpoint = 'http://'.$host.'/v1'; + $endpoint = 'http://' . $host . '/v1'; try { $credentials = $migration->getAttribute('credentials', []); @@ -463,7 +469,7 @@ class Migrations extends Action $migration->setAttribute('status', 'failed'); $migration->setAttribute('stage', 'finished'); - call_user_func($this->logError, $th, 'appwrite-worker', 'appwrite-queue-'.self::getName(), [ + call_user_func($this->logError, $th, 'appwrite-worker', 'appwrite-queue-' . self::getName(), [ 'migrationId' => $migration->getId(), 'source' => $migration->getAttribute('source') ?? '', 'destination' => $migration->getAttribute('destination') ?? '', @@ -474,7 +480,7 @@ class Migrations extends Action $this->updateMigrationDocument($migration, $project, $queueForRealtime); if ($migration->getAttribute('status', '') === 'failed') { - Console::error('Migration('.$migration->getSequence().':'.$migration->getId().') failed, Project('.$this->project->getSequence().':'.$this->project->getId().')'); + Console::error('Migration(' . $migration->getSequence() . ':' . $migration->getId() . ') failed, Project(' . $this->project->getSequence() . ':' . $this->project->getId() . ')'); $sourceErrors = $source?->getErrors() ?? []; $destinationErrors = $destination?->getErrors() ?? []; @@ -500,8 +506,9 @@ class Migrations extends Action foreach ($aggregatedResources as $resource) { $this->processMigrationResourceStats( $resource, - $queueForStatsUsage, + $usage, $project, + $publisherForUsage, $migration->getAttribute('source'), $authorization, $migration->getAttribute('resourceId') @@ -802,7 +809,7 @@ class Migrations extends Action return $errors; } - private function processMigrationResourceStats(array $resources, StatsUsage $queueForStatsUsage, Document $projectDocument, string $source, Authorization $authorization, ?string $resourceId) + private function processMigrationResourceStats(array $resources, Context $usage, Document $projectDocument, UsagePublisher $publisherForUsage, string $source, Authorization $authorization, ?string $resourceId) { $resourceName = $resources['name']; $count = $resources['count']; @@ -819,11 +826,11 @@ class Migrations extends Action switch ($resourceName) { case ResourceDatabase::getName(): - $queueForStatsUsage->addMetric(METRIC_DATABASES, $count); + $usage->addMetric(METRIC_DATABASES, $count); break; case ResourceTable::getName(): - $queueForStatsUsage + $usage ->addMetric(METRIC_COLLECTIONS, $count) ->addMetric( str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_COLLECTIONS), @@ -832,7 +839,7 @@ class Migrations extends Action break; case ResourceRow::getName(): - $queueForStatsUsage + $usage ->addMetric( str_replace( ['{databaseInternalId}','{collectionInternalId}'], @@ -852,7 +859,12 @@ class Migrations extends Action break; } - $queueForStatsUsage->setProject($projectDocument)->trigger(); - $queueForStatsUsage->reset(); + $message = new UsageMessage( + project: $projectDocument, + metrics: $usage->getMetrics(), + reduce: $usage->getReduce() + ); + $publisherForUsage->enqueue($message); + $usage->reset(); } } diff --git a/src/Appwrite/Platform/Workers/Webhooks.php b/src/Appwrite/Platform/Workers/Webhooks.php index 4855a1d4d8..fce3c7b149 100644 --- a/src/Appwrite/Platform/Workers/Webhooks.php +++ b/src/Appwrite/Platform/Workers/Webhooks.php @@ -3,8 +3,10 @@ namespace Appwrite\Platform\Workers; use Appwrite\Event\Mail; -use Appwrite\Event\StatsUsage; +use Appwrite\Event\Message\Usage as UsageMessage; +use Appwrite\Event\Publisher\Usage as UsagePublisher; use Appwrite\Template\Template; +use Appwrite\Usage\Context as UsageContext; use Exception; use Utopia\Database\Database; use Utopia\Database\Document; @@ -35,7 +37,7 @@ class Webhooks extends Action ->inject('project') ->inject('dbForPlatform') ->inject('queueForMails') - ->inject('queueForStatsUsage') + ->inject('publisherForUsage') ->inject('log') ->inject('plan') ->callback($this->action(...)); @@ -46,13 +48,13 @@ class Webhooks extends Action * @param Document $project * @param Database $dbForPlatform * @param Mail $queueForMails - * @param StatsUsage $queueForStatsUsage + * @param UsagePublisher $publisherForUsage * @param Log $log * @param array $plan * @return void * @throws Exception */ - public function action(Message $message, Document $project, Database $dbForPlatform, Mail $queueForMails, StatsUsage $queueForStatsUsage, Log $log, array $plan): void + public function action(Message $message, Document $project, Database $dbForPlatform, Mail $queueForMails, UsagePublisher $publisherForUsage, Log $log, array $plan): void { $this->errors = []; $payload = $message->getPayload() ?? []; @@ -71,7 +73,7 @@ class Webhooks extends Action foreach ($project->getAttribute('webhooks', []) as $webhook) { if (array_intersect($webhook->getAttribute('events', []), $events)) { - $this->execute($events, $webhookPayload, $webhook, $user, $project, $dbForPlatform, $queueForMails, $queueForStatsUsage, $plan); + $this->execute($events, $webhookPayload, $webhook, $user, $project, $dbForPlatform, $queueForMails, $publisherForUsage, $plan); } } @@ -91,7 +93,7 @@ class Webhooks extends Action * @param array $plan * @return void */ - private function execute(array $events, string $payload, Document $webhook, Document $user, Document $project, Database $dbForPlatform, Mail $queueForMails, StatsUsage $queueForStatsUsage, array $plan): void + private function execute(array $events, string $payload, Document $webhook, Document $user, Document $project, Database $dbForPlatform, Mail $queueForMails, UsagePublisher $publisherForUsage, array $plan): void { if ($webhook->getAttribute('enabled') !== true) { return; @@ -180,26 +182,23 @@ class Webhooks extends Action $dbForPlatform->purgeCachedDocument('projects', $project->getId()); $this->errors[] = $logs; - $queueForStatsUsage + $usage = (new UsageContext()) ->addMetric(METRIC_WEBHOOKS_FAILED, 1) - ->addMetric(str_replace('{webhookInternalId}', $webhook->getSequence(), METRIC_WEBHOOK_ID_FAILED), 1) - ; - - + ->addMetric(str_replace('{webhookInternalId}', $webhook->getSequence(), METRIC_WEBHOOK_ID_FAILED), 1); } else { $dbForPlatform->updateDocument('webhooks', $webhook->getId(), new Document([ 'attempts' => 0, ])); $dbForPlatform->purgeCachedDocument('projects', $project->getId()); - $queueForStatsUsage + $usage = (new UsageContext()) ->addMetric(METRIC_WEBHOOKS_SENT, 1) - ->addMetric(str_replace('{webhookInternalId}', $webhook->getSequence(), METRIC_WEBHOOK_ID_SENT), 1) - ; + ->addMetric(str_replace('{webhookInternalId}', $webhook->getSequence(), METRIC_WEBHOOK_ID_SENT), 1); } - $queueForStatsUsage - ->setProject($project) - ->trigger(); + $publisherForUsage->enqueue(new UsageMessage( + project: $project, + metrics: $usage->getMetrics(), + )); } /** diff --git a/src/Appwrite/Usage/Context.php b/src/Appwrite/Usage/Context.php new file mode 100644 index 0000000000..7283cff836 --- /dev/null +++ b/src/Appwrite/Usage/Context.php @@ -0,0 +1,74 @@ +metrics[] = [ + 'key' => $key, + 'value' => $value, + ]; + + return $this; + } + + /** + * Add a document to reduce + */ + public function addReduce(Document $document): self + { + $this->reduce[] = $document; + + return $this; + } + + /** + * Get all metrics + * + * @return array + */ + public function getMetrics(): array + { + return $this->metrics; + } + + /** + * Get all reduce documents + * + * @return array + */ + public function getReduce(): array + { + return $this->reduce; + } + + /** + * Check if context is empty + */ + public function isEmpty(): bool + { + return empty($this->metrics) && empty($this->reduce); + } + + /** + * Reset the context + */ + public function reset(): self + { + $this->metrics = []; + $this->reduce = []; + + return $this; + } +} diff --git a/tests/e2e/Services/Migrations/MigrationsBase.php b/tests/e2e/Services/Migrations/MigrationsBase.php index d5fe7753a4..0d992c472e 100644 --- a/tests/e2e/Services/Migrations/MigrationsBase.php +++ b/tests/e2e/Services/Migrations/MigrationsBase.php @@ -1651,9 +1651,9 @@ trait MigrationsBase }, 30_000, 500); // Check that email was sent with download link - $lastEmail = $this->getLastEmail(); - $this->assertNotEmpty($lastEmail); - $this->assertEquals('Your CSV export is ready', $lastEmail['subject']); + $lastEmail = $this->getLastEmail(probe: function ($email) { + $this->assertEquals('Your CSV export is ready', $email['subject']); + }); $this->assertStringContainsStringIgnoringCase('Your data export has been completed successfully', $lastEmail['text']); // Extract download URL from email HTML