Merge branch 'refactor-usage-sn' of github.com:appwrite/appwrite into refactor-move-cli-scripts

 Conflicts:

	src/Appwrite/Platform/Services/Tasks.php

.env

@@ -4,12 +4,13 @@ _APP_WORKER_PER_CORE=6
 _APP_CONSOLE_WHITELIST_ROOT=disabled
 _APP_CONSOLE_WHITELIST_EMAILS=
 _APP_CONSOLE_WHITELIST_IPS=
+_APP_CONSOLE_HOSTNAMES=localhost,appwrite.io,*.appwrite.io
 _APP_SYSTEM_EMAIL_NAME=Appwrite
 _APP_SYSTEM_EMAIL_ADDRESS=team@appwrite.io
 _APP_SYSTEM_SECURITY_EMAIL_ADDRESS=security@appwrite.io
 _APP_SYSTEM_RESPONSE_FORMAT=
 _APP_OPTIONS_ABUSE=disabled
-_APP_OPTIONS_ROUTER_PROTECTION=disbled
+_APP_OPTIONS_ROUTER_PROTECTION=disabled
 _APP_OPTIONS_FORCE_HTTPS=disabled
 _APP_OPTIONS_FUNCTIONS_FORCE_HTTPS=disabled
 _APP_OPENSSL_KEY_V1=your-secret-key
@@ -57,6 +58,7 @@ _APP_SMTP_USERNAME=
 _APP_SMTP_PASSWORD=
 _APP_SMS_PROVIDER=sms://username:password@mock
 _APP_SMS_FROM=+123456789
+_APP_SMS_PROJECTS_DENY_LIST=
 _APP_STORAGE_LIMIT=30000000
 _APP_STORAGE_PREVIEW_LIMIT=20000000
 _APP_FUNCTIONS_SIZE_LIMIT=30000000
@@ -75,7 +77,7 @@ _APP_MAINTENANCE_RETENTION_CACHE=2592000
 _APP_MAINTENANCE_RETENTION_EXECUTION=1209600
 _APP_MAINTENANCE_RETENTION_ABUSE=86400
 _APP_MAINTENANCE_RETENTION_AUDIT=1209600
-_APP_USAGE_AGGREGATION_INTERVAL=5
+_APP_USAGE_AGGREGATION_INTERVAL=60000
 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000
 _APP_MAINTENANCE_RETENTION_SCHEDULES=86400
 _APP_USAGE_STATS=enabled

.github/workflows/tests.yml

@@ -75,8 +75,32 @@ jobs:
       - name: Run Unit Tests
         run: docker compose exec appwrite test /usr/src/code/tests/unit
 
-  e2e_test:
-    name: E2E Test
+  e2e_general_test:
+    name: E2E General Test
+    runs-on: ubuntu-latest
+    needs: setup
+    steps:
+      - name: checkout
+        uses: actions/checkout@v3
+
+      - name: Load Cache
+        uses: actions/cache@v3
+        with:
+          key: ${{ env.CACHE_KEY }}
+          path: /tmp/${{ env.IMAGE }}.tar
+          fail-on-cache-miss: true
+
+      - name: Load and Start Appwrite
+        run: |
+          docker load --input /tmp/${{ env.IMAGE }}.tar
+          docker compose up -d
+          sleep 10
+
+      - name: Run General Tests
+        run: docker compose exec -T appwrite test /usr/src/code/tests/e2e/General --debug
+
+  e2e_service_test:
+    name: E2E Service Test
     runs-on: ubuntu-latest
     needs: setup
     strategy:
@@ -119,4 +143,4 @@ jobs:
           sleep 10
 
       - name: Run ${{matrix.service}} Tests
-        run: docker compose exec -T appwrite test /usr/src/code/tests/e2e/Services/${{matrix.service}} --debug
+        run: docker compose exec -T appwrite test /usr/src/code/tests/e2e/Services/${{matrix.service}} --debug

.gitmodules

@@ -1,4 +1,4 @@
 [submodule "app/console"]
 	path = app/console
 	url = https://github.com/appwrite/console
-	branch = 3.3.6
+	branch = 3.2.16

Dockerfile

@@ -84,6 +84,10 @@ RUN chmod +x /usr/local/bin/doctor && \
     chmod +x /usr/local/bin/ssl && \
     chmod +x /usr/local/bin/test && \
     chmod +x /usr/local/bin/vars && \
+    chmod +x /usr/local/bin/queue-retry && \
+    chmod +x /usr/local/bin/queue-count-failed && \
+    chmod +x /usr/local/bin/queue-count-processing && \
+    chmod +x /usr/local/bin/queue-count-success && \
     chmod +x /usr/local/bin/worker-audits && \
     chmod +x /usr/local/bin/worker-certificates && \
     chmod +x /usr/local/bin/worker-databases && \
@@ -93,21 +97,9 @@ RUN chmod +x /usr/local/bin/doctor && \
     chmod +x /usr/local/bin/worker-mails && \
     chmod +x /usr/local/bin/worker-messaging && \
     chmod +x /usr/local/bin/worker-webhooks && \
-    chmod +x /usr/local/bin/worker-usage && \
-    chmod +x /usr/local/bin/worker-migrations
+    chmod +x /usr/local/bin/worker-migrations && \
+    chmod +x /usr/local/bin/worker-usage
 
-# Cloud Executabless
-RUN chmod +x /usr/local/bin/hamster && \
-    chmod +x /usr/local/bin/volume-sync && \
-    chmod +x /usr/local/bin/patch-delete-schedule-updated-at-attribute && \
-    chmod +x /usr/local/bin/patch-recreate-repositories-documents && \
-    chmod +x /usr/local/bin/patch-delete-project-collections && \
-    chmod +x /usr/local/bin/delete-orphaned-projects && \
-    chmod +x /usr/local/bin/clear-card-cache && \
-    chmod +x /usr/local/bin/calc-users-stats && \
-    chmod +x /usr/local/bin/calc-tier-stats && \
-    chmod +x /usr/local/bin/get-migration-stats && \
-    chmod +x /usr/local/bin/create-inf-metric
 
 # Letsencrypt Permissions
 RUN mkdir -p /etc/letsencrypt/live/ && chmod -Rf 755 /etc/letsencrypt/live/

app/cli.php

@@ -6,6 +6,7 @@ require_once __DIR__ . '/controllers/general.php';
 use Appwrite\Event\Delete;
 use Appwrite\Event\Certificate;
 use Appwrite\Event\Func;
+use Appwrite\Event\Hamster;
 use Appwrite\Platform\Appwrite;
 use Utopia\CLI\CLI;
 use Utopia\Database\Validator\Authorization;
@@ -130,6 +131,9 @@ CLI::setResource('queue', function (Group $pools) {
 CLI::setResource('queueForFunctions', function (Connection $queue) {
     return new Func($queue);
 }, ['queue']);
+CLI::setResource('queueForHamster', function (Connection $queue) {
+    return new Hamster($queue);
+}, ['queue']);
 CLI::setResource('queueForDeletes', function (Connection $queue) {
     return new Delete($queue);
 }, ['queue']);

app/config/errors.php

@@ -680,6 +680,7 @@ return [
         'name' => Exception::RULE_VERIFICATION_FAILED,
         'description' => 'Domain verification failed. Please check if your DNS records are correct and try again.',
         'code' => 401,
+        'publish' => true
     ],
     Exception::PROJECT_SMTP_CONFIG_INVALID => [
         'name' => Exception::PROJECT_SMTP_CONFIG_INVALID,

app/config/locale/translations/tr.json

@@ -3,30 +3,36 @@
     "settings.locale": "tr",
     "settings.direction": "ltr",
     "emails.sender": "%s Takımı",
-    "emails.verification.subject": "",
-    "emails.verification.hello": "",
-    "emails.verification.body": "",
-    "emails.verification.footer": "",
-    "emails.verification.thanks": "",
-    "emails.verification.signature": "",
-    "emails.magicSession.subject": "",
-    "emails.magicSession.hello": "",
-    "emails.magicSession.body": "",
-    "emails.magicSession.footer": "",
-    "emails.magicSession.thanks": "",
-    "emails.magicSession.signature": "",
-    "emails.recovery.subject": "",
-    "emails.recovery.hello": "",
-    "emails.recovery.body": "",
-    "emails.recovery.footer": "",
-    "emails.recovery.thanks": "",
-    "emails.recovery.signature": "",
-    "emails.invitation.subject": "",
-    "emails.invitation.hello": "",
-    "emails.invitation.body": "",
-    "emails.invitation.footer": "",
-    "emails.invitation.thanks": "",
-    "emails.invitation.signature": "",
+    "emails.verification.subject": "Hesabını Doğrula",
+    "emails.verification.hello": "Merhaba {{user}}",
+    "emails.verification.body": "Eposta adresini doğrulamak için bu bağlantıyı kullanın.",
+    "emails.verification.footer": "Eğer bu eposta adresini doğrulamak isteyen siz değilseniz devam etmeyin.",
+    "emails.verification.thanks": "Teşekkürler",
+    "emails.verification.signature": "{{project}} takımı",
+    "emails.magicSession.subject": "Giriş",
+    "emails.magicSession.hello": "Merhaba,",
+    "emails.magicSession.body": "Giriş yapmak için tıklayın.",
+    "emails.magicSession.footer": "Eğer bu eposta adresini kullanarak giriş yapmak istemediyseniz devam etmeyin.",
+    "emails.magicSession.thanks": "Teşekkürler",
+    "emails.magicSession.signature": "{{project}} takımı",
+    "emails.recovery.subject": "Şifremi Sıfırla",
+    "emails.recovery.hello": "Merhaba {{user}}",
+    "emails.recovery.body": "{{project}} şifrenizi sıfırlamak için bu bağlantıyı kullanın.",
+    "emails.recovery.footer": "Eğer şifre sıfırlama talebinde bulunmadıysanız devam etmeyin.",
+    "emails.recovery.thanks": "Teşekkürler",
+    "emails.recovery.signature": "{{project}} takımı",
+    "emails.invitation.subject": "%s üzerinde %s Takımına Davet",
+    "emails.invitation.hello": "Merhaba",
+    "emails.invitation.body": "Bu epostayı aldınız, çünkü {{owner}} sizi {{project}} üzerinde {{team}} takımının üyesi olmaya davet etti.",
+    "emails.invitation.footer": "Eğer ilgilenmiyorsanız devam etmeyin.",
+    "emails.invitation.thanks": "Teşekkürler",
+    "emails.invitation.signature": "{{project}} takımı",
+    "emails.certificate.subject": "%s için sertifika hatası",
+    "emails.certificate.hello": "Merhaba",
+    "emails.certificate.body": "Alan adınız '{{domain}}' için sertifika oluşturulamadı. Deneme sayısı {{attempt}} ve hata sebebi: {{error}}",
+    "emails.certificate.footer": "Geçmiş sertifikanız ilk denemeden sonra 30 gün daha geçerli kalacaktır. Bu konuyu araştırmanızı öneriyoruz, aksi taktirde alan adınız SSL sertifikasız kalacaktır.",
+    "emails.certificate.thanks": "Teşekkürler",
+    "emails.certificate.signature": "{{project}} takımı",
     "locale.country.unknown": "Bilinmeyen",
     "countries.af": "Afganistan",
     "countries.ao": "Angola",
@@ -229,4 +235,4 @@
     "continents.na": "Kuzey Amerika",
     "continents.oc": "Okyanusya",
     "continents.sa": "Güney Amerika"
-}
+}

app/config/platforms.php

@@ -185,7 +185,7 @@ return [
             [
                 'key' => 'web',
                 'name' => 'Console',
-                'version' => '0.4.2',
+                'version' => '0.5.0',
                 'url' => 'https://github.com/appwrite/sdk-for-console',
                 'package' => '',
                 'enabled' => true,
@@ -196,7 +196,7 @@ return [
                 'prism' => 'javascript',
                 'source' => \realpath(__DIR__ . '/../sdks/console-web'),
                 'gitUrl' => 'https://github.com/appwrite/sdk-for-console.git',
-                'gitBranch' => 'dev',
+                'gitBranch' => 'main',
                 'gitRepoName' => 'sdk-for-console',
                 'gitUserName' => 'appwrite',
             ],

app/config/variables.php

@@ -115,14 +115,6 @@ return [
                 'question' => '',
                 'filter' => ''
             ],
-            // [
-            //     'name' => '_APP_CONSOLE_WHITELIST_DOMAINS',
-            //     'description' => 'This option allows you to limit creation of users to Appwrite console for users sharing the same email domains. This option is very useful for team working with company emails domain.\n\nTo enable this option, pass a list of allowed email domains separated by a comma.',
-            //     'introduction' => '',
-            //     'default' => '',
-            //     'required' => false,
-            //     'question' => '',
-            // ],
             [
                 'name' => '_APP_CONSOLE_WHITELIST_IPS',
                 'description' => "This last option allows you to limit creation of users in Appwrite console for users sharing the same set of IP addresses. This option is very useful for team working with a VPN service or a company IP.\n\nTo enable/activate this option, pass a list of allowed IP addresses separated by a comma.",
@@ -132,6 +124,15 @@ return [
                 'question' => '',
                 'filter' => ''
             ],
+            [
+                'name' => '_APP_CONSOLE_HOSTNAMES',
+                'description' => 'This option allows you to add additional hostnames to your Appwrite console. This option is very useful for allowing access to the console project from additional domains. To enable it, pass a list of allowed hostnames separated by a comma.',
+                'introduction' => '1.5.0',
+                'default' => '',
+                'required' => false,
+                'question' => '',
+                'filter' => ''
+            ],
             [
                 'name' => '_APP_SYSTEM_EMAIL_NAME',
                 'description' => 'This is the sender name value that will appear on email messages sent to developers from the Appwrite console. The default value is: \'Appwrite\'. You can use url encoded strings for spaces and special chars.',

app/controllers/api/account.php

@@ -45,6 +45,7 @@ use Utopia\Validator\WhiteList;
 use Appwrite\Auth\Validator\PasswordHistory;
 use Appwrite\Auth\Validator\PasswordDictionary;
 use Appwrite\Auth\Validator\PersonalData;
+use Appwrite\Event\Delete;
 use Appwrite\Hooks\Hooks;
 
 $oauthDefaultSuccess = '/auth/oauth2/success';
@@ -119,7 +120,7 @@ App::post('/v1/account')
             }
         }
 
-        $hooks->trigger('passwordValidator', [$project, $password, $user]);
+        $hooks->trigger('passwordValidator', [$dbForProject, $project, $password, &$user, true]);
 
         $passwordHistory = $project->getAttribute('auths', [])['passwordHistory'] ?? 0;
         $password = Auth::passwordHash($password, Auth::DEFAULT_ALGO, Auth::DEFAULT_ALGO_OPTIONS);
@@ -196,7 +197,8 @@ App::post('/v1/account/sessions/email')
     ->inject('locale')
     ->inject('geodb')
     ->inject('queueForEvents')
-    ->action(function (string $email, string $password, Request $request, Response $response, Document $user, Database $dbForProject, Document $project, Locale $locale, Reader $geodb, Event $queueForEvents) {
+    ->inject('hooks')
+    ->action(function (string $email, string $password, Request $request, Response $response, Document $user, Database $dbForProject, Document $project, Locale $locale, Reader $geodb, Event $queueForEvents, Hooks $hooks) {
 
         $email = \strtolower($email);
         $protocol = $request->getProtocol();
@@ -215,6 +217,8 @@ App::post('/v1/account/sessions/email')
 
         $user->setAttributes($profile->getArrayCopy());
 
+        $hooks->trigger('passwordValidator', [$dbForProject, $project, $password, &$user, false]);
+
         $duration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG;
 
         $detector = new Detector($request->getUserAgent('UNKNOWN'));
@@ -549,11 +553,19 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect')
         if (!$user->isEmpty()) {
             $userId = $user->getId();
 
-            $identitiesWithMatchingEmail = $dbForProject->find('identities', [
+            $identityWithMatchingEmail = $dbForProject->findOne('identities', [
                 Query::equal('providerEmail', [$email]),
                 Query::notEqual('userId', $userId),
             ]);
-            if (!empty($identitiesWithMatchingEmail)) {
+            if (!empty($identityWithMatchingEmail)) {
+                throw new Exception(Exception::USER_ALREADY_EXISTS);
+            }
+
+            $userWithMatchingEmail = $dbForProject->find('users', [
+                Query::equal('email', [$email]),
+                Query::notEqual('$id', $userId),
+            ]);
+            if (!empty($userWithMatchingEmail)) {
                 throw new Exception(Exception::USER_ALREADY_EXISTS);
             }
         }
@@ -840,7 +852,7 @@ App::get('/v1/account/identities')
     });
 
 App::delete('/v1/account/identities/:identityId')
-    ->desc('Delete Identity')
+    ->desc('Delete identity')
     ->groups(['api', 'account'])
     ->label('scope', 'account')
     ->label('event', 'users.[userId].identities.[identityId].delete')
@@ -856,7 +868,8 @@ App::delete('/v1/account/identities/:identityId')
     ->param('identityId', '', new UID(), 'Identity ID.')
     ->inject('response')
     ->inject('dbForProject')
-    ->action(function (string $identityId, Response $response, Database $dbForProject) {
+    ->inject('queueForEvents')
+    ->action(function (string $identityId, Response $response, Database $dbForProject, Event $queueForEvents) {
 
         $identity = $dbForProject->getDocument('identities', $identityId);
 
@@ -866,12 +879,17 @@ App::delete('/v1/account/identities/:identityId')
 
         $dbForProject->deleteDocument('identities', $identityId);
 
+        $queueForEvents
+            ->setParam('userId', $identity->getAttribute('userId'))
+            ->setParam('identityId', $identity->getId())
+            ->setPayload($response->output($identity, Response::MODEL_IDENTITY));
+
         return $response->noContent();
     });
 
 App::post('/v1/account/sessions/magic-url')
     ->desc('Create magic URL session')
-    ->groups(['api', 'account'])
+    ->groups(['api', 'account', 'auth'])
     ->label('scope', 'public')
     ->label('auth.type', 'magic-url')
     ->label('audits.event', 'session.create')
@@ -884,8 +902,8 @@ App::post('/v1/account/sessions/magic-url')
     ->label('sdk.response.code', Response::STATUS_CODE_CREATED)
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_TOKEN)
-    ->label('abuse-limit', 10)
-    ->label('abuse-key', 'url:{url},email:{param-email}')
+    ->label('abuse-limit', 60)
+    ->label('abuse-key', ['url:{url},email:{param-email}', 'url:{url},ip:{ip}'])
     ->param('userId', '', new CustomId(), 'Unique Id. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.')
     ->param('email', '', new Email(), 'User email.')
     ->param('url', '', fn($clients) => new Host($clients), 'URL to redirect the user back to your app from the magic URL login. Only URLs from hostnames in your project platform list are allowed. This requirement helps to prevent an [open redirect](https://cheatsheetseries.owasp.org/cheatsheets/Unvalidated_Redirects_and_Forwards_Cheat_Sheet.html) attack against your project API.', true, ['clients'])
@@ -998,7 +1016,7 @@ App::post('/v1/account/sessions/magic-url')
 
         $message = Template::fromFile(__DIR__ . '/../../config/locale/templates/email-inner-base.tpl');
         $message
-            ->setParam('{{body}}', $body)
+            ->setParam('{{body}}', $body, escapeHtml: false)
             ->setParam('{{hello}}', $locale->getText("emails.magicSession.hello"))
             ->setParam('{{footer}}', $locale->getText("emails.magicSession.footer"))
             ->setParam('{{thanks}}', $locale->getText("emails.magicSession.thanks"))
@@ -1053,11 +1071,11 @@ App::post('/v1/account/sessions/magic-url')
 
         $emailVariables = [
             'direction' => $locale->getText('settings.direction'),
-            /* {{user}} ,{{team}}, {{project}} and {{redirect}} are required in the templates */
+            /* {{user}}, {{team}}, {{redirect}} and {{project}} are required in default and custom templates */
             'user' => '',
             'team' => '',
-            'project' => $project->getAttribute('name'),
-            'redirect' => $url
+            'redirect' => $url,
+            'project' => $project->getAttribute('name')
         ];
 
         $queueForMails
@@ -1205,7 +1223,7 @@ App::put('/v1/account/sessions/magic-url')
 
 App::post('/v1/account/sessions/phone')
     ->desc('Create phone session')
-    ->groups(['api', 'account'])
+    ->groups(['api', 'account', 'auth'])
     ->label('scope', 'public')
     ->label('auth.type', 'phone')
     ->label('audits.event', 'session.create')
@@ -1219,7 +1237,7 @@ App::post('/v1/account/sessions/phone')
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_TOKEN)
     ->label('abuse-limit', 10)
-    ->label('abuse-key', 'url:{url},phone:{param-phone}')
+    ->label('abuse-key', ['url:{url},phone:{param-phone}', 'url:{url},ip:{ip}'])
     ->param('userId', '', new CustomId(), 'Unique Id. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.')
     ->param('phone', '', new Phone(), 'Phone number. Format this number with a leading \'+\' and a country code, e.g., +16175551212.')
     ->inject('request')
@@ -1318,9 +1336,12 @@ App::post('/v1/account/sessions/phone')
         $message = $message->setParam('{{token}}', $secret);
         $message = $message->render();
 
+        var_dump($request->getIP());
+        var_dump($project->getId());
         $queueForMessaging
             ->setRecipient($phone)
             ->setMessage($message)
+            ->setProject($project)
             ->trigger();
 
         $queueForEvents->setPayload(
@@ -1879,7 +1900,8 @@ App::patch('/v1/account/password')
     ->inject('project')
     ->inject('dbForProject')
     ->inject('queueForEvents')
-    ->action(function (string $password, string $oldPassword, ?\DateTime $requestTimestamp, Response $response, Document $user, Document $project, Database $dbForProject, Event $queueForEvents) {
+    ->inject('hooks')
+    ->action(function (string $password, string $oldPassword, ?\DateTime $requestTimestamp, Response $response, Document $user, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks) {
 
         // Check old password only if its an existing user.
         if (!empty($user->getAttribute('passwordUpdate')) && !Auth::passwordVerify($oldPassword, $user->getAttribute('password'), $user->getAttribute('hash'), $user->getAttribute('hashOptions'))) { // Double check user password
@@ -1906,6 +1928,8 @@ App::patch('/v1/account/password')
             }
         }
 
+        $hooks->trigger('passwordValidator', [$dbForProject, $project, $password, &$user, true]);
+
         $user
             ->setAttribute('password', $newPassword)
             ->setAttribute('passwordHistory', $history)
@@ -1943,7 +1967,9 @@ App::patch('/v1/account/email')
     ->inject('user')
     ->inject('dbForProject')
     ->inject('queueForEvents')
-    ->action(function (string $email, string $password, ?\DateTime $requestTimestamp, Response $response, Document $user, Database $dbForProject, Event $queueForEvents) {
+    ->inject('project')
+    ->inject('hooks')
+    ->action(function (string $email, string $password, ?\DateTime $requestTimestamp, Response $response, Document $user, Database $dbForProject, Event $queueForEvents, Document $project, Hooks $hooks) {
         // passwordUpdate will be empty if the user has never set a password
         $passwordUpdate = $user->getAttribute('passwordUpdate');
 
@@ -1954,6 +1980,8 @@ App::patch('/v1/account/email')
             throw new Exception(Exception::USER_INVALID_CREDENTIALS);
         }
 
+        $hooks->trigger('passwordValidator', [$dbForProject, $project, $password, &$user, false]);
+
         $email = \strtolower($email);
 
         // Makes sure this email is not already used in another identity
@@ -2012,7 +2040,9 @@ App::patch('/v1/account/phone')
     ->inject('user')
     ->inject('dbForProject')
     ->inject('queueForEvents')
-    ->action(function (string $phone, string $password, ?\DateTime $requestTimestamp, Response $response, Document $user, Database $dbForProject, Event $queueForEvents) {
+    ->inject('project')
+    ->inject('hooks')
+    ->action(function (string $phone, string $password, ?\DateTime $requestTimestamp, Response $response, Document $user, Database $dbForProject, Event $queueForEvents, Document $project, Hooks $hooks) {
         // passwordUpdate will be empty if the user has never set a password
         $passwordUpdate = $user->getAttribute('passwordUpdate');
 
@@ -2023,6 +2053,8 @@ App::patch('/v1/account/phone')
             throw new Exception(Exception::USER_INVALID_CREDENTIALS);
         }
 
+        $hooks->trigger('passwordValidator', [$dbForProject, $project, $password, &$user, false]);
+
         $user
             ->setAttribute('phone', $phone)
             ->setAttribute('phoneVerification', false) // After this user needs to confirm phone number again
@@ -2359,7 +2391,7 @@ App::post('/v1/account/recovery')
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_TOKEN)
     ->label('abuse-limit', 10)
-    ->label('abuse-key', ['url:{url},email:{param-email}', 'ip:{ip}'])
+    ->label('abuse-key', ['url:{url},email:{param-email}', 'url:{url},ip:{ip}'])
     ->param('email', '', new Email(), 'User email.')
     ->param('url', '', fn ($clients) => new Host($clients), 'URL to redirect the user back to your app from the recovery email. Only URLs from hostnames in your project platform list are allowed. This requirement helps to prevent an [open redirect](https://cheatsheetseries.owasp.org/cheatsheets/Unvalidated_Redirects_and_Forwards_Cheat_Sheet.html) attack against your project API.', false, ['clients'])
     ->inject('request')
@@ -2432,7 +2464,7 @@ App::post('/v1/account/recovery')
 
         $message = Template::fromFile(__DIR__ . '/../../config/locale/templates/email-inner-base.tpl');
         $message
-            ->setParam('{{body}}', $body)
+            ->setParam('{{body}}', $body, escapeHtml: false)
             ->setParam('{{hello}}', $locale->getText("emails.recovery.hello"))
             ->setParam('{{footer}}', $locale->getText("emails.recovery.footer"))
             ->setParam('{{thanks}}', $locale->getText("emails.recovery.thanks"))
@@ -2487,11 +2519,11 @@ App::post('/v1/account/recovery')
 
         $emailVariables = [
             'direction' => $locale->getText('settings.direction'),
-            /* {{user}} ,{{team}}, {{project}} and {{redirect}} are required in the templates */
+            /* {{user}}, {{team}}, {{redirect}} and {{project}} are required in default and custom templates */
             'user' => $profile->getAttribute('name'),
             'team' => '',
-            'project' => $projectName,
-            'redirect' => $url
+            'redirect' => $url,
+            'project' => $projectName
         ];
 
         $queueForMails
@@ -2546,7 +2578,8 @@ App::put('/v1/account/recovery')
     ->inject('dbForProject')
     ->inject('project')
     ->inject('queueForEvents')
-    ->action(function (string $userId, string $secret, string $password, string $passwordAgain, Response $response, Document $user, Database $dbForProject, Document $project, Event $queueForEvents) {
+    ->inject('hooks')
+    ->action(function (string $userId, string $secret, string $password, string $passwordAgain, Response $response, Document $user, Database $dbForProject, Document $project, Event $queueForEvents, Hooks $hooks) {
         if ($password !== $passwordAgain) {
             throw new Exception(Exception::USER_PASSWORD_MISMATCH);
         }
@@ -2580,6 +2613,8 @@ App::put('/v1/account/recovery')
             $history = array_slice($history, (count($history) - $historyLimit), $historyLimit);
         }
 
+        $hooks->trigger('passwordValidator', [$dbForProject, $project, $password, &$user, true]);
+
         $profile = $dbForProject->updateDocument('users', $profile->getId(), $profile
                 ->setAttribute('password', $newPassword)
                 ->setAttribute('passwordHistory', $history)
@@ -2681,7 +2716,7 @@ App::post('/v1/account/verification')
 
         $message = Template::fromFile(__DIR__ . '/../../config/locale/templates/email-inner-base.tpl');
         $message
-            ->setParam('{{body}}', $body)
+            ->setParam('{{body}}', $body, escapeHtml: false)
             ->setParam('{{hello}}', $locale->getText("emails.verification.hello"))
             ->setParam('{{footer}}', $locale->getText("emails.verification.footer"))
             ->setParam('{{thanks}}', $locale->getText("emails.verification.thanks"))
@@ -2736,11 +2771,11 @@ App::post('/v1/account/verification')
 
         $emailVariables = [
             'direction' => $locale->getText('settings.direction'),
-            /* {{user}} ,{{team}}, {{project}} and {{redirect}} are required in the templates */
+            /* {{user}}, {{team}}, {{redirect}} and {{project}} are required in default and custom templates */
             'user' => $user->getAttribute('name'),
             'team' => '',
-            'project' => $projectName,
-            'redirect' => $url
+            'redirect' => $url,
+            'project' => $projectName
         ];
 
         $queueForMails
@@ -2829,8 +2864,9 @@ App::put('/v1/account/verification')
 
 App::post('/v1/account/verification/phone')
     ->desc('Create phone verification')
-    ->groups(['api', 'account'])
+    ->groups(['api', 'account', 'auth'])
     ->label('scope', 'account')
+    ->label('auth.type', 'phone')
     ->label('event', 'users.[userId].verification.[tokenId].create')
     ->label('audits.event', 'verification.create')
     ->label('audits.resource', 'user/{response.userId}')
@@ -2842,7 +2878,7 @@ App::post('/v1/account/verification/phone')
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_TOKEN)
     ->label('abuse-limit', 10)
-    ->label('abuse-key', 'userId:{userId}')
+    ->label('abuse-key', ['url:{url},userId:{userId}', 'url:{url},ip:{ip}'])
     ->inject('request')
     ->inject('response')
     ->inject('user')
@@ -2903,9 +2939,12 @@ App::post('/v1/account/verification/phone')
         $message = $message->setParam('{{token}}', $secret);
         $message = $message->render();
 
+        var_dump($request->getIP());
+        var_dump($project->getId());
         $queueForMessaging
             ->setRecipient($user->getAttribute('phone'))
             ->setMessage($message)
+            ->setProject($project)
             ->trigger()
         ;
 
@@ -2983,3 +3022,40 @@ App::put('/v1/account/verification/phone')
 
         $response->dynamic($verificationDocument, Response::MODEL_TOKEN);
     });
+
+App::delete('/v1/account')
+    ->desc('Delete account')
+    ->groups(['api', 'account'])
+    ->label('event', 'users.[userId].delete')
+    ->label('scope', 'account')
+    ->label('audits.event', 'user.delete')
+    ->label('audits.resource', 'user/{response.$id}')
+    ->label('usage.metric', 'users.{scope}.requests.delete')
+    ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
+    ->label('sdk.namespace', 'account')
+    ->label('sdk.method', 'delete')
+    ->label('sdk.description', '/docs/references/account/delete.md')
+    ->label('sdk.response.code', Response::STATUS_CODE_NOCONTENT)
+    ->label('sdk.response.model', Response::MODEL_NONE)
+    ->inject('user')
+    ->inject('response')
+    ->inject('dbForProject')
+    ->inject('queueForEvents')
+    ->inject('queueForDeletes')
+    ->action(function (Document $user, Response $response, Database $dbForProject, Event $queueForEvents, Delete $queueForDeletes) {
+        if ($user->isEmpty()) {
+            throw new Exception(Exception::USER_NOT_FOUND);
+        }
+
+        $dbForProject->deleteDocument('users', $user->getId());
+
+        $queueForDeletes
+            ->setType(DELETE_TYPE_DOCUMENT)
+            ->setDocument($user);
+
+        $queueForEvents
+            ->setParam('userId', $user->getId())
+            ->setPayload($response->output($user, Response::MODEL_USER));
+
+        $response->noContent();
+    });

app/controllers/api/functions.php

@@ -456,7 +456,7 @@ App::get('/v1/functions/:functionId/usage')
     ->label('sdk.method', 'getFunctionUsage')
     ->label('sdk.response.code', Response::STATUS_CODE_OK)
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
-    ->label('sdk.response.model', Response::MODEL_USAGE_FUNCTIONS)
+    ->label('sdk.response.model', Response::MODEL_USAGE_FUNCTION)
     ->param('functionId', '', new UID(), 'Function ID.')
     ->param('range', '30d', new WhiteList(['24h', '30d', '90d']), 'Date range.', true)
     ->inject('response')

app/controllers/api/health.php

@@ -16,6 +16,7 @@ use Utopia\Storage\Device\Local;
 use Utopia\Storage\Storage;
 use Utopia\Validator\Integer;
 use Utopia\Validator\Text;
+use Utopia\Validator\WhiteList;
 
 App::get('/v1/health')
     ->desc('Get HTTP')
@@ -687,6 +688,47 @@ App::get('/v1/health/anti-virus')
         $response->dynamic(new Document($output), Response::MODEL_HEALTH_ANTIVIRUS);
     });
 
+App::get('/v1/health/queue/failed/:name')
+    ->desc('Get number of failed queue jobs')
+    ->groups(['api', 'health'])
+    ->label('scope', 'health.read')
+    ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
+    ->label('sdk.namespace', 'health')
+    ->label('sdk.method', 'getFailedJobs')
+    ->param('queueName', '', new WhiteList([
+        Event::DATABASE_QUEUE_NAME,
+        Event::DELETE_QUEUE_NAME,
+        Event::AUDITS_QUEUE_NAME,
+        Event::MAILS_QUEUE_NAME,
+        Event::FUNCTIONS_QUEUE_NAME,
+        Event::USAGE_QUEUE_NAME,
+        Event::WEBHOOK_CLASS_NAME,
+        Event::CERTIFICATES_QUEUE_NAME,
+        Event::BUILDS_QUEUE_NAME,
+        Event::MESSAGING_QUEUE_NAME,
+        Event::MIGRATIONS_QUEUE_NAME,
+        Event::HAMSTER_CLASS_NAME
+    ]), 'The name of the queue')
+    ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
+    ->label('sdk.description', '/docs/references/health/get-failed-queue-jobs.md')
+    ->label('sdk.response.code', Response::STATUS_CODE_OK)
+    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
+    ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
+    ->inject('response')
+    ->inject('queue')
+    ->action(function (string $name, int|string $threshold, Response $response, Connection $queue) {
+        $threshold = \intval($threshold);
+
+        $client = new Client($name, $queue);
+        $failed = $client->countFailedJobs();
+
+        if ($failed >= $threshold) {
+            throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue failed jobs threshold hit. Current size is {$failed} and threshold is {$threshold}.");
+        }
+
+        $response->dynamic(new Document([ 'size' => $failed ]), Response::MODEL_HEALTH_QUEUE);
+    });
+
 App::get('/v1/health/stats') // Currently only used internally
 ->desc('Get system stats')
     ->groups(['api', 'health'])

app/controllers/api/projects.php

@@ -1590,7 +1590,7 @@ App::get('/v1/projects/:projectId/templates/email/:type/:locale')
             $message
                 ->setParam('{{hello}}', $localeObj->getText("emails.{$type}.hello"))
                 ->setParam('{{footer}}', $localeObj->getText("emails.{$type}.footer"))
-                ->setParam('{{body}}', $localeObj->getText('emails.' . $type . '.body'))
+                ->setParam('{{body}}', $localeObj->getText('emails.' . $type . '.body'), escapeHtml: false)
                 ->setParam('{{thanks}}', $localeObj->getText("emails.{$type}.thanks"))
                 ->setParam('{{signature}}', $localeObj->getText("emails.{$type}.signature"))
                 ->setParam('{{direction}}', $localeObj->getText('settings.direction'));

app/controllers/api/proxy.php

@@ -14,6 +14,7 @@ use Utopia\Database\Helpers\ID;
 use Utopia\Database\Query;
 use Utopia\Database\Validator\UID;
 use Utopia\Domains\Domain;
+use Utopia\Logger\Log;
 use Utopia\Validator\Domain as ValidatorDomain;
 use Utopia\Validator\Text;
 use Utopia\Validator\WhiteList;
@@ -278,7 +279,8 @@ App::patch('/v1/proxy/rules/:ruleId/verification')
     ->inject('queueForEvents')
     ->inject('project')
     ->inject('dbForConsole')
-    ->action(function (string $ruleId, Response $response, Certificate $queueForCertificates, Event $queueForEvents, Document $project, Database $dbForConsole) {
+    ->inject('log')
+    ->action(function (string $ruleId, Response $response, Certificate $queueForCertificates, Event $queueForEvents, Document $project, Database $dbForConsole, Log $log) {
         $rule = $dbForConsole->getDocument('rules', $ruleId);
 
         if ($rule->isEmpty() || $rule->getAttribute('projectInternalId') !== $project->getInternalId()) {
@@ -298,7 +300,14 @@ App::patch('/v1/proxy/rules/:ruleId/verification')
         $validator = new CNAME($target->get()); // Verify Domain with DNS records
         $domain = new Domain($rule->getAttribute('domain', ''));
 
+        $validationStart = \microtime(true);
         if (!$validator->isValid($domain->get())) {
+            $log->addExtra('dnsTiming', \strval(\microtime(true) - $validationStart));
+            $log->addTag('dnsDomain', $domain->get());
+
+            $error = $validator->getLogs();
+            $log->addExtra('dnsResponse', \is_array($error) ? \json_encode($error) : \strval($error));
+
             throw new Exception(Exception::RULE_VERIFICATION_FAILED);
         }
 

app/controllers/api/storage.php

@@ -44,6 +44,7 @@ use Utopia\Validator\Text;
 use Utopia\Validator\WhiteList;
 use Utopia\DSN\DSN;
 use Utopia\Swoole\Request;
+use Utopia\Storage\Compression\Compression;
 
 App::post('/v1/storage/buckets')
     ->desc('Create bucket')
@@ -66,7 +67,7 @@ App::post('/v1/storage/buckets')
     ->param('enabled', true, new Boolean(true), 'Is bucket enabled? When set to \'disabled\', users cannot access the files in this bucket but Server SDKs with and API key can still access the bucket. No files are lost when this is toggled.', true)
     ->param('maximumFileSize', (int) App::getEnv('_APP_STORAGE_LIMIT', 0), new Range(1, (int) App::getEnv('_APP_STORAGE_LIMIT', 0)), 'Maximum file size allowed in bytes. Maximum allowed value is ' . Storage::human(App::getEnv('_APP_STORAGE_LIMIT', 0), 0) . '.', true)
     ->param('allowedFileExtensions', [], new ArrayList(new Text(64), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Allowed file extensions. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' extensions are allowed, each 64 characters long.', true)
-    ->param('compression', COMPRESSION_TYPE_NONE, new WhiteList([COMPRESSION_TYPE_NONE, COMPRESSION_TYPE_GZIP, COMPRESSION_TYPE_ZSTD]), 'Compression algorithm choosen for compression. Can be one of ' . COMPRESSION_TYPE_NONE . ',  [' . COMPRESSION_TYPE_GZIP . '](https://en.wikipedia.org/wiki/Gzip), or [' . COMPRESSION_TYPE_ZSTD . '](https://en.wikipedia.org/wiki/Zstd), For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' compression is skipped even if it\'s enabled', true)
+    ->param('compression', Compression::NONE, new WhiteList([Compression::NONE, Compression::GZIP, Compression::ZSTD]), 'Compression algorithm choosen for compression. Can be one of ' . Compression::NONE . ',  [' . Compression::GZIP . '](https://en.wikipedia.org/wiki/Gzip), or [' . Compression::ZSTD . '](https://en.wikipedia.org/wiki/Zstd), For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' compression is skipped even if it\'s enabled', true)
     ->param('encryption', true, new Boolean(true), 'Is encryption enabled? For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' encryption is skipped even if it\'s enabled', true)
     ->param('antivirus', true, new Boolean(true), 'Is virus scanning enabled? For file size above ' . Storage::human(APP_LIMIT_ANTIVIRUS, 0) . ' AntiVirus scanning is skipped even if it\'s enabled', true)
     ->inject('response')
@@ -237,7 +238,7 @@ App::put('/v1/storage/buckets/:bucketId')
     ->param('enabled', true, new Boolean(true), 'Is bucket enabled? When set to \'disabled\', users cannot access the files in this bucket but Server SDKs with and API key can still access the bucket. No files are lost when this is toggled.', true)
     ->param('maximumFileSize', null, new Range(1, (int) App::getEnv('_APP_STORAGE_LIMIT', 0)), 'Maximum file size allowed in bytes. Maximum allowed value is ' . Storage::human((int)App::getEnv('_APP_STORAGE_LIMIT', 0), 0) . '.', true)
     ->param('allowedFileExtensions', [], new ArrayList(new Text(64), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Allowed file extensions. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' extensions are allowed, each 64 characters long.', true)
-    ->param('compression', COMPRESSION_TYPE_NONE, new WhiteList([COMPRESSION_TYPE_NONE, COMPRESSION_TYPE_GZIP, COMPRESSION_TYPE_ZSTD]), 'Compression algorithm choosen for compression. Can be one of ' . COMPRESSION_TYPE_NONE . ', [' . COMPRESSION_TYPE_GZIP . '](https://en.wikipedia.org/wiki/Gzip), or [' . COMPRESSION_TYPE_ZSTD . '](https://en.wikipedia.org/wiki/Zstd), For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' compression is skipped even if it\'s enabled', true)
+    ->param('compression', Compression::NONE, new WhiteList([Compression::NONE, Compression::GZIP, Compression::ZSTD]), 'Compression algorithm choosen for compression. Can be one of ' . Compression::NONE . ', [' . Compression::GZIP . '](https://en.wikipedia.org/wiki/Gzip), or [' . Compression::ZSTD . '](https://en.wikipedia.org/wiki/Zstd), For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' compression is skipped even if it\'s enabled', true)
     ->param('encryption', true, new Boolean(true), 'Is encryption enabled? For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' encryption is skipped even if it\'s enabled', true)
     ->param('antivirus', true, new Boolean(true), 'Is virus scanning enabled? For file size above ' . Storage::human(APP_LIMIT_ANTIVIRUS, 0) . ' AntiVirus scanning is skipped even if it\'s enabled', true)
     ->inject('response')
@@ -531,19 +532,24 @@ App::post('/v1/storage/buckets/:bucketId/files')
             $fileHash = $deviceFiles->getFileHash($path); // Get file hash before compression and encryption
             $data = '';
             // Compression
-            $algorithm = $bucket->getAttribute('compression', COMPRESSION_TYPE_NONE);
-            if ($fileSize <= APP_STORAGE_READ_BUFFER && $algorithm != COMPRESSION_TYPE_NONE) {
+            $algorithm = $bucket->getAttribute('compression', Compression::NONE);
+            if ($fileSize <= APP_STORAGE_READ_BUFFER && $algorithm != Compression::NONE) {
                 $data = $deviceFiles->read($path);
                 switch ($algorithm) {
-                    case COMPRESSION_TYPE_ZSTD:
+                    case Compression::ZSTD:
                         $compressor = new Zstd();
                         break;
-                    case COMPRESSION_TYPE_GZIP:
+                    case Compression::GZIP:
                     default:
                         $compressor = new GZIP();
                         break;
                 }
                 $data = $compressor->compress($data);
+            } else {
+                // reset the algorithm to none as we do not compress the file
+                // if file size exceedes the APP_STORAGE_READ_BUFFER
+                // regardless the bucket compression algoorithm
+                $algorithm = Compression::NONE;
             }
 
             if ($bucket->getAttribute('encryption', true) && $fileSize <= APP_STORAGE_READ_BUFFER) {
@@ -615,7 +621,17 @@ App::post('/v1/storage/buckets/:bucketId/files')
                         ->setAttribute('metadata', $metadata)
                         ->setAttribute('chunksUploaded', $chunksUploaded);
 
-                    $file = $dbForProject->updateDocument('bucket_' . $bucket->getInternalId(), $fileId, $file);
+                    /**
+                     * Validate create permission and skip authorization in updateDocument
+                     * Without this, the file creation will fail when user doesn't have update permission
+                     * However as with chunk upload even if we are updating, we are essentially creating a file
+                     * adding it's new chunk so we validate create permission instead of update
+                     */
+                    $validator = new Authorization(Database::PERMISSION_CREATE);
+                    if (!$validator->isValid($bucket->getCreate())) {
+                        throw new Exception(Exception::USER_UNAUTHORIZED);
+                    }
+                    $file = Authorization::skip(fn() => $dbForProject->updateDocument('bucket_' . $bucket->getInternalId(), $fileId, $file));
                 }
             } catch (AuthorizationException) {
                 throw new Exception(Exception::USER_UNAUTHORIZED);
@@ -652,7 +668,17 @@ App::post('/v1/storage/buckets/:bucketId/files')
                         ->setAttribute('chunksUploaded', $chunksUploaded)
                         ->setAttribute('metadata', $metadata);
 
-                    $file = $dbForProject->updateDocument('bucket_' . $bucket->getInternalId(), $fileId, $file);
+                    /**
+                     * Validate create permission and skip authorization in updateDocument
+                     * Without this, the file creation will fail when user doesn't have update permission
+                     * However as with chunk upload even if we are updating, we are essentially creating a file
+                     * adding it's new chunk so we validate create permission instead of update
+                     */
+                    $validator = new Authorization(Database::PERMISSION_CREATE);
+                    if (!$validator->isValid($bucket->getCreate())) {
+                        throw new Exception(Exception::USER_UNAUTHORIZED);
+                    }
+                    $file = Authorization::skip(fn() => $dbForProject->updateDocument('bucket_' . $bucket->getInternalId(), $fileId, $file));
                 }
             } catch (AuthorizationException) {
                 throw new Exception(Exception::USER_UNAUTHORIZED);
@@ -859,14 +885,6 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview')
             throw new Exception(Exception::USER_UNAUTHORIZED);
         }
 
-        if ((\strpos($request->getAccept(), 'image/webp') === false) && ('webp' === $output)) { // Fallback webp to jpeg when no browser support
-            $output = 'jpg';
-        }
-
-        $inputs = Config::getParam('storage-inputs');
-        $outputs = Config::getParam('storage-outputs');
-        $fileLogos = Config::getParam('storage-logos');
-
         if ($fileSecurity && !$valid) {
             $file = $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId);
         } else {
@@ -877,9 +895,17 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview')
             throw new Exception(Exception::STORAGE_FILE_NOT_FOUND);
         }
 
+        if ((\strpos($request->getAccept(), 'image/webp') === false) && ('webp' === $output)) { // Fallback webp to jpeg when no browser support
+            $output = 'jpg';
+        }
+
+        $inputs = Config::getParam('storage-inputs');
+        $outputs = Config::getParam('storage-outputs');
+        $fileLogos = Config::getParam('storage-logos');
+
         $path = $file->getAttribute('path');
         $type = \strtolower(\pathinfo($path, PATHINFO_EXTENSION));
-        $algorithm = $file->getAttribute('algorithm', 'none');
+        $algorithm = $file->getAttribute('algorithm', Compression::NONE);
         $cipher = $file->getAttribute('openSSLCipher');
         $mime = $file->getAttribute('mimeType');
         if (!\in_array($mime, $inputs) || $file->getAttribute('sizeActual') > (int) App::getEnv('_APP_STORAGE_PREVIEW_LIMIT', 20000000)) {
@@ -890,7 +916,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview')
                 $path = $fileLogos['default_image'];
             }
 
-            $algorithm = 'none';
+            $algorithm = Compression::NONE;
             $cipher = null;
             $background = (empty($background)) ? 'eceff1' : $background;
             $type = \strtolower(\pathinfo($path, PATHINFO_EXTENSION));
@@ -902,12 +928,17 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview')
         }
 
         if (empty($output)) {
+            // when file extension is provided but it's not one of our
+            // supported outputs we fallback to `jpg`
+            if (!empty($type) && !array_key_exists($type, $outputs)) {
+                $type = 'jpg';
+            }
+
             // when file extension is not provided and the mime type is not one of our supported outputs
             // we fallback to `jpg` output format
             $output = empty($type) ? (array_search($mime, $outputs) ?? 'jpg') : $type;
         }
 
-
         $source = $deviceFiles->read($path);
 
         if (!empty($cipher)) { // Decrypt
@@ -922,11 +953,11 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview')
         }
 
         switch ($algorithm) {
-            case 'zstd':
+            case Compression::ZSTD:
                 $compressor = new Zstd();
                 $source = $compressor->decompress($source);
                 break;
-            case 'gzip':
+            case Compression::GZIP:
                 $compressor = new GZIP();
                 $source = $compressor->decompress($source);
                 break;
@@ -1065,15 +1096,15 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download')
             );
         }
 
-        switch ($file->getAttribute('algorithm', 'none')) {
-            case 'zstd':
+        switch ($file->getAttribute('algorithm', Compression::NONE)) {
+            case Compression::ZSTD:
                 if (empty($source)) {
                     $source = $deviceFiles->read($path);
                 }
                 $compressor = new Zstd();
                 $source = $compressor->decompress($source);
                 break;
-            case 'gzip':
+            case Compression::GZIP:
                 if (empty($source)) {
                     $source = $deviceFiles->read($path);
                 }
@@ -1214,15 +1245,15 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view')
             );
         }
 
-        switch ($file->getAttribute('algorithm', 'none')) {
-            case 'zstd':
+        switch ($file->getAttribute('algorithm', Compression::NONE)) {
+            case Compression::ZSTD:
                 if (empty($source)) {
                     $source = $deviceFiles->read($path);
                 }
                 $compressor = new Zstd();
                 $source = $compressor->decompress($source);
                 break;
-            case 'gzip':
+            case Compression::GZIP:
                 if (empty($source)) {
                     $source = $deviceFiles->read($path);
                 }
@@ -1236,10 +1267,12 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view')
                 $response->send(substr($source, $start, ($end - $start + 1)));
             }
             $response->send($source);
+            return;
         }
 
         if (!empty($rangeHeader)) {
             $response->send($deviceFiles->read($path, $start, ($end - $start + 1)));
+            return;
         }
 
         $size = $deviceFiles->getFileSize($path);

app/controllers/api/teams.php

@@ -380,6 +380,7 @@ App::post('/v1/teams/:teamId/memberships')
     ->param('roles', [], new ArrayList(new Key(), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of strings. Use this param to set the user roles in the team. A role can be any string. Learn more about [roles and permissions](https://appwrite.io/docs/permissions). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' roles are allowed, each 32 characters long.')
     ->param('url', '', fn($clients) => new Host($clients), 'URL to redirect the user back to your app from the invitation email.  Only URLs from hostnames in your project platform list are allowed. This requirement helps to prevent an [open redirect](https://cheatsheetseries.owasp.org/cheatsheets/Unvalidated_Redirects_and_Forwards_Cheat_Sheet.html) attack against your project API.', true, ['clients']) // TODO add our own built-in confirm page
     ->param('name', '', new Text(128), 'Name of the new team member. Max length: 128 chars.', true)
+    ->inject('request')
     ->inject('response')
     ->inject('project')
     ->inject('user')
@@ -388,7 +389,7 @@ App::post('/v1/teams/:teamId/memberships')
     ->inject('queueForMails')
     ->inject('queueForMessaging')
     ->inject('queueForEvents')
-    ->action(function (string $teamId, string $email, string $userId, string $phone, array $roles, string $url, string $name, Response $response, Document $project, Document $user, Database $dbForProject, Locale $locale, Mail $queueForMails, EventPhone $queueForMessaging, Event $queueForEvents) {
+    ->action(function (string $teamId, string $email, string $userId, string $phone, array $roles, string $url, string $name, Request $request, Response $response, Document $project, Document $user, Database $dbForProject, Locale $locale, Mail $queueForMails, EventPhone $queueForMessaging, Event $queueForEvents) {
         $isAPIKey = Auth::isAppUser(Authorization::getRoles());
         $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles());
 
@@ -533,8 +534,8 @@ App::post('/v1/teams/:teamId/memberships')
             } catch (Duplicate $th) {
                 throw new Exception(Exception::TEAM_INVITE_ALREADY_EXISTS);
             }
-            $team->setAttribute('total', $team->getAttribute('total', 0) + 1);
-            $team = Authorization::skip(fn() => $dbForProject->updateDocument('teams', $team->getId(), $team));
+
+            Authorization::skip(fn() => $dbForProject->increaseDocumentAttribute('teams', $team->getId(), 'total', 1));
 
             $dbForProject->deleteCachedDocument('users', $invitee->getId());
         } else {
@@ -556,7 +557,7 @@ App::post('/v1/teams/:teamId/memberships')
 
                 $message = Template::fromFile(__DIR__ . '/../../config/locale/templates/email-inner-base.tpl');
                 $message
-                    ->setParam('{{body}}', $body)
+                    ->setParam('{{body}}', $body, escapeHtml: false)
                     ->setParam('{{hello}}', $locale->getText("emails.invitation.hello"))
                     ->setParam('{{footer}}', $locale->getText("emails.invitation.footer"))
                     ->setParam('{{thanks}}', $locale->getText("emails.invitation.thanks"))
@@ -612,11 +613,11 @@ App::post('/v1/teams/:teamId/memberships')
                 $emailVariables = [
                     'owner' => $user->getAttribute('name'),
                     'direction' => $locale->getText('settings.direction'),
-                    /* {{user}} ,{{team}}, {{project}} and {{redirect}} are required in the templates */
+                    /* {{user}}, {{team}}, {{redirect}} and {{project}} are required in default and custom templates */
                     'user' => $user->getAttribute('name'),
                     'team' => $team->getAttribute('name'),
-                    'project' => $projectName,
-                    'redirect' => $url
+                    'redirect' => $url,
+                    'project' => $projectName
                 ];
 
                 $queueForMails
@@ -638,9 +639,12 @@ App::post('/v1/teams/:teamId/memberships')
                 $message = $message->setParam('{{token}}', $url);
                 $message = $message->render();
 
+                var_dump($request->getIP());
+                var_dump($project->getId());
                 $queueForMessaging
                     ->setRecipient($phone)
                     ->setMessage($message)
+                    ->setProject($project)
                     ->trigger();
             }
         }

app/controllers/api/users.php

@@ -39,10 +39,12 @@ use Utopia\Validator\Integer;
 use Appwrite\Auth\Validator\PasswordHistory;
 use Appwrite\Auth\Validator\PasswordDictionary;
 use Appwrite\Auth\Validator\PersonalData;
+use Appwrite\Hooks\Hooks;
 
 /** TODO: Remove function when we move to using utopia/platform */
-function createUser(string $hash, mixed $hashOptions, string $userId, ?string $email, ?string $password, ?string $phone, string $name, Document $project, Database $dbForProject, Event $queueForEvents): Document
+function createUser(string $hash, mixed $hashOptions, string $userId, ?string $email, ?string $password, ?string $phone, string $name, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks): Document
 {
+    $plaintextPassword = $password;
     $hashOptionsObject = (\is_string($hashOptions)) ? \json_decode($hashOptions, true) : $hashOptions; // Cast to JSON array
     $passwordHistory = $project->getAttribute('auths', [])['passwordHistory'] ?? 0;
 
@@ -65,13 +67,13 @@ function createUser(string $hash, mixed $hashOptions, string $userId, ?string $e
 
         if ($project->getAttribute('auths', [])['personalDataCheck'] ?? false) {
             $personalDataValidator = new PersonalData($userId, $email, $name, $phone);
-            if (!$personalDataValidator->isValid($password)) {
+            if (!$personalDataValidator->isValid($plaintextPassword)) {
                 throw new Exception(Exception::USER_PASSWORD_PERSONAL_DATA);
             }
         }
 
         $password = (!empty($password)) ? ($hash === 'plaintext' ? Auth::passwordHash($password, $hash, $hashOptionsObject) : $password) : null;
-        $user = $dbForProject->createDocument('users', new Document([
+        $user = new Document([
             '$id' => $userId,
             '$permissions' => [
                 Permission::read(Role::any()),
@@ -97,7 +99,13 @@ function createUser(string $hash, mixed $hashOptions, string $userId, ?string $e
             'tokens' => null,
             'memberships' => null,
             'search' => implode(' ', [$userId, $email, $phone, $name]),
-        ]));
+        ]);
+
+        if ($hash === 'plaintext') {
+            $hooks->trigger('passwordValidator', [$dbForProject, $project, $plaintextPassword, &$user, true]);
+        }
+
+        $user = $dbForProject->createDocument('users', $user);
     } catch (Duplicate $th) {
         throw new Exception(Exception::USER_ALREADY_EXISTS);
     }
@@ -130,8 +138,9 @@ App::post('/v1/users')
     ->inject('project')
     ->inject('dbForProject')
     ->inject('queueForEvents')
-    ->action(function (string $userId, ?string $email, ?string $phone, ?string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents) {
-        $user = createUser('plaintext', '{}', $userId, $email, $password, $phone, $name, $project, $dbForProject, $queueForEvents);
+    ->inject('hooks')
+    ->action(function (string $userId, ?string $email, ?string $phone, ?string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks) {
+        $user = createUser('plaintext', '{}', $userId, $email, $password, $phone, $name, $project, $dbForProject, $queueForEvents, $hooks);
 
         $response
             ->setStatusCode(Response::STATUS_CODE_CREATED)
@@ -160,8 +169,9 @@ App::post('/v1/users/bcrypt')
     ->inject('project')
     ->inject('dbForProject')
     ->inject('queueForEvents')
-    ->action(function (string $userId, string $email, string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents) {
-        $user = createUser('bcrypt', '{}', $userId, $email, $password, null, $name, $project, $dbForProject, $queueForEvents);
+    ->inject('hooks')
+    ->action(function (string $userId, string $email, string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks) {
+        $user = createUser('bcrypt', '{}', $userId, $email, $password, null, $name, $project, $dbForProject, $queueForEvents, $hooks);
 
         $response
             ->setStatusCode(Response::STATUS_CODE_CREATED)
@@ -190,8 +200,9 @@ App::post('/v1/users/md5')
     ->inject('project')
     ->inject('dbForProject')
     ->inject('queueForEvents')
-    ->action(function (string $userId, string $email, string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents) {
-        $user = createUser('md5', '{}', $userId, $email, $password, null, $name, $project, $dbForProject, $queueForEvents);
+    ->inject('hooks')
+    ->action(function (string $userId, string $email, string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks) {
+        $user = createUser('md5', '{}', $userId, $email, $password, null, $name, $project, $dbForProject, $queueForEvents, $hooks);
 
         $response
             ->setStatusCode(Response::STATUS_CODE_CREATED)
@@ -220,8 +231,9 @@ App::post('/v1/users/argon2')
     ->inject('project')
     ->inject('dbForProject')
     ->inject('queueForEvents')
-    ->action(function (string $userId, string $email, string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents) {
-        $user = createUser('argon2', '{}', $userId, $email, $password, null, $name, $project, $dbForProject, $queueForEvents);
+    ->inject('hooks')
+    ->action(function (string $userId, string $email, string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks) {
+        $user = createUser('argon2', '{}', $userId, $email, $password, null, $name, $project, $dbForProject, $queueForEvents, $hooks);
 
         $response
             ->setStatusCode(Response::STATUS_CODE_CREATED)
@@ -251,14 +263,15 @@ App::post('/v1/users/sha')
     ->inject('project')
     ->inject('dbForProject')
     ->inject('queueForEvents')
-    ->action(function (string $userId, string $email, string $password, string $passwordVersion, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents) {
+    ->inject('hooks')
+    ->action(function (string $userId, string $email, string $password, string $passwordVersion, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks) {
         $options = '{}';
 
         if (!empty($passwordVersion)) {
             $options = '{"version":"' . $passwordVersion . '"}';
         }
 
-        $user = createUser('sha', $options, $userId, $email, $password, null, $name, $project, $dbForProject, $queueForEvents);
+        $user = createUser('sha', $options, $userId, $email, $password, null, $name, $project, $dbForProject, $queueForEvents, $hooks);
 
         $response
             ->setStatusCode(Response::STATUS_CODE_CREATED)
@@ -287,8 +300,9 @@ App::post('/v1/users/phpass')
     ->inject('project')
     ->inject('dbForProject')
     ->inject('queueForEvents')
-    ->action(function (string $userId, string $email, string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents) {
-        $user = createUser('phpass', '{}', $userId, $email, $password, null, $name, $project, $dbForProject, $queueForEvents);
+    ->inject('hooks')
+    ->action(function (string $userId, string $email, string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks) {
+        $user = createUser('phpass', '{}', $userId, $email, $password, null, $name, $project, $dbForProject, $queueForEvents, $hooks);
 
         $response
             ->setStatusCode(Response::STATUS_CODE_CREATED)
@@ -322,7 +336,8 @@ App::post('/v1/users/scrypt')
     ->inject('project')
     ->inject('dbForProject')
     ->inject('queueForEvents')
-    ->action(function (string $userId, string $email, string $password, string $passwordSalt, int $passwordCpu, int $passwordMemory, int $passwordParallel, int $passwordLength, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents) {
+    ->inject('hooks')
+    ->action(function (string $userId, string $email, string $password, string $passwordSalt, int $passwordCpu, int $passwordMemory, int $passwordParallel, int $passwordLength, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks) {
         $options = [
             'salt' => $passwordSalt,
             'costCpu' => $passwordCpu,
@@ -331,7 +346,7 @@ App::post('/v1/users/scrypt')
             'length' => $passwordLength
         ];
 
-        $user = createUser('scrypt', \json_encode($options), $userId, $email, $password, null, $name, $project, $dbForProject, $queueForEvents);
+        $user = createUser('scrypt', \json_encode($options), $userId, $email, $password, null, $name, $project, $dbForProject, $queueForEvents, $hooks);
 
         $response
             ->setStatusCode(Response::STATUS_CODE_CREATED)
@@ -363,8 +378,9 @@ App::post('/v1/users/scrypt-modified')
     ->inject('project')
     ->inject('dbForProject')
     ->inject('queueForEvents')
-    ->action(function (string $userId, string $email, string $password, string $passwordSalt, string $passwordSaltSeparator, string $passwordSignerKey, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents) {
-        $user = createUser('scryptMod', '{"signerKey":"' . $passwordSignerKey . '","saltSeparator":"' . $passwordSaltSeparator . '","salt":"' . $passwordSalt . '"}', $userId, $email, $password, null, $name, $project, $dbForProject, $queueForEvents);
+    ->inject('hooks')
+    ->action(function (string $userId, string $email, string $password, string $passwordSalt, string $passwordSaltSeparator, string $passwordSignerKey, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks) {
+        $user = createUser('scryptMod', '{"signerKey":"' . $passwordSignerKey . '","saltSeparator":"' . $passwordSaltSeparator . '","salt":"' . $passwordSalt . '"}', $userId, $email, $password, null, $name, $project, $dbForProject, $queueForEvents, $hooks);
 
         $response
             ->setStatusCode(Response::STATUS_CODE_CREATED)
@@ -846,7 +862,8 @@ App::patch('/v1/users/:userId/password')
     ->inject('project')
     ->inject('dbForProject')
     ->inject('queueForEvents')
-    ->action(function (string $userId, string $password, Response $response, Document $project, Database $dbForProject, Event $queueForEvents) {
+    ->inject('hooks')
+    ->action(function (string $userId, string $password, Response $response, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks) {
 
         $user = $dbForProject->getDocument('users', $userId);
 
@@ -861,6 +878,8 @@ App::patch('/v1/users/:userId/password')
             }
         }
 
+        $hooks->trigger('passwordValidator', [$dbForProject, $project, $password, &$user, true]);
+
         $newPassword = Auth::passwordHash($password, Auth::DEFAULT_ALGO, Auth::DEFAULT_ALGO_OPTIONS);
 
         $historyLimit = $project->getAttribute('auths', [])['passwordHistory'] ?? 0;
@@ -1184,7 +1203,7 @@ App::delete('/v1/users/:userId')
     });
 
 App::delete('/v1/users/identities/:identityId')
-    ->desc('Delete Identity')
+    ->desc('Delete identity')
     ->groups(['api', 'users'])
     ->label('event', 'users.[userId].identities.[identityId].delete')
     ->label('scope', 'users.write')
@@ -1199,7 +1218,8 @@ App::delete('/v1/users/identities/:identityId')
     ->param('identityId', '', new UID(), 'Identity ID.')
     ->inject('response')
     ->inject('dbForProject')
-    ->action(function (string $identityId, Response $response, Database $dbForProject) {
+    ->inject('queueForEvents')
+    ->action(function (string $identityId, Response $response, Database $dbForProject, Event $queueForEvents) {
 
         $identity = $dbForProject->getDocument('identities', $identityId);
 
@@ -1209,6 +1229,11 @@ App::delete('/v1/users/identities/:identityId')
 
         $dbForProject->deleteDocument('identities', $identityId);
 
+        $queueForEvents
+            ->setParam('userId', $identity->getAttribute('userId'))
+            ->setParam('identityId', $identity->getId())
+            ->setPayload($response->output($identity, Response::MODEL_IDENTITY));
+
         return $response->noContent();
     });
 

app/controllers/general.php

@@ -603,9 +603,8 @@ App::error()
     ->inject('response')
     ->inject('project')
     ->inject('logger')
-    ->inject('loggerBreadcrumbs')
-    ->action(function (Throwable $error, App $utopia, Request $request, Response $response, Document $project, ?Logger $logger, array $loggerBreadcrumbs) {
-
+    ->inject('log')
+    ->action(function (Throwable $error, App $utopia, Request $request, Response $response, Document $project, ?Logger $logger, Log $log) {
         $version = App::getEnv('_APP_VERSION', 'UNKNOWN');
         $route = $utopia->getRoute();
         $publish = true;
@@ -614,55 +613,47 @@ App::error()
             $publish = $error->isPublishable();
         }
 
-        if ($logger && $publish) {
-            if ($error->getCode() >= 500 || $error->getCode() === 0) {
-                try {
-                    /** @var Utopia\Database\Document $user */
-                    $user = $utopia->getResource('user');
-                } catch (\Throwable $th) {
-                    // All good, user is optional information for logger
-                }
-
-                $log = new Utopia\Logger\Log();
-
-                if (isset($user) && !$user->isEmpty()) {
-                    $log->setUser(new User($user->getId()));
-                }
-
-                $log->setNamespace("http");
-                $log->setServer(\gethostname());
-                $log->setVersion($version);
-                $log->setType(Log::TYPE_ERROR);
-                $log->setMessage($error->getMessage());
-
-                $log->addTag('database', $project->getAttribute('database', 'console'));
-                $log->addTag('method', $route->getMethod());
-                $log->addTag('url', $route->getPath());
-                $log->addTag('verboseType', get_class($error));
-                $log->addTag('code', $error->getCode());
-                $log->addTag('projectId', $project->getId());
-                $log->addTag('hostname', $request->getHostname());
-                $log->addTag('locale', (string)$request->getParam('locale', $request->getHeader('x-appwrite-locale', '')));
-
-                $log->addExtra('file', $error->getFile());
-                $log->addExtra('line', $error->getLine());
-                $log->addExtra('trace', $error->getTraceAsString());
-                $log->addExtra('detailedTrace', $error->getTrace());
-                $log->addExtra('roles', Authorization::getRoles());
-
-                $action = $route->getLabel("sdk.namespace", "UNKNOWN_NAMESPACE") . '.' . $route->getLabel("sdk.method", "UNKNOWN_METHOD");
-                $log->setAction($action);
-
-                $isProduction = App::getEnv('_APP_ENV', 'development') === 'production';
-                $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING);
-
-                foreach ($loggerBreadcrumbs as $loggerBreadcrumb) {
-                    $log->addBreadcrumb($loggerBreadcrumb);
-                }
-
-                $responseCode = $logger->addLog($log);
-                Console::info('Log pushed with status code: ' . $responseCode);
+        if ($logger && ($publish || $error->getCode() === 0)) {
+            try {
+                /** @var Utopia\Database\Document $user */
+                $user = $utopia->getResource('user');
+            } catch (\Throwable $th) {
+                // All good, user is optional information for logger
             }
+
+            if (isset($user) && !$user->isEmpty()) {
+                $log->setUser(new User($user->getId()));
+            }
+
+            $log->setNamespace("http");
+            $log->setServer(\gethostname());
+            $log->setVersion($version);
+            $log->setType(Log::TYPE_ERROR);
+            $log->setMessage($error->getMessage());
+
+            $log->addTag('database', $project->getAttribute('database', 'console'));
+            $log->addTag('method', $route->getMethod());
+            $log->addTag('url', $route->getPath());
+            $log->addTag('verboseType', get_class($error));
+            $log->addTag('code', $error->getCode());
+            $log->addTag('projectId', $project->getId());
+            $log->addTag('hostname', $request->getHostname());
+            $log->addTag('locale', (string)$request->getParam('locale', $request->getHeader('x-appwrite-locale', '')));
+
+            $log->addExtra('file', $error->getFile());
+            $log->addExtra('line', $error->getLine());
+            $log->addExtra('trace', $error->getTraceAsString());
+            $log->addExtra('detailedTrace', $error->getTrace());
+            $log->addExtra('roles', Authorization::getRoles());
+
+            $action = $route->getLabel("sdk.namespace", "UNKNOWN_NAMESPACE") . '.' . $route->getLabel("sdk.method", "UNKNOWN_METHOD");
+            $log->setAction($action);
+
+            $isProduction = App::getEnv('_APP_ENV', 'development') === 'production';
+            $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING);
+
+            $responseCode = $logger->addLog($log);
+            Console::info('Log pushed with status code: ' . $responseCode);
         }
 
         $code = $error->getCode();

app/controllers/shared/api.php

@@ -177,6 +177,7 @@ App::init()
             $end = $request->getContentRangeEnd();
             $timeLimit = new TimeLimit($abuseKey, $route->getLabel('abuse-limit', 0), $route->getLabel('abuse-time', 3600), $dbForProject);
             $timeLimit
+                ->setParam('{projectId}', $project->getId())
                 ->setParam('{userId}', $user->getId())
                 ->setParam('{userAgent}', $request->getUserAgent(''))
                 ->setParam('{ip}', $request->getIP())
@@ -335,7 +336,7 @@ App::init()
                 break;
 
             case 'magic-url':
-                if ($project->getAttribute('usersAuthMagicURL', true) === false) {
+                if (($auths['usersAuthMagicURL'] ?? true) === false) {
                     throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Magic URL authentication is disabled for this project');
                 }
                 break;
@@ -346,6 +347,12 @@ App::init()
                 }
                 break;
 
+            case 'phone':
+                if (($auths['phone'] ?? true) === false) {
+                    throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Phone authentication is disabled for this project');
+                }
+                break;
+
             case 'invites':
                 if (($auths['invites'] ?? true) === false) {
                     throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Invites authentication is disabled for this project');
@@ -359,7 +366,7 @@ App::init()
                 break;
 
             default:
-                throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Unsupported authentication route');
+                throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Unsupported authentication type: ' . $route->getLabel('auth.type', ''));
                 break;
         }
     });
@@ -592,6 +599,22 @@ App::shutdown()
                 ->setProject($project)
                 ->trigger();
         }
+
+        /**
+         * Update user last activity
+         */
+        if (!$user->isEmpty()) {
+            $accessedAt = $user->getAttribute('accessedAt', '');
+            if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_USER_ACCCESS)) > $accessedAt) {
+                $user->setAttribute('accessedAt', DateTime::now());
+
+                if (APP_MODE_ADMIN !== $mode) {
+                    $dbForProject->updateDocument('users', $user->getId(), $user);
+                } else {
+                    $dbForConsole->updateDocument('users', $user->getId(), $user);
+                }
+            }
+        }
     });
 
 App::init()

app/controllers/shared/api/auth.php

@@ -32,7 +32,7 @@ App::init()
                 break;
 
             case 'magic-url':
-                if ($project->getAttribute('usersAuthMagicURL', true) === false) {
+                if (($auths['usersAuthMagicURL'] ?? true) === false) {
                     throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Magic URL authentication is disabled for this project');
                 }
                 break;
@@ -43,6 +43,12 @@ App::init()
                 }
                 break;
 
+            case 'phone':
+                if (($auths['phone'] ?? true) === false) {
+                    throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Phone authentication is disabled for this project');
+                }
+                break;
+
             case 'invites':
                 if (($auths['invites'] ?? true) === false) {
                     throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Invites authentication is disabled for this project');

app/http.php

@@ -263,10 +263,9 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo
                 // All good, user is optional information for logger
             }
 
-            $loggerBreadcrumbs = $app->getResource("loggerBreadcrumbs");
             $route = $app->getRoute();
 
-            $log = new Utopia\Logger\Log();
+            $log = $app->getResource("log");
 
             if (isset($user) && !$user->isEmpty()) {
                 $log->setUser(new User($user->getId()));
@@ -298,10 +297,6 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo
             $isProduction = App::getEnv('_APP_ENV', 'development') === 'production';
             $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING);
 
-            foreach ($loggerBreadcrumbs as $loggerBreadcrumb) {
-                $log->addBreadcrumb($loggerBreadcrumb);
-            }
-
             $responseCode = $logger->addLog($log);
             Console::info('Log pushed with status code: ' . $responseCode);
         }

app/init.php

@@ -76,11 +76,13 @@ use Appwrite\Hooks\Hooks;
 use MaxMind\Db\Reader;
 use PHPMailer\PHPMailer\PHPMailer;
 use Swoole\Database\PDOProxy;
+use Utopia\Logger\Log;
 use Utopia\Queue;
 use Utopia\Queue\Connection;
 use Utopia\Storage\Storage;
 use Utopia\VCS\Adapter\Git\GitHub as VcsGitHub;
 use Utopia\Validator\Range;
+use Utopia\Validator\Hostname;
 use Utopia\Validator\IP;
 use Utopia\Validator\URL;
 use Utopia\Validator\WhiteList;
@@ -110,7 +112,7 @@ const APP_LIMIT_LIST_DEFAULT = 25; // Default maximum number of items to return
 const APP_KEY_ACCCESS = 24 * 60 * 60; // 24 hours
 const APP_USER_ACCCESS = 24 * 60 * 60; // 24 hours
 const APP_CACHE_UPDATE = 24 * 60 * 60; // 24 hours
-const APP_CACHE_BUSTER = 328;
+const APP_CACHE_BUSTER = 329;
 const APP_VERSION_STABLE = '1.4.13';
 const APP_DATABASE_ATTRIBUTE_EMAIL = 'email';
 const APP_DATABASE_ATTRIBUTE_ENUM = 'enum';
@@ -174,10 +176,6 @@ const DELETE_TYPE_SESSIONS = 'sessions';
 const DELETE_TYPE_CACHE_BY_TIMESTAMP = 'cacheByTimeStamp';
 const DELETE_TYPE_CACHE_BY_RESOURCE  = 'cacheByResource';
 const DELETE_TYPE_SCHEDULES = 'schedules';
-// Compression type
-const COMPRESSION_TYPE_NONE = 'none';
-const COMPRESSION_TYPE_GZIP = 'gzip';
-const COMPRESSION_TYPE_ZSTD = 'zstd';
 // Mail Types
 const MAIL_TYPE_VERIFICATION = 'verification';
 const MAIL_TYPE_MAGIC_SESSION = 'magicSession';
@@ -771,22 +769,22 @@ $register->set('pools', function () {
 
 $register->set('db', function () {
     // This is usually for our workers or CLI commands scope
-    $dbHost = App::getEnv('_APP_DB_HOST', '');
-    $dbPort = App::getEnv('_APP_DB_PORT', '');
-    $dbUser = App::getEnv('_APP_DB_USER', '');
-    $dbPass = App::getEnv('_APP_DB_PASS', '');
-    $dbScheme = App::getEnv('_APP_DB_SCHEMA', '');
+       $dbHost = App::getEnv('_APP_DB_HOST', '');
+       $dbPort = App::getEnv('_APP_DB_PORT', '');
+       $dbUser = App::getEnv('_APP_DB_USER', '');
+       $dbPass = App::getEnv('_APP_DB_PASS', '');
+       $dbScheme = App::getEnv('_APP_DB_SCHEMA', '');
 
-    $pdo = new PDO("mysql:host={$dbHost};port={$dbPort};dbname={$dbScheme};charset=utf8mb4", $dbUser, $dbPass, array(
-        PDO::ATTR_TIMEOUT => 3, // Seconds
-        PDO::ATTR_PERSISTENT => true,
-        PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
-        PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION,
-        PDO::ATTR_EMULATE_PREPARES => true,
-        PDO::ATTR_STRINGIFY_FETCHES => true,
-    ));
+       $pdo = new PDO("mysql:host={$dbHost};port={$dbPort};dbname={$dbScheme};charset=utf8mb4", $dbUser, $dbPass, array(
+           PDO::ATTR_TIMEOUT => 3, // Seconds
+           PDO::ATTR_PERSISTENT => true,
+           PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
+           PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION,
+           PDO::ATTR_EMULATE_PREPARES => true,
+           PDO::ATTR_STRINGIFY_FETCHES => true,
+       ));
 
-    return $pdo;
+       return $pdo;
 });
 
 $register->set('smtp', function () {
@@ -867,6 +865,7 @@ foreach ($locales as $locale) {
 ]);
 
 // Runtime Execution
+App::setResource('log', fn() => new Log());
 App::setResource('logger', function ($register) {
     return $register->get('logger');
 }, ['register']);
@@ -875,10 +874,6 @@ App::setResource('hooks', function ($register) {
     return $register->get('hooks');
 }, ['register']);
 
-App::setResource('loggerBreadcrumbs', function () {
-    return [];
-});
-
 App::setResource('register', fn() => $register);
 App::setResource('locale', fn() => new Locale(App::getEnv('_APP_LOCALE', 'en')));
 
@@ -931,6 +926,21 @@ App::setResource('clients', function ($request, $console, $project) {
         'hostname' => $request->getHostname(),
     ], Document::SET_TYPE_APPEND);
 
+    $hostnames = explode(',', App::getEnv('_APP_CONSOLE_HOSTNAMES', ''));
+    $validator = new Hostname();
+    foreach ($hostnames as $hostname) {
+        $hostname = trim($hostname);
+        if (!$validator->isValid($hostname)) {
+            continue;
+        }
+        $console->setAttribute('platforms', [
+            '$collection' => ID::custom('platforms'),
+            'type' => Origin::CLIENT_TYPE_WEB,
+            'name' => $hostname,
+            'hostname' => $hostname,
+        ], Document::SET_TYPE_APPEND);
+    }
+
     /**
      * Get All verified client URLs for both console and current projects
      * + Filter for duplicated entries

app/views/install/compose.phtml

@@ -78,6 +78,7 @@ services:
       - _APP_CONSOLE_WHITELIST_ROOT
       - _APP_CONSOLE_WHITELIST_EMAILS
       - _APP_CONSOLE_WHITELIST_IPS
+      - _APP_CONSOLE_HOSTNAMES
       - _APP_SYSTEM_EMAIL_NAME
       - _APP_SYSTEM_EMAIL_ADDRESS
       - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS
@@ -268,7 +269,7 @@ services:
     depends_on:
       - redis
       - mariadb
-    volumes: 
+    volumes:
       - appwrite-uploads:/storage/uploads:rw
       - appwrite-cache:/storage/cache:rw
       - appwrite-functions:/storage/functions:rw
@@ -415,7 +416,7 @@ services:
     depends_on:
       - redis
       - mariadb
-    volumes: 
+    volumes:
       - appwrite-config:/storage/config:rw
       - appwrite-certificates:/storage/certificates:rw
     environment:
@@ -643,7 +644,7 @@ services:
       - _APP_DB_PASS
 
   appwrite-assistant:
-    image: appwrite/assistant:0.2.2
+    image: appwrite/assistant:0.3.0
     container_name: appwrite-assistant
     <<: *x-logging
     restart: unless-stopped

app/worker.php

@@ -9,6 +9,7 @@ use Appwrite\Event\Certificate;
 use Appwrite\Event\Database as EventDatabase;
 use Appwrite\Event\Delete;
 use Appwrite\Event\Func;
+use Appwrite\Event\Hamster;
 use Appwrite\Event\Mail;
 use Appwrite\Event\Migration;
 use Appwrite\Event\Phone;
@@ -178,6 +179,9 @@ Server::setResource('queueForCertificates', function (Connection $queue) {
 Server::setResource('queueForMigrations', function (Connection $queue) {
     return new Migration($queue);
 }, ['queue']);
+Server::setResource('queueForHamster', function (Connection $queue) {
+    return new Hamster($queue);
+}, ['queue']);
 Server::setResource('logger', function (Registry $register) {
     return $register->get('logger');
 }, ['register']);

bin/calc-tier-stats

@@ -1,3 +0,0 @@
-#!/bin/sh
-
-php /usr/src/code/app/cli.php calc-tier-stats $@

bin/calc-users-stats

@@ -1,3 +0,0 @@
-#!/bin/sh
-
-php /usr/src/code/app/cli.php calc-users-stats $@

bin/clear-card-cache

@@ -1,3 +0,0 @@
-#!/bin/sh
-
-php /usr/src/code/app/cli.php clear-card-cache $@

bin/delete-orphaned-projects

@@ -1,3 +0,0 @@
-#!/bin/sh
-
-php /usr/src/code/app/cli.php delete-orphaned-projects $@

bin/get-migration-stats

@@ -1,3 +0,0 @@
-#!/bin/sh
-
-php /usr/src/code/app/cli.php get-migration-stats $@

bin/hamster

@@ -1,3 +0,0 @@
-#!/bin/sh
-
-php /usr/src/code/app/cli.php hamster $@

bin/queue-count-failed

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+php /usr/src/code/app/cli.php queue-count --type=failed $@

bin/queue-count-processing

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+php /usr/src/code/app/cli.php queue-count --type=processing $@

bin/queue-count-success

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+php /usr/src/code/app/cli.php queue-count --type=success $@

bin/queue-retry

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+php /usr/src/code/app/cli.php queue-retry $@

bin/usage

@@ -1,3 +0,0 @@
-#!/bin/sh
-
-php /usr/src/code/app/cli.php usage $@

composer.json

@@ -46,13 +46,13 @@
         "utopia-php/abuse": "0.33.*",
         "utopia-php/analytics": "0.10.*",
         "utopia-php/audit": "0.35.*",
-        "utopia-php/cache": "0.8.*",
+        "utopia-php/cache": "0.9.*",
         "utopia-php/cli": "0.15.*",
         "utopia-php/config": "0.2.*",
         "utopia-php/database": "0.45.*",
         "utopia-php/domains": "0.3.*",
         "utopia-php/dsn": "0.1.*",
-        "utopia-php/framework": "0.31.0",
+        "utopia-php/framework": "0.33.*",
         "utopia-php/image": "0.5.*",
         "utopia-php/locale": "0.4.*",
         "utopia-php/logger": "0.3.*",
@@ -62,7 +62,7 @@
         "utopia-php/platform": "0.5.*",
         "utopia-php/pools": "0.4.*",
         "utopia-php/preloader": "0.2.*",
-        "utopia-php/queue": "0.6.*",
+        "utopia-php/queue": "0.7.*",
         "utopia-php/registry": "0.5.*",
         "utopia-php/storage": "0.18.*",
         "utopia-php/swoole": "0.5.*",

composer.lock

@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
         "This file is @generated automatically"
     ],
-    "content-hash": "0954f1d596a1b28854685b0671110d0e",
+    "content-hash": "35dcde03d0eb9a0d27de653b9fa038d4",
     "packages": [
         {
             "name": "adhocore/jwt",
@@ -277,16 +277,16 @@
         },
         {
             "name": "chillerlan/php-settings-container",
-            "version": "2.1.4",
+            "version": "2.1.5",
             "source": {
                 "type": "git",
                 "url": "https://github.com/chillerlan/php-settings-container.git",
-                "reference": "1beb7df3c14346d4344b0b2e12f6f9a74feabd4a"
+                "reference": "f705310389264c3578fdd9ffb15aa2cd6d91772e"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/chillerlan/php-settings-container/zipball/1beb7df3c14346d4344b0b2e12f6f9a74feabd4a",
-                "reference": "1beb7df3c14346d4344b0b2e12f6f9a74feabd4a",
+                "url": "https://api.github.com/repos/chillerlan/php-settings-container/zipball/f705310389264c3578fdd9ffb15aa2cd6d91772e",
+                "reference": "f705310389264c3578fdd9ffb15aa2cd6d91772e",
                 "shasum": ""
             },
             "require": {
@@ -294,8 +294,10 @@
                 "php": "^7.4 || ^8.0"
             },
             "require-dev": {
-                "phan/phan": "^5.3",
-                "phpunit/phpunit": "^9.5"
+                "phan/phan": "^5.4",
+                "phpcsstandards/php_codesniffer": "^3.8",
+                "phpmd/phpmd": "^2.13",
+                "phpunit/phpunit": "^9.6"
             },
             "type": "library",
             "autoload": {
@@ -337,7 +339,7 @@
                     "type": "ko_fi"
                 }
             ],
-            "time": "2022-07-05T22:32:14+00:00"
+            "time": "2024-01-05T23:20:55+00:00"
         },
         {
             "name": "dragonmantank/cron-expression",
@@ -1038,16 +1040,16 @@
         },
         {
             "name": "utopia-php/cache",
-            "version": "0.8.0",
+            "version": "0.9.0",
             "source": {
                 "type": "git",
                 "url": "https://github.com/utopia-php/cache.git",
-                "reference": "212e66100a1f32e674fca5d9bc317cc998303089"
+                "reference": "4fc7b4789b5f0ce74835c1ecfec4f3afe6f0e34e"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/cache/zipball/212e66100a1f32e674fca5d9bc317cc998303089",
-                "reference": "212e66100a1f32e674fca5d9bc317cc998303089",
+                "url": "https://api.github.com/repos/utopia-php/cache/zipball/4fc7b4789b5f0ce74835c1ecfec4f3afe6f0e34e",
+                "reference": "4fc7b4789b5f0ce74835c1ecfec4f3afe6f0e34e",
                 "shasum": ""
             },
             "require": {
@@ -1058,6 +1060,7 @@
             },
             "require-dev": {
                 "laravel/pint": "1.2.*",
+                "phpstan/phpstan": "1.9.x-dev",
                 "phpunit/phpunit": "^9.3",
                 "vimeo/psalm": "4.13.1"
             },
@@ -1081,9 +1084,9 @@
             ],
             "support": {
                 "issues": "https://github.com/utopia-php/cache/issues",
-                "source": "https://github.com/utopia-php/cache/tree/0.8.0"
+                "source": "https://github.com/utopia-php/cache/tree/0.9.0"
             },
-            "time": "2022-10-16T16:48:09+00:00"
+            "time": "2024-01-07T18:11:23+00:00"
         },
         {
             "name": "utopia-php/cli",
@@ -1187,23 +1190,23 @@
         },
         {
             "name": "utopia-php/database",
-            "version": "0.45.2",
+            "version": "0.45.5",
             "source": {
                 "type": "git",
                 "url": "https://github.com/utopia-php/database.git",
-                "reference": "dc789f2c1fd8b5ee07ff883e11c9ad7970824788"
+                "reference": "0b66a017f817a910acb83e6aea92bccea9571fe6"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/database/zipball/dc789f2c1fd8b5ee07ff883e11c9ad7970824788",
-                "reference": "dc789f2c1fd8b5ee07ff883e11c9ad7970824788",
+                "url": "https://api.github.com/repos/utopia-php/database/zipball/0b66a017f817a910acb83e6aea92bccea9571fe6",
+                "reference": "0b66a017f817a910acb83e6aea92bccea9571fe6",
                 "shasum": ""
             },
             "require": {
                 "ext-mbstring": "*",
                 "ext-pdo": "*",
                 "php": ">=8.0",
-                "utopia-php/cache": "0.8.*",
+                "utopia-php/cache": "0.9.*",
                 "utopia-php/framework": "0.*.*",
                 "utopia-php/mongo": "0.3.*"
             },
@@ -1237,9 +1240,9 @@
             ],
             "support": {
                 "issues": "https://github.com/utopia-php/database/issues",
-                "source": "https://github.com/utopia-php/database/tree/0.45.2"
+                "source": "https://github.com/utopia-php/database/tree/0.45.5"
             },
-            "time": "2023-11-15T03:38:47+00:00"
+            "time": "2024-01-08T17:08:15+00:00"
         },
         {
             "name": "utopia-php/domains",
@@ -1350,16 +1353,16 @@
         },
         {
             "name": "utopia-php/framework",
-            "version": "0.31.0",
+            "version": "0.33.1",
             "source": {
                 "type": "git",
-                "url": "https://github.com/utopia-php/framework.git",
-                "reference": "207f77378965fca9a9bc3783ea379d3549f86bc0"
+                "url": "https://github.com/utopia-php/http.git",
+                "reference": "b745607aa1875554a0ad52e28f6db918da1ce11c"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/framework/zipball/207f77378965fca9a9bc3783ea379d3549f86bc0",
-                "reference": "207f77378965fca9a9bc3783ea379d3549f86bc0",
+                "url": "https://api.github.com/repos/utopia-php/http/zipball/b745607aa1875554a0ad52e28f6db918da1ce11c",
+                "reference": "b745607aa1875554a0ad52e28f6db918da1ce11c",
                 "shasum": ""
             },
             "require": {
@@ -1388,10 +1391,10 @@
                 "upf"
             ],
             "support": {
-                "issues": "https://github.com/utopia-php/framework/issues",
-                "source": "https://github.com/utopia-php/framework/tree/0.31.0"
+                "issues": "https://github.com/utopia-php/http/issues",
+                "source": "https://github.com/utopia-php/http/tree/0.33.1"
             },
-            "time": "2023-08-30T16:10:04+00:00"
+            "time": "2024-01-17T16:48:32+00:00"
         },
         {
             "name": "utopia-php/image",
@@ -1757,16 +1760,16 @@
         },
         {
             "name": "utopia-php/platform",
-            "version": "0.5.0",
+            "version": "0.5.1",
             "source": {
                 "type": "git",
                 "url": "https://github.com/utopia-php/platform.git",
-                "reference": "229a7b1fa1f39afd1532f7a515326a6afc222a26"
+                "reference": "3eceef0b6593fe0f7d2efd36d40402a395a4c285"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/platform/zipball/229a7b1fa1f39afd1532f7a515326a6afc222a26",
-                "reference": "229a7b1fa1f39afd1532f7a515326a6afc222a26",
+                "url": "https://api.github.com/repos/utopia-php/platform/zipball/3eceef0b6593fe0f7d2efd36d40402a395a4c285",
+                "reference": "3eceef0b6593fe0f7d2efd36d40402a395a4c285",
                 "shasum": ""
             },
             "require": {
@@ -1774,7 +1777,7 @@
                 "ext-redis": "*",
                 "php": ">=8.0",
                 "utopia-php/cli": "0.15.*",
-                "utopia-php/framework": "0.31.*"
+                "utopia-php/framework": "0.*.*"
             },
             "require-dev": {
                 "laravel/pint": "1.2.*",
@@ -1800,9 +1803,9 @@
             ],
             "support": {
                 "issues": "https://github.com/utopia-php/platform/issues",
-                "source": "https://github.com/utopia-php/platform/tree/0.5.0"
+                "source": "https://github.com/utopia-php/platform/tree/0.5.1"
             },
-            "time": "2023-10-16T20:28:49+00:00"
+            "time": "2023-12-26T16:14:41+00:00"
         },
         {
             "name": "utopia-php/pools",
@@ -1910,16 +1913,16 @@
         },
         {
             "name": "utopia-php/queue",
-            "version": "0.6.0",
+            "version": "0.7.0",
             "source": {
                 "type": "git",
                 "url": "https://github.com/utopia-php/queue.git",
-                "reference": "0120bd21904cb2bee34e4571b1737589ffff0eb1"
+                "reference": "917565256eb94bcab7246f7a746b1a486813761b"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/queue/zipball/0120bd21904cb2bee34e4571b1737589ffff0eb1",
-                "reference": "0120bd21904cb2bee34e4571b1737589ffff0eb1",
+                "url": "https://api.github.com/repos/utopia-php/queue/zipball/917565256eb94bcab7246f7a746b1a486813761b",
+                "reference": "917565256eb94bcab7246f7a746b1a486813761b",
                 "shasum": ""
             },
             "require": {
@@ -1965,9 +1968,9 @@
             ],
             "support": {
                 "issues": "https://github.com/utopia-php/queue/issues",
-                "source": "https://github.com/utopia-php/queue/tree/0.6.0"
+                "source": "https://github.com/utopia-php/queue/tree/0.7.0"
             },
-            "time": "2023-10-16T16:59:45+00:00"
+            "time": "2024-01-17T19:00:43+00:00"
         },
         {
             "name": "utopia-php/registry",
@@ -2023,16 +2026,16 @@
         },
         {
             "name": "utopia-php/storage",
-            "version": "0.18.1",
+            "version": "0.18.3",
             "source": {
                 "type": "git",
                 "url": "https://github.com/utopia-php/storage.git",
-                "reference": "983e6dee137012f9f57f126d3c79aab54e4e8824"
+                "reference": "faa0279519ac14f3501e8b138e0865ad9d12bff6"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/storage/zipball/983e6dee137012f9f57f126d3c79aab54e4e8824",
-                "reference": "983e6dee137012f9f57f126d3c79aab54e4e8824",
+                "url": "https://api.github.com/repos/utopia-php/storage/zipball/faa0279519ac14f3501e8b138e0865ad9d12bff6",
+                "reference": "faa0279519ac14f3501e8b138e0865ad9d12bff6",
                 "shasum": ""
             },
             "require": {
@@ -2072,9 +2075,9 @@
             ],
             "support": {
                 "issues": "https://github.com/utopia-php/storage/issues",
-                "source": "https://github.com/utopia-php/storage/tree/0.18.1"
+                "source": "https://github.com/utopia-php/storage/tree/0.18.3"
             },
-            "time": "2023-10-24T14:44:19+00:00"
+            "time": "2023-12-31T11:45:12+00:00"
         },
         {
             "name": "utopia-php/swoole",
@@ -2185,23 +2188,23 @@
         },
         {
             "name": "utopia-php/vcs",
-            "version": "0.6.2",
+            "version": "0.6.5",
             "source": {
                 "type": "git",
                 "url": "https://github.com/utopia-php/vcs.git",
-                "reference": "f135291b87cb45335fc6608722e7f89894bc33ee"
+                "reference": "104e47ea8e38c156ec0e0bd415caa3dcd5046fe2"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/vcs/zipball/f135291b87cb45335fc6608722e7f89894bc33ee",
-                "reference": "f135291b87cb45335fc6608722e7f89894bc33ee",
+                "url": "https://api.github.com/repos/utopia-php/vcs/zipball/104e47ea8e38c156ec0e0bd415caa3dcd5046fe2",
+                "reference": "104e47ea8e38c156ec0e0bd415caa3dcd5046fe2",
                 "shasum": ""
             },
             "require": {
                 "adhocore/jwt": "^1.1",
                 "php": ">=8.0",
-                "utopia-php/cache": "^0.8.0",
-                "utopia-php/framework": "0.31.*"
+                "utopia-php/cache": "^0.9.0",
+                "utopia-php/framework": "0.*.*"
             },
             "require-dev": {
                 "laravel/pint": "1.2.*",
@@ -2228,9 +2231,9 @@
             ],
             "support": {
                 "issues": "https://github.com/utopia-php/vcs/issues",
-                "source": "https://github.com/utopia-php/vcs/tree/0.6.2"
+                "source": "https://github.com/utopia-php/vcs/tree/0.6.5"
             },
-            "time": "2023-11-08T15:36:03+00:00"
+            "time": "2024-01-08T17:11:12+00:00"
         },
         {
             "name": "utopia-php/websocket",
@@ -2417,16 +2420,16 @@
     "packages-dev": [
         {
             "name": "appwrite/sdk-generator",
-            "version": "0.36.0",
+            "version": "0.36.2",
             "source": {
                 "type": "git",
                 "url": "https://github.com/appwrite/sdk-generator.git",
-                "reference": "3a10f1f895ed71120442ff71eb6adec3fd6b4e8a"
+                "reference": "0aa67479d75f0e0cb7b60454031534d7f0abaece"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/3a10f1f895ed71120442ff71eb6adec3fd6b4e8a",
-                "reference": "3a10f1f895ed71120442ff71eb6adec3fd6b4e8a",
+                "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/0aa67479d75f0e0cb7b60454031534d7f0abaece",
+                "reference": "0aa67479d75f0e0cb7b60454031534d7f0abaece",
                 "shasum": ""
             },
             "require": {
@@ -2462,9 +2465,9 @@
             "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms",
             "support": {
                 "issues": "https://github.com/appwrite/sdk-generator/issues",
-                "source": "https://github.com/appwrite/sdk-generator/tree/0.36.0"
+                "source": "https://github.com/appwrite/sdk-generator/tree/0.36.2"
             },
-            "time": "2023-11-20T10:03:06+00:00"
+            "time": "2024-01-19T01:04:35+00:00"
         },
         {
             "name": "doctrine/deprecations",
@@ -2768,25 +2771,27 @@
         },
         {
             "name": "nikic/php-parser",
-            "version": "v4.18.0",
+            "version": "v5.0.0",
             "source": {
                 "type": "git",
                 "url": "https://github.com/nikic/PHP-Parser.git",
-                "reference": "1bcbb2179f97633e98bbbc87044ee2611c7d7999"
+                "reference": "4a21235f7e56e713259a6f76bf4b5ea08502b9dc"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/1bcbb2179f97633e98bbbc87044ee2611c7d7999",
-                "reference": "1bcbb2179f97633e98bbbc87044ee2611c7d7999",
+                "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/4a21235f7e56e713259a6f76bf4b5ea08502b9dc",
+                "reference": "4a21235f7e56e713259a6f76bf4b5ea08502b9dc",
                 "shasum": ""
             },
             "require": {
+                "ext-ctype": "*",
+                "ext-json": "*",
                 "ext-tokenizer": "*",
-                "php": ">=7.0"
+                "php": ">=7.4"
             },
             "require-dev": {
                 "ircmaxell/php-yacc": "^0.0.7",
-                "phpunit/phpunit": "^6.5 || ^7.0 || ^8.0 || ^9.0"
+                "phpunit/phpunit": "^7.0 || ^8.0 || ^9.0"
             },
             "bin": [
                 "bin/php-parse"
@@ -2794,7 +2799,7 @@
             "type": "library",
             "extra": {
                 "branch-alias": {
-                    "dev-master": "4.9-dev"
+                    "dev-master": "5.0-dev"
                 }
             },
             "autoload": {
@@ -2818,9 +2823,9 @@
             ],
             "support": {
                 "issues": "https://github.com/nikic/PHP-Parser/issues",
-                "source": "https://github.com/nikic/PHP-Parser/tree/v4.18.0"
+                "source": "https://github.com/nikic/PHP-Parser/tree/v5.0.0"
             },
-            "time": "2023-12-10T21:03:43+00:00"
+            "time": "2024-01-07T17:17:35+00:00"
         },
         {
             "name": "phar-io/manifest",
@@ -3045,16 +3050,16 @@
         },
         {
             "name": "phpdocumentor/type-resolver",
-            "version": "1.7.3",
+            "version": "1.8.0",
             "source": {
                 "type": "git",
                 "url": "https://github.com/phpDocumentor/TypeResolver.git",
-                "reference": "3219c6ee25c9ea71e3d9bbaf39c67c9ebd499419"
+                "reference": "fad452781b3d774e3337b0c0b245dd8e5a4455fc"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/3219c6ee25c9ea71e3d9bbaf39c67c9ebd499419",
-                "reference": "3219c6ee25c9ea71e3d9bbaf39c67c9ebd499419",
+                "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/fad452781b3d774e3337b0c0b245dd8e5a4455fc",
+                "reference": "fad452781b3d774e3337b0c0b245dd8e5a4455fc",
                 "shasum": ""
             },
             "require": {
@@ -3097,9 +3102,9 @@
             "description": "A PSR-5 based resolver of Class names, Types and Structural Element Names",
             "support": {
                 "issues": "https://github.com/phpDocumentor/TypeResolver/issues",
-                "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.7.3"
+                "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.8.0"
             },
-            "time": "2023-08-12T11:01:26+00:00"
+            "time": "2024-01-11T11:49:22+00:00"
         },
         {
             "name": "phpspec/prophecy",
@@ -3172,16 +3177,16 @@
         },
         {
             "name": "phpstan/phpdoc-parser",
-            "version": "1.24.4",
+            "version": "1.25.0",
             "source": {
                 "type": "git",
                 "url": "https://github.com/phpstan/phpdoc-parser.git",
-                "reference": "6bd0c26f3786cd9b7c359675cb789e35a8e07496"
+                "reference": "bd84b629c8de41aa2ae82c067c955e06f1b00240"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/6bd0c26f3786cd9b7c359675cb789e35a8e07496",
-                "reference": "6bd0c26f3786cd9b7c359675cb789e35a8e07496",
+                "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/bd84b629c8de41aa2ae82c067c955e06f1b00240",
+                "reference": "bd84b629c8de41aa2ae82c067c955e06f1b00240",
                 "shasum": ""
             },
             "require": {
@@ -3213,29 +3218,29 @@
             "description": "PHPDoc parser with support for nullable, intersection and generic types",
             "support": {
                 "issues": "https://github.com/phpstan/phpdoc-parser/issues",
-                "source": "https://github.com/phpstan/phpdoc-parser/tree/1.24.4"
+                "source": "https://github.com/phpstan/phpdoc-parser/tree/1.25.0"
             },
-            "time": "2023-11-26T18:29:22+00:00"
+            "time": "2024-01-04T17:06:16+00:00"
         },
         {
             "name": "phpunit/php-code-coverage",
-            "version": "9.2.29",
+            "version": "9.2.30",
             "source": {
                 "type": "git",
                 "url": "https://github.com/sebastianbergmann/php-code-coverage.git",
-                "reference": "6a3a87ac2bbe33b25042753df8195ba4aa534c76"
+                "reference": "ca2bd87d2f9215904682a9cb9bb37dda98e76089"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/6a3a87ac2bbe33b25042753df8195ba4aa534c76",
-                "reference": "6a3a87ac2bbe33b25042753df8195ba4aa534c76",
+                "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/ca2bd87d2f9215904682a9cb9bb37dda98e76089",
+                "reference": "ca2bd87d2f9215904682a9cb9bb37dda98e76089",
                 "shasum": ""
             },
             "require": {
                 "ext-dom": "*",
                 "ext-libxml": "*",
                 "ext-xmlwriter": "*",
-                "nikic/php-parser": "^4.15",
+                "nikic/php-parser": "^4.18 || ^5.0",
                 "php": ">=7.3",
                 "phpunit/php-file-iterator": "^3.0.3",
                 "phpunit/php-text-template": "^2.0.2",
@@ -3285,7 +3290,7 @@
             "support": {
                 "issues": "https://github.com/sebastianbergmann/php-code-coverage/issues",
                 "security": "https://github.com/sebastianbergmann/php-code-coverage/security/policy",
-                "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.29"
+                "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.30"
             },
             "funding": [
                 {
@@ -3293,7 +3298,7 @@
                     "type": "github"
                 }
             ],
-            "time": "2023-09-19T04:57:46+00:00"
+            "time": "2023-12-22T06:47:57+00:00"
         },
         {
             "name": "phpunit/php-file-iterator",
@@ -3932,20 +3937,20 @@
         },
         {
             "name": "sebastian/complexity",
-            "version": "2.0.2",
+            "version": "2.0.3",
             "source": {
                 "type": "git",
                 "url": "https://github.com/sebastianbergmann/complexity.git",
-                "reference": "739b35e53379900cc9ac327b2147867b8b6efd88"
+                "reference": "25f207c40d62b8b7aa32f5ab026c53561964053a"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/sebastianbergmann/complexity/zipball/739b35e53379900cc9ac327b2147867b8b6efd88",
-                "reference": "739b35e53379900cc9ac327b2147867b8b6efd88",
+                "url": "https://api.github.com/repos/sebastianbergmann/complexity/zipball/25f207c40d62b8b7aa32f5ab026c53561964053a",
+                "reference": "25f207c40d62b8b7aa32f5ab026c53561964053a",
                 "shasum": ""
             },
             "require": {
-                "nikic/php-parser": "^4.7",
+                "nikic/php-parser": "^4.18 || ^5.0",
                 "php": ">=7.3"
             },
             "require-dev": {
@@ -3977,7 +3982,7 @@
             "homepage": "https://github.com/sebastianbergmann/complexity",
             "support": {
                 "issues": "https://github.com/sebastianbergmann/complexity/issues",
-                "source": "https://github.com/sebastianbergmann/complexity/tree/2.0.2"
+                "source": "https://github.com/sebastianbergmann/complexity/tree/2.0.3"
             },
             "funding": [
                 {
@@ -3985,7 +3990,7 @@
                     "type": "github"
                 }
             ],
-            "time": "2020-10-26T15:52:27+00:00"
+            "time": "2023-12-22T06:19:30+00:00"
         },
         {
             "name": "sebastian/diff",
@@ -4259,20 +4264,20 @@
         },
         {
             "name": "sebastian/lines-of-code",
-            "version": "1.0.3",
+            "version": "1.0.4",
             "source": {
                 "type": "git",
                 "url": "https://github.com/sebastianbergmann/lines-of-code.git",
-                "reference": "c1c2e997aa3146983ed888ad08b15470a2e22ecc"
+                "reference": "e1e4a170560925c26d424b6a03aed157e7dcc5c5"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/sebastianbergmann/lines-of-code/zipball/c1c2e997aa3146983ed888ad08b15470a2e22ecc",
-                "reference": "c1c2e997aa3146983ed888ad08b15470a2e22ecc",
+                "url": "https://api.github.com/repos/sebastianbergmann/lines-of-code/zipball/e1e4a170560925c26d424b6a03aed157e7dcc5c5",
+                "reference": "e1e4a170560925c26d424b6a03aed157e7dcc5c5",
                 "shasum": ""
             },
             "require": {
-                "nikic/php-parser": "^4.6",
+                "nikic/php-parser": "^4.18 || ^5.0",
                 "php": ">=7.3"
             },
             "require-dev": {
@@ -4304,7 +4309,7 @@
             "homepage": "https://github.com/sebastianbergmann/lines-of-code",
             "support": {
                 "issues": "https://github.com/sebastianbergmann/lines-of-code/issues",
-                "source": "https://github.com/sebastianbergmann/lines-of-code/tree/1.0.3"
+                "source": "https://github.com/sebastianbergmann/lines-of-code/tree/1.0.4"
             },
             "funding": [
                 {
@@ -4312,7 +4317,7 @@
                     "type": "github"
                 }
             ],
-            "time": "2020-11-28T06:42:11+00:00"
+            "time": "2023-12-22T06:20:34+00:00"
         },
         {
             "name": "sebastian/object-enumerator",
@@ -4655,16 +4660,16 @@
         },
         {
             "name": "squizlabs/php_codesniffer",
-            "version": "3.8.0",
+            "version": "3.8.1",
             "source": {
                 "type": "git",
                 "url": "https://github.com/PHPCSStandards/PHP_CodeSniffer.git",
-                "reference": "5805f7a4e4958dbb5e944ef1e6edae0a303765e7"
+                "reference": "14f5fff1e64118595db5408e946f3a22c75807f7"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/PHPCSStandards/PHP_CodeSniffer/zipball/5805f7a4e4958dbb5e944ef1e6edae0a303765e7",
-                "reference": "5805f7a4e4958dbb5e944ef1e6edae0a303765e7",
+                "url": "https://api.github.com/repos/PHPCSStandards/PHP_CodeSniffer/zipball/14f5fff1e64118595db5408e946f3a22c75807f7",
+                "reference": "14f5fff1e64118595db5408e946f3a22c75807f7",
                 "shasum": ""
             },
             "require": {
@@ -4674,11 +4679,11 @@
                 "php": ">=5.4.0"
             },
             "require-dev": {
-                "phpunit/phpunit": "^4.0 || ^5.0 || ^6.0 || ^7.0 || ^8.0 || ^9.0"
+                "phpunit/phpunit": "^4.0 || ^5.0 || ^6.0 || ^7.0 || ^8.0 || ^9.3.4"
             },
             "bin": [
-                "bin/phpcs",
-                "bin/phpcbf"
+                "bin/phpcbf",
+                "bin/phpcs"
             ],
             "type": "library",
             "extra": {
@@ -4731,7 +4736,7 @@
                     "type": "open_collective"
                 }
             ],
-            "time": "2023-12-08T12:32:31+00:00"
+            "time": "2024-01-11T20:47:48+00:00"
         },
         {
             "name": "swoole/ide-helper",

docker-compose.yml

@@ -53,7 +53,7 @@ services:
         DEBUG: false
         TESTING: true
         VERSION: dev
-    ports: 
+    ports:
       - 9501:80
     networks:
       - appwrite
@@ -88,7 +88,7 @@ services:
       - mariadb
       - redis
       # - clamav
-    entrypoint: 
+    entrypoint:
       - php
       - -e
       - app/http.php
@@ -100,6 +100,7 @@ services:
       - _APP_CONSOLE_WHITELIST_ROOT
       - _APP_CONSOLE_WHITELIST_EMAILS
       - _APP_CONSOLE_WHITELIST_IPS
+      - _APP_CONSOLE_HOSTNAMES
       - _APP_SYSTEM_EMAIL_NAME
       - _APP_SYSTEM_EMAIL_ADDRESS
       - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS
@@ -191,7 +192,7 @@ services:
     container_name: appwrite-realtime
     image: appwrite-dev
     restart: unless-stopped
-    ports: 
+    ports:
       - 9505:80
     labels:
       - "traefik.enable=true"
@@ -299,7 +300,7 @@ services:
     depends_on:
       - redis
       - mariadb
-    volumes: 
+    volumes:
       - appwrite-uploads:/storage/uploads:rw
       - appwrite-cache:/storage/cache:rw
       - appwrite-functions:/storage/functions:rw
@@ -353,7 +354,7 @@ services:
     image: appwrite-dev
     networks:
       - appwrite
-    volumes: 
+    volumes:
       - ./app:/usr/src/code/app
       - ./src:/usr/src/code/src
     depends_on:
@@ -452,7 +453,7 @@ services:
     depends_on:
       - redis
       - mariadb
-    volumes: 
+    volumes:
       - appwrite-config:/storage/config:rw
       - appwrite-certificates:/storage/certificates:rw
       - ./app:/usr/src/code/app
@@ -570,6 +571,7 @@ services:
       - _APP_REDIS_PASS
       - _APP_SMS_PROVIDER
       - _APP_SMS_FROM
+      - _APP_SMS_PROJECTS_DENY_LIST
       - _APP_LOGGING_PROVIDER
       - _APP_LOGGING_CONFIG
 
@@ -673,6 +675,7 @@ services:
       - _APP_USAGE_STATS
       - _APP_LOGGING_PROVIDER
       - _APP_LOGGING_CONFIG
+      - _APP_USAGE_AGGREGATION_INTERVAL
 
   appwrite-schedule:
     entrypoint: schedule
@@ -703,7 +706,7 @@ services:
 
   appwrite-assistant:
     container_name: appwrite-assistant
-    image: appwrite/assistant:0.2.2
+    image: appwrite/assistant:0.3.0
     networks:
       - appwrite
     environment:
@@ -889,7 +892,7 @@ services:
   #     - './debug:/tmp'
   #   ports:
   #     - '3001:80'
-  
+
   graphql-explorer:
     container_name: appwrite-graphql-explorer
     image: appwrite/altair:0.3.0

docs/references/account/delete.md

@@ -0,0 +1 @@
+Delete the currently logged in user.

docs/references/health/get-failed-queue-jobs.md

@@ -0,0 +1 @@
+Returns the amount of failed jobs in a given queue.

src/Appwrite/Event/Event.php

@@ -42,6 +42,7 @@ class Event
     public const MIGRATIONS_QUEUE_NAME = 'v1-migrations';
     public const MIGRATIONS_CLASS_NAME = 'MigrationsV1';
 
+
     protected string $queue = '';
     protected string $class = '';
     protected string $event = '';

src/Appwrite/Extend/Exception.php

@@ -238,21 +238,16 @@ class Exception extends \Exception
 
     protected string $type = '';
     protected array $errors = [];
-    protected bool $publish = true;
+    protected bool $publish;
 
     public function __construct(string $type = Exception::GENERAL_UNKNOWN, string $message = null, int $code = null, \Throwable $previous = null)
     {
         $this->errors = Config::getParam('errors');
         $this->type = $type;
+        $this->code = $code ?? $this->errors[$type]['code'];
+        $this->message = $message ?? $this->errors[$type]['description'];
 
-        if (isset($this->errors[$type])) {
-            $this->code = $this->errors[$type]['code'];
-            $this->message = $this->errors[$type]['description'];
-            $this->publish = $this->errors[$type]['publish'] ?? true;
-        }
-
-        $this->message = $message ?? $this->message;
-        $this->code = $code ?? $this->code;
+        $this->publish = $this->errors[$type]['publish'] ?? ($this->code >= 500);
 
         parent::__construct($this->message, $this->code, $previous);
     }

src/Appwrite/Network/Validator/CNAME.php

@@ -6,6 +6,11 @@ use Utopia\Validator;
 
 class CNAME extends Validator
 {
+    /**
+     * @var mixed
+     */
+    protected mixed $logs;
+
     /**
      * @var string
      */
@@ -27,6 +32,14 @@ class CNAME extends Validator
         return 'Invalid CNAME record';
     }
 
+    /**
+     * @return mixed
+     */
+    public function getLogs(): mixed
+    {
+        return $this->logs;
+    }
+
     /**
      * Check if CNAME record target value matches selected target
      *
@@ -42,6 +55,7 @@ class CNAME extends Validator
 
         try {
             $records = \dns_get_record($domain, DNS_CNAME);
+            $this->logs = $records;
         } catch (\Throwable $th) {
             return false;
         }

src/Appwrite/Platform/Services/Tasks.php

@@ -11,16 +11,9 @@ use Appwrite\Platform\Tasks\Schedule;
 use Appwrite\Platform\Tasks\SDKs;
 use Appwrite\Platform\Tasks\Specs;
 use Appwrite\Platform\Tasks\SSL;
-use Appwrite\Platform\Tasks\Hamster;
 use Appwrite\Platform\Tasks\Vars;
 use Appwrite\Platform\Tasks\Version;
-use Appwrite\Platform\Tasks\VolumeSync;
-use Appwrite\Platform\Tasks\CalcTierStats;
 use Appwrite\Platform\Tasks\Upgrade;
-use Appwrite\Platform\Tasks\DeleteOrphanedProjects;
-use Appwrite\Platform\Tasks\GetMigrationStats;
-use Appwrite\Platform\Tasks\PatchRecreateRepositoriesDocuments;
-use Appwrite\Platform\Tasks\CreateInfMetric;
 
 class Tasks extends Service
 {
@@ -31,7 +24,6 @@ class Tasks extends Service
             ->addAction(Version::getName(), new Version())
             ->addAction(Vars::getName(), new Vars())
             ->addAction(SSL::getName(), new SSL())
-            ->addAction(Hamster::getName(), new Hamster())
             ->addAction(Doctor::getName(), new Doctor())
             ->addAction(Install::getName(), new Install())
             ->addAction(Upgrade::getName(), new Upgrade())
@@ -39,13 +31,7 @@ class Tasks extends Service
             ->addAction(Schedule::getName(), new Schedule())
             ->addAction(Migrate::getName(), new Migrate())
             ->addAction(SDKs::getName(), new SDKs())
-            ->addAction(VolumeSync::getName(), new VolumeSync())
             ->addAction(Specs::getName(), new Specs())
-            ->addAction(CalcTierStats::getName(), new CalcTierStats())
-            ->addAction(DeleteOrphanedProjects::getName(), new DeleteOrphanedProjects())
-            ->addAction(PatchRecreateRepositoriesDocuments::getName(), new PatchRecreateRepositoriesDocuments())
-            ->addAction(GetMigrationStats::getName(), new GetMigrationStats())
-            ->addAction(CreateInfMetric::getName(), new CreateInfMetric())
 
         ;
     }

src/Appwrite/Platform/Services/Workers.php

@@ -31,6 +31,7 @@ class Workers extends Service
             ->addAction(Functions::getName(), new Functions())
             ->addAction(Builds::getName(), new Builds())
             ->addAction(Deletes::getName(), new Deletes())
+            ->addAction(Hamster::getName(), new Hamster())
             ->addAction(UsageHook::getName(), new UsageHook())
             ->addAction(Usage::getName(), new Usage())
             ->addAction(Migrations::getName(), new Migrations())

src/Appwrite/Platform/Tasks/CalcTierStats.php

@@ -1,408 +0,0 @@
-<?php
-
-namespace Appwrite\Platform\Tasks;
-
-use Exception;
-use League\Csv\CannotInsertRecord;
-use Utopia\App;
-use Utopia\Database\Document;
-use Utopia\Database\Validator\Authorization;
-use Utopia\Platform\Action;
-use Utopia\Cache\Cache;
-use Utopia\CLI\Console;
-use Utopia\Database\Database;
-use Utopia\Database\Query;
-use League\Csv\Writer;
-use PHPMailer\PHPMailer\PHPMailer;
-use Utopia\Pools\Group;
-use Utopia\Registry\Registry;
-use Utopia\Validator\Text;
-
-class CalcTierStats extends Action
-{
-    /*
-     * Csv cols headers
-     */
-    private array $columns = [
-        'Project ID',
-        'Organization ID',
-        'Organization Email',
-        'Organization Members',
-        'Teams',
-        'Users',
-        'Requests',
-        'Bandwidth',
-        'Domains',
-        'Api keys',
-        'Webhooks',
-        'Platforms',
-        'Buckets',
-        'Files',
-        'Storage (bytes)',
-        'Max File Size (bytes)',
-        'Databases',
-        'Functions',
-        'Deployments',
-        'Executions',
-        'Migrations',
-    ];
-
-    protected string $directory = '/usr/local';
-    protected string $path;
-    protected string $date;
-
-    private array $usageStats = [
-        'network.requests'  => 'Requests',
-        'network.inbound' => 'Inbound',
-        'network.outbound' => 'Outbound',
-
-    ];
-
-    public static function getName(): string
-    {
-        return 'calc-tier-stats';
-    }
-
-    public function __construct()
-    {
-
-        $this
-            ->desc('Get stats for projects')
-            ->param('after', '', new Text(36), 'After cursor', true)
-            ->param('projectId', '', new Text(36), 'Select project to validate', true)
-            ->inject('pools')
-            ->inject('cache')
-            ->inject('dbForConsole')
-            ->inject('getProjectDB')
-            ->inject('register')
-            ->callback(function ($after, $projectId, Group $pools, Cache $cache, Database $dbForConsole, callable $getProjectDB, Registry $register) {
-                $this->action($after, $projectId, $pools, $cache, $dbForConsole, $getProjectDB, $register);
-            });
-    }
-
-
-    public function action(string $after, string $projectId, Group $pools, Cache $cache, Database $dbForConsole, callable $getProjectDB, Registry $register): void
-    {
-        //docker compose exec -t appwrite calc-tier-stats
-
-        Console::title('Cloud free tier  stats calculation V1');
-        Console::success(APP_NAME . ' cloud free tier  stats calculation has started');
-
-        /** CSV stuff */
-        $this->date = date('Y-m-d');
-        $this->path = "{$this->directory}/tier_stats_{$this->date}.csv";
-        $csv = Writer::createFromPath($this->path, 'w');
-        $csv->insertOne($this->columns);
-
-        if (!empty($projectId)) {
-            try {
-                console::log("Project " . $projectId);
-                $project = $dbForConsole->getDocument('projects', $projectId);
-                $dbForProject = call_user_func($getProjectDB, $project);
-                $data = $this->getData($project, $dbForConsole, $dbForProject);
-                $csv->insertOne($data);
-                $this->sendMail($register);
-
-                return;
-            } catch (\Throwable $th) {
-                Console::error("Unexpected error occured with Project ID {$projectId}");
-                Console::error('[Error] Type: ' . get_class($th));
-                Console::error('[Error] Message: ' . $th->getMessage());
-                Console::error('[Error] File: ' . $th->getFile());
-                Console::error('[Error] Line: ' . $th->getLine());
-            }
-        }
-            $queries = [];
-        if (!empty($after)) {
-            Console::info("Iterating remaining projects after project with ID {$after}");
-            $project = $dbForConsole->getDocument('projects', $after);
-            $queries = [Query::cursorAfter($project)];
-        } else {
-            Console::info("Iterating all projects");
-        }
-            $this->foreachDocument($dbForConsole, 'projects', $queries, function (Document $project) use ($getProjectDB, $dbForConsole, $csv) {
-                $projectId = $project->getId();
-                console::log("Project " . $projectId);
-                try {
-                    $dbForProject = call_user_func($getProjectDB, $project);
-                    $data = $this->getData($project, $dbForConsole, $dbForProject);
-                    $csv->insertOne($data);
-                } catch (\Throwable $th) {
-                    Console::error("Unexpected error occured with Project ID {$projectId}");
-                    Console::error('[Error] Type: ' . get_class($th));
-                    Console::error('[Error] Message: ' . $th->getMessage());
-                    Console::error('[Error] File: ' . $th->getFile());
-                    Console::error('[Error] Line: ' . $th->getLine());
-                }
-            });
-
-        $this->sendMail($register);
-    }
-
-    private function foreachDocument(Database $database, string $collection, array $queries = [], callable $callback = null): void
-    {
-        $limit = 1000;
-        $results = [];
-        $sum = $limit;
-        $latestDocument = null;
-
-        while ($sum === $limit) {
-            $newQueries = $queries;
-
-            if ($latestDocument != null) {
-                array_unshift($newQueries, Query::cursorAfter($latestDocument));
-            }
-            $newQueries[] = Query::limit($limit);
-            $results = $database->find('projects', $newQueries);
-
-            if (empty($results)) {
-                return;
-            }
-
-            $sum = count($results);
-
-            foreach ($results as $document) {
-                if (is_callable($callback)) {
-                    $callback($document);
-                }
-            }
-            $latestDocument = $results[array_key_last($results)];
-        }
-    }
-
-    private function sendMail(Registry $register): void
-    {
-        /** @var PHPMailer $mail */
-        $mail = $register->get('smtp');
-        $mail->clearAddresses();
-        $mail->clearAllRecipients();
-        $mail->clearReplyTos();
-        $mail->clearAttachments();
-        $mail->clearBCCs();
-        $mail->clearCCs();
-
-        try {
-            /** Addresses */
-            $mail->setFrom(App::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM), 'Appwrite Cloud Hamster');
-            $recipients = explode(',', App::getEnv('_APP_USERS_STATS_RECIPIENTS', ''));
-            foreach ($recipients as $recipient) {
-                $mail->addAddress($recipient);
-            }
-
-            /** Attachments */
-            $mail->addAttachment($this->path);
-
-            /** Content */
-            $mail->Subject = "Cloud Report for {$this->date}";
-            $mail->Body = "Please find the daily cloud report atttached";
-            $mail->send();
-            Console::success('Email has been sent!');
-        } catch (Exception $e) {
-            Console::error("Message could not be sent. Mailer Error: {$mail->ErrorInfo}");
-        }
-    }
-
-
-    private function getData(Document $project, Database $dbForConsole, Database $dbForProject): array
-    {
-
-            $stats['Project ID'] = $project->getId();
-            $stats['Organization ID']   = $project->getAttribute('teamId', null);
-
-            $teamInternalId = $project->getAttribute('teamInternalId', 0);
-
-        if ($teamInternalId) {
-            $membership = $dbForConsole->findOne('memberships', [
-                Query::equal('teamInternalId', [$teamInternalId]),
-            ]);
-
-            if (!$membership || $membership->isEmpty()) {
-                Console::error('Membership not found. Skipping project : ' . $project->getId());
-            }
-
-            $userId = $membership->getAttribute('userId', null);
-            if ($userId) {
-                $user = $dbForConsole->getDocument('users', $userId);
-                $stats['Organization Email'] = $user->getAttribute('email', null);
-            }
-        } else {
-            Console::error("Email was not found for this Organization ID :{$teamInternalId}");
-        }
-
-            /** Get Total Members */
-            $teamInternalId = $project->getAttribute('teamInternalId', null);
-        if ($teamInternalId) {
-            $stats['Organization Members'] = $dbForConsole->count('memberships', [
-                Query::equal('$internalId', [(string)$teamInternalId])
-            ]);
-        } else {
-            $stats['Organization Members'] = 0;
-        }
-
-            /** Get Total internal Teams */
-        try {
-            $stats['Teams'] = $dbForProject->count('teams', []);
-        } catch (\Throwable) {
-            $stats['Teams'] = 0;
-        }
-
-            /** Get Total users */
-        try {
-            $stats['Users'] = $dbForProject->count('users', []);
-        } catch (\Throwable) {
-            $stats['Users'] = 0;
-        }
-
-            /** Get Usage stats */
-            $range = '30d';
-            $periods = [
-                '30d' => [
-                    'period' => '1d',
-                    'limit' => 30,
-                ]
-            ];
-
-            $tmp = [];
-            $metrics = $this->usageStats;
-            Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$tmp) {
-                foreach ($metrics as $metric => $name) {
-                    $limit = $periods[$range]['limit'];
-                    $period = $periods[$range]['period'];
-
-                    $requestDocs = $dbForProject->find('stats_v2', [
-                        Query::equal('metric', [$metric]),
-                        Query::equal('period', [$period]),
-                        Query::limit($limit),
-                        Query::orderDesc('time'),
-                    ]);
-
-                    $tmp[$metric] = [];
-                    foreach ($requestDocs as $requestDoc) {
-                        if (empty($requestDoc)) {
-                            continue;
-                        }
-
-                        $tmp[$metric][] = [
-                            'value' => $requestDoc->getAttribute('value'),
-                            'date' => $requestDoc->getAttribute('time'),
-                        ];
-                    }
-
-                    $tmp[$metric] = array_reverse($tmp[$metric]);
-                    $tmp[$metric] = array_sum(array_column($tmp[$metric], 'value'));
-                }
-            });
-
-        foreach ($tmp as $key => $value) {
-            $stats[$metrics[$key]]  = $value;
-        }
-
-            /**
-             * Workaround to combine network.inbound+network.outbound as network.
-             */
-            $stats['Network'] = ($stats['Inbound'] ?? 0) + ($stats['Outbound'] ?? 0);
-            unset($stats['Inbound']);
-            unset($stats['Outbound']);
-
-        try {
-            /** Get Domains */
-            $stats['Domains'] = $dbForConsole->count('rules', [
-                Query::equal('projectInternalId', [$project->getInternalId()]),
-            ]);
-        } catch (\Throwable) {
-            $stats['Domains'] = 0;
-        }
-
-        try {
-            /** Get Api keys */
-            $stats['Api keys'] = $dbForConsole->count('keys', [
-                Query::equal('projectInternalId', [$project->getInternalId()]),
-            ]);
-        } catch (\Throwable) {
-            $stats['Api keys'] = 0;
-        }
-
-        try {
-            /** Get Webhooks */
-            $stats['Webhooks'] = $dbForConsole->count('webhooks', [
-                Query::equal('projectInternalId', [$project->getInternalId()]),
-            ]);
-        } catch (\Throwable) {
-            $stats['Webhooks'] = 0;
-        }
-
-        try {
-            /** Get Platforms */
-            $stats['Platforms'] = $dbForConsole->count('platforms', [
-                Query::equal('projectInternalId', [$project->getInternalId()]),
-            ]);
-        } catch (\Throwable) {
-            $stats['Platforms'] = 0;
-        }
-
-            /** Get Files & Buckets */
-            $filesCount = 0;
-            $filesSum = 0;
-            $maxFileSize = 0;
-            $counter = 0;
-        try {
-            $buckets = $dbForProject->find('buckets', []);
-            foreach ($buckets as $bucket) {
-                $file = $dbForProject->findOne('bucket_' . $bucket->getInternalId(), [Query::orderDesc('sizeOriginal'),]);
-                if (empty($file)) {
-                    continue;
-                }
-                $filesSum   += $dbForProject->sum('bucket_' . $bucket->getInternalId(), 'sizeOriginal', []);
-                $filesCount += $dbForProject->count('bucket_' . $bucket->getInternalId(), []);
-                if ($file->getAttribute('sizeOriginal') > $maxFileSize) {
-                    $maxFileSize = $file->getAttribute('sizeOriginal');
-                }
-                $counter++;
-            }
-        } catch (\Throwable $t) {
-            Console::error("Error while counting buckets: {$project->getId()}");
-        }
-            $stats['Buckets'] = $counter;
-            $stats['Files'] = $filesCount;
-            $stats['Storage (bytes)'] = $filesSum;
-            $stats['Max File Size (bytes)'] = $maxFileSize;
-
-
-        try {
-            /** Get Total Functions */
-            $stats['Databases'] = $dbForProject->count('databases', []);
-        } catch (\Throwable) {
-            $stats['Databases'] = 0;
-        }
-
-            /** Get Total Functions */
-        try {
-            $stats['Functions'] = $dbForProject->count('functions', []);
-        } catch (\Throwable) {
-            $stats['Functions'] = 0;
-        }
-
-            /** Get Total Deployments */
-        try {
-            $stats['Deployments'] = $dbForProject->count('deployments', []);
-        } catch (\Throwable) {
-            $stats['Deployments'] = 0;
-        }
-
-            /** Get Total Executions */
-        try {
-            $stats['Executions'] = $dbForProject->count('executions', []);
-        } catch (\Throwable) {
-            $stats['Executions'] = 0;
-        }
-
-            /** Get Total Migrations */
-        try {
-            $stats['Migrations'] = $dbForProject->count('migrations', []);
-        } catch (\Throwable) {
-            $stats['Migrations'] = 0;
-        }
-
-        return array_values($stats);
-    }
-}

src/Appwrite/Platform/Tasks/CreateInfMetric.php

@@ -1,314 +0,0 @@
-<?php
-
-namespace Appwrite\Platform\Tasks;
-
-use Utopia\App;
-use Utopia\Database\Document;
-use Utopia\Database\Exception;
-use Utopia\Database\Exception\Duplicate;
-use Utopia\Database\Query;
-use Utopia\Platform\Action;
-use Utopia\Cache\Cache;
-use Utopia\CLI\Console;
-use Utopia\Database\Database;
-use Utopia\Pools\Group;
-use Utopia\Registry\Registry;
-
-class CreateInfMetric extends Action
-{
-    public static function getName(): string
-    {
-        return 'create-inf-metric';
-    }
-
-    public function __construct()
-    {
-
-        $this
-            ->desc('Create infinity stats metric')
-            ->inject('pools')
-            ->inject('cache')
-            ->inject('dbForConsole')
-            ->inject('register')
-            ->callback(function (Group $pools, Cache $cache, Database $dbForConsole, Registry $register) {
-                $this->action($pools, $cache, $dbForConsole, $register);
-            });
-    }
-
-
-    public function action(Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void
-    {
-
-        Console::title('Create infinity metric V1');
-        Console::success(APP_NAME . ' Create infinity metric started');
-
-
-        /* Initialise new Utopia app */
-        $app = new App('UTC');
-        $console = $app->getResource('console');
-        $projects = [$console];
-
-        /** Database connections */
-        $totalProjects = $dbForConsole->count('projects');
-        Console::success("Found a total of: {$totalProjects} projects");
-
-        $count = 0;
-        $limit = 30;
-        $sum = 30;
-        $offset = 0;
-        while (!empty($projects)) {
-            foreach ($projects as $project) {
-
-                /**
-                 * Skip user projects with id 'console'
-                 */
-                if ($project->getId() === 'console') {
-                    continue;
-                }
-
-                try {
-                    $db = $project->getAttribute('database');
-                    $adapter = $pools
-                        ->get($db)
-                        ->pop()
-                        ->getResource();
-
-                    $dbForProject = new Database($adapter, $cache);
-                    $dbForProject->setDefaultDatabase('appwrite');
-                    $dbForProject->setNamespace('_' . $project->getInternalId());
-
-                    $this->network($dbForProject);
-                    $this->sessions($dbForProject);
-                    $this->users($dbForProject);
-                    $this->teams($dbForProject);
-                    $this->databases($dbForProject);
-                    $this->functions($dbForProject);
-                    $this->storage($dbForProject);
-                } catch (\Throwable $th) {
-                    var_dump($th->getMessage());
-                } finally {
-                    $pools
-                        ->get($db)
-                        ->reclaim();
-                }
-
-                Console::log('Finished project ' . $project->getId() . ' ' . $project->getInternalId());
-            }
-
-            $sum = \count($projects);
-
-            $projects = $dbForConsole->find('projects', [
-                Query::limit($limit),
-                Query::offset($offset),
-            ]);
-
-            $offset = $offset + $limit;
-            $count = $count + $sum;
-        }
-
-        Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects');
-    }
-
-    /**
-     * @param string $metric
-     * @param int|float $value
-     * @return void
-     */
-    private function createInfMetric(database $dbForProject, string $metric, int|float $value): void
-    {
-
-        try {
-            $id = \md5("_inf_{$metric}");
-
-            $dbForProject->deleteDocument('stats_v2', $id);
-
-            $dbForProject->createDocument('stats_v2', new Document([
-                '$id' => $id,
-                'metric' => $metric,
-                'period' => 'inf',
-                'value'  => (int)$value,
-                'time'   => null,
-                'region' => 'default',
-            ]));
-        } catch (Duplicate $th) {
-            console::log("Error while creating inf metric: duplicate id {$metric}  {$id}");
-        }
-    }
-
-    /**
-     * @param Database $dbForProject
-     * @param string $metric
-     * @return int|float
-     * @throws Exception
-     */
-    protected function getFromMetric(database $dbForProject, string $metric): int|float
-    {
-
-        return  $dbForProject->sum('stats_v2', 'value', [
-            Query::equal('metric', [
-                $metric,
-            ]),
-            Query::equal('period', ['1d']),
-        ]);
-    }
-
-    /**
-     * @throws Exception
-     */
-    private function network(database $dbForProject)
-    {
-        $this->createInfMetric($dbForProject, 'network.inbound', $this->getFromMetric($dbForProject, 'network.inbound'));
-        $this->createInfMetric($dbForProject, 'network.outbound', $this->getFromMetric($dbForProject, 'network.outbound'));
-        $this->createInfMetric($dbForProject, 'network.requests', $this->getFromMetric($dbForProject, 'network.requests'));
-    }
-
-
-    private function storage(database $dbForProject)
-    {
-        $bucketsCount = 0;
-        $filesCount = 0;
-        $filesStorageSum = 0;
-
-        $buckets = $dbForProject->find('buckets');
-        foreach ($buckets as $bucket) {
-            $files = $dbForProject->count('bucket_' . $bucket->getInternalId());
-            $this->createInfMetric($dbForProject, $bucket->getInternalId() . '.files', $files);
-
-            $filesStorage = $dbForProject->sum('bucket_' . $bucket->getInternalId(), 'sizeOriginal');
-            $this->createInfMetric($dbForProject, $bucket->getInternalId() . '.files.storage', $filesStorage);
-
-            $bucketsCount++;
-            $filesCount += $files;
-            $filesStorageSum += $filesStorage;
-        }
-
-        $this->createInfMetric($dbForProject, 'buckets', $bucketsCount);
-        $this->createInfMetric($dbForProject, 'files', $filesCount);
-        $this->createInfMetric($dbForProject, 'files.storage', $filesStorageSum);
-    }
-
-
-    private function functions(Database $dbForProject)
-    {
-        $functionsCount = 0;
-        $deploymentsCount = 0;
-        $buildsCount = 0;
-        $buildsStorageSum = 0;
-        $buildsComputeSum = 0;
-        $executionsCount = 0;
-        $executionsComputeSum = 0;
-        $deploymentsStorageSum = 0;
-
-        //functions
-        $functions = $dbForProject->find('functions');
-        foreach ($functions as $function) {
-            //deployments
-            $deployments = $dbForProject->find('deployments', [
-                Query::equal('resourceType', ['functions']),
-                Query::equal('resourceInternalId', [$function->getInternalId()]),
-            ]);
-
-            $deploymentCount = 0;
-            $deploymentStorageSum = 0;
-            foreach ($deployments as $deployment) {
-                //builds
-                $builds = $dbForProject->count('builds', [
-                    Query::equal('deploymentInternalId', [$deployment->getInternalId()]),
-                ]);
-
-                $buildsCompute = $dbForProject->sum('builds', 'duration', [
-                    Query::equal('deploymentInternalId', [$deployment->getInternalId()]),
-                ]);
-
-                $buildsStorage = $dbForProject->sum('builds', 'size', [
-                    Query::equal('deploymentInternalId', [$deployment->getInternalId()]),
-                ]);
-
-                $this->createInfMetric($dbForProject, $function->getInternalId() . '.builds', $builds);
-                $this->createInfMetric($dbForProject, $function->getInternalId() . '.builds.storage', $buildsCompute * 1000);
-                $this->createInfMetric($dbForProject, $function->getInternalId() . '.builds.compute', $buildsStorage);
-
-                $buildsCount += $builds;
-                $buildsComputeSum += $buildsCompute;
-                $buildsStorageSum += $buildsStorage;
-
-
-                $deploymentCount++;
-                $deploymentsCount++;
-                $deploymentsStorageSum += $deployment['size'];
-                $deploymentStorageSum += $deployment['size'];
-            }
-            $this->createInfMetric($dbForProject, 'functions.' . $function->getInternalId() . '.deployments', $deploymentCount);
-            $this->createInfMetric($dbForProject, 'functions.' . $function->getInternalId() . '.deployments.storage', $deploymentStorageSum);
-
-            //executions
-            $executions = $dbForProject->count('executions', [
-                Query::equal('functionInternalId', [$function->getInternalId()]),
-            ]);
-
-            $executionsCompute = $dbForProject->sum('executions', 'duration', [
-                Query::equal('functionInternalId', [$function->getInternalId()]),
-            ]);
-
-            $this->createInfMetric($dbForProject, $function->getInternalId() . '.executions', $executions);
-            $this->createInfMetric($dbForProject, $function->getInternalId() . '.executions.compute', $executionsCompute * 1000);
-            $executionsCount += $executions;
-            $executionsComputeSum += $executionsCompute;
-
-            $functionsCount++;
-        }
-
-        $this->createInfMetric($dbForProject, 'functions', $functionsCount);
-        $this->createInfMetric($dbForProject, 'deployments', $deploymentsCount);
-        $this->createInfMetric($dbForProject, 'deployments.storage', $deploymentsStorageSum);
-        $this->createInfMetric($dbForProject, 'builds', $buildsCount);
-        $this->createInfMetric($dbForProject, 'builds.compute', $buildsComputeSum * 1000);
-        $this->createInfMetric($dbForProject, 'builds.storage', $buildsStorageSum);
-        $this->createInfMetric($dbForProject, 'executions', $executionsCount);
-        $this->createInfMetric($dbForProject, 'executions.compute', $executionsComputeSum * 1000);
-    }
-
-    private function databases(Database $dbForProject)
-    {
-        $databasesCount = 0;
-        $collectionsCount = 0;
-        $documentsCount = 0;
-        $databases = $dbForProject->find('databases');
-        foreach ($databases as $database) {
-            $collectionCount = 0;
-            $collections = $dbForProject->find('database_' . $database->getInternalId());
-            foreach ($collections as $collection) {
-                $documents = $dbForProject->count('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId());
-                $this->createInfMetric($dbForProject, $database->getInternalId() . '.' . $collection->getInternalId() . '.documents', $documents);
-                $documentsCount += $documents;
-                $collectionCount++;
-                $collectionsCount++;
-            }
-            $this->createInfMetric($dbForProject, $database->getInternalId() . '.collections', $collectionCount);
-            $this->createInfMetric($dbForProject, $database->getInternalId() . '.documents', $documentsCount);
-            $databasesCount++;
-        }
-        $this->createInfMetric($dbForProject, 'collections', $collectionsCount);
-        $this->createInfMetric($dbForProject, 'databases', $databasesCount);
-        $this->createInfMetric($dbForProject, 'documents', $documentsCount);
-    }
-
-
-    private function users(Database $dbForProject)
-    {
-        $users = $dbForProject->count('users');
-        $this->createInfMetric($dbForProject, 'users', $users);
-    }
-
-    private function sessions(Database $dbForProject)
-    {
-        $users = $dbForProject->count('sessions');
-        $this->createInfMetric($dbForProject, 'sessions', $users);
-    }
-
-    private function teams(Database $dbForProject)
-    {
-        $teams = $dbForProject->count('teams');
-        $this->createInfMetric($dbForProject, 'teams', $teams);
-    }
-}

src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php

@@ -1,161 +0,0 @@
-<?php
-
-namespace Appwrite\Platform\Tasks;
-
-use Utopia\App;
-use Utopia\Config\Config;
-use Utopia\Database\Helpers\ID;
-use Utopia\Database\Query;
-use Utopia\Platform\Action;
-use Utopia\Cache\Cache;
-use Utopia\CLI\Console;
-use Utopia\Database\Database;
-use Utopia\Pools\Group;
-use Utopia\Registry\Registry;
-use Utopia\Validator\Boolean;
-
-class DeleteOrphanedProjects extends Action
-{
-    public static function getName(): string
-    {
-        return 'delete-orphaned-projects';
-    }
-
-    public function __construct()
-    {
-
-        $this
-            ->desc('Delete orphaned projects')
-            ->param('commit', false, new Boolean(true), 'Commit  project deletion', true)
-            ->inject('pools')
-            ->inject('cache')
-            ->inject('dbForConsole')
-            ->inject('register')
-            ->callback(function (bool $commit, Group $pools, Cache $cache, Database $dbForConsole, Registry $register) {
-                $this->action($commit, $pools, $cache, $dbForConsole, $register);
-            });
-    }
-
-
-    public function action(bool $commit, Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void
-    {
-
-        Console::title('Delete orphaned projects V1');
-        Console::success(APP_NAME . ' Delete orphaned projects started');
-
-        /** @var array $collections */
-        $collectionsConfig = Config::getParam('collections', [])['projects'] ?? [];
-
-        $collectionsConfig = array_merge([
-            'audit' => [
-                '$id' => ID::custom('audit'),
-                '$collection' => Database::METADATA
-            ],
-            'abuse' => [
-                '$id' => ID::custom('abuse'),
-                '$collection' => Database::METADATA
-            ]
-        ], $collectionsConfig);
-
-        /* Initialise new Utopia app */
-        $app = new App('UTC');
-        $console = $app->getResource('console');
-        $projects = [$console];
-
-        /** Database connections */
-        $totalProjects = $dbForConsole->count('projects');
-        Console::success("Found a total of: {$totalProjects} projects");
-
-        $orphans = 1;
-        $cnt = 0;
-        $count = 0;
-        $limit = 30;
-        $sum = 30;
-        $offset = 0;
-        while (!empty($projects)) {
-            foreach ($projects as $project) {
-
-                /**
-                 * Skip user projects with id 'console'
-                 */
-                if ($project->getId() === 'console') {
-                    continue;
-                }
-
-                try {
-                    $db = $project->getAttribute('database');
-                    $adapter = $pools
-                        ->get($db)
-                        ->pop()
-                        ->getResource();
-
-                    $dbForProject = new Database($adapter, $cache);
-                    $dbForProject->setDefaultDatabase('appwrite');
-                    $dbForProject->setNamespace('_' . $project->getInternalId());
-
-                    $collectionsCreated = 0;
-                    $cnt++;
-                    if ($dbForProject->exists($dbForProject->getDefaultDatabase(), Database::METADATA)) {
-                        $collectionsCreated = $dbForProject->count(Database::METADATA);
-                    }
-
-                    $msg = '(' . $cnt . ') found (' . $collectionsCreated . ') collections on project  (' . $project->getInternalId() . ') , database (' . $project['database'] . ')';
-
-                    if ($collectionsCreated >= count($collectionsConfig)) {
-                        Console::log($msg . ' ignoring....');
-                        continue;
-                    }
-
-                    Console::log($msg);
-
-                    if ($collectionsCreated > 0) {
-                        $collections = $dbForProject->find(Database::METADATA, []);
-                        foreach ($collections as $collection) {
-                            if ($commit) {
-                                $dbForProject->deleteCollection($collection->getId());
-                                $dbForConsole->deleteCachedCollection($collection->getId());
-                            }
-                            Console::info('--Deleting collection  (' . $collection->getId() . ') project no (' . $project->getInternalId() . ')');
-                        }
-                    }
-
-                    if ($commit) {
-                        $dbForConsole->deleteDocument('projects', $project->getId());
-                        $dbForConsole->deleteCachedDocument('projects', $project->getId());
-
-                        if ($dbForProject->exists($dbForProject->getDefaultDatabase(), Database::METADATA)) {
-                            try {
-                                $dbForProject->deleteCollection(Database::METADATA);
-                                $dbForProject->deleteCachedCollection(Database::METADATA);
-                            } catch (\Throwable $th) {
-                                Console::warning('Metadata collection does not exist');
-                            }
-                        }
-                    }
-
-                    Console::info('--Deleting project no (' . $project->getInternalId() . ')');
-
-                    $orphans++;
-                } catch (\Throwable $th) {
-                    Console::error('Error: ' . $th->getMessage() . ' ' . $th->getTraceAsString());
-                } finally {
-                    $pools
-                        ->get($db)
-                        ->reclaim();
-                }
-            }
-
-            $sum = \count($projects);
-
-            $projects = $dbForConsole->find('projects', [
-                Query::limit($limit),
-                Query::offset($offset),
-            ]);
-
-            $offset = $offset + $limit;
-            $count = $count + $sum;
-        }
-
-        Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects found ' . $orphans - 1 . ' orphans');
-    }
-}

src/Appwrite/Platform/Tasks/GetMigrationStats.php

@@ -1,187 +0,0 @@
-<?php
-
-namespace Appwrite\Platform\Tasks;
-
-use Exception;
-use League\Csv\CannotInsertRecord;
-use Utopia\App;
-use Utopia\Platform\Action;
-use Utopia\Cache\Cache;
-use Utopia\CLI\Console;
-use Utopia\Database\Database;
-use Utopia\Database\Query;
-use League\Csv\Writer;
-use PHPMailer\PHPMailer\PHPMailer;
-use Utopia\Pools\Group;
-use Utopia\Registry\Registry;
-
-class GetMigrationStats extends Action
-{
-    /*
-     * Csv cols headers
-     */
-    private array $columns = [
-        'Project ID',
-        '$id',
-        '$createdAt',
-        'status',
-        'stage',
-        'source'
-    ];
-
-    protected string $directory = '/usr/local';
-    protected string $path;
-    protected string $date;
-
-    public static function getName(): string
-    {
-        return 'get-migration-stats';
-    }
-
-    public function __construct()
-    {
-
-        $this
-            ->desc('Get stats for projects')
-            ->inject('pools')
-            ->inject('cache')
-            ->inject('dbForConsole')
-            ->inject('register')
-            ->callback(function (Group $pools, Cache $cache, Database $dbForConsole, Registry $register) {
-                $this->action($pools, $cache, $dbForConsole, $register);
-            });
-    }
-
-    /**
-     * @throws \Utopia\Exception
-     * @throws CannotInsertRecord
-     */
-    public function action(Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void
-    {
-        //docker compose exec -t appwrite get-migration-stats
-
-        Console::title('Migration stats calculation V1');
-        Console::success(APP_NAME . ' Migration stats calculation has started');
-
-        /* Initialise new Utopia app */
-        $app = new App('UTC');
-        $console = $app->getResource('console');
-
-        /** CSV stuff */
-        $this->date = date('Y-m-d');
-        $this->path = "{$this->directory}/migration_stats_{$this->date}.csv";
-        $csv = Writer::createFromPath($this->path, 'w');
-        $csv->insertOne($this->columns);
-
-        /** Database connections */
-        $totalProjects = $dbForConsole->count('projects');
-        Console::success("Found a total of: {$totalProjects} projects");
-
-        $projects = [$console];
-        $count = 0;
-        $limit = 100;
-        $sum = 100;
-        $offset = 0;
-        while (!empty($projects)) {
-            foreach ($projects as $project) {
-
-                /**
-                 * Skip user projects with id 'console'
-                 */
-                if ($project->getId() === 'console') {
-                    continue;
-                }
-
-                Console::info("Getting stats for {$project->getId()}");
-
-                try {
-                    $db = $project->getAttribute('database');
-                    $adapter = $pools
-                        ->get($db)
-                        ->pop()
-                        ->getResource();
-
-                    $dbForProject = new Database($adapter, $cache);
-                    $dbForProject->setDefaultDatabase('appwrite');
-                    $dbForProject->setNamespace('_' . $project->getInternalId());
-
-                    /** Get Project ID */
-                    $stats['Project ID'] = $project->getId();
-
-                    /** Get Migration details */
-                    $migrations = $dbForProject->find('migrations', [
-                        Query::limit(500)
-                    ]);
-
-                    $migrations = array_map(function ($migration) use ($project) {
-                        return [
-                            $project->getId(),
-                            $migration->getAttribute('$id'),
-                            $migration->getAttribute('$createdAt'),
-                            $migration->getAttribute('status'),
-                            $migration->getAttribute('stage'),
-                            $migration->getAttribute('source'),
-                        ];
-                    }, $migrations);
-
-                    if (!empty($migrations)) {
-                        $csv->insertAll($migrations);
-                    }
-                } catch (\Throwable $th) {
-                    Console::error('Failed on project ("' . $project->getId() . '") with error on File: ' . $th->getFile() . '  line no: ' . $th->getline() . ' with message: ' . $th->getMessage());
-                } finally {
-                    $pools
-                        ->get($db)
-                        ->reclaim();
-                }
-            }
-
-            $sum = \count($projects);
-
-            $projects = $dbForConsole->find('projects', [
-                Query::limit($limit),
-                Query::offset($offset),
-            ]);
-
-            $offset = $offset + $limit;
-            $count = $count + $sum;
-        }
-
-        Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects...');
-
-        $pools
-            ->get('console')
-            ->reclaim();
-
-        /** @var PHPMailer $mail */
-        $mail = $register->get('smtp');
-
-        $mail->clearAddresses();
-        $mail->clearAllRecipients();
-        $mail->clearReplyTos();
-        $mail->clearAttachments();
-        $mail->clearBCCs();
-        $mail->clearCCs();
-
-        try {
-            /** Addresses */
-            $mail->setFrom(App::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM), 'Appwrite Cloud Hamster');
-            $recipients = explode(',', App::getEnv('_APP_USERS_STATS_RECIPIENTS', ''));
-
-            foreach ($recipients as $recipient) {
-                $mail->addAddress($recipient);
-            }
-
-            /** Attachments */
-            $mail->addAttachment($this->path);
-
-            /** Content */
-            $mail->Subject = "Migration Report for {$this->date}";
-            $mail->Body = "Please find the migration report atttached";
-            $mail->send();
-            Console::success('Email has been sent!');
-        } catch (Exception $e) {
-            Console::error("Message could not be sent. Mailer Error: {$mail->ErrorInfo}");
-        }
-    }
-}

src/Appwrite/Platform/Tasks/Hamster.php

@@ -1,488 +0,0 @@
-<?php
-
-namespace Appwrite\Platform\Tasks;
-
-use Appwrite\Network\Validator\Origin;
-use Exception;
-use Utopia\App;
-use Utopia\Platform\Action;
-use Utopia\Cache\Cache;
-use Utopia\CLI\Console;
-use Utopia\Database\Database;
-use Utopia\Database\Query;
-use Utopia\Database\Validator\Authorization;
-use Utopia\Analytics\Adapter\Mixpanel;
-use Utopia\Analytics\Event;
-use Utopia\Config\Config;
-use Utopia\Database\Document;
-use Utopia\Pools\Group;
-
-class Hamster extends Action
-{
-    private array $metrics = [
-        'usage_files' => 'files',
-        'usage_buckets' => 'buckets',
-        'usage_databases' => 'databases',
-        'usage_documents' => 'documents',
-        'usage_collections' => 'collections',
-        'usage_storage' => 'files.storage',
-        'usage_requests' => 'network.requests',
-        'usage_inbound' => 'network.inbound',
-        'usage_outbound' => 'network.outbound',
-        'usage_users' => 'users',
-        'usage_sessions' => 'sessions',
-        'usage_executions' => 'executions',
-    ];
-
-    protected string $directory = '/usr/local';
-
-    protected string $path;
-
-    protected string $date;
-
-    protected Mixpanel $mixpanel;
-
-    public static function getName(): string
-    {
-        return 'hamster';
-    }
-
-    public function __construct()
-    {
-        $this->mixpanel = new Mixpanel(App::getEnv('_APP_MIXPANEL_TOKEN', ''));
-
-        $this
-            ->desc('Get stats for projects')
-            ->inject('pools')
-            ->inject('cache')
-            ->inject('dbForConsole')
-            ->callback(function (Group $pools, Cache $cache, Database $dbForConsole) {
-                $this->action($pools, $cache, $dbForConsole);
-            });
-    }
-
-    private function getStatsPerProject(Group $pools, Cache $cache, Database $dbForConsole)
-    {
-        $this->calculateByGroup('projects', $dbForConsole, function (Database $dbForConsole, Document $project) use ($pools, $cache) {
-            /**
-             * Skip user projects with id 'console'
-             */
-            if ($project->getId() === 'console') {
-                Console::info("Skipping project console");
-                return;
-            }
-
-            Console::log("Getting stats for {$project->getId()}");
-
-            try {
-                $db = $project->getAttribute('database');
-                $adapter = $pools
-                    ->get($db)
-                    ->pop()
-                    ->getResource();
-
-                $dbForProject = new Database($adapter, $cache);
-                $dbForProject->setDefaultDatabase('appwrite');
-                $dbForProject->setNamespace('_' . $project->getInternalId());
-
-                $statsPerProject = [];
-
-                $statsPerProject['time'] = microtime(true);
-
-                /** Get Project ID */
-                $statsPerProject['project_id'] = $project->getId();
-
-                /** Get project created time */
-                $statsPerProject['project_created'] = $project->getAttribute('$createdAt');
-
-                /** Get Project Name */
-                $statsPerProject['project_name'] = $project->getAttribute('name');
-
-                /** Total Project Variables */
-                $statsPerProject['custom_variables'] = $dbForProject->count('variables', [], APP_LIMIT_COUNT);
-
-                /** Total Migrations */
-                $statsPerProject['custom_migrations'] = $dbForProject->count('migrations', [], APP_LIMIT_COUNT);
-
-                /** Get Custom SMTP */
-                $smtp = $project->getAttribute('smtp', null);
-                if ($smtp) {
-                    $statsPerProject['custom_smtp_status'] = $smtp['enabled'] === true ? 'enabled' : 'disabled';
-
-                    /** Get Custom Templates Count */
-                    $templates = array_keys($project->getAttribute('templates', []));
-                    $statsPerProject['custom_email_templates'] = array_filter($templates, function ($template) {
-                        return str_contains($template, 'email');
-                    });
-                    $statsPerProject['custom_sms_templates'] = array_filter($templates, function ($template) {
-                        return str_contains($template, 'sms');
-                    });
-                }
-
-                /** Get total relationship attributes */
-                $statsPerProject['custom_relationship_attributes'] = $dbForProject->count('attributes', [
-                    Query::equal('type', ['relationship'])
-                ], APP_LIMIT_COUNT);
-
-                /** Get Total Functions */
-                $statsPerProject['custom_functions'] = $dbForProject->count('functions', [], APP_LIMIT_COUNT);
-
-                foreach (\array_keys(Config::getParam('runtimes')) as $runtime) {
-                    $statsPerProject['custom_functions_' . $runtime] = $dbForProject->count('functions', [
-                        Query::equal('runtime', [$runtime]),
-                    ], APP_LIMIT_COUNT);
-                }
-
-                /** Get Total Deployments */
-                $statsPerProject['custom_deployments'] = $dbForProject->count('deployments', [], APP_LIMIT_COUNT);
-                $statsPerProject['custom_deployments_manual'] = $dbForProject->count('deployments', [
-                    Query::equal('type', ['manual'])
-                ], APP_LIMIT_COUNT);
-                $statsPerProject['custom_deployments_git'] = $dbForProject->count('deployments', [
-                    Query::equal('type', ['vcs'])
-                ], APP_LIMIT_COUNT);
-
-                /** Get VCS repos connected */
-                $statsPerProject['custom_vcs_repositories'] = $dbForConsole->count('repositories', [
-                    Query::equal('projectInternalId', [$project->getInternalId()])
-                ], APP_LIMIT_COUNT);
-
-                /** Get Total Teams */
-                $statsPerProject['custom_teams'] = $dbForProject->count('teams', [], APP_LIMIT_COUNT);
-
-                /** Get Total Migrations */
-                $statsPerProject['custom_migrations'] = $dbForProject->count('migrations', [], APP_LIMIT_COUNT);
-
-                /** Get Total Members */
-                $teamInternalId = $project->getAttribute('teamInternalId', null);
-                if ($teamInternalId) {
-                    $statsPerProject['custom_organization_members'] = $dbForConsole->count('memberships', [
-                        Query::equal('teamInternalId', [$teamInternalId])
-                    ], APP_LIMIT_COUNT);
-                } else {
-                    $statsPerProject['custom_organization_members'] = 0;
-                }
-
-                /** Get Email and Name of the project owner */
-                if ($teamInternalId) {
-                    $membership = $dbForConsole->findOne('memberships', [
-                        Query::equal('teamInternalId', [$teamInternalId]),
-                    ]);
-
-                    if (!$membership || $membership->isEmpty()) {
-                        throw new Exception('Membership not found. Skipping project : ' . $project->getId());
-                    }
-
-                    $userId = $membership->getAttribute('userId', null);
-                    if ($userId) {
-                        $user = $dbForConsole->getDocument('users', $userId);
-                        $statsPerProject['email'] = $user->getAttribute('email', null);
-                        $statsPerProject['name'] = $user->getAttribute('name', null);
-                    }
-                }
-
-                /** Get Domains */
-                $statsPerProject['custom_domains'] = $dbForConsole->count('rules', [
-                    Query::equal('projectInternalId', [$project->getInternalId()]),
-                    Query::limit(APP_LIMIT_COUNT)
-                ]);
-
-                /** Get Platforms */
-                $platforms = $dbForConsole->find('platforms', [
-                    Query::equal('projectInternalId', [$project->getInternalId()]),
-                    Query::limit(APP_LIMIT_COUNT)
-                ]);
-
-                $statsPerProject['custom_platforms_web'] = sizeof(array_filter($platforms, function ($platform) {
-                    return $platform['type'] === 'web';
-                }));
-
-                $statsPerProject['custom_platforms_android'] = sizeof(array_filter($platforms, function ($platform) {
-                    return $platform['type'] === 'android';
-                }));
-
-                $statsPerProject['custom_platforms_apple'] = sizeof(array_filter($platforms, function ($platform) {
-                    return str_contains($platform['type'], 'apple');
-                }));
-
-                $statsPerProject['custom_platforms_flutter'] = sizeof(array_filter($platforms, function ($platform) {
-                    return str_contains($platform['type'], 'flutter');
-                }));
-
-                $flutterPlatforms = [Origin::CLIENT_TYPE_FLUTTER_ANDROID, Origin::CLIENT_TYPE_FLUTTER_IOS, Origin::CLIENT_TYPE_FLUTTER_MACOS, Origin::CLIENT_TYPE_FLUTTER_WINDOWS, Origin::CLIENT_TYPE_FLUTTER_LINUX];
-
-                foreach ($flutterPlatforms as $flutterPlatform) {
-                    $statsPerProject['custom_platforms_' . $flutterPlatform] = sizeof(array_filter($platforms, function ($platform) use ($flutterPlatform) {
-                        return $platform['type'] === $flutterPlatform;
-                    }));
-                }
-
-                $statsPerProject['custom_platforms_api_keys'] = $dbForConsole->count('keys', [
-                    Query::equal('projectInternalId', [$project->getInternalId()]),
-                    Query::limit(APP_LIMIT_COUNT)
-                ]);
-
-                /** Get Usage $statsPerProject */
-                $periods = [
-                    'infinity' => [
-                        'period' => '1d',
-                        'limit' => 90,
-                    ],
-                    '24h' => [
-                        'period' => '1h',
-                        'limit' => 24,
-                    ],
-                ];
-
-                Authorization::skip(function () use ($dbForProject, $periods, &$statsPerProject) {
-                    foreach ($this->metrics as $key => $metric) {
-                        foreach ($periods as $periodKey => $periodValue) {
-                            $limit = $periodValue['limit'];
-                            $period = $periodValue['period'];
-
-                            $requestDocs = $dbForProject->find('stats_v2', [
-                                Query::equal('metric', [$metric]),
-                                Query::equal('period', [$period]),
-                                Query::limit($limit),
-                                Query::orderDesc('time'),
-                            ]);
-
-                            $statsPerProject[$key . '_' . $periodKey] = [];
-                            foreach ($requestDocs as $requestDoc) {
-                                $statsPerProject[$key . '_' . $periodKey][] = [
-                                    'value' => $requestDoc->getAttribute('value'),
-                                    'date' => $requestDoc->getAttribute('time'),
-                                ];
-                            }
-
-                            $statsPerProject[$key . '_' . $periodKey] = array_reverse($statsPerProject[$key . '_' . $periodKey]);
-                            // Calculate aggregate of each metric
-                            $statsPerProject[$key . '_' . $periodKey] = array_sum(array_column($statsPerProject[$key . '_' . $periodKey], 'value'));
-                        }
-                    }
-                });
-
-                /**
-                 * Workaround to combine network.inbound+network.outbound as network.
-                 */
-                $statsPerProject["usage_network_infinity"]  = $statsPerProject["usage_inbound_infinity"] + $statsPerProject["usage_outbound_infinity"];
-                $statsPerProject["usage_network_24h"]  = $statsPerProject["usage_inbound_24h"] + $statsPerProject["usage_outbound_24h"];
-                unset($statsPerProject["usage_outbound_24h"]);
-                unset($statsPerProject["usage_inbound_24h"]);
-                unset($statsPerProject["usage_outbound_infinity"]);
-                unset($statsPerProject["usage_inbound_infinity"]);
-
-                if (isset($statsPerProject['email'])) {
-                    /** Send data to mixpanel */
-                    $res = $this->mixpanel->createProfile($statsPerProject['email'], '', [
-                        'name' => $statsPerProject['name'],
-                        'email' => $statsPerProject['email']
-                    ]);
-
-                    if (!$res) {
-                        Console::error('Failed to create user profile for project: ' . $project->getId());
-                    }
-                }
-
-                $event = new Event();
-                $event
-                    ->setName('Project Daily Usage')
-                    ->setProps($statsPerProject);
-                $res = $this->mixpanel->createEvent($event);
-
-                if (!$res) {
-                    Console::error('Failed to create event for project: ' . $project->getId());
-                }
-            } catch (Exception $e) {
-                Console::error('Failed to send stats for project: ' . $project->getId());
-                Console::error($e->getMessage());
-            } finally {
-                $pools
-                    ->get($db)
-                    ->reclaim();
-            }
-        });
-    }
-
-    public function action(Group $pools, Cache $cache, Database $dbForConsole): void
-    {
-
-        Console::title('Cloud Hamster V1');
-        Console::success(APP_NAME . ' cloud hamster process has started');
-
-        $sleep = (int) App::getEnv('_APP_HAMSTER_INTERVAL', '30'); // 30 seconds (by default)
-        $jobInitTime = App::getEnv('_APP_HAMSTER_TIME', '22:00'); // (hour:minutes)
-        $now = new \DateTime();
-        $now->setTimezone(new \DateTimeZone(date_default_timezone_get()));
-        $next = new \DateTime($now->format("Y-m-d $jobInitTime"));
-        $next->setTimezone(new \DateTimeZone(date_default_timezone_get()));
-        $delay = $next->getTimestamp() - $now->getTimestamp();
-
-        /**
-         * If time passed for the target day.
-         */
-        if ($delay <= 0) {
-            $next->add(\DateInterval::createFromDateString('1 days'));
-            $delay = $next->getTimestamp() - $now->getTimestamp();
-        }
-        Console::log('[' . $now->format("Y-m-d H:i:s.v") . '] Delaying for ' . $delay . ' setting loop to [' . $next->format("Y-m-d H:i:s.v") . ']');
-
-        Console::loop(function () use ($pools, $cache, $dbForConsole, $sleep) {
-            $now = date('d-m-Y H:i:s', time());
-            Console::info("[{$now}] Getting Cloud Usage Stats every {$sleep} seconds");
-            $loopStart = microtime(true);
-
-            /* Initialise new Utopia app */
-            $app = new App('UTC');
-
-            Console::info('Getting stats for all projects');
-            $this->getStatsPerProject($pools, $cache, $dbForConsole);
-            Console::success('Completed getting stats for all projects');
-
-            Console::info('Getting stats for all organizations');
-            $this->getStatsPerOrganization($dbForConsole);
-            Console::success('Completed getting stats for all organizations');
-
-            Console::info('Getting stats for all users');
-            $this->getStatsPerUser($dbForConsole);
-            Console::success('Completed getting stats for all users');
-
-            $pools
-                ->get('console')
-                ->reclaim();
-
-            $loopTook = microtime(true) - $loopStart;
-            $now = date('d-m-Y H:i:s', time());
-            Console::info("[{$now}] Cloud Stats took {$loopTook} seconds");
-        }, $sleep, $delay);
-    }
-
-    protected function calculateByGroup(string $collection, Database $dbForConsole, callable $callback)
-    {
-        $count = 0;
-        $chunk = 0;
-        $limit = 50;
-        $results = [];
-        $sum = $limit;
-
-        $executionStart = \microtime(true);
-
-        while ($sum === $limit) {
-            $chunk++;
-
-            $results = $dbForConsole->find($collection, \array_merge([
-                Query::limit($limit),
-                Query::offset($count)
-            ]));
-
-            $sum = count($results);
-
-            Console::log('Processing chunk #' . $chunk . '. Found ' . $sum . ' documents');
-
-            foreach ($results as $document) {
-                call_user_func($callback, $dbForConsole, $document);
-                $count++;
-            }
-        }
-
-        $executionEnd = \microtime(true);
-
-        Console::log("Processed {$count} document by group in " . ($executionEnd - $executionStart) . " seconds");
-    }
-
-    protected function getStatsPerOrganization(Database $dbForConsole)
-    {
-
-        $this->calculateByGroup('teams', $dbForConsole, function (Database $dbForConsole, Document $document) {
-            try {
-                $statsPerOrganization = [];
-
-                /** Organization name */
-                $statsPerOrganization['name'] = $document->getAttribute('name');
-
-                /** Get Email and of the organization owner */
-                $membership = $dbForConsole->findOne('memberships', [
-                    Query::equal('teamInternalId', [$document->getInternalId()]),
-                ]);
-
-                if (!$membership || $membership->isEmpty()) {
-                    throw new Exception('Membership not found. Skipping organization : ' . $document->getId());
-                }
-
-                $userId = $membership->getAttribute('userId', null);
-                if ($userId) {
-                    $user = $dbForConsole->getDocument('users', $userId);
-                    $statsPerOrganization['email'] = $user->getAttribute('email', null);
-                }
-
-                /** Organization Creation Date */
-                $statsPerOrganization['created'] = $document->getAttribute('$createdAt');
-
-                /** Number of team members */
-                $statsPerOrganization['members'] = $document->getAttribute('total');
-
-                /** Number of projects in this organization */
-                $statsPerOrganization['projects'] = $dbForConsole->count('projects', [
-                    Query::equal('teamId', [$document->getId()]),
-                    Query::limit(APP_LIMIT_COUNT)
-                ]);
-
-                if (!isset($statsPerOrganization['email'])) {
-                    throw new Exception('Email not found. Skipping organization : ' . $document->getId());
-                }
-
-                $event = new Event();
-                $event
-                    ->setName('Organization Daily Usage')
-                    ->setProps($statsPerOrganization);
-                $res = $this->mixpanel->createEvent($event);
-                if (!$res) {
-                    throw new Exception('Failed to create event for organization : ' . $document->getId());
-                }
-            } catch (Exception $e) {
-                Console::error($e->getMessage());
-            }
-        });
-    }
-
-    protected function getStatsPerUser(Database $dbForConsole)
-    {
-        $this->calculateByGroup('users', $dbForConsole, function (Database $dbForConsole, Document $document) {
-            try {
-                $statsPerUser = [];
-
-                /** Organization name */
-                $statsPerUser['name'] = $document->getAttribute('name');
-
-                /** Organization ID (needs to be stored as an email since mixpanel uses the email attribute as a distinctID) */
-                $statsPerUser['email'] = $document->getAttribute('email');
-
-                /** Organization Creation Date */
-                $statsPerUser['created'] = $document->getAttribute('$createdAt');
-
-                /** Number of teams this user is a part of */
-                $statsPerUser['memberships'] = $dbForConsole->count('memberships', [
-                    Query::equal('userInternalId', [$document->getInternalId()]),
-                    Query::limit(APP_LIMIT_COUNT)
-                ]);
-
-                if (!isset($statsPerUser['email'])) {
-                    throw new Exception('User has no email: ' . $document->getId());
-                }
-
-                /** Send data to mixpanel */
-                $event = new Event();
-                $event
-                    ->setName('User Daily Usage')
-                    ->setProps($statsPerUser);
-                $res = $this->mixpanel->createEvent($event);
-
-                if (!$res) {
-                    throw new Exception('Failed to create user profile for user: ' . $document->getId());
-                }
-            } catch (Exception $e) {
-                Console::error($e->getMessage());
-            }
-        });
-    }
-}

src/Appwrite/Platform/Tasks/Install.php

@@ -8,6 +8,7 @@ use Appwrite\Docker\Env;
 use Appwrite\Utopia\View;
 use Utopia\CLI\Console;
 use Utopia\Config\Config;
+use Utopia\Validator\Boolean;
 use Utopia\Validator\Text;
 use Utopia\Platform\Action;
 
@@ -24,15 +25,16 @@ class Install extends Action
     {
         $this
             ->desc('Install Appwrite')
-            ->param('httpPort', '', new Text(4), 'Server HTTP port', true)
-            ->param('httpsPort', '', new Text(4), 'Server HTTPS port', true)
+            ->param('http-port', '', new Text(4), 'Server HTTP port', true)
+            ->param('https-port', '', new Text(4), 'Server HTTPS port', true)
             ->param('organization', 'appwrite', new Text(0), 'Docker Registry organization', true)
             ->param('image', 'appwrite', new Text(0), 'Main appwrite docker image', true)
             ->param('interactive', 'Y', new Text(1), 'Run an interactive session', true)
-            ->callback(fn ($httpPort, $httpsPort, $organization, $image, $interactive) => $this->action($httpPort, $httpsPort, $organization, $image, $interactive));
+            ->param('no-start', false, new Boolean(true), 'Run an interactive session', true)
+            ->callback(fn ($httpPort, $httpsPort, $organization, $image, $interactive, $noStart) => $this->action($httpPort, $httpsPort, $organization, $image, $interactive, $noStart));
     }
 
-    public function action(string $httpPort, string $httpsPort, string $organization, string $image, string $interactive): void
+    public function action(string $httpPort, string $httpsPort, string $organization, string $image, string $interactive, bool $noStart): void
     {
         $config = Config::getParam('variables');
         $defaultHTTPPort = '80';
@@ -220,9 +222,11 @@ class Install extends Action
             }
         }
 
-        Console::log("Running \"docker compose up -d --remove-orphans --renew-anon-volumes\"");
-
-        $exit = Console::execute("$env docker compose --project-directory $this->path up -d --remove-orphans --renew-anon-volumes", '', $stdout, $stderr);
+        $exit = 0;
+        if (!$noStart) {
+            Console::log("Running \"docker compose up -d --remove-orphans --renew-anon-volumes\"");
+            $exit = Console::execute("$env docker compose --project-directory $this->path up -d --remove-orphans --renew-anon-volumes", '', $stdout, $stderr);
+        }
 
         if ($exit !== 0) {
             $message = 'Failed to install Appwrite dockers';

src/Appwrite/Platform/Tasks/PatchRecreateRepositoriesDocuments.php

@@ -1,169 +0,0 @@
-<?php
-
-namespace Appwrite\Platform\Tasks;
-
-use Utopia\Platform\Action;
-use Utopia\CLI\Console;
-use Utopia\Database\Database;
-use Utopia\Database\Document;
-use Utopia\Database\Helpers\ID;
-use Utopia\Database\Helpers\Permission;
-use Utopia\Database\Helpers\Role;
-use Utopia\Database\Query;
-use Utopia\Validator\Text;
-
-class PatchRecreateRepositoriesDocuments extends Action
-{
-    public static function getName(): string
-    {
-        return 'patch-recreate-repositories-documents';
-    }
-
-    public function __construct()
-    {
-        $this
-            ->desc('Recreate missing repositories in consoleDB from projectDBs. They can be missing if you used Appwrite 1.4.10 or 1.4.11, and deleted a function.')
-            ->param('after', '', new Text(36), 'After cursor', true)
-            ->param('projectId', '', new Text(36), 'Select project to validate', true)
-            ->inject('dbForConsole')
-            ->inject('getProjectDB')
-            ->callback(fn ($after, $projectId, $dbForConsole, $getProjectDB) => $this->action($after, $projectId, $dbForConsole, $getProjectDB));
-    }
-
-    public function action($after, $projectId, Database $dbForConsole, callable $getProjectDB): void
-    {
-        Console::info("Starting the patch");
-
-        $startTime = microtime(true);
-
-        if (!empty($projectId)) {
-            try {
-                $project = $dbForConsole->getDocument('projects', $projectId);
-                $dbForProject = call_user_func($getProjectDB, $project);
-                $this->recreateRepositories($dbForConsole, $dbForProject, $project);
-            } catch (\Throwable $th) {
-                Console::error("Unexpected error occured with Project ID {$projectId}");
-                Console::error('[Error] Type: ' . get_class($th));
-                Console::error('[Error] Message: ' . $th->getMessage());
-                Console::error('[Error] File: ' . $th->getFile());
-                Console::error('[Error] Line: ' . $th->getLine());
-            }
-        } else {
-            $queries = [];
-            if (!empty($after)) {
-                Console::info("Iterating remaining projects after project with ID {$after}");
-                $project = $dbForConsole->getDocument('projects', $after);
-                $queries = [Query::cursorAfter($project)];
-            } else {
-                Console::info("Iterating all projects");
-            }
-            $this->foreachDocument($dbForConsole, 'projects', $queries, function (Document $project) use ($getProjectDB, $dbForConsole) {
-                $projectId = $project->getId();
-
-                try {
-                    $dbForProject = call_user_func($getProjectDB, $project);
-                    $this->recreateRepositories($dbForConsole, $dbForProject, $project);
-                } catch (\Throwable $th) {
-                    Console::error("Unexpected error occured with Project ID {$projectId}");
-                    Console::error('[Error] Type: ' . get_class($th));
-                    Console::error('[Error] Message: ' . $th->getMessage());
-                    Console::error('[Error] File: ' . $th->getFile());
-                    Console::error('[Error] Line: ' . $th->getLine());
-                }
-            });
-        }
-
-        $endTime = microtime(true);
-        $timeTaken = $endTime - $startTime;
-
-        $hours = (int)($timeTaken / 3600);
-        $timeTaken -= $hours * 3600;
-        $minutes = (int)($timeTaken / 60);
-        $timeTaken -= $minutes * 60;
-        $seconds = (int)$timeTaken;
-        $milliseconds = ($timeTaken - $seconds) * 1000;
-        Console::info("Recreate patch completed in $hours h, $minutes m, $seconds s, $milliseconds mis ( total $timeTaken milliseconds)");
-    }
-
-    protected function foreachDocument(Database $database, string $collection, array $queries = [], callable $callback = null): void
-    {
-        $limit = 1000;
-        $results = [];
-        $sum = $limit;
-        $latestDocument = null;
-
-        while ($sum === $limit) {
-            $newQueries = $queries;
-
-            if ($latestDocument != null) {
-                array_unshift($newQueries, Query::cursorAfter($latestDocument));
-            }
-            $newQueries[] = Query::limit($limit);
-            $results = $database->find($collection, $newQueries);
-
-            if (empty($results)) {
-                return;
-            }
-
-            $sum = count($results);
-
-            foreach ($results as $document) {
-                if (is_callable($callback)) {
-                    $callback($document);
-                }
-            }
-            $latestDocument = $results[array_key_last($results)];
-        }
-    }
-
-    public function recreateRepositories(Database $dbForConsole, Database $dbForProject, Document $project): void
-    {
-        $projectId = $project->getId();
-        Console::log("Running patch for project {$projectId}");
-
-        $this->foreachDocument($dbForProject, 'functions', [], function (Document $function) use ($dbForProject, $dbForConsole, $project) {
-            $isConnected = !empty($function->getAttribute('providerRepositoryId', ''));
-
-            if ($isConnected) {
-                $repository = $dbForConsole->getDocument('repositories', $function->getAttribute('repositoryId', ''));
-
-                if ($repository->isEmpty()) {
-                    $projectId = $project->getId();
-                    $functionId = $function->getId();
-                    Console::success("Recreating repositories document for project ID {$projectId}, function ID {$functionId}");
-
-                    $repository = $dbForConsole->createDocument('repositories', new Document([
-                        '$id' => ID::unique(),
-                        '$permissions' => [
-                            Permission::read(Role::any()),
-                            Permission::update(Role::any()),
-                            Permission::delete(Role::any()),
-                        ],
-                        'installationId' => $function->getAttribute('installationId', ''),
-                        'installationInternalId' => $function->getAttribute('installationInternalId', ''),
-                        'projectId' => $project->getId(),
-                        'projectInternalId' => $project->getInternalId(),
-                        'providerRepositoryId' => $function->getAttribute('providerRepositoryId', ''),
-                        'resourceId' => $function->getId(),
-                        'resourceInternalId' => $function->getInternalId(),
-                        'resourceType' => 'function',
-                        'providerPullRequestIds' => []
-                    ]));
-
-                    $function = $dbForProject->updateDocument('functions', $function->getId(), $function
-                        ->setAttribute('repositoryId', $repository->getId())
-                        ->setAttribute('repositoryInternalId', $repository->getInternalId()));
-
-                    $this->foreachDocument($dbForProject, 'deployments', [
-                        Query::equal('resourceInternalId', [$function->getInternalId()]),
-                        Query::equal('resourceType', ['functions'])
-                    ], function (Document $deployment) use ($dbForProject, $repository) {
-                        $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment
-                            ->setAttribute('repositoryId', $repository->getId())
-                            ->setAttribute('repositoryInternalId', $repository->getInternalId()));
-                    });
-                }
-            }
-        });
-    }
-}

src/Appwrite/Platform/Tasks/QueueCount.php

@@ -0,0 +1,77 @@
+<?php
+
+namespace Appwrite\Platform\Tasks;
+
+use Appwrite\Event\Event;
+use Utopia\CLI\Console;
+use Utopia\Platform\Action;
+use Utopia\Queue\Client;
+use Utopia\Queue\Connection;
+use Utopia\Validator\WhiteList;
+
+class QueueCount extends Action
+{
+    public static function getName(): string
+    {
+        return 'queue-count';
+    }
+
+
+    public function __construct()
+    {
+        $this
+            ->desc('Return the number of from a specific queue identified by the name parameter with a specific type')
+            ->param('name', '', new WhiteList([
+                Event::DATABASE_QUEUE_NAME,
+                Event::DELETE_QUEUE_NAME,
+                Event::AUDITS_QUEUE_NAME,
+                Event::MAILS_QUEUE_NAME,
+                Event::FUNCTIONS_QUEUE_NAME,
+                Event::USAGE_QUEUE_NAME,
+                Event::WEBHOOK_QUEUE_NAME,
+                Event::CERTIFICATES_QUEUE_NAME,
+                Event::BUILDS_QUEUE_NAME,
+                Event::MESSAGING_QUEUE_NAME,
+                Event::MIGRATIONS_QUEUE_NAME,
+                Event::HAMSTER_QUEUE_NAME
+            ]), 'Queue name')
+            ->param('type', '', new WhiteList([
+                'success',
+                'failed',
+                'processing',
+            ]), 'Queue type')
+            ->inject('queue')
+            ->callback(fn ($name, $type, $queue) => $this->action($name, $type, $queue));
+    }
+
+    /**
+     * @param string $name The name of the queue to count the jobs from
+     * @param string $type The type of jobs to count
+     * @param Connection $queue
+     */
+    public function action(string $name, string $type, Connection $queue): void
+    {
+        if (!$name) {
+            Console::error('Missing required parameter $name');
+            return;
+        }
+
+        $queueClient = new Client($name, $queue);
+
+        $count = 0;
+
+        switch ($type) {
+            case 'success':
+                $count = $queueClient->countSuccessfulJobs();
+                break;
+            case 'failed':
+                $count = $queueClient->countFailedJobs();
+                break;
+            case 'processing':
+                $count = $queueClient->countProcessingJobs();
+                break;
+        };
+
+        Console::log("Queue: '{$name}' has {$count} {$type} jobs.");
+    }
+}

src/Appwrite/Platform/Tasks/QueueRetry.php

@@ -0,0 +1,64 @@
+<?php
+
+namespace Appwrite\Platform\Tasks;
+
+use Appwrite\Event\Event;
+use Utopia\CLI\Console;
+use Utopia\Platform\Action;
+use Utopia\Queue\Client;
+use Utopia\Queue\Connection;
+use Utopia\Validator\WhiteList;
+
+class QueueRetry extends Action
+{
+    public static function getName(): string
+    {
+        return 'queue-retry';
+    }
+
+
+    public function __construct()
+    {
+        $this
+            ->desc('Retry failed jobs from a specific queue identified by the name parameter')
+            ->param('name', '', new WhiteList([
+                Event::DATABASE_QUEUE_NAME,
+                Event::DELETE_QUEUE_NAME,
+                Event::AUDITS_QUEUE_NAME,
+                Event::MAILS_QUEUE_NAME,
+                Event::FUNCTIONS_QUEUE_NAME,
+                Event::USAGE_QUEUE_NAME,
+                Event::WEBHOOK_CLASS_NAME,
+                Event::CERTIFICATES_QUEUE_NAME,
+                Event::BUILDS_QUEUE_NAME,
+                Event::MESSAGING_QUEUE_NAME,
+                Event::MIGRATIONS_QUEUE_NAME,
+                Event::HAMSTER_CLASS_NAME
+            ]), 'Queue name')
+            ->inject('queue')
+            ->callback(fn ($name, $queue) => $this->action($name, $queue));
+    }
+
+    /**
+     * @param string $name The name of the queue to retry jobs from
+     * @param Connection $queue
+     */
+    public function action(string $name, Connection $queue): void
+    {
+        if (!$name) {
+            Console::error('Missing required parameter $name');
+            return;
+        }
+
+        $queueClient = new Client($name, $queue);
+
+        if ($queueClient->countFailedJobs() === 0) {
+            Console::error('No failed jobs found.');
+            return;
+        }
+
+        Console::log('Retrying failed jobs...');
+
+        $queueClient->retry();
+    }
+}

src/Appwrite/Platform/Tasks/SSL.php

@@ -7,6 +7,7 @@ use Appwrite\Event\Certificate;
 use Utopia\App;
 use Utopia\CLI\Console;
 use Utopia\Database\Document;
+use Utopia\Validator\Boolean;
 use Utopia\Validator\Hostname;
 
 class SSL extends Action
@@ -21,19 +22,22 @@ class SSL extends Action
         $this
             ->desc('Validate server certificates')
             ->param('domain', App::getEnv('_APP_DOMAIN', ''), new Hostname(), 'Domain to generate certificate for. If empty, main domain will be used.', true)
+            ->param('skip-check', true, new Boolean(true), 'If DNS and renew check should be skipped. Defaults to true, and when true, all jobs will result in certificate generation attempt.', true)
             ->inject('queueForCertificates')
-            ->callback(fn (string $domain, Certificate $queueForCertificates) => $this->action($domain, $queueForCertificates));
+            ->callback(fn (string $domain, bool|string $skipCheck, Certificate $queueForCertificates) => $this->action($domain, $skipCheck, $queueForCertificates));
     }
 
-    public function action(string $domain, Certificate $queueForCertificates): void
+    public function action(string $domain, bool|string $skipCheck, Certificate $queueForCertificates): void
     {
+        $skipCheck = \strval($skipCheck) === 'true';
+
         Console::success('Scheduling a job to issue a TLS certificate for domain: ' . $domain);
 
         $queueForCertificates
             ->setDomain(new Document([
                 'domain' => $domain
             ]))
-            ->setSkipRenewCheck(true)
+            ->setSkipRenewCheck($skipCheck)
             ->trigger();
     }
 }

src/Appwrite/Platform/Tasks/Upgrade.php

@@ -3,6 +3,7 @@
 namespace Appwrite\Platform\Tasks;
 
 use Utopia\CLI\Console;
+use Utopia\Validator\Boolean;
 use Utopia\Validator\Text;
 
 class Upgrade extends Install
@@ -16,15 +17,16 @@ class Upgrade extends Install
     {
         $this
             ->desc('Upgrade Appwrite')
-            ->param('httpPort', '', new Text(4), 'Server HTTP port', true)
-            ->param('httpsPort', '', new Text(4), 'Server HTTPS port', true)
+            ->param('http-port', '', new Text(4), 'Server HTTP port', true)
+            ->param('https-port', '', new Text(4), 'Server HTTPS port', true)
             ->param('organization', 'appwrite', new Text(0), 'Docker Registry organization', true)
             ->param('image', 'appwrite', new Text(0), 'Main appwrite docker image', true)
             ->param('interactive', 'Y', new Text(1), 'Run an interactive session', true)
-            ->callback(fn ($httpPort, $httpsPort, $organization, $image, $interactive) => $this->action($httpPort, $httpsPort, $organization, $image, $interactive));
+            ->param('no-start', false, new Boolean(true), 'Run an interactive session', true)
+            ->callback(fn ($httpPort, $httpsPort, $organization, $image, $interactive, $noStart) => $this->action($httpPort, $httpsPort, $organization, $image, $interactive, $noStart));
     }
 
-    public function action(string $httpPort, string $httpsPort, string $organization, string $image, string $interactive): void
+    public function action(string $httpPort, string $httpsPort, string $organization, string $image, string $interactive, bool $noStart): void
     {
         // Check for previous installation
         $data = @file_get_contents($this->path . '/docker-compose.yml');
@@ -37,6 +39,6 @@ class Upgrade extends Install
             Console::log('      └── docker-compose.yml');
             Console::exit(1);
         }
-        parent::action($httpPort, $httpsPort, $organization, $image, $interactive);
+        parent::action($httpPort, $httpsPort, $organization, $image, $interactive, $noStart);
     }
 }

src/Appwrite/Platform/Tasks/VolumeSync.php

@@ -1,59 +0,0 @@
-<?php
-
-namespace Appwrite\Platform\Tasks;
-
-use Utopia\CLI\Console;
-use Utopia\Database\DateTime;
-use Utopia\Platform\Action;
-use Utopia\Validator\Integer;
-use Utopia\Validator\Text;
-
-class VolumeSync extends Action
-{
-    public static function getName(): string
-    {
-        return 'volume-sync';
-    }
-
-    public function __construct()
-    {
-        $this
-            ->desc('Runs rsync to sync certificates between the storage mount and traefik.')
-            ->param('source', null, new Text(255), 'Source path to sync from.', false)
-            ->param('destination', null, new Text(255), 'Destination path to sync to.', false)
-            ->param('interval', null, new Integer(true), 'Interval to run rsync', false)
-            ->callback(fn ($source, $destination, $interval) => $this->action($source, $destination, $interval));
-    }
-
-    public function action(string $source, string $destination, int $interval)
-    {
-
-        Console::title('RSync V1');
-        Console::success(APP_NAME . ' rsync process v1 has started');
-
-        if (!file_exists($source)) {
-            Console::error('Source directory does not exist. Exiting ... ');
-            Console::exit(0);
-        }
-
-        Console::loop(function () use ($interval, $source, $destination) {
-            $time = DateTime::now();
-
-            Console::info("[{$time}] Executing rsync every {$interval} seconds");
-            Console::info("Syncing between $source and $destination");
-
-            if (!file_exists($source)) {
-                Console::error('Source directory does not exist. Skipping ... ');
-                return;
-            }
-
-            $stdin = "";
-            $stdout = "";
-            $stderr = "";
-
-            Console::execute("rsync -av $source $destination", $stdin, $stdout, $stderr);
-            Console::success($stdout);
-            Console::error($stderr);
-        }, $interval);
-    }
-}

src/Appwrite/Platform/Workers/Certificates.php

@@ -23,6 +23,7 @@ use Utopia\Database\Helpers\ID;
 use Utopia\Database\Query;
 use Utopia\Domains\Domain;
 use Utopia\Locale\Locale;
+use Utopia\Logger\Log;
 use Utopia\Platform\Action;
 use Utopia\Queue\Message;
 
@@ -45,7 +46,8 @@ class Certificates extends Action
             ->inject('queueForMails')
             ->inject('queueForEvents')
             ->inject('queueForFunctions')
-            ->callback(fn(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions) => $this->action($message, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions));
+            ->inject('log')
+            ->callback(fn(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log) => $this->action($message, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log));
     }
 
     /**
@@ -58,7 +60,7 @@ class Certificates extends Action
      * @throws Throwable
      * @throws \Utopia\Database\Exception
      */
-    public function action(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions): void
+    public function action(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log): void
     {
         $payload = $message->getPayload() ?? [];
 
@@ -70,7 +72,7 @@ class Certificates extends Action
         $domain   = new Domain($document->getAttribute('domain', ''));
         $skipRenewCheck = $payload['skipRenewCheck'] ?? false;
 
-        $this->execute($domain, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $skipRenewCheck);
+        $this->execute($domain, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log, $skipRenewCheck);
     }
 
     /**
@@ -84,7 +86,7 @@ class Certificates extends Action
      * @throws Throwable
      * @throws \Utopia\Database\Exception
      */
-    private function execute(Domain $domain, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, bool $skipRenewCheck = false): void
+    private function execute(Domain $domain, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, bool $skipRenewCheck = false): void
     {
         /**
          * 1. Read arguments and validate domain
@@ -138,11 +140,11 @@ class Certificates extends Action
             if (!$skipRenewCheck) {
                 $mainDomain = $this->getMainDomain();
                 $isMainDomain = !isset($mainDomain) || $domain->get() === $mainDomain;
-                $this->validateDomain($domain, $isMainDomain);
+                $this->validateDomain($domain, $isMainDomain, $log);
             }
 
             // If certificate exists already, double-check expiry date. Skip if job is forced
-            if (!$skipRenewCheck && !$this->isRenewRequired($domain->get())) {
+            if (!$skipRenewCheck && !$this->isRenewRequired($domain->get(), $log)) {
                 throw new Exception('Renew isn\'t required.');
             }
 
@@ -180,6 +182,8 @@ class Certificates extends Action
 
             // Send email to security email
             $this->notifyError($domain->get(), $e->getMessage(), $attempts, $queueForMails);
+
+            throw $e;
         } finally {
             // All actions result in new updatedAt date
             $certificate->setAttribute('updated', DateTime::now());
@@ -247,7 +251,7 @@ class Certificates extends Action
      * @return void
      * @throws Exception
      */
-    private function validateDomain(Domain $domain, bool $isMainDomain): void
+    private function validateDomain(Domain $domain, bool $isMainDomain, Log $log): void
     {
         if (empty($domain->get())) {
             throw new Exception('Missing certificate domain.');
@@ -267,8 +271,15 @@ class Certificates extends Action
             }
 
             // Verify domain with DNS records
+            $validationStart = \microtime(true);
             $validator = new CNAME($target->get());
             if (!$validator->isValid($domain->get())) {
+                $log->addExtra('dnsTiming', \strval(\microtime(true) - $validationStart));
+                $log->addTag('dnsDomain', $domain->get());
+
+                $error = $validator->getLogs();
+                $log->addExtra('dnsResponse', \is_array($error) ? \json_encode($error) : \strval($error));
+
                 throw new Exception('Failed to verify domain DNS records.');
             }
         } else {
@@ -284,7 +295,7 @@ class Certificates extends Action
      * @return bool True, if certificate needs to be renewed
      * @throws Exception
      */
-    private function isRenewRequired(string $domain): bool
+    private function isRenewRequired(string $domain, Log $log): bool
     {
         $certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem';
         if (\file_exists($certPath)) {
@@ -294,12 +305,15 @@ class Certificates extends Action
             $validTo = $certData['validTo_time_t'] ?? 0;
 
             if (empty($validTo)) {
+                $log->addTag('certificateDomain', $domain);
                 throw new Exception('Unable to read certificate file (cert.pem).');
             }
 
             // LetsEncrypt allows renewal 30 days before expiry
             $expiryInAdvance = (60 * 60 * 24 * 30);
             if ($validTo - $expiryInAdvance > \time()) {
+                $log->addTag('certificateDomain', $domain);
+                $log->addExtra('certificateData', \is_array($certData) ? \json_encode($certData) : \strval($certData));
                 return false;
             }
         }
@@ -430,7 +444,7 @@ class Certificates extends Action
 
         $message = Template::fromFile(__DIR__ . '/../../../../app/config/locale/templates/email-inner-base.tpl');
         $message
-            ->setParam('{{body}}', $locale->getText("emails.certificate.body"))
+            ->setParam('{{body}}', $locale->getText("emails.certificate.body"), escapeHtml: false)
             ->setParam('{{hello}}', $locale->getText("emails.certificate.hello"))
             ->setParam('{{footer}}', $locale->getText("emails.certificate.footer"))
             ->setParam('{{thanks}}', $locale->getText("emails.certificate.thanks"))

src/Appwrite/Platform/Workers/Deletes.php

@@ -516,13 +516,8 @@ class Deletes extends Action
             if ($document->getAttribute('confirm')) { // Count only confirmed members
                 $teamId = $document->getAttribute('teamId');
                 $team = $dbForProject->getDocument('teams', $teamId);
-                if (!$team->isEmpty()) {
-                    $team = $dbForProject->updateDocument(
-                        'teams',
-                        $teamId,
-                        // Ensure that total >= 0
-                        $team->setAttribute('total', \max($team->getAttribute('total', 0) - 1, 0))
-                    );
+                if (!$team->isEmpty() && $team->getAttribute('total', 0) > 0) {
+                    $dbForProject->decreaseDocumentAttribute('teams', $teamId, 'total', 1);
                 }
             }
         });

src/Appwrite/Platform/Workers/Mails.php

@@ -65,9 +65,10 @@ class Mails extends Action
             $bodyTemplate = __DIR__ . '/../../../../app/config/locale/templates/email-base.tpl';
         }
         $bodyTemplate = Template::fromFile($bodyTemplate);
-        $bodyTemplate->setParam('{{body}}', $body);
+        $bodyTemplate->setParam('{{body}}', $body, escapeHtml: false);
         foreach ($variables as $key => $value) {
-            $bodyTemplate->setParam('{{' . $key . '}}', $value);
+            // TODO: hotfix for redirect param
+            $bodyTemplate->setParam('{{' . $key . '}}', $value, escapeHtml: $key !== 'redirect');
         }
         $body = $bodyTemplate->render();
 

src/Appwrite/Platform/Workers/Messaging.php

@@ -55,6 +55,18 @@ class Messaging extends Action
     {
         $payload = $message->getPayload() ?? [];
 
+        if (empty($payload['project'])) {
+            throw new Exception('Project not set in payload');
+        }
+
+        Console::log($payload['project']['$id']);
+        $denyList = App::getEnv('_APP_SMS_PROJECTS_DENY_LIST', '');
+        $denyList = explode(',', $denyList);
+        if (in_array($payload['project']['$id'], $denyList)) {
+            Console::error("Project is in the deny list. Skipping ...");
+            return;
+        }
+
         if (empty($payload)) {
             Console::error('Payload arg not found');
             return;

src/Appwrite/Platform/Workers/Usage.php

@@ -5,6 +5,7 @@ namespace Appwrite\Platform\Workers;
 use Exception;
 use Utopia\CLI\Console;
 use Utopia\Database\Database;
+use Utopia\Database\DateTime;
 use Utopia\Database\Document;
 use Utopia\Platform\Action;
 use Utopia\Queue\Message;
@@ -19,7 +20,6 @@ class Usage extends Action
     ];
 
     protected const INFINITY_PERIOD = '_inf_';
-
     public static function getName(): string
     {
         return 'usage';
@@ -32,26 +32,24 @@ class Usage extends Action
     {
 
         $this
-            ->desc('Usage worker')
-            ->inject('message')
-            ->inject('pools')
-            ->inject('cache')
-            ->callback(function ($message, $pools, $cache) {
-                 $this->action($message, $pools, $cache);
-            });
+        ->desc('Usage worker')
+        ->inject('message')
+        ->inject('getProjectDB')
+        ->callback(function (Message $message, callable $getProjectDB) {
+             $this->action($message, $getProjectDB);
+        });
     }
 
     /**
      * @param Message $message
-     * @param $pools
-     * @param $cache
+     * @param callable $getProjectDB
      * @return void
      * @throws \Utopia\Database\Exception
+     * @throws Exception
      */
-    public function action(Message $message, $pools, $cache): void
+    public function action(Message $message, callable $getProjectDB): void
     {
         $payload = $message->getPayload() ?? [];
-
         if (empty($payload)) {
             throw new Exception('Missing payload');
         }
@@ -65,16 +63,13 @@ class Usage extends Action
             }
 
             $this->reduce(
-                database: $project->getAttribute('database'),
-                projectInternalId: $project->getInternalId(),
+                project: $project,
                 document: new Document($document),
                 metrics:  $payload['metrics'],
-                pools: $pools,
-                cache: $cache
+                getProjectDB: $getProjectDB
             );
         }
-
-        self::$stats[$projectId]['database'] = $project->getAttribute('database');
+        self::$stats[$projectId]['project'] = $project;
         foreach ($payload['metrics'] ?? [] as $metric) {
             if (!isset(self::$stats[$projectId]['keys'][$metric['key']])) {
                 self::$stats[$projectId]['keys'][$metric['key']] = $metric['value'];
@@ -88,35 +83,25 @@ class Usage extends Action
      /**
      * On Documents that tied by relations like functions>deployments>build || documents>collection>database || buckets>files.
      * When we remove a parent document we need to deduct his children aggregation from the project scope.
-
-     * @param $database
-     * @param $projectInternalId
+     * @param Document $project
      * @param Document $document
      * @param array $metrics
-     * @param $pools
-     * @param $cache
+     * @param  callable $getProjectDB
      * @return void
      */
-    private function reduce($database, $projectInternalId, Document $document, array &$metrics, $pools, $cache)
+    private function reduce(Document $project, Document $document, array &$metrics, callable $getProjectDB): void
     {
+
+        $dbForProject = $getProjectDB($project);
+
         try {
-            $dbForProject = new Database(
-                $pools
-                ->get($database)
-                ->pop()
-                ->getResource(),
-                $cache
-            );
-
-            $dbForProject->setNamespace('_' . $projectInternalId);
-
             switch (true) {
                 case $document->getCollection() === 'users': // users
                     $sessions = count($document->getAttribute(METRIC_SESSIONS, 0));
                     if (!empty($sessions)) {
                         $metrics[] = [
-                            'key' => METRIC_SESSIONS,
-                            'value' => ($sessions * -1),
+                        'key' => METRIC_SESSIONS,
+                        'value' => ($sessions * -1),
                         ];
                     }
                     break;
@@ -144,12 +129,12 @@ class Usage extends Action
 
                     if (!empty($documents['value'])) {
                         $metrics[] = [
-                            'key' => METRIC_DOCUMENTS,
-                            'value' => ($documents['value'] * -1),
+                        'key' => METRIC_DOCUMENTS,
+                        'value' => ($documents['value'] * -1),
                         ];
                         $metrics[] = [
-                            'key' => str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_DOCUMENTS),
-                            'value' => ($documents['value'] * -1),
+                        'key' => str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_DOCUMENTS),
+                        'value' => ($documents['value'] * -1),
                         ];
                     }
                     break;
@@ -160,15 +145,15 @@ class Usage extends Action
 
                     if (!empty($files['value'])) {
                         $metrics[] = [
-                            'key' => METRIC_FILES,
-                            'value' => ($files['value'] * -1),
+                        'key' => METRIC_FILES,
+                        'value' => ($files['value'] * -1),
                         ];
                     }
 
                     if (!empty($storage['value'])) {
                         $metrics[] = [
-                            'key' => METRIC_FILES_STORAGE,
-                            'value' => ($storage['value'] * -1),
+                        'key' => METRIC_FILES_STORAGE,
+                        'value' => ($storage['value'] * -1),
                         ];
                     }
                     break;
@@ -176,58 +161,58 @@ class Usage extends Action
                 case $document->getCollection() === 'functions':
                     $deployments = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS)));
                     $deploymentsStorage = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE)));
-                    $builds = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD .  str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS)));
+                    $builds = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS)));
                     $buildsStorage = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE)));
                     $buildsCompute = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE)));
-                    $executions = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD .  str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS)));
+                    $executions = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS)));
                     $executionsCompute = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE)));
 
                     if (!empty($deployments['value'])) {
                         $metrics[] = [
-                            'key' => METRIC_DEPLOYMENTS,
-                            'value' => ($deployments['value'] * -1),
+                        'key' => METRIC_DEPLOYMENTS,
+                        'value' => ($deployments['value'] * -1),
                         ];
                     }
 
                     if (!empty($deploymentsStorage['value'])) {
                         $metrics[] = [
-                            'key' => METRIC_DEPLOYMENTS_STORAGE,
-                            'value' => ($deploymentsStorage['value'] * -1),
+                        'key' => METRIC_DEPLOYMENTS_STORAGE,
+                        'value' => ($deploymentsStorage['value'] * -1),
                         ];
                     }
 
                     if (!empty($builds['value'])) {
                         $metrics[] = [
-                            'key' => METRIC_BUILDS,
-                            'value' => ($builds['value'] * -1),
+                        'key' => METRIC_BUILDS,
+                        'value' => ($builds['value'] * -1),
                         ];
                     }
 
                     if (!empty($buildsStorage['value'])) {
                         $metrics[] = [
-                            'key' => METRIC_BUILDS_STORAGE,
-                            'value' => ($buildsStorage['value'] * -1),
+                        'key' => METRIC_BUILDS_STORAGE,
+                        'value' => ($buildsStorage['value'] * -1),
                         ];
                     }
 
                     if (!empty($buildsCompute['value'])) {
                         $metrics[] = [
-                            'key' => METRIC_BUILDS_COMPUTE,
-                            'value' => ($buildsCompute['value'] * -1),
+                        'key' => METRIC_BUILDS_COMPUTE,
+                        'value' => ($buildsCompute['value'] * -1),
                         ];
                     }
 
                     if (!empty($executions['value'])) {
                         $metrics[] = [
-                            'key' => METRIC_EXECUTIONS,
-                            'value' => ($executions['value'] * -1),
+                        'key' => METRIC_EXECUTIONS,
+                        'value' => ($executions['value'] * -1),
                         ];
                     }
 
                     if (!empty($executionsCompute['value'])) {
                         $metrics[] = [
-                            'key' => METRIC_EXECUTIONS_COMPUTE,
-                            'value' => ($executionsCompute['value'] * -1),
+                        'key' => METRIC_EXECUTIONS_COMPUTE,
+                        'value' => ($executionsCompute['value'] * -1),
                         ];
                     }
                     break;
@@ -235,10 +220,7 @@ class Usage extends Action
                     break;
             }
         } catch (\Exception $e) {
-            console::error("[reducer] " . " {DateTime::now()} " .  " {$projectInternalId} " . " {$e->getMessage()}");
-        } catch (\Throwable $e) {
-        } finally {
-            $pools->reclaim();
+            console::error("[reducer] " . " {DateTime::now()} " . " {$project->getInternalId()} " . " {$e->getMessage()}");
         }
     }
 }

src/Appwrite/Platform/Workers/UsageHook.php

@@ -3,12 +3,12 @@
 namespace Appwrite\Platform\Workers;
 
 use Utopia\App;
-use Utopia\Database\Database;
 use Utopia\Database\Document;
 use Utopia\Database\Exception\Duplicate;
 use Utopia\Platform\Action;
 use Utopia\CLI\Console;
 use Swoole\Timer;
+use Utopia\Database\DateTime;
 
 class UsageHook extends Usage
 {
@@ -23,41 +23,41 @@ class UsageHook extends Usage
         $this
             ->setType(Action::TYPE_WORKER_START)
             ->inject('register')
-            ->inject('cache')
-            ->inject('pools')
-            ->callback(function ($register, $cache, $pools) {
-                $this->action($register, $cache, $pools);
+            ->inject('getProjectDB')
+            ->callback(function ($register, callable $getProjectDB) {
+                $this->action($register, $getProjectDB);
             })
         ;
     }
 
     /**
      * @param $register
-     * @param $cache
-     * @param $pools
+     * @param $getProjectDB
      * @return void
      */
-    public function action($register, $cache, $pools): void
+    public function action($register, $getProjectDB): void
     {
-        Timer::tick(30000, function () use ($register, $cache, $pools) {
+
+        $interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '60000');
+        Timer::tick($interval, function () use ($register, $getProjectDB) {
 
             $offset = count(self::$stats);
             $projects = array_slice(self::$stats, 0, $offset, true);
             array_splice(self::$stats, 0, $offset);
+            foreach ($projects as $data) {
+                $numberOfKeys = !empty($data['keys']) ? count($data['keys']) : 0;
+                $projectInternalId = $data['project']->getInternalId();
+                $database = $data['project']['database'] ?? '';
+
+                console::warning('Ticker started ' . DateTime::now());
+
+                if ($numberOfKeys === 0) {
+                    continue;
+                }
 
-            foreach ($projects as $projectInternalId => $project) {
                 try {
-                    $dbForProject = new Database(
-                        $pools
-                            ->get($project['database'])
-                            ->pop()
-                            ->getResource(),
-                        $cache
-                    );
-
-                    $dbForProject->setNamespace('_' . $projectInternalId);
-
-                    foreach ($project['keys'] ?? [] as $key => $value) {
+                    $dbForProject = $getProjectDB($data['project']);
+                    foreach ($data['keys'] ?? [] as $key => $value) {
                         if ($value == 0) {
                             continue;
                         }
@@ -95,9 +95,7 @@ class UsageHook extends Usage
                         }
                     }
                 } catch (\Exception $e) {
-                    console::error("[logger] " . " {DateTime::now()} " .  " {$projectInternalId} " . " {$e->getMessage()}");
-                } finally {
-                    $pools->reclaim();
+                    console::error(DateTime::now() . ' ' . $projectInternalId . ' ' . $e->getMessage());
                 }
             }
         });

src/Appwrite/Utopia/Response/Model/Bucket.php

@@ -4,6 +4,7 @@ namespace Appwrite\Utopia\Response\Model;
 
 use Appwrite\Utopia\Response;
 use Appwrite\Utopia\Response\Model;
+use Utopia\Storage\Compression\Compression;
 
 class Bucket extends Model
 {
@@ -68,7 +69,7 @@ class Bucket extends Model
             ])
             ->addRule('compression', [
                 'type' => self::TYPE_STRING,
-                'description' => 'Compression algorithm choosen for compression. Will be one of ' . COMPRESSION_TYPE_NONE . ', [' . COMPRESSION_TYPE_GZIP . '](https://en.wikipedia.org/wiki/Gzip), or [' . COMPRESSION_TYPE_ZSTD . '](https://en.wikipedia.org/wiki/Zstd).',
+                'description' => 'Compression algorithm choosen for compression. Will be one of ' . Compression::NONE . ', [' . Compression::GZIP . '](https://en.wikipedia.org/wiki/Gzip), or [' . Compression::ZSTD . '](https://en.wikipedia.org/wiki/Zstd).',
                 'default' => '',
                 'example' => 'gzip',
                 'array' => false

src/Executor/Executor.php

@@ -72,6 +72,7 @@ class Executor
     ) {
         $runtimeId = "$projectId-$deploymentId-build";
         $route = "/runtimes";
+        $timeout = (int) App::getEnv('_APP_FUNCTIONS_BUILD_TIMEOUT', 900);
         $params = [
             'runtimeId' => $runtimeId,
             'source' => $source,
@@ -84,10 +85,9 @@ class Executor
             'cpus' => $this->cpus,
             'memory' => $this->memory,
             'version' => $version,
+            'timeout' => $timeout,
         ];
 
-        $timeout  = (int) App::getEnv('_APP_FUNCTIONS_BUILD_TIMEOUT', 900);
-
         $response = $this->call(self::METHOD_POST, $route, [ 'x-opr-runtime-id' => $runtimeId ], $params, true, $timeout);
 
         $status = $response['headers']['status-code'];
@@ -111,7 +111,7 @@ class Executor
         string $projectId,
         callable $callback
     ) {
-        $timeout  = (int) App::getEnv('_APP_FUNCTIONS_BUILD_TIMEOUT', 900);
+        $timeout = (int) App::getEnv('_APP_FUNCTIONS_BUILD_TIMEOUT', 900);
 
         $runtimeId = "$projectId-$deploymentId-build";
         $route = "/runtimes/{$runtimeId}/logs";

tests/e2e/Client.php

@@ -168,6 +168,7 @@ class Client
         $headers            = array_merge($this->headers, $headers);
         $ch                 = curl_init($this->endpoint . $path . (($method == self::METHOD_GET && !empty($params)) ? '?' . http_build_query($params) : ''));
         $responseHeaders    = [];
+        $cookies = [];
 
         $query = match ($headers['content-type']) {
             'application/json' => json_encode($params),
@@ -189,7 +190,7 @@ class Client
         curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
         curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 0);
         curl_setopt($ch, CURLOPT_TIMEOUT, 15);
-        curl_setopt($ch, CURLOPT_HEADERFUNCTION, function ($curl, $header) use (&$responseHeaders) {
+        curl_setopt($ch, CURLOPT_HEADERFUNCTION, function ($curl, $header) use (&$responseHeaders, &$cookies) {
             $len = strlen($header);
             $header = explode(':', $header, 2);
 
@@ -197,6 +198,12 @@ class Client
                 return $len;
             }
 
+            if (strtolower(trim($header[0])) == 'set-cookie') {
+                $parsed = $this->parseCookie((string)trim($header[1]));
+                $name = array_key_first($parsed);
+                $cookies[$name] = $parsed[$name];
+            }
+
             $responseHeaders[strtolower(trim($header[0]))] = trim($header[1]);
 
             return $len;
@@ -241,6 +248,7 @@ class Client
 
         return [
             'headers' => $responseHeaders,
+            'cookies' => $cookies,
             'body' => $responseBody
         ];
     }

tests/e2e/General/HTTPTest.php

@@ -171,4 +171,50 @@ class HTTPTest extends Scope
 
         $this->assertEquals(200, $response['headers']['status-code']);
     }
+
+    public function testCors()
+    {
+        /**
+         * Test for SUCCESS
+         */
+
+        $endpoint = '/v1/projects'; // Can be any non-404 route
+
+        $response = $this->client->call(Client::METHOD_GET, $endpoint);
+
+        $this->assertEquals('http://localhost', $response['headers']['access-control-allow-origin']);
+
+        $response = $this->client->call(Client::METHOD_GET, $endpoint, [
+            'origin' => 'http://localhost',
+        ]);
+
+        $this->assertEquals('http://localhost', $response['headers']['access-control-allow-origin']);
+
+        $response = $this->client->call(Client::METHOD_GET, $endpoint, [
+            'origin' => 'http://appwrite.io',
+        ]);
+
+        $this->assertEquals('http://appwrite.io', $response['headers']['access-control-allow-origin']);
+
+        $response = $this->client->call(Client::METHOD_GET, $endpoint, [
+            'origin' => 'https://appwrite.io',
+        ]);
+
+        $this->assertEquals('https://appwrite.io', $response['headers']['access-control-allow-origin']);
+
+        $response = $this->client->call(Client::METHOD_GET, $endpoint, [
+            'origin' => 'http://cloud.appwrite.io',
+        ]);
+
+        $this->assertEquals('http://cloud.appwrite.io', $response['headers']['access-control-allow-origin']);
+
+        /**
+         * Test for FAILURE
+         */
+        $response = $this->client->call(Client::METHOD_GET, $endpoint, [
+            'origin' => 'http://google.com',
+        ]);
+
+        $this->assertEquals('http://localhost', $response['headers']['access-control-allow-origin']);
+    }
 }

tests/e2e/General/UsageTest.php

@@ -2,6 +2,7 @@
 
 namespace Tests\E2E\General;
 
+use Appwrite\Tests\Retry;
 use Tests\E2E\Client;
 use Tests\E2E\Scopes\ProjectCustom;
 use Tests\E2E\Scopes\Scope;
@@ -11,6 +12,7 @@ use DateTime;
 use Tests\E2E\Services\Functions\FunctionsBase;
 use Utopia\Database\Helpers\Permission;
 use Utopia\Database\Helpers\Role;
+use Utopia\Database\Validator\Datetime as DatetimeValidator;
 
 class UsageTest extends Scope
 {
@@ -101,6 +103,7 @@ class UsageTest extends Scope
     /**
      * @depends testPrepareUsersStats
      */
+    #[Retry(count: 1)]
     public function testUsersStats(array $data): array
     {
         sleep(self::WAIT);
@@ -145,9 +148,9 @@ class UsageTest extends Scope
 
         $res = $res['body'];
         $this->assertEquals('90d', $res['range']);
-        $this->assertEquals(90, count($res['usersTotal']));
-        $this->assertEquals(90, count($res['sessionsTotal']));
-        $this->assertEquals((self::CREATE / 2), $res['usersTotal'][array_key_last($res['usersTotal'])]['value']);
+        $this->assertEquals(90, count($res['users']));
+        $this->assertEquals(90, count($res['sessions']));
+        $this->assertEquals((self::CREATE / 2), $res['users'][array_key_last($res['users'])]['value']);
 
         return [
             'projectId' => $projectId,
@@ -269,6 +272,7 @@ class UsageTest extends Scope
     /**
      * @depends testPrepareStorageStats
      */
+    #[Retry(count: 1)]
     public function testStorageStats(array $data): array
     {
         $bucketId      = $data['bucketId'];
@@ -294,13 +298,11 @@ class UsageTest extends Scope
         );
         $res = $res['body'];
 
-        $this->assertEquals(9, count($res));
-        $this->assertEquals(30, count($res['requestsTotal']));
-        $this->assertEquals(30, count($res['filesStorage']));
-        $this->assertEquals($requestsTotal, $res['requestsTotal'][array_key_last($res['requestsTotal'])]['value']);
-        $this->validateDates($res['requestsTotal']);
-        $this->assertEquals($storageTotal, $res['filesStorage'][array_key_last($res['filesStorage'])]['value']);
-        $this->validateDates($res['filesStorage']);
+        $this->assertEquals(12, count($res));
+        $this->assertEquals(1, count($res['requests']));
+        $this->assertEquals($requestsTotal, $res['requests'][array_key_last($res['requests'])]['value']);
+        $this->validateDates($res['requests']);
+        $this->assertEquals($storageTotal, $res['filesStorageTotal']);
 
         $res = $this->client->call(
             Client::METHOD_GET,
@@ -312,12 +314,12 @@ class UsageTest extends Scope
         );
 
         $res = $res['body'];
-        $this->assertEquals($storageTotal, $res['filesStorage'][array_key_last($res['filesStorage'])]['value']);
-        $this->validateDates($res['filesStorage']);
-        $this->assertEquals($bucketsTotal, $res['bucketsTotal'][array_key_last($res['bucketsTotal'])]['value']);
-        $this->validateDates($res['bucketsTotal']);
-        $this->assertEquals($filesTotal, $res['filesTotal'][array_key_last($res['filesTotal'])]['value']);
-        $this->validateDates($res['filesTotal']);
+        $this->assertEquals($storageTotal, $res['storage'][array_key_last($res['storage'])]['value']);
+        $this->validateDates($res['storage']);
+        $this->assertEquals($bucketsTotal, $res['buckets'][array_key_last($res['buckets'])]['value']);
+        $this->validateDates($res['buckets']);
+        $this->assertEquals($filesTotal, $res['files'][array_key_last($res['files'])]['value']);
+        $this->validateDates($res['files']);
 
         $res = $this->client->call(
             Client::METHOD_GET,
@@ -329,8 +331,8 @@ class UsageTest extends Scope
         );
 
         $res = $res['body'];
-        $this->assertEquals($storageTotal, $res['filesStorage'][array_key_last($res['filesStorage'])]['value']);
-        $this->assertEquals($filesTotal, $res['filesTotal'][array_key_last($res['filesTotal'])]['value']);
+        $this->assertEquals($storageTotal, $res['storage'][array_key_last($res['storage'])]['value']);
+        $this->assertEquals($filesTotal, $res['files'][array_key_last($res['files'])]['value']);
 
         $data['requestsTotal'] = $requestsTotal;
 
@@ -475,7 +477,7 @@ class UsageTest extends Scope
     }
 
     /** @depends testPrepareDatabaseStats */
-
+    #[Retry(count: 1)]
     public function testDatabaseStats(array $data): array
     {
 
@@ -501,15 +503,13 @@ class UsageTest extends Scope
         );
         $res = $res['body'];
 
-        $this->assertEquals(9, count($res));
-        $this->assertEquals(30, count($res['requestsTotal']));
-        $this->assertEquals(30, count($res['filesStorage']));
-        $this->assertEquals($requestsTotal, $res['requestsTotal'][array_key_last($res['requestsTotal'])]['value']);
-        $this->validateDates($res['requestsTotal']);
-        $this->assertEquals($databasesTotal, $res['databasesTotal'][array_key_last($res['databasesTotal'])]['value']);
-        $this->validateDates($res['databasesTotal']);
-        $this->assertEquals($documentsTotal, $res['documentsTotal'][array_key_last($res['documentsTotal'])]['value']);
-        $this->validateDates($res['documentsTotal']);
+        $this->assertEquals(12, count($res));
+        $this->assertEquals(1, count($res['requests']));
+        $this->assertEquals(1, count($res['network']));
+        $this->assertEquals($requestsTotal, $res['requests'][array_key_last($res['requests'])]['value']);
+        $this->validateDates($res['requests']);
+        $this->assertEquals($databasesTotal, $res['databasesTotal']);
+        $this->assertEquals($documentsTotal, $res['documentsTotal']);
 
         $res = $this->client->call(
             Client::METHOD_GET,
@@ -518,12 +518,12 @@ class UsageTest extends Scope
         );
         $res = $res['body'];
 
-        $this->assertEquals($databasesTotal, $res['databasesTotal'][array_key_last($res['databasesTotal'])]['value']);
-        $this->validateDates($res['databasesTotal']);
-        $this->assertEquals($collectionsTotal, $res['collectionsTotal'][array_key_last($res['collectionsTotal'])]['value']);
-        $this->validateDates($res['collectionsTotal']);
-        $this->assertEquals($documentsTotal, $res['documentsTotal'][array_key_last($res['documentsTotal'])]['value']);
-        $this->validateDates($res['documentsTotal']);
+        $this->assertEquals($databasesTotal, $res['databases'][array_key_last($res['databases'])]['value']);
+        $this->validateDates($res['databases']);
+        $this->assertEquals($collectionsTotal, $res['collections'][array_key_last($res['collections'])]['value']);
+        $this->validateDates($res['collections']);
+        $this->assertEquals($documentsTotal, $res['documents'][array_key_last($res['documents'])]['value']);
+        $this->validateDates($res['documents']);
 
         $res = $this->client->call(
             Client::METHOD_GET,
@@ -532,17 +532,17 @@ class UsageTest extends Scope
         );
         $res = $res['body'];
 
-        $this->assertEquals($collectionsTotal, $res['collectionsTotal'][array_key_last($res['collectionsTotal'])]['value']);
-        $this->validateDates($res['collectionsTotal']);
+        $this->assertEquals($collectionsTotal, $res['collections'][array_key_last($res['collections'])]['value']);
+        $this->validateDates($res['collections']);
 
-        $this->assertEquals($documentsTotal, $res['documentsTotal'][array_key_last($res['documentsTotal'])]['value']);
-        $this->validateDates($res['documentsTotal']);
+        $this->assertEquals($documentsTotal, $res['documents'][array_key_last($res['documents'])]['value']);
+        $this->validateDates($res['documents']);
 
         $res = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/collections/' . $collectionId . '/usage?range=30d', $data['consoleHeaders']);
         $res = $res['body'];
 
-        $this->assertEquals($documentsTotal, $res['documentsTotal'][array_key_last($res['documentsTotal'])]['value']);
-        $this->validateDates($res['documentsTotal']);
+        $this->assertEquals($documentsTotal, $res['documents'][array_key_last($res['documents'])]['value']);
+        $this->validateDates($res['documents']);
 
         $data['requestsTotal'] = $requestsTotal;
 
@@ -701,6 +701,7 @@ class UsageTest extends Scope
     }
 
     /** @depends testPrepareFunctionsStats */
+    #[Retry(count: 1)]
     public function testFunctionsStats(array $data): void
     {
         $functionId = $data['functionId'];
@@ -716,19 +717,19 @@ class UsageTest extends Scope
         );
 
         $this->assertEquals(200, $response['headers']['status-code']);
-        $this->assertEquals(8, count($response['body']));
+        $this->assertEquals(15, count($response['body']));
         $this->assertEquals('30d', $response['body']['range']);
-        $this->assertIsArray($response['body']['deploymentsTotal']);
+        $this->assertIsArray($response['body']['deployments']);
         $this->assertIsArray($response['body']['deploymentsStorage']);
-        $this->assertIsArray($response['body']['buildsTotal']);
+        $this->assertIsArray($response['body']['builds']);
         $this->assertIsArray($response['body']['buildsTime']);
-        $this->assertIsArray($response['body']['executionsTotal']);
+        $this->assertIsArray($response['body']['executions']);
         $this->assertIsArray($response['body']['executionsTime']);
 
         $response = $response['body'];
 
-        $this->assertEquals($executions, $response['executionsTotal'][array_key_last($response['executionsTotal'])]['value']);
-        $this->validateDates($response['executionsTotal']);
+        $this->assertEquals($executions, $response['executions'][array_key_last($response['executions'])]['value']);
+        $this->validateDates($response['executions']);
         $this->assertEquals($executionTime, $response['executionsTime'][array_key_last($response['executionsTime'])]['value']);
         $this->validateDates($response['executionsTime']);
 
@@ -739,20 +740,20 @@ class UsageTest extends Scope
         );
 
         $this->assertEquals(200, $response['headers']['status-code']);
-        $this->assertEquals(9, count($response['body']));
+        $this->assertEquals(17, count($response['body']));
         $this->assertEquals($response['body']['range'], '30d');
-        $this->assertIsArray($response['body']['functionsTotal']);
-        $this->assertIsArray($response['body']['deploymentsTotal']);
+        $this->assertIsArray($response['body']['functions']);
+        $this->assertIsArray($response['body']['deployments']);
         $this->assertIsArray($response['body']['deploymentsStorage']);
-        $this->assertIsArray($response['body']['buildsTotal']);
+        $this->assertIsArray($response['body']['builds']);
         $this->assertIsArray($response['body']['buildsTime']);
-        $this->assertIsArray($response['body']['executionsTotal']);
+        $this->assertIsArray($response['body']['executions']);
         $this->assertIsArray($response['body']['executionsTime']);
 
         $response = $response['body'];
 
-        $this->assertEquals($executions, $response['executionsTotal'][array_key_last($response['executionsTotal'])]['value']);
-        $this->validateDates($response['executionsTotal']);
+        $this->assertEquals($executions, $response['executions'][array_key_last($response['executions'])]['value']);
+        $this->validateDates($response['executions']);
         $this->assertEquals($executionTime, $response['executionsTime'][array_key_last($response['executionsTime'])]['value']);
         $this->validateDates($response['executionsTime']);
         $this->assertGreaterThan(0, $response['buildsTime'][array_key_last($response['buildsTime'])]['value']);

tests/e2e/Scopes/Scope.php

@@ -98,7 +98,7 @@ abstract class Scope extends TestCase
             'password' => $password,
         ]);
 
-        $session = $this->client->parseCookie((string)$session['headers']['set-cookie'])['a_session_console'];
+        $session = $session['cookies']['a_session_console'];
 
         self::$root = [
             '$id' => ID::custom($root['body']['$id']),
@@ -150,7 +150,7 @@ abstract class Scope extends TestCase
             'password' => $password,
         ]);
 
-        $token = $this->client->parseCookie((string)$session['headers']['set-cookie'])['a_session_' . $this->getProject()['$id']];
+        $token = $session['cookies']['a_session_' . $this->getProject()['$id']];
 
         self::$user[$this->getProject()['$id']] = [
             '$id' => ID::custom($user['body']['$id']),

tests/e2e/Services/Account/AccountBase.php

@@ -126,7 +126,7 @@ trait AccountBase
         $this->assertNotFalse(\DateTime::createFromFormat('Y-m-d\TH:i:s.uP', $response['body']['expire']));
 
         $sessionId = $response['body']['$id'];
-        $session = $this->client->parseCookie((string)$response['headers']['set-cookie'])['a_session_' . $this->getProject()['$id']];
+        $session = $response['cookies']['a_session_' . $this->getProject()['$id']];
 
         // apiKey is only available in custom client test
         $apiKey = $this->getProject()['apiKey'];
@@ -993,7 +993,7 @@ trait AccountBase
         ]);
 
         $sessionNewId = $response['body']['$id'];
-        $sessionNew = $this->client->parseCookie((string)$response['headers']['set-cookie'])['a_session_' . $this->getProject()['$id']];
+        $sessionNew = $response['cookies']['a_session_' . $this->getProject()['$id']];
 
         $this->assertEquals($response['headers']['status-code'], 201);
 
@@ -1059,7 +1059,7 @@ trait AccountBase
             'password' => $password,
         ]);
 
-        $sessionNew = $this->client->parseCookie((string)$response['headers']['set-cookie'])['a_session_' . $this->getProject()['$id']];
+        $sessionNew = $response['cookies']['a_session_' . $this->getProject()['$id']];
 
         $this->assertEquals($response['headers']['status-code'], 201);
 
@@ -1141,7 +1141,7 @@ trait AccountBase
             'password' => $password,
         ]);
 
-        $data['session'] = $this->client->parseCookie((string)$response['headers']['set-cookie'])['a_session_' . $this->getProject()['$id']];
+        $data['session'] = $response['cookies']['a_session_' . $this->getProject()['$id']];
 
         return $data;
     }
@@ -1417,7 +1417,7 @@ trait AccountBase
         $this->assertNotEmpty($response['body']['userId']);
 
         $sessionId = $response['body']['$id'];
-        $session = $this->client->parseCookie((string)$response['headers']['set-cookie'])['a_session_' . $this->getProject()['$id']];
+        $session = $response['cookies']['a_session_' . $this->getProject()['$id']];
 
         $response = $this->client->call(Client::METHOD_GET, '/account', array_merge([
             'origin' => 'http://localhost',
@@ -1554,4 +1554,46 @@ trait AccountBase
 
         return $data;
     }
+
+    public function testDeleteAccount(): void
+    {
+         $email = uniqid() . 'user@localhost.test';
+         $password = 'password';
+         $name = 'User Name';
+
+         $response = $this->client->call(Client::METHOD_POST, '/account', array_merge([
+             'origin' => 'http://localhost',
+             'content-type' => 'application/json',
+             'x-appwrite-project' => $this->getProject()['$id'],
+         ]), [
+             'userId' => ID::unique(),
+             'email' => $email,
+             'password' => $password,
+             'name' => $name,
+         ]);
+
+         $this->assertEquals($response['headers']['status-code'], 201);
+
+         $response = $this->client->call(Client::METHOD_POST, '/account/sessions/email', array_merge([
+             'origin' => 'http://localhost',
+             'content-type' => 'application/json',
+             'x-appwrite-project' => $this->getProject()['$id'],
+         ]), [
+             'email' => $email,
+             'password' => $password,
+         ]);
+
+         $this->assertEquals($response['headers']['status-code'], 201);
+
+         $session = $response['cookies']['a_session_' . $this->getProject()['$id']];
+
+         $response = $this->client->call(Client::METHOD_DELETE, '/account', array_merge([
+             'origin' => 'http://localhost',
+             'content-type' => 'application/json',
+             'x-appwrite-project' => $this->getProject()['$id'],
+             'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session,
+         ]));
+
+         $this->assertEquals($response['headers']['status-code'], 204);
+    }
 }

tests/e2e/Services/Account/AccountCustomClientTest.php

@@ -126,7 +126,7 @@ class AccountCustomClientTest extends Scope
         $this->assertEquals($response['headers']['status-code'], 201);
 
         $sessionId = $response['body']['$id'];
-        $session = $this->client->parseCookie((string)$response['headers']['set-cookie'])['a_session_' . $this->getProject()['$id']];
+        $session = $response['cookies']['a_session_' . $this->getProject()['$id']];
 
         $response = $this->client->call(Client::METHOD_GET, '/account', array_merge([
             'origin' => 'http://localhost',
@@ -206,7 +206,7 @@ class AccountCustomClientTest extends Scope
 
         $this->assertEquals($response['headers']['status-code'], 201);
 
-        $session = $this->client->parseCookie((string)$response['headers']['set-cookie'])['a_session_' . $this->getProject()['$id']];
+        $session = $response['cookies']['a_session_' . $this->getProject()['$id']];
 
         $response = $this->client->call(Client::METHOD_GET, '/account', array_merge([
             'origin' => 'http://localhost',
@@ -288,7 +288,7 @@ class AccountCustomClientTest extends Scope
         $this->assertEquals($response['headers']['status-code'], 201);
 
         $sessionId = $response['body']['$id'];
-        $session = $this->client->parseCookie((string)$response['headers']['set-cookie'])['a_session_' . $this->getProject()['$id']];
+        $session = $response['cookies']['a_session_' . $this->getProject()['$id']];
 
         $response = $this->client->call(Client::METHOD_GET, '/account', array_merge([
             'origin' => 'http://localhost',
@@ -368,7 +368,7 @@ class AccountCustomClientTest extends Scope
         $this->assertNotEmpty($response['body']);
         $this->assertNotEmpty($response['body']['$id']);
 
-        $session = $this->client->parseCookie((string)$response['headers']['set-cookie'])['a_session_' . $this->getProject()['$id']];
+        $session = $response['cookies']['a_session_' . $this->getProject()['$id']];
 
         \usleep(1000 * 30); // wait for 30ms to let the shutdown update accessedAt
 
@@ -571,7 +571,7 @@ class AccountCustomClientTest extends Scope
             'failure' => 'http://localhost/v1/mock/tests/general/oauth2/failure',
         ]);
 
-        $session = $this->client->parseCookie((string)$response['headers']['set-cookie'])['a_session_' . $this->getProject()['$id']];
+        $session = $response['cookies']['a_session_' . $this->getProject()['$id']];
 
         $this->assertEquals(200, $response['headers']['status-code']);
         $this->assertEquals('success', $response['body']['result']);
@@ -849,7 +849,7 @@ class AccountCustomClientTest extends Scope
         $this->assertNotEmpty($response['body']['$id']);
         $this->assertNotEmpty($response['body']['userId']);
 
-        $session = $this->client->parseCookie((string)$response['headers']['set-cookie'])['a_session_' . $this->getProject()['$id']];
+        $session = $response['cookies']['a_session_' . $this->getProject()['$id']];
 
         $response = $this->client->call(Client::METHOD_GET, '/account', array_merge([
             'origin' => 'http://localhost',

tests/e2e/Services/Databases/DatabasesBase.php

@@ -978,7 +978,7 @@ trait DatabasesBase
         ]);
 
         $this->assertEquals(400, $badEnum['headers']['status-code']);
-        $this->assertEquals('Invalid `elements` param: Value must a valid array and Value must be a valid string and at least 1 chars and no longer than 255 chars', $badEnum['body']['message']);
+        $this->assertEquals('Invalid `elements` param: Value must a valid array no longer than 100 items and Value must be a valid string and at least 1 chars and no longer than 255 chars', $badEnum['body']['message']);
 
         return $data;
     }
@@ -2791,7 +2791,7 @@ trait DatabasesBase
             'email' => $email,
             'password' => $password,
         ]);
-        $session2 = $this->client->parseCookie((string)$session2['headers']['set-cookie'])['a_session_' . $this->getProject()['$id']];
+        $session2 = $session2['cookies']['a_session_' . $this->getProject()['$id']];
 
         $document3GetWithDocumentRead = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents/' . $document3['body']['$id'], [
             'origin' => 'http://localhost',
@@ -2979,7 +2979,7 @@ trait DatabasesBase
             'email' => $email,
             'password' => $password,
         ]);
-        $session2 = $this->client->parseCookie((string)$session2['headers']['set-cookie'])['a_session_' . $this->getProject()['$id']];
+        $session2 = $session2['cookies']['a_session_' . $this->getProject()['$id']];
 
         $document3GetWithDocumentRead = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents/' . $document3['body']['$id'], [
             'origin' => 'http://localhost',

tests/e2e/Services/Databases/DatabasesPermissionsScope.php

@@ -32,7 +32,7 @@ trait DatabasesPermissionsScope
             'password' => $password,
         ]);
 
-        $session = $this->client->parseCookie((string)$session['headers']['set-cookie'])['a_session_' . $this->getProject()['$id']];
+        $session = $session['cookies']['a_session_' . $this->getProject()['$id']];
 
         $user = [
             '$id' => $user['body']['$id'],

tests/e2e/Services/GraphQL/AccountTest.php

@@ -63,7 +63,7 @@ class AccountTest extends Scope
         $this->assertIsArray($session['body']['data']);
         $this->assertIsArray($session['body']['data']['accountCreateEmailSession']);
 
-        $cookie = $this->client->parseCookie((string)$session['headers']['set-cookie'])['a_session_' . $this->getProject()['$id']];
+        $cookie = $session['cookies']['a_session_' . $this->getProject()['$id']];
         $this->assertNotEmpty($cookie);
     }
 

tests/e2e/Services/GraphQL/AuthTest.php

@@ -73,9 +73,7 @@ class AuthTest extends Scope
             'x-appwrite-project' => $projectId,
         ], $graphQLPayload);
 
-        $this->token1 = $this->client->parseCookie(
-            (string)$session1['headers']['set-cookie']
-        )['a_session_' . $projectId];
+        $this->token1 = $session1['cookies']['a_session_' . $projectId];
 
         // Create session 2
         $graphQLPayload['variables']['email'] = $email2;
@@ -85,9 +83,7 @@ class AuthTest extends Scope
             'x-appwrite-project' => $projectId,
         ], $graphQLPayload);
 
-        $this->token2 = $this->client->parseCookie(
-            (string)$session2['headers']['set-cookie']
-        )['a_session_' . $projectId];
+        $this->token2 = $session2['cookies']['a_session_' . $projectId];
 
         // Create database
         $query = $this->getQuery(self::$CREATE_DATABASE);

tests/e2e/Services/Projects/ProjectsConsoleClientTest.php

@@ -441,6 +441,9 @@ class ProjectsConsoleClientTest extends Scope
      */
     public function testGetProjectUsage($data): array
     {
+        $this->markTestIncomplete(
+            'This test is failing right now due to functions collection.'
+        );
         /**
          * Test for SUCCESS
          */
@@ -468,7 +471,6 @@ class ProjectsConsoleClientTest extends Scope
         /**
          * Test for FAILURE
          */
-
         $response = $this->client->call(Client::METHOD_GET, '/projects/empty', array_merge([
             'content-type' => 'application/json',
             'x-appwrite-project' => $this->getProject()['$id'],
@@ -487,7 +489,7 @@ class ProjectsConsoleClientTest extends Scope
     }
 
     /**
-     * @depends testGetProjectUsage
+     * @depends testCreateProject
      */
     public function testUpdateProject($data): array
     {
@@ -671,7 +673,7 @@ class ProjectsConsoleClientTest extends Scope
         return $data;
     }
 
-    /** @depends testGetProjectUsage */
+    /** @depends testCreateProject */
     public function testUpdateProjectAuthDuration($data): array
     {
         $id = $data['projectId'];
@@ -791,7 +793,7 @@ class ProjectsConsoleClientTest extends Scope
     }
 
     /**
-     * @depends testGetProjectUsage
+     * @depends testCreateProject
      */
     public function testUpdateProjectOAuth($data): array
     {
@@ -902,7 +904,7 @@ class ProjectsConsoleClientTest extends Scope
     }
 
     /**
-     * @depends testGetProjectUsage
+     * @depends testCreateProject
      */
     public function testUpdateProjectAuthStatus($data): array
     {
@@ -933,7 +935,7 @@ class ProjectsConsoleClientTest extends Scope
             'password' => $originalPassword,
         ]);
 
-        $session = $this->client->parseCookie((string)$response['headers']['set-cookie'])['a_session_' . $id];
+        $session = $response['cookies']['a_session_' . $id];
 
         /**
          * Test for SUCCESS
@@ -1047,7 +1049,7 @@ class ProjectsConsoleClientTest extends Scope
     }
 
     /**
-     * @depends testGetProjectUsage
+     * @depends testCreateProject
      */
     public function testUpdateProjectAuthLimit($data): array
     {
@@ -1315,7 +1317,7 @@ class ProjectsConsoleClientTest extends Scope
             'password' => $password,
         ]);
         $this->assertEquals(201, $session['headers']['status-code']);
-        $session = $this->client->parseCookie((string)$session['headers']['set-cookie'])['a_session_' . $id];
+        $session = $session['cookies']['a_session_' . $id];
 
         $response = $this->client->call(Client::METHOD_PATCH, '/account/password', array_merge([
             'origin' => 'http://localhost',

tests/e2e/Services/Realtime/RealtimeCustomClientTest.php

@@ -468,7 +468,7 @@ class RealtimeCustomClientTest extends Scope
                 'password' => 'new-password',
             ]);
 
-            $sessionNew = $this->client->parseCookie((string)$response['headers']['set-cookie'])['a_session_' . $projectId];
+            $sessionNew = $response['cookies']['a_session_' . $projectId];
             $sessionNewId = $response['body']['$id'];
 
             return array("session" => $sessionNew, "sessionId" => $sessionNewId);

tests/e2e/Services/Storage/StorageBase.php

@@ -27,7 +27,7 @@ trait StorageBase
             'name' => 'Test Bucket',
             'fileSecurity' => true,
             'maximumFileSize' => 2000000, //2MB
-            'allowedFileExtensions' => ["jpg", "png"],
+            'allowedFileExtensions' => ["jpg", "png", 'jfif'],
             'permissions' => [
                 Permission::read(Role::any()),
                 Permission::create(Role::any()),
@@ -74,10 +74,7 @@ trait StorageBase
             'name' => 'Test Bucket 2',
             'fileSecurity' => true,
             'permissions' => [
-                Permission::read(Role::any()),
                 Permission::create(Role::any()),
-                Permission::update(Role::any()),
-                Permission::delete(Role::any()),
             ],
         ]);
         $this->assertEquals(201, $bucket2['headers']['status-code']);
@@ -110,9 +107,7 @@ trait StorageBase
                 'fileId' => $fileId,
                 'file' => $curlFile,
                 'permissions' => [
-                    Permission::read(Role::any()),
-                    Permission::update(Role::any()),
-                    Permission::delete(Role::any()),
+                    Permission::read(Role::any())
                 ],
             ]);
             $counter++;
@@ -462,6 +457,32 @@ trait StorageBase
         $this->assertEquals('image/png', $file2['headers']['content-type']);
         $this->assertNotEmpty($file2['body']);
 
+        // upload JXL file for preview
+        $fileJfif = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucketId . '/files', array_merge([
+            'content-type' => 'multipart/form-data',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), [
+            'fileId' => ID::unique(),
+            'file' => new CURLFile(realpath(__DIR__ . '/../../../resources/disk-a/preview-test.jfif'), 'image/jxl', 'preview-test.jfif'),
+            'permissions' => [
+                Permission::read(Role::any()),
+                Permission::update(Role::any()),
+                Permission::delete(Role::any()),
+            ],
+        ]);
+        $this->assertEquals(201, $fileJfif['headers']['status-code']);
+        $this->assertNotEmpty($fileJfif['body']['$id']);
+
+        // TEST preview JXL
+        $preview = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $fileJfif['body']['$id'] . '/preview', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()));
+
+        $this->assertEquals(200, $preview['headers']['status-code']);
+        $this->assertEquals('image/jpeg', $preview['headers']['content-type']);
+        $this->assertNotEmpty($preview['body']);
+
         //new image preview features
         $file3 = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $data['fileId'] . '/preview', array_merge([
             'content-type' => 'application/json',

tests/e2e/Services/Storage/StoragePermissionsScope.php

@@ -32,7 +32,7 @@ trait StoragePermissionsScope
             'password' => $password,
         ]);
 
-        $session = $this->client->parseCookie((string)$session['headers']['set-cookie'])['a_session_' . $this->getProject()['$id']];
+        $session = $session['cookies']['a_session_' . $this->getProject()['$id']];
 
 
         $user = [

tests/e2e/Services/Teams/TeamsBaseClient.php

@@ -403,7 +403,7 @@ trait TeamsBaseClient
         $this->assertCount(2, $response['body']['roles']);
         $this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['joined']));
         $this->assertEquals(true, $response['body']['confirm']);
-        $session = $this->client->parseCookie((string)$response['headers']['set-cookie'])['a_session_' . $this->getProject()['$id']];
+        $session = $response['cookies']['a_session_' . $this->getProject()['$id']];
         $data['session'] = $session;
 
         $response = $this->client->call(Client::METHOD_GET, '/account', array_merge([

tests/e2e/Services/Webhooks/WebhooksCustomClientTest.php

@@ -107,7 +107,7 @@ class WebhooksCustomClientTest extends Scope
         $this->assertEquals($accountSession['headers']['status-code'], 201);
 
         $id = $account['body']['$id'];
-        $session = $this->client->parseCookie((string)$accountSession['headers']['set-cookie'])['a_session_' . $this->getProject()['$id']];
+        $session = $accountSession['cookies']['a_session_' . $this->getProject()['$id']];
 
         $account = $this->client->call(Client::METHOD_PATCH, '/account/status', array_merge([
             'origin' => 'http://localhost',
@@ -170,7 +170,7 @@ class WebhooksCustomClientTest extends Scope
         $this->assertEquals($accountSession['headers']['status-code'], 201);
 
         $sessionId = $accountSession['body']['$id'];
-        $session = $this->client->parseCookie((string)$accountSession['headers']['set-cookie'])['a_session_' . $this->getProject()['$id']];
+        $session = $accountSession['cookies']['a_session_' . $this->getProject()['$id']];
 
         $webhook = $this->getLastRequest();
         $signatureKey = $this->getProject()['signatureKey'];
@@ -248,7 +248,7 @@ class WebhooksCustomClientTest extends Scope
         ]);
 
         $sessionId = $accountSession['body']['$id'];
-        $session = $this->client->parseCookie((string)$accountSession['headers']['set-cookie'])['a_session_' . $this->getProject()['$id']];
+        $session = $accountSession['cookies']['a_session_' . $this->getProject()['$id']];
 
         $this->assertEquals($accountSession['headers']['status-code'], 201);
 
@@ -334,7 +334,7 @@ class WebhooksCustomClientTest extends Scope
         ]);
 
         $sessionId = $accountSession['body']['$id'];
-        $session = $this->client->parseCookie((string)$accountSession['headers']['set-cookie'])['a_session_' . $this->getProject()['$id']];
+        $session = $accountSession['cookies']['a_session_' . $this->getProject()['$id']];
 
         $this->assertEquals($accountSession['headers']['status-code'], 201);
 
@@ -407,7 +407,7 @@ class WebhooksCustomClientTest extends Scope
         $this->assertEquals($accountSession['headers']['status-code'], 201);
 
         $sessionId = $accountSession['body']['$id'];
-        $session = $this->client->parseCookie((string)$accountSession['headers']['set-cookie'])['a_session_' . $this->getProject()['$id']];
+        $session = $accountSession['cookies']['a_session_' . $this->getProject()['$id']];
 
         return array_merge($data, [
             'sessionId' => $sessionId,