Merge branch 'main' into LauraDuRy-patch-1

.env

@@ -58,6 +58,7 @@ _APP_SMTP_USERNAME=
 _APP_SMTP_PASSWORD=
 _APP_SMS_PROVIDER=sms://username:password@mock
 _APP_SMS_FROM=+123456789
+_APP_SMS_PROJECTS_DENY_LIST=
 _APP_STORAGE_LIMIT=30000000
 _APP_STORAGE_PREVIEW_LIMIT=20000000
 _APP_FUNCTIONS_SIZE_LIMIT=30000000
@@ -72,11 +73,12 @@ _APP_EXECUTOR_SECRET=your-secret-key
 _APP_EXECUTOR_HOST=http://proxy/v1
 _APP_FUNCTIONS_RUNTIMES=php-8.0,node-18.0,python-3.9,ruby-3.1
 _APP_MAINTENANCE_INTERVAL=86400
+_APP_MAINTENANCE_DELAY=
 _APP_MAINTENANCE_RETENTION_CACHE=2592000
 _APP_MAINTENANCE_RETENTION_EXECUTION=1209600
 _APP_MAINTENANCE_RETENTION_ABUSE=86400
 _APP_MAINTENANCE_RETENTION_AUDIT=1209600
-_APP_USAGE_AGGREGATION_INTERVAL=60000
+_APP_USAGE_AGGREGATION_INTERVAL=30
 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000
 _APP_MAINTENANCE_RETENTION_SCHEDULES=86400
 _APP_USAGE_STATS=enabled

Dockerfile

@@ -84,6 +84,10 @@ RUN chmod +x /usr/local/bin/doctor && \
     chmod +x /usr/local/bin/ssl && \
     chmod +x /usr/local/bin/test && \
     chmod +x /usr/local/bin/vars && \
+    chmod +x /usr/local/bin/queue-retry && \
+    chmod +x /usr/local/bin/queue-count-failed && \
+    chmod +x /usr/local/bin/queue-count-processing && \
+    chmod +x /usr/local/bin/queue-count-success && \
     chmod +x /usr/local/bin/worker-audits && \
     chmod +x /usr/local/bin/worker-certificates && \
     chmod +x /usr/local/bin/worker-databases && \
@@ -95,7 +99,8 @@ RUN chmod +x /usr/local/bin/doctor && \
     chmod +x /usr/local/bin/worker-webhooks && \
     chmod +x /usr/local/bin/worker-migrations && \
     chmod +x /usr/local/bin/worker-hamster && \
-    chmod +x /usr/local/bin/worker-usage
+    chmod +x /usr/local/bin/worker-usage && \
+    chmod +x /usr/local/bin/worker-usage-dump
 
 
 # Cloud Executabless

app/config/collections.php

@@ -34,6 +34,28 @@ $commonCollections = [
                 'array' => false,
                 'filters' => [],
             ],
+            [
+                '$id' => 'resourceType',
+                'type' => Database::VAR_STRING,
+                'format' => '',
+                'size' => 255,
+                'signed' => true,
+                'required' => false,
+                'default' => null,
+                'array' => false,
+                'filters' => [],
+            ],
+            [
+                '$id' => ID::custom('mimeType'),
+                'type' => Database::VAR_STRING,
+                'format' => '',
+                'size' => 255, // https://tools.ietf.org/html/rfc4288#section-4.2
+                'signed' => true,
+                'required' => false,
+                'default' => null,
+                'array' => false,
+                'filters' => [],
+            ],
             [
                 '$id' => 'accessedAt',
                 'type' => Database::VAR_DATETIME,
@@ -1327,10 +1349,10 @@ $commonCollections = [
         ]
     ],
 
-    'stats_v2' => [
+    'stats' => [
         '$collection' => ID::custom(Database::METADATA),
-        '$id' => ID::custom('stats_v2'),
-        'name' => 'stats_v2',
+        '$id' => ID::custom('stats'),
+        'name' => 'stats',
         'attributes' => [
             [
                 '$id' => ID::custom('metric'),

app/config/errors.php

@@ -251,6 +251,11 @@ return [
         'description' => 'User phone is already verified',
         'code' => 409
     ],
+    Exception::USER_DELETION_PROHIBITED => [
+        'name' => Exception::USER_DELETION_PROHIBITED,
+        'description' => 'User deletion is not allowed for users with active memberships. Please delete all confirmed memberships before deleting the account.',
+        'code' => 400
+    ],
 
     /** Teams */
     Exception::TEAM_NOT_FOUND => [
@@ -680,6 +685,7 @@ return [
         'name' => Exception::RULE_VERIFICATION_FAILED,
         'description' => 'Domain verification failed. Please check if your DNS records are correct and try again.',
         'code' => 401,
+        'publish' => true
     ],
     Exception::PROJECT_SMTP_CONFIG_INVALID => [
         'name' => Exception::PROJECT_SMTP_CONFIG_INVALID,
@@ -767,10 +773,22 @@ return [
     ],
 
     /** Health */
-    Exception::QUEUE_SIZE_EXCEEDED => [
-        'name' => Exception::QUEUE_SIZE_EXCEEDED,
+    Exception::HEALTH_QUEUE_SIZE_EXCEEDED => [
+        'name' => Exception::HEALTH_QUEUE_SIZE_EXCEEDED,
         'description' => 'Queue size threshold hit.',
         'code' => 503,
         'publish' => false
     ],
+
+    Exception::HEALTH_CERTIFICATE_EXPIRED => [
+        'name' => Exception::HEALTH_CERTIFICATE_EXPIRED,
+        'description' => 'The SSL certificate for the specified domain has expired and is no longer valid.',
+        'code' => 404,
+    ],
+
+    Exception::HEALTH_INVALID_HOST => [
+        'name' => Exception::HEALTH_INVALID_HOST,
+        'description' => 'Failed to establish a connection to the specified domain. Please verify the domain name and ensure that the server is running and accessible.',
+        'code' => 404,
+    ],
 ];

app/config/variables.php

@@ -948,6 +948,15 @@ return [
                 'question' => '',
                 'filter' => ''
             ],
+            [
+                'name' => '_APP_MAINTENANCE_DELAY',
+                'description' => 'Delay value containing the number of seconds that the Appwrite maintenance process should wait before executing system cleanups and optimizations. The default value is 0 seconds.',
+                'introduction' => '1.5.0',
+                'default' => '0',
+                'required' => false,
+                'question' => '',
+                'filter' => ''
+            ],
             [
                 'name' => '_APP_MAINTENANCE_RETENTION_CACHE',
                 'description' => 'The maximum duration (in seconds) upto which to retain cached files. The default value is 2592000 seconds (30 days).',

app/controllers/api/account.php

@@ -889,7 +889,7 @@ App::delete('/v1/account/identities/:identityId')
 
 App::post('/v1/account/sessions/magic-url')
     ->desc('Create magic URL session')
-    ->groups(['api', 'account'])
+    ->groups(['api', 'account', 'auth'])
     ->label('scope', 'public')
     ->label('auth.type', 'magic-url')
     ->label('audits.event', 'session.create')
@@ -902,8 +902,8 @@ App::post('/v1/account/sessions/magic-url')
     ->label('sdk.response.code', Response::STATUS_CODE_CREATED)
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_TOKEN)
-    ->label('abuse-limit', 10)
-    ->label('abuse-key', 'url:{url},email:{param-email}')
+    ->label('abuse-limit', 60)
+    ->label('abuse-key', ['url:{url},email:{param-email}', 'url:{url},ip:{ip}'])
     ->param('userId', '', new CustomId(), 'Unique Id. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.')
     ->param('email', '', new Email(), 'User email.')
     ->param('url', '', fn($clients) => new Host($clients), 'URL to redirect the user back to your app from the magic URL login. Only URLs from hostnames in your project platform list are allowed. This requirement helps to prevent an [open redirect](https://cheatsheetseries.owasp.org/cheatsheets/Unvalidated_Redirects_and_Forwards_Cheat_Sheet.html) attack against your project API.', true, ['clients'])
@@ -1223,7 +1223,7 @@ App::put('/v1/account/sessions/magic-url')
 
 App::post('/v1/account/sessions/phone')
     ->desc('Create phone session')
-    ->groups(['api', 'account'])
+    ->groups(['api', 'account', 'auth'])
     ->label('scope', 'public')
     ->label('auth.type', 'phone')
     ->label('audits.event', 'session.create')
@@ -1237,7 +1237,7 @@ App::post('/v1/account/sessions/phone')
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_TOKEN)
     ->label('abuse-limit', 10)
-    ->label('abuse-key', 'url:{url},phone:{param-phone}')
+    ->label('abuse-key', ['url:{url},phone:{param-phone}', 'url:{url},ip:{ip}'])
     ->param('userId', '', new CustomId(), 'Unique Id. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.')
     ->param('phone', '', new Phone(), 'Phone number. Format this number with a leading \'+\' and a country code, e.g., +16175551212.')
     ->inject('request')
@@ -1339,6 +1339,7 @@ App::post('/v1/account/sessions/phone')
         $queueForMessaging
             ->setRecipient($phone)
             ->setMessage($message)
+            ->setProject($project)
             ->trigger();
 
         $queueForEvents->setPayload(
@@ -1790,7 +1791,7 @@ App::get('/v1/account/logs')
         }
 
         $response->dynamic(new Document([
-            'total' => $audit->countLogsByUser($user->getId()),
+            'total' => $audit->countLogsByUser($user->getInternalId()),
             'logs' => $output,
         ]), Response::MODEL_LOG_LIST);
     });
@@ -2388,7 +2389,7 @@ App::post('/v1/account/recovery')
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_TOKEN)
     ->label('abuse-limit', 10)
-    ->label('abuse-key', ['url:{url},email:{param-email}', 'ip:{ip}'])
+    ->label('abuse-key', ['url:{url},email:{param-email}', 'url:{url},ip:{ip}'])
     ->param('email', '', new Email(), 'User email.')
     ->param('url', '', fn ($clients) => new Host($clients), 'URL to redirect the user back to your app from the recovery email. Only URLs from hostnames in your project platform list are allowed. This requirement helps to prevent an [open redirect](https://cheatsheetseries.owasp.org/cheatsheets/Unvalidated_Redirects_and_Forwards_Cheat_Sheet.html) attack against your project API.', false, ['clients'])
     ->inject('request')
@@ -2861,8 +2862,9 @@ App::put('/v1/account/verification')
 
 App::post('/v1/account/verification/phone')
     ->desc('Create phone verification')
-    ->groups(['api', 'account'])
+    ->groups(['api', 'account', 'auth'])
     ->label('scope', 'account')
+    ->label('auth.type', 'phone')
     ->label('event', 'users.[userId].verification.[tokenId].create')
     ->label('audits.event', 'verification.create')
     ->label('audits.resource', 'user/{response.userId}')
@@ -2874,7 +2876,7 @@ App::post('/v1/account/verification/phone')
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_TOKEN)
     ->label('abuse-limit', 10)
-    ->label('abuse-key', 'userId:{userId}')
+    ->label('abuse-key', ['url:{url},userId:{userId}', 'url:{url},ip:{ip}'])
     ->inject('request')
     ->inject('response')
     ->inject('user')
@@ -2938,6 +2940,7 @@ App::post('/v1/account/verification/phone')
         $queueForMessaging
             ->setRecipient($user->getAttribute('phone'))
             ->setMessage($message)
+            ->setProject($project)
             ->trigger()
         ;
 
@@ -3031,15 +3034,27 @@ App::delete('/v1/account')
     ->label('sdk.response.code', Response::STATUS_CODE_NOCONTENT)
     ->label('sdk.response.model', Response::MODEL_NONE)
     ->inject('user')
+    ->inject('project')
     ->inject('response')
     ->inject('dbForProject')
     ->inject('queueForEvents')
     ->inject('queueForDeletes')
-    ->action(function (Document $user, Response $response, Database $dbForProject, Event $queueForEvents, Delete $queueForDeletes) {
+    ->action(function (Document $user, Document $project, Response $response, Database $dbForProject, Event $queueForEvents, Delete $queueForDeletes) {
         if ($user->isEmpty()) {
             throw new Exception(Exception::USER_NOT_FOUND);
         }
 
+        if ($project->getId() === 'console') {
+            // get all memberships
+            $memberships = $user->getAttribute('memberships', []);
+            foreach ($memberships as $membership) {
+                // prevent deletion if at least one active membership
+                if ($membership->getAttribute('confirm', false)) {
+                    throw new Exception(Exception::USER_DELETION_PROHIBITED);
+                }
+            }
+        }
+
         $dbForProject->deleteDocument('users', $user->getId());
 
         $queueForDeletes

app/controllers/api/databases.php

@@ -3563,7 +3563,7 @@ App::get('/v1/databases/usage')
 
         Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
             foreach ($metrics as $metric) {
-                $result =  $dbForProject->findOne('stats_v2', [
+                $result =  $dbForProject->findOne('stats', [
                     Query::equal('metric', [$metric]),
                     Query::equal('period', ['inf'])
                 ]);
@@ -3571,7 +3571,7 @@ App::get('/v1/databases/usage')
                 $stats[$metric]['total'] = $result['value'] ?? 0;
                 $limit = $days['limit'];
                 $period = $days['period'];
-                $results = $dbForProject->find('stats_v2', [
+                $results = $dbForProject->find('stats', [
                     Query::equal('metric', [$metric]),
                     Query::equal('period', [$period]),
                     Query::limit($limit),
@@ -3647,7 +3647,7 @@ App::get('/v1/databases/:databaseId/usage')
 
         Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
             foreach ($metrics as $metric) {
-                $result =  $dbForProject->findOne('stats_v2', [
+                $result =  $dbForProject->findOne('stats', [
                     Query::equal('metric', [$metric]),
                     Query::equal('period', ['inf'])
                 ]);
@@ -3655,7 +3655,7 @@ App::get('/v1/databases/:databaseId/usage')
                 $stats[$metric]['total'] = $result['value'] ?? 0;
                 $limit = $days['limit'];
                 $period = $days['period'];
-                $results = $dbForProject->find('stats_v2', [
+                $results = $dbForProject->find('stats', [
                     Query::equal('metric', [$metric]),
                     Query::equal('period', [$period]),
                     Query::limit($limit),
@@ -3733,7 +3733,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/usage')
 
         Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
             foreach ($metrics as $metric) {
-                $result =  $dbForProject->findOne('stats_v2', [
+                $result =  $dbForProject->findOne('stats', [
                     Query::equal('metric', [$metric]),
                     Query::equal('period', ['inf'])
                 ]);
@@ -3741,7 +3741,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/usage')
                 $stats[$metric]['total'] = $result['value'] ?? 0;
                 $limit = $days['limit'];
                 $period = $days['period'];
-                $results = $dbForProject->find('stats_v2', [
+                $results = $dbForProject->find('stats', [
                     Query::equal('metric', [$metric]),
                     Query::equal('period', [$period]),
                     Query::limit($limit),

app/controllers/api/functions.php

@@ -484,7 +484,7 @@ App::get('/v1/functions/:functionId/usage')
 
         Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
             foreach ($metrics as $metric) {
-                $result =  $dbForProject->findOne('stats_v2', [
+                $result =  $dbForProject->findOne('stats', [
                     Query::equal('metric', [$metric]),
                     Query::equal('period', ['inf'])
                 ]);
@@ -492,7 +492,7 @@ App::get('/v1/functions/:functionId/usage')
                 $stats[$metric]['total'] = $result['value'] ?? 0;
                 $limit = $days['limit'];
                 $period = $days['period'];
-                $results = $dbForProject->find('stats_v2', [
+                $results = $dbForProject->find('stats', [
                     Query::equal('metric', [$metric]),
                     Query::equal('period', [$period]),
                     Query::limit($limit),
@@ -576,7 +576,7 @@ App::get('/v1/functions/usage')
 
         Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
             foreach ($metrics as $metric) {
-                $result =  $dbForProject->findOne('stats_v2', [
+                $result =  $dbForProject->findOne('stats', [
                     Query::equal('metric', [$metric]),
                     Query::equal('period', ['inf'])
                 ]);
@@ -584,7 +584,7 @@ App::get('/v1/functions/usage')
                 $stats[$metric]['total'] = $result['value'] ?? 0;
                 $limit = $days['limit'];
                 $period = $days['period'];
-                $results = $dbForProject->find('stats_v2', [
+                $results = $dbForProject->find('stats', [
                     Query::equal('metric', [$metric]),
                     Query::equal('period', [$period]),
                     Query::limit($limit),

app/controllers/api/health.php

@@ -7,6 +7,7 @@ use Appwrite\Utopia\Response;
 use Utopia\App;
 use Utopia\Config\Config;
 use Utopia\Database\Document;
+use Utopia\Domains\Validator\PublicDomain;
 use Utopia\Pools\Group;
 use Utopia\Queue\Client;
 use Utopia\Queue\Connection;
@@ -14,8 +15,11 @@ use Utopia\Registry\Registry;
 use Utopia\Storage\Device;
 use Utopia\Storage\Device\Local;
 use Utopia\Storage\Storage;
+use Utopia\Validator\Domain;
 use Utopia\Validator\Integer;
+use Utopia\Validator\Multiple;
 use Utopia\Validator\Text;
+use Utopia\Validator\WhiteList;
 
 App::get('/v1/health')
     ->desc('Get HTTP')
@@ -355,7 +359,7 @@ App::get('/v1/health/queue/webhooks')
         $size = $client->getQueueSize();
 
         if ($size >= $threshold) {
-            throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
+            throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
         }
 
         $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
@@ -382,12 +386,62 @@ App::get('/v1/health/queue/logs')
         $size = $client->getQueueSize();
 
         if ($size >= $threshold) {
-            throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
+            throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
         }
 
         $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
     }, ['response']);
 
+App::get('/v1/health/certificate')
+    ->desc('Get the SSL certificate for a domain')
+    ->groups(['api', 'health'])
+    ->label('scope', 'health.read')
+    ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
+    ->label('sdk.namespace', 'health')
+    ->label('sdk.method', 'getCertificate')
+    ->label('sdk.description', '/docs/references/health/get-certificate.md')
+    ->label('sdk.response.code', Response::STATUS_CODE_OK)
+    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
+    ->label('sdk.response.model', Response::MODEL_HEALTH_CERTIFICATE)
+    ->param('domain', null, new Multiple([new Domain(), new PublicDomain()]), Multiple::TYPE_STRING, 'Domain name')
+    ->inject('response')
+    ->action(function (string $domain, Response $response) {
+        if (filter_var($domain, FILTER_VALIDATE_URL)) {
+            $domain = parse_url($domain, PHP_URL_HOST);
+        }
+
+        $sslContext = stream_context_create([
+            "ssl" => [
+                "capture_peer_cert" => true
+            ]
+        ]);
+        $sslSocket = stream_socket_client("ssl://" . $domain . ":443", $errno, $errstr, 30, STREAM_CLIENT_CONNECT, $sslContext);
+        if (!$sslSocket) {
+            throw new Exception(Exception::HEALTH_INVALID_HOST);
+        }
+
+        $streamContextParams = stream_context_get_params($sslSocket);
+        $peerCertificate = $streamContextParams['options']['ssl']['peer_certificate'];
+        $certificatePayload = openssl_x509_parse($peerCertificate);
+
+
+        $sslExpiration = $certificatePayload['validTo_time_t'];
+        $status = $sslExpiration < time() ? 'fail' : 'pass';
+
+        if ($status === 'fail') {
+            throw new Exception(Exception::HEALTH_CERTIFICATE_EXPIRED);
+        }
+
+        $response->dynamic(new Document([
+            'name' => $certificatePayload['name'],
+            'subjectSN' => $certificatePayload['subject']['CN'],
+            'issuerOrganisation' => $certificatePayload['issuer']['O'],
+            'validFrom' => $certificatePayload['validFrom_time_t'],
+            'validTo' => $certificatePayload['validTo_time_t'],
+            'signatureTypeSN' => $certificatePayload['signatureTypeSN'],
+        ]), Response::MODEL_HEALTH_CERTIFICATE);
+    }, ['response']);
+
 App::get('/v1/health/queue/certificates')
     ->desc('Get certificates queue')
     ->groups(['api', 'health'])
@@ -409,7 +463,7 @@ App::get('/v1/health/queue/certificates')
         $size = $client->getQueueSize();
 
         if ($size >= $threshold) {
-            throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
+            throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
         }
 
         $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
@@ -436,7 +490,7 @@ App::get('/v1/health/queue/builds')
         $size = $client->getQueueSize();
 
         if ($size >= $threshold) {
-            throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
+            throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
         }
 
         $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
@@ -464,7 +518,7 @@ App::get('/v1/health/queue/databases')
         $size = $client->getQueueSize();
 
         if ($size >= $threshold) {
-            throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
+            throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
         }
 
         $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
@@ -491,7 +545,7 @@ App::get('/v1/health/queue/deletes')
         $size = $client->getQueueSize();
 
         if ($size >= $threshold) {
-            throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
+            throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
         }
 
         $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
@@ -518,7 +572,7 @@ App::get('/v1/health/queue/mails')
         $size = $client->getQueueSize();
 
         if ($size >= $threshold) {
-            throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
+            throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
         }
 
         $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
@@ -545,7 +599,7 @@ App::get('/v1/health/queue/messaging')
         $size = $client->getQueueSize();
 
         if ($size >= $threshold) {
-            throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
+            throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
         }
 
         $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
@@ -572,7 +626,7 @@ App::get('/v1/health/queue/migrations')
         $size = $client->getQueueSize();
 
         if ($size >= $threshold) {
-            throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
+            throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
         }
 
         $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
@@ -599,7 +653,7 @@ App::get('/v1/health/queue/functions')
         $size = $client->getQueueSize();
 
         if ($size >= $threshold) {
-            throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
+            throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
         }
 
         $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
@@ -687,6 +741,47 @@ App::get('/v1/health/anti-virus')
         $response->dynamic(new Document($output), Response::MODEL_HEALTH_ANTIVIRUS);
     });
 
+App::get('/v1/health/queue/failed/:name')
+    ->desc('Get number of failed queue jobs')
+    ->groups(['api', 'health'])
+    ->label('scope', 'health.read')
+    ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
+    ->label('sdk.namespace', 'health')
+    ->label('sdk.method', 'getFailedJobs')
+    ->param('name', '', new WhiteList([
+        Event::DATABASE_QUEUE_NAME,
+        Event::DELETE_QUEUE_NAME,
+        Event::AUDITS_QUEUE_NAME,
+        Event::MAILS_QUEUE_NAME,
+        Event::FUNCTIONS_QUEUE_NAME,
+        Event::USAGE_QUEUE_NAME,
+        Event::WEBHOOK_CLASS_NAME,
+        Event::CERTIFICATES_QUEUE_NAME,
+        Event::BUILDS_QUEUE_NAME,
+        Event::MESSAGING_QUEUE_NAME,
+        Event::MIGRATIONS_QUEUE_NAME,
+        Event::HAMSTER_CLASS_NAME
+    ]), 'The name of the queue')
+    ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
+    ->label('sdk.description', '/docs/references/health/get-failed-queue-jobs.md')
+    ->label('sdk.response.code', Response::STATUS_CODE_OK)
+    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
+    ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
+    ->inject('response')
+    ->inject('queue')
+    ->action(function (string $name, int|string $threshold, Response $response, Connection $queue) {
+        $threshold = \intval($threshold);
+
+        $client = new Client($name, $queue);
+        $failed = $client->countFailedJobs();
+
+        if ($failed >= $threshold) {
+            throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue failed jobs threshold hit. Current size is {$failed} and threshold is {$threshold}.");
+        }
+
+        $response->dynamic(new Document([ 'size' => $failed ]), Response::MODEL_HEALTH_QUEUE);
+    });
+
 App::get('/v1/health/stats') // Currently only used internally
 ->desc('Get system stats')
     ->groups(['api', 'health'])

app/controllers/api/project.php

@@ -73,7 +73,7 @@ App::get('/v1/project/usage')
 
         Authorization::skip(function () use ($dbForProject, $firstDay, $lastDay, $period, $metrics, &$total, &$stats) {
             foreach ($metrics['total'] as $metric) {
-                $result = $dbForProject->findOne('stats_v2', [
+                $result = $dbForProject->findOne('stats', [
                     Query::equal('metric', [$metric]),
                     Query::equal('period', ['inf'])
                 ]);
@@ -81,7 +81,7 @@ App::get('/v1/project/usage')
             }
 
             foreach ($metrics['period'] as $metric) {
-                $results = $dbForProject->find('stats_v2', [
+                $results = $dbForProject->find('stats', [
                     Query::equal('metric', [$metric]),
                     Query::equal('period', [$period]),
                     Query::greaterThanEqual('time', $firstDay),
@@ -116,7 +116,7 @@ App::get('/v1/project/usage')
             $id = $function->getId();
             $name = $function->getAttribute('name');
             $metric = str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS);
-            $value = $dbForProject->findOne('stats_v2', [
+            $value = $dbForProject->findOne('stats', [
                 Query::equal('metric', [$metric]),
                 Query::equal('period', ['inf'])
             ]);
@@ -132,7 +132,7 @@ App::get('/v1/project/usage')
             $id = $bucket->getId();
             $name = $bucket->getAttribute('name');
             $metric = str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE);
-            $value = $dbForProject->findOne('stats_v2', [
+            $value = $dbForProject->findOne('stats', [
                 Query::equal('metric', [$metric]),
                 Query::equal('period', ['inf'])
             ]);

app/controllers/api/proxy.php

@@ -14,6 +14,7 @@ use Utopia\Database\Helpers\ID;
 use Utopia\Database\Query;
 use Utopia\Database\Validator\UID;
 use Utopia\Domains\Domain;
+use Utopia\Logger\Log;
 use Utopia\Validator\Domain as ValidatorDomain;
 use Utopia\Validator\Text;
 use Utopia\Validator\WhiteList;
@@ -278,7 +279,8 @@ App::patch('/v1/proxy/rules/:ruleId/verification')
     ->inject('queueForEvents')
     ->inject('project')
     ->inject('dbForConsole')
-    ->action(function (string $ruleId, Response $response, Certificate $queueForCertificates, Event $queueForEvents, Document $project, Database $dbForConsole) {
+    ->inject('log')
+    ->action(function (string $ruleId, Response $response, Certificate $queueForCertificates, Event $queueForEvents, Document $project, Database $dbForConsole, Log $log) {
         $rule = $dbForConsole->getDocument('rules', $ruleId);
 
         if ($rule->isEmpty() || $rule->getAttribute('projectInternalId') !== $project->getInternalId()) {
@@ -298,7 +300,14 @@ App::patch('/v1/proxy/rules/:ruleId/verification')
         $validator = new CNAME($target->get()); // Verify Domain with DNS records
         $domain = new Domain($rule->getAttribute('domain', ''));
 
+        $validationStart = \microtime(true);
         if (!$validator->isValid($domain->get())) {
+            $log->addExtra('dnsTiming', \strval(\microtime(true) - $validationStart));
+            $log->addTag('dnsDomain', $domain->get());
+
+            $error = $validator->getLogs();
+            $log->addExtra('dnsResponse', \is_array($error) ? \json_encode($error) : \strval($error));
+
             throw new Exception(Exception::RULE_VERIFICATION_FAILED);
         }
 

app/controllers/api/storage.php

@@ -1467,6 +1467,7 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId')
         if ($deviceDeleted) {
             $queueForDeletes
                 ->setType(DELETE_TYPE_CACHE_BY_RESOURCE)
+                ->setResourceType('bucket/' . $bucket->getId())
                 ->setResource('file/' . $fileId)
             ;
 
@@ -1524,7 +1525,7 @@ App::get('/v1/storage/usage')
         $total = [];
         Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats, &$total) {
             foreach ($metrics as $metric) {
-                $result =  $dbForProject->findOne('stats_v2', [
+                $result =  $dbForProject->findOne('stats', [
                     Query::equal('metric', [$metric]),
                     Query::equal('period', ['inf'])
                 ]);
@@ -1532,7 +1533,7 @@ App::get('/v1/storage/usage')
                 $stats[$metric]['total'] = $result['value'] ?? 0;
                 $limit = $days['limit'];
                 $period = $days['period'];
-                $results = $dbForProject->find('stats_v2', [
+                $results = $dbForProject->find('stats', [
                     Query::equal('metric', [$metric]),
                     Query::equal('period', [$period]),
                     Query::limit($limit),
@@ -1609,7 +1610,7 @@ App::get('/v1/storage/:bucketId/usage')
 
         Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats, &$total) {
             foreach ($metrics as $metric) {
-                $result =  $dbForProject->findOne('stats_v2', [
+                $result =  $dbForProject->findOne('stats', [
                     Query::equal('metric', [$metric]),
                     Query::equal('period', ['inf'])
                 ]);
@@ -1617,7 +1618,7 @@ App::get('/v1/storage/:bucketId/usage')
                 $stats[$metric]['total'] = $result['value'] ?? 0;
                 $limit = $days['limit'];
                 $period = $days['period'];
-                $results = $dbForProject->find('stats_v2', [
+                $results = $dbForProject->find('stats', [
                     Query::equal('metric', [$metric]),
                     Query::equal('period', [$period]),
                     Query::limit($limit),

app/controllers/api/teams.php

@@ -533,8 +533,8 @@ App::post('/v1/teams/:teamId/memberships')
             } catch (Duplicate $th) {
                 throw new Exception(Exception::TEAM_INVITE_ALREADY_EXISTS);
             }
-            $team->setAttribute('total', $team->getAttribute('total', 0) + 1);
-            $team = Authorization::skip(fn() => $dbForProject->updateDocument('teams', $team->getId(), $team));
+
+            Authorization::skip(fn() => $dbForProject->increaseDocumentAttribute('teams', $team->getId(), 'total', 1));
 
             $dbForProject->deleteCachedDocument('users', $invitee->getId());
         } else {
@@ -641,6 +641,7 @@ App::post('/v1/teams/:teamId/memberships')
                 $queueForMessaging
                     ->setRecipient($phone)
                     ->setMessage($message)
+                    ->setProject($project)
                     ->trigger();
             }
         }
@@ -966,7 +967,7 @@ App::patch('/v1/teams/:teamId/memberships/:membershipId/status')
 
         $dbForProject->deleteCachedDocument('users', $user->getId());
 
-        $team = Authorization::skip(fn() => $dbForProject->updateDocument('teams', $team->getId(), $team->setAttribute('total', $team->getAttribute('total', 0) + 1)));
+        Authorization::skip(fn() => $dbForProject->increaseDocumentAttribute('teams', $team->getId(), 'total', 1));
 
         $queueForEvents
             ->setParam('teamId', $team->getId())
@@ -1046,8 +1047,7 @@ App::delete('/v1/teams/:teamId/memberships/:membershipId')
         $dbForProject->deleteCachedDocument('users', $user->getId());
 
         if ($membership->getAttribute('confirm')) { // Count only confirmed members
-            $team->setAttribute('total', \max($team->getAttribute('total', 0) - 1, 0));
-            Authorization::skip(fn() => $dbForProject->updateDocument('teams', $team->getId(), $team));
+            Authorization::skip(fn() => $dbForProject->decreaseDocumentAttribute('teams', $team->getId(), 'total', 1, 0));
         }
 
         $queueForEvents

app/controllers/api/users.php

@@ -9,6 +9,7 @@ use Appwrite\Event\Event;
 use Appwrite\Network\Validator\Email;
 use Appwrite\Utopia\Database\Validator\CustomId;
 use Appwrite\Utopia\Database\Validator\Queries\Identities;
+use Utopia\CLI\Console;
 use Utopia\Database\Validator\Queries;
 use Appwrite\Utopia\Database\Validator\Queries\Users;
 use Utopia\Database\Validator\Query\Limit;
@@ -141,7 +142,6 @@ App::post('/v1/users')
     ->inject('hooks')
     ->action(function (string $userId, ?string $email, ?string $phone, ?string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks) {
         $user = createUser('plaintext', '{}', $userId, $email, $password, $phone, $name, $project, $dbForProject, $queueForEvents, $hooks);
-
         $response
             ->setStatusCode(Response::STATUS_CODE_CREATED)
             ->dynamic($user, Response::MODEL_USER);
@@ -615,6 +615,9 @@ App::get('/v1/users/:userId/logs')
 
             $output[$i] = new Document([
                 'event' => $log['event'],
+                'userId' => ID::custom($log['data']['userId']),
+                'userEmail' => $log['data']['userEmail'] ?? null,
+                'userName' => $log['data']['userName'] ?? null,
                 'ip' => $log['ip'],
                 'time' => $log['time'],
                 'osCode' => $os['osCode'],
@@ -643,7 +646,7 @@ App::get('/v1/users/:userId/logs')
         }
 
         $response->dynamic(new Document([
-            'total' => $audit->countLogsByUser($user->getId()),
+            'total' => $audit->countLogsByUser($user->getInternalId()),
             'logs' => $output,
         ]), Response::MODEL_LOG_LIST);
     });
@@ -1263,7 +1266,7 @@ App::get('/v1/users/usage')
 
         Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
             foreach ($metrics as $count => $metric) {
-                $result =  $dbForProject->findOne('stats_v2', [
+                $result =  $dbForProject->findOne('stats', [
                     Query::equal('metric', [$metric]),
                     Query::equal('period', ['inf'])
                 ]);
@@ -1271,7 +1274,7 @@ App::get('/v1/users/usage')
                 $stats[$metric]['total'] = $result['value'] ?? 0;
                 $limit = $days['limit'];
                 $period = $days['period'];
-                $results = $dbForProject->find('stats_v2', [
+                $results = $dbForProject->find('stats', [
                     Query::equal('metric', [$metric]),
                     Query::equal('period', [$period]),
                     Query::limit($limit),

app/controllers/general.php

@@ -603,66 +603,58 @@ App::error()
     ->inject('response')
     ->inject('project')
     ->inject('logger')
-    ->inject('loggerBreadcrumbs')
-    ->action(function (Throwable $error, App $utopia, Request $request, Response $response, Document $project, ?Logger $logger, array $loggerBreadcrumbs) {
-
+    ->inject('log')
+    ->action(function (Throwable $error, App $utopia, Request $request, Response $response, Document $project, ?Logger $logger, Log $log) {
         $version = App::getEnv('_APP_VERSION', 'UNKNOWN');
         $route = $utopia->getRoute();
-        $publish = true;
 
         if ($error instanceof AppwriteException) {
             $publish = $error->isPublishable();
+        } else {
+            $publish = $error->getCode() === 0 || $error->getCode() >= 500;
         }
 
-        if ($logger && $publish) {
-            if ($error->getCode() >= 500 || $error->getCode() === 0) {
-                try {
-                    /** @var Utopia\Database\Document $user */
-                    $user = $utopia->getResource('user');
-                } catch (\Throwable $th) {
-                    // All good, user is optional information for logger
-                }
-
-                $log = new Utopia\Logger\Log();
-
-                if (isset($user) && !$user->isEmpty()) {
-                    $log->setUser(new User($user->getId()));
-                }
-
-                $log->setNamespace("http");
-                $log->setServer(\gethostname());
-                $log->setVersion($version);
-                $log->setType(Log::TYPE_ERROR);
-                $log->setMessage($error->getMessage());
-
-                $log->addTag('database', $project->getAttribute('database', 'console'));
-                $log->addTag('method', $route->getMethod());
-                $log->addTag('url', $route->getPath());
-                $log->addTag('verboseType', get_class($error));
-                $log->addTag('code', $error->getCode());
-                $log->addTag('projectId', $project->getId());
-                $log->addTag('hostname', $request->getHostname());
-                $log->addTag('locale', (string)$request->getParam('locale', $request->getHeader('x-appwrite-locale', '')));
-
-                $log->addExtra('file', $error->getFile());
-                $log->addExtra('line', $error->getLine());
-                $log->addExtra('trace', $error->getTraceAsString());
-                $log->addExtra('detailedTrace', $error->getTrace());
-                $log->addExtra('roles', Authorization::getRoles());
-
-                $action = $route->getLabel("sdk.namespace", "UNKNOWN_NAMESPACE") . '.' . $route->getLabel("sdk.method", "UNKNOWN_METHOD");
-                $log->setAction($action);
-
-                $isProduction = App::getEnv('_APP_ENV', 'development') === 'production';
-                $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING);
-
-                foreach ($loggerBreadcrumbs as $loggerBreadcrumb) {
-                    $log->addBreadcrumb($loggerBreadcrumb);
-                }
-
-                $responseCode = $logger->addLog($log);
-                Console::info('Log pushed with status code: ' . $responseCode);
+        if ($logger && ($publish || $error->getCode() === 0)) {
+            try {
+                /** @var Utopia\Database\Document $user */
+                $user = $utopia->getResource('user');
+            } catch (\Throwable $th) {
+                // All good, user is optional information for logger
             }
+
+            if (isset($user) && !$user->isEmpty()) {
+                $log->setUser(new User($user->getId()));
+            }
+
+            $log->setNamespace("http");
+            $log->setServer(\gethostname());
+            $log->setVersion($version);
+            $log->setType(Log::TYPE_ERROR);
+            $log->setMessage($error->getMessage());
+
+            $log->addTag('database', $project->getAttribute('database', 'console'));
+            $log->addTag('method', $route->getMethod());
+            $log->addTag('url', $route->getPath());
+            $log->addTag('verboseType', get_class($error));
+            $log->addTag('code', $error->getCode());
+            $log->addTag('projectId', $project->getId());
+            $log->addTag('hostname', $request->getHostname());
+            $log->addTag('locale', (string)$request->getParam('locale', $request->getHeader('x-appwrite-locale', '')));
+
+            $log->addExtra('file', $error->getFile());
+            $log->addExtra('line', $error->getLine());
+            $log->addExtra('trace', $error->getTraceAsString());
+            $log->addExtra('detailedTrace', $error->getTrace());
+            $log->addExtra('roles', Authorization::getRoles());
+
+            $action = $route->getLabel("sdk.namespace", "UNKNOWN_NAMESPACE") . '.' . $route->getLabel("sdk.method", "UNKNOWN_METHOD");
+            $log->setAction($action);
+
+            $isProduction = App::getEnv('_APP_ENV', 'development') === 'production';
+            $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING);
+
+            $responseCode = $logger->addLog($log);
+            Console::info('Log pushed with status code: ' . $responseCode);
         }
 
         $code = $error->getCode();

app/controllers/shared/api.php

@@ -177,6 +177,7 @@ App::init()
             $end = $request->getContentRangeEnd();
             $timeLimit = new TimeLimit($abuseKey, $route->getLabel('abuse-limit', 0), $route->getLabel('abuse-time', 3600), $dbForProject);
             $timeLimit
+                ->setParam('{projectId}', $project->getId())
                 ->setParam('{userId}', $user->getId())
                 ->setParam('{userAgent}', $request->getUserAgent(''))
                 ->setParam('{ip}', $request->getIP())
@@ -251,24 +252,22 @@ App::init()
         ;
 
         $useCache = $route->getLabel('cache', false);
-
         if ($useCache) {
             $key = md5($request->getURI() . implode('*', $request->getParams()) . '*' . APP_CACHE_BUSTER);
+            $cacheLog  = Authorization::skip(fn () => $dbForProject->getDocument('cache', $key));
             $cache = new Cache(
                 new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $project->getId())
             );
             $timestamp = 60 * 60 * 24 * 30;
             $data = $cache->load($key, $timestamp);
 
-            if (!empty($data)) {
-                $data = json_decode($data, true);
-                $parts = explode('/', $data['resourceType']);
+            if (!empty($data) && !$cacheLog->isEmpty()) {
+                $parts = explode('/', $cacheLog->getAttribute('resourceType'));
                 $type = $parts[0] ?? null;
 
                 if ($type === 'bucket') {
                     $bucketId = $parts[1] ?? null;
-
-                    $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId));
+                    $bucket   = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId));
 
                     $isAPIKey = Auth::isAppUser(Authorization::getRoles());
                     $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles());
@@ -280,11 +279,12 @@ App::init()
                     $fileSecurity = $bucket->getAttribute('fileSecurity', false);
                     $validator = new Authorization(Database::PERMISSION_READ);
                     $valid = $validator->isValid($bucket->getRead());
+
                     if (!$fileSecurity && !$valid) {
                         throw new Exception(Exception::USER_UNAUTHORIZED);
                     }
 
-                    $parts = explode('/', $data['resource']);
+                    $parts = explode('/', $cacheLog->getAttribute('resource'));
                     $fileId = $parts[1] ?? null;
 
                     if ($fileSecurity && !$valid) {
@@ -301,8 +301,8 @@ App::init()
                 $response
                     ->addHeader('Expires', \date('D, d M Y H:i:s', \time() + $timestamp) . ' GMT')
                     ->addHeader('X-Appwrite-Cache', 'hit')
-                    ->setContentType($data['contentType'])
-                    ->send(base64_decode($data['payload']))
+                    ->setContentType($cacheLog->getAttribute('mimeType'))
+                    ->send($data)
                 ;
             } else {
                 $response->addHeader('X-Appwrite-Cache', 'miss');
@@ -335,7 +335,7 @@ App::init()
                 break;
 
             case 'magic-url':
-                if ($project->getAttribute('usersAuthMagicURL', true) === false) {
+                if (($auths['usersAuthMagicURL'] ?? true) === false) {
                     throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Magic URL authentication is disabled for this project');
                 }
                 break;
@@ -346,6 +346,12 @@ App::init()
                 }
                 break;
 
+            case 'phone':
+                if (($auths['phone'] ?? true) === false) {
+                    throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Phone authentication is disabled for this project');
+                }
+                break;
+
             case 'invites':
                 if (($auths['invites'] ?? true) === false) {
                     throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Invites authentication is disabled for this project');
@@ -359,7 +365,7 @@ App::init()
                 break;
 
             default:
-                throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Unsupported authentication route');
+                throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Unsupported authentication type: ' . $route->getLabel('auth.type', ''));
                 break;
         }
     });
@@ -527,7 +533,6 @@ App::shutdown()
         if ($useCache) {
             $resource = $resourceType = null;
             $data = $response->getPayload();
-
             if (!empty($data['payload'])) {
                 $pattern = $route->getLabel('cache.resource', null);
                 if (!empty($pattern)) {
@@ -539,22 +544,17 @@ App::shutdown()
                     $resourceType = $parseLabel($pattern, $responsePayload, $requestParams, $user);
                 }
 
-                $key = md5($request->getURI() . implode('*', $request->getParams()) . '*' . APP_CACHE_BUSTER);
-                $data = json_encode([
-                    'resourceType' => $resourceType,
-                    'resource' => $resource,
-                    'contentType' => $response->getContentType(),
-                    'payload' => base64_encode($data['payload']),
-                ]) ;
-
-                $signature = md5($data);
-                $cacheLog  = Authorization::skip(fn () => $dbForProject->getDocument('cache', $key));
+                $key = md5($request->getURI() . '*' . implode('*', $request->getParams())) . '*' . APP_CACHE_BUSTER;
+                $signature = md5($data['payload']);
+                $cacheLog  =  Authorization::skip(fn () => $dbForProject->getDocument('cache', $key));
                 $accessedAt = $cacheLog->getAttribute('accessedAt', '');
                 $now = DateTime::now();
                 if ($cacheLog->isEmpty()) {
                     Authorization::skip(fn () => $dbForProject->createDocument('cache', new Document([
                     '$id' => $key,
                     'resource' => $resource,
+                    'resourceType' => $resourceType,
+                    'mimeType' => $response->getContentType(),
                     'accessedAt' => $now,
                     'signature' => $signature,
                     ])));
@@ -567,7 +567,7 @@ App::shutdown()
                     $cache = new Cache(
                         new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $project->getId())
                     );
-                    $cache->save($key, $data);
+                    $cache->save($key, $data['payload']);
                 }
             }
         }

app/controllers/shared/api/auth.php

@@ -32,7 +32,7 @@ App::init()
                 break;
 
             case 'magic-url':
-                if ($project->getAttribute('usersAuthMagicURL', true) === false) {
+                if (($auths['usersAuthMagicURL'] ?? true) === false) {
                     throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Magic URL authentication is disabled for this project');
                 }
                 break;
@@ -43,6 +43,12 @@ App::init()
                 }
                 break;
 
+            case 'phone':
+                if (($auths['phone'] ?? true) === false) {
+                    throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Phone authentication is disabled for this project');
+                }
+                break;
+
             case 'invites':
                 if (($auths['invites'] ?? true) === false) {
                     throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Invites authentication is disabled for this project');

app/http.php

@@ -263,10 +263,9 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo
                 // All good, user is optional information for logger
             }
 
-            $loggerBreadcrumbs = $app->getResource("loggerBreadcrumbs");
             $route = $app->getRoute();
 
-            $log = new Utopia\Logger\Log();
+            $log = $app->getResource("log");
 
             if (isset($user) && !$user->isEmpty()) {
                 $log->setUser(new User($user->getId()));
@@ -298,10 +297,6 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo
             $isProduction = App::getEnv('_APP_ENV', 'development') === 'production';
             $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING);
 
-            foreach ($loggerBreadcrumbs as $loggerBreadcrumb) {
-                $log->addBreadcrumb($loggerBreadcrumb);
-            }
-
             $responseCode = $logger->addLog($log);
             Console::info('Log pushed with status code: ' . $responseCode);
         }

app/init.php

@@ -76,6 +76,7 @@ use Appwrite\Hooks\Hooks;
 use MaxMind\Db\Reader;
 use PHPMailer\PHPMailer\PHPMailer;
 use Swoole\Database\PDOProxy;
+use Utopia\Logger\Log;
 use Utopia\Queue;
 use Utopia\Queue\Connection;
 use Utopia\Storage\Storage;
@@ -111,7 +112,7 @@ const APP_LIMIT_LIST_DEFAULT = 25; // Default maximum number of items to return
 const APP_KEY_ACCCESS = 24 * 60 * 60; // 24 hours
 const APP_USER_ACCCESS = 24 * 60 * 60; // 24 hours
 const APP_CACHE_UPDATE = 24 * 60 * 60; // 24 hours
-const APP_CACHE_BUSTER = 329;
+const APP_CACHE_BUSTER = 330;
 const APP_VERSION_STABLE = '1.4.13';
 const APP_DATABASE_ATTRIBUTE_EMAIL = 'email';
 const APP_DATABASE_ATTRIBUTE_ENUM = 'enum';
@@ -194,6 +195,7 @@ const FUNCTION_ALLOWLIST_HEADERS_RESPONSE = ['content-type', 'content-length'];
 // Usage metrics
 const METRIC_TEAMS = 'teams';
 const METRIC_USERS = 'users';
+const METRIC_MESSAGES  = 'messages';
 const METRIC_SESSIONS  = 'sessions';
 const METRIC_DATABASES = 'databases';
 const METRIC_COLLECTIONS = 'collections';
@@ -864,6 +866,7 @@ foreach ($locales as $locale) {
 ]);
 
 // Runtime Execution
+App::setResource('log', fn() => new Log());
 App::setResource('logger', function ($register) {
     return $register->get('logger');
 }, ['register']);
@@ -872,10 +875,6 @@ App::setResource('hooks', function ($register) {
     return $register->get('hooks');
 }, ['register']);
 
-App::setResource('loggerBreadcrumbs', function () {
-    return [];
-});
-
 App::setResource('register', fn() => $register);
 App::setResource('locale', fn() => new Locale(App::getEnv('_APP_LOCALE', 'en')));
 

app/views/install/compose.phtml

@@ -143,6 +143,7 @@ services:
       - _APP_LOGGING_PROVIDER
       - _APP_LOGGING_CONFIG
       - _APP_MAINTENANCE_INTERVAL
+      - _APP_MAINTENANCE_DELAY
       - _APP_MAINTENANCE_RETENTION_EXECUTION
       - _APP_MAINTENANCE_RETENTION_CACHE
       - _APP_MAINTENANCE_RETENTION_ABUSE

app/worker.php

@@ -14,6 +14,7 @@ use Appwrite\Event\Mail;
 use Appwrite\Event\Migration;
 use Appwrite\Event\Phone;
 use Appwrite\Event\Usage;
+use Appwrite\Event\UsageDump;
 use Appwrite\Platform\Appwrite;
 use Swoole\Runtime;
 use Utopia\App;
@@ -146,6 +147,9 @@ Server::setResource('log', fn() => new Log());
 Server::setResource('queueForUsage', function (Connection $queue) {
     return new Usage($queue);
 }, ['queue']);
+Server::setResource('queueForUsageDump', function (Connection $queue) {
+    return new UsageDump($queue);
+}, ['queue']);
 Server::setResource('queue', function (Group $pools) {
     return $pools->get('queue')->pop()->getResource();
 }, ['pools']);
@@ -299,12 +303,9 @@ $worker
         Console::error('[Error] Line: ' . $error->getLine());
     });
 
-try {
-    $workerStart = $worker->getWorkerStart();
-} catch (\Throwable $error) {
-    $worker->workerStart();
-} finally {
-    Console::info("Worker $workerName  started");
-}
+$worker->workerStart()
+    ->action(function () use ($workerName) {
+        Console::info("Worker $workerName  started");
+    });
 
 $worker->start();

bin/queue-count-failed

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+php /usr/src/code/app/cli.php queue-count --type=failed $@

bin/queue-count-processing

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+php /usr/src/code/app/cli.php queue-count --type=processing $@

bin/queue-count-success

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+php /usr/src/code/app/cli.php queue-count --type=success $@

bin/queue-retry

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+php /usr/src/code/app/cli.php queue-retry $@

bin/worker-usage-dump

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+php /usr/src/code/app/worker.php usage-dump $@

composer.json

@@ -50,7 +50,7 @@
         "utopia-php/cli": "0.15.*",
         "utopia-php/config": "0.2.*",
         "utopia-php/database": "0.45.*",
-        "utopia-php/domains": "0.3.*",
+        "utopia-php/domains": "0.5.*",
         "utopia-php/dsn": "0.1.*",
         "utopia-php/framework": "0.33.*",
         "utopia-php/image": "0.5.*",
@@ -62,7 +62,7 @@
         "utopia-php/platform": "0.5.*",
         "utopia-php/pools": "0.4.*",
         "utopia-php/preloader": "0.2.*",
-        "utopia-php/queue": "0.6.*",
+        "utopia-php/queue": "0.7.*",
         "utopia-php/registry": "0.5.*",
         "utopia-php/storage": "0.18.*",
         "utopia-php/swoole": "0.5.*",

composer.lock

@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
         "This file is @generated automatically"
     ],
-    "content-hash": "1b3fd261ed93452413b8e6ba77dbab55",
+    "content-hash": "fd03f97115d752d1a94b533ccf570109",
     "packages": [
         {
             "name": "adhocore/jwt",
@@ -815,16 +815,16 @@
         },
         {
             "name": "symfony/polyfill-php80",
-            "version": "v1.28.0",
+            "version": "v1.29.0",
             "source": {
                 "type": "git",
                 "url": "https://github.com/symfony/polyfill-php80.git",
-                "reference": "6caa57379c4aec19c0a12a38b59b26487dcfe4b5"
+                "reference": "87b68208d5c1188808dd7839ee1e6c8ec3b02f1b"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/symfony/polyfill-php80/zipball/6caa57379c4aec19c0a12a38b59b26487dcfe4b5",
-                "reference": "6caa57379c4aec19c0a12a38b59b26487dcfe4b5",
+                "url": "https://api.github.com/repos/symfony/polyfill-php80/zipball/87b68208d5c1188808dd7839ee1e6c8ec3b02f1b",
+                "reference": "87b68208d5c1188808dd7839ee1e6c8ec3b02f1b",
                 "shasum": ""
             },
             "require": {
@@ -832,9 +832,6 @@
             },
             "type": "library",
             "extra": {
-                "branch-alias": {
-                    "dev-main": "1.28-dev"
-                },
                 "thanks": {
                     "name": "symfony/polyfill",
                     "url": "https://github.com/symfony/polyfill"
@@ -878,7 +875,7 @@
                 "shim"
             ],
             "support": {
-                "source": "https://github.com/symfony/polyfill-php80/tree/v1.28.0"
+                "source": "https://github.com/symfony/polyfill-php80/tree/v1.29.0"
             },
             "funding": [
                 {
@@ -894,7 +891,7 @@
                     "type": "tidelift"
                 }
             ],
-            "time": "2023-01-26T09:26:14+00:00"
+            "time": "2024-01-29T20:11:03+00:00"
         },
         {
             "name": "utopia-php/abuse",
@@ -1190,16 +1187,16 @@
         },
         {
             "name": "utopia-php/database",
-            "version": "0.45.5",
+            "version": "0.45.6",
             "source": {
                 "type": "git",
                 "url": "https://github.com/utopia-php/database.git",
-                "reference": "0b66a017f817a910acb83e6aea92bccea9571fe6"
+                "reference": "c7cc6d57683a4c13d9772dbeea343adb72c443fd"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/database/zipball/0b66a017f817a910acb83e6aea92bccea9571fe6",
-                "reference": "0b66a017f817a910acb83e6aea92bccea9571fe6",
+                "url": "https://api.github.com/repos/utopia-php/database/zipball/c7cc6d57683a4c13d9772dbeea343adb72c443fd",
+                "reference": "c7cc6d57683a4c13d9772dbeea343adb72c443fd",
                 "shasum": ""
             },
             "require": {
@@ -1240,22 +1237,22 @@
             ],
             "support": {
                 "issues": "https://github.com/utopia-php/database/issues",
-                "source": "https://github.com/utopia-php/database/tree/0.45.5"
+                "source": "https://github.com/utopia-php/database/tree/0.45.6"
             },
-            "time": "2024-01-08T17:08:15+00:00"
+            "time": "2024-02-01T02:33:43+00:00"
         },
         {
             "name": "utopia-php/domains",
-            "version": "0.3.2",
+            "version": "0.5.0",
             "source": {
                 "type": "git",
                 "url": "https://github.com/utopia-php/domains.git",
-                "reference": "aaa8c9a96c69ccb397997b1f4f2299c66f77eefb"
+                "reference": "bf07f60326f8389f378ddf6fcde86217e5cfe18c"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/domains/zipball/aaa8c9a96c69ccb397997b1f4f2299c66f77eefb",
-                "reference": "aaa8c9a96c69ccb397997b1f4f2299c66f77eefb",
+                "url": "https://api.github.com/repos/utopia-php/domains/zipball/bf07f60326f8389f378ddf6fcde86217e5cfe18c",
+                "reference": "bf07f60326f8389f378ddf6fcde86217e5cfe18c",
                 "shasum": ""
             },
             "require": {
@@ -1300,9 +1297,9 @@
             ],
             "support": {
                 "issues": "https://github.com/utopia-php/domains/issues",
-                "source": "https://github.com/utopia-php/domains/tree/0.3.2"
+                "source": "https://github.com/utopia-php/domains/tree/0.5.0"
             },
-            "time": "2023-07-19T16:39:24+00:00"
+            "time": "2024-01-03T22:04:27+00:00"
         },
         {
             "name": "utopia-php/dsn",
@@ -1353,16 +1350,16 @@
         },
         {
             "name": "utopia-php/framework",
-            "version": "0.33.0",
+            "version": "0.33.2",
             "source": {
                 "type": "git",
                 "url": "https://github.com/utopia-php/http.git",
-                "reference": "e3ff6b933082d57b48e7c4267bb605c0bf2250fd"
+                "reference": "b1423ca3e3b61c6c4c2e619d2cb80672809a19f3"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/http/zipball/e3ff6b933082d57b48e7c4267bb605c0bf2250fd",
-                "reference": "e3ff6b933082d57b48e7c4267bb605c0bf2250fd",
+                "url": "https://api.github.com/repos/utopia-php/http/zipball/b1423ca3e3b61c6c4c2e619d2cb80672809a19f3",
+                "reference": "b1423ca3e3b61c6c4c2e619d2cb80672809a19f3",
                 "shasum": ""
             },
             "require": {
@@ -1392,9 +1389,9 @@
             ],
             "support": {
                 "issues": "https://github.com/utopia-php/http/issues",
-                "source": "https://github.com/utopia-php/http/tree/0.33.0"
+                "source": "https://github.com/utopia-php/http/tree/0.33.2"
             },
-            "time": "2024-01-08T13:30:27+00:00"
+            "time": "2024-01-31T10:35:59+00:00"
         },
         {
             "name": "utopia-php/image",
@@ -1913,16 +1910,16 @@
         },
         {
             "name": "utopia-php/queue",
-            "version": "0.6.0",
+            "version": "0.7.0",
             "source": {
                 "type": "git",
                 "url": "https://github.com/utopia-php/queue.git",
-                "reference": "0120bd21904cb2bee34e4571b1737589ffff0eb1"
+                "reference": "917565256eb94bcab7246f7a746b1a486813761b"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/queue/zipball/0120bd21904cb2bee34e4571b1737589ffff0eb1",
-                "reference": "0120bd21904cb2bee34e4571b1737589ffff0eb1",
+                "url": "https://api.github.com/repos/utopia-php/queue/zipball/917565256eb94bcab7246f7a746b1a486813761b",
+                "reference": "917565256eb94bcab7246f7a746b1a486813761b",
                 "shasum": ""
             },
             "require": {
@@ -1968,9 +1965,9 @@
             ],
             "support": {
                 "issues": "https://github.com/utopia-php/queue/issues",
-                "source": "https://github.com/utopia-php/queue/tree/0.6.0"
+                "source": "https://github.com/utopia-php/queue/tree/0.7.0"
             },
-            "time": "2023-10-16T16:59:45+00:00"
+            "time": "2024-01-17T19:00:43+00:00"
         },
         {
             "name": "utopia-php/registry",
@@ -2420,16 +2417,16 @@
     "packages-dev": [
         {
             "name": "appwrite/sdk-generator",
-            "version": "0.36.0",
+            "version": "0.36.2",
             "source": {
                 "type": "git",
                 "url": "https://github.com/appwrite/sdk-generator.git",
-                "reference": "3a10f1f895ed71120442ff71eb6adec3fd6b4e8a"
+                "reference": "0aa67479d75f0e0cb7b60454031534d7f0abaece"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/3a10f1f895ed71120442ff71eb6adec3fd6b4e8a",
-                "reference": "3a10f1f895ed71120442ff71eb6adec3fd6b4e8a",
+                "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/0aa67479d75f0e0cb7b60454031534d7f0abaece",
+                "reference": "0aa67479d75f0e0cb7b60454031534d7f0abaece",
                 "shasum": ""
             },
             "require": {
@@ -2465,22 +2462,22 @@
             "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms",
             "support": {
                 "issues": "https://github.com/appwrite/sdk-generator/issues",
-                "source": "https://github.com/appwrite/sdk-generator/tree/0.36.0"
+                "source": "https://github.com/appwrite/sdk-generator/tree/0.36.2"
             },
-            "time": "2023-11-20T10:03:06+00:00"
+            "time": "2024-01-19T01:04:35+00:00"
         },
         {
             "name": "doctrine/deprecations",
-            "version": "1.1.2",
+            "version": "1.1.3",
             "source": {
                 "type": "git",
                 "url": "https://github.com/doctrine/deprecations.git",
-                "reference": "4f2d4f2836e7ec4e7a8625e75c6aa916004db931"
+                "reference": "dfbaa3c2d2e9a9df1118213f3b8b0c597bb99fab"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/doctrine/deprecations/zipball/4f2d4f2836e7ec4e7a8625e75c6aa916004db931",
-                "reference": "4f2d4f2836e7ec4e7a8625e75c6aa916004db931",
+                "url": "https://api.github.com/repos/doctrine/deprecations/zipball/dfbaa3c2d2e9a9df1118213f3b8b0c597bb99fab",
+                "reference": "dfbaa3c2d2e9a9df1118213f3b8b0c597bb99fab",
                 "shasum": ""
             },
             "require": {
@@ -2512,9 +2509,9 @@
             "homepage": "https://www.doctrine-project.org/",
             "support": {
                 "issues": "https://github.com/doctrine/deprecations/issues",
-                "source": "https://github.com/doctrine/deprecations/tree/1.1.2"
+                "source": "https://github.com/doctrine/deprecations/tree/1.1.3"
             },
-            "time": "2023-09-27T20:04:15+00:00"
+            "time": "2024-01-30T19:34:25+00:00"
         },
         {
             "name": "doctrine/instantiator",
@@ -4772,16 +4769,16 @@
         },
         {
             "name": "symfony/polyfill-ctype",
-            "version": "v1.28.0",
+            "version": "v1.29.0",
             "source": {
                 "type": "git",
                 "url": "https://github.com/symfony/polyfill-ctype.git",
-                "reference": "ea208ce43cbb04af6867b4fdddb1bdbf84cc28cb"
+                "reference": "ef4d7e442ca910c4764bce785146269b30cb5fc4"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/ea208ce43cbb04af6867b4fdddb1bdbf84cc28cb",
-                "reference": "ea208ce43cbb04af6867b4fdddb1bdbf84cc28cb",
+                "url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/ef4d7e442ca910c4764bce785146269b30cb5fc4",
+                "reference": "ef4d7e442ca910c4764bce785146269b30cb5fc4",
                 "shasum": ""
             },
             "require": {
@@ -4795,9 +4792,6 @@
             },
             "type": "library",
             "extra": {
-                "branch-alias": {
-                    "dev-main": "1.28-dev"
-                },
                 "thanks": {
                     "name": "symfony/polyfill",
                     "url": "https://github.com/symfony/polyfill"
@@ -4834,7 +4828,7 @@
                 "portable"
             ],
             "support": {
-                "source": "https://github.com/symfony/polyfill-ctype/tree/v1.28.0"
+                "source": "https://github.com/symfony/polyfill-ctype/tree/v1.29.0"
             },
             "funding": [
                 {
@@ -4850,20 +4844,20 @@
                     "type": "tidelift"
                 }
             ],
-            "time": "2023-01-26T09:26:14+00:00"
+            "time": "2024-01-29T20:11:03+00:00"
         },
         {
             "name": "symfony/polyfill-mbstring",
-            "version": "v1.28.0",
+            "version": "v1.29.0",
             "source": {
                 "type": "git",
                 "url": "https://github.com/symfony/polyfill-mbstring.git",
-                "reference": "42292d99c55abe617799667f454222c54c60e229"
+                "reference": "9773676c8a1bb1f8d4340a62efe641cf76eda7ec"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/42292d99c55abe617799667f454222c54c60e229",
-                "reference": "42292d99c55abe617799667f454222c54c60e229",
+                "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/9773676c8a1bb1f8d4340a62efe641cf76eda7ec",
+                "reference": "9773676c8a1bb1f8d4340a62efe641cf76eda7ec",
                 "shasum": ""
             },
             "require": {
@@ -4877,9 +4871,6 @@
             },
             "type": "library",
             "extra": {
-                "branch-alias": {
-                    "dev-main": "1.28-dev"
-                },
                 "thanks": {
                     "name": "symfony/polyfill",
                     "url": "https://github.com/symfony/polyfill"
@@ -4917,7 +4908,7 @@
                 "shim"
             ],
             "support": {
-                "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.28.0"
+                "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.29.0"
             },
             "funding": [
                 {
@@ -4933,7 +4924,7 @@
                     "type": "tidelift"
                 }
             ],
-            "time": "2023-07-28T09:04:16+00:00"
+            "time": "2024-01-29T20:11:03+00:00"
         },
         {
             "name": "textalk/websocket",

docker-compose.yml

@@ -571,6 +571,7 @@ services:
       - _APP_REDIS_PASS
       - _APP_SMS_PROVIDER
       - _APP_SMS_FROM
+      - _APP_SMS_PROJECTS_DENY_LIST
       - _APP_LOGGING_PROVIDER
       - _APP_LOGGING_CONFIG
 
@@ -644,6 +645,7 @@ services:
       - _APP_MAINTENANCE_RETENTION_AUDIT
       - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY
       - _APP_MAINTENANCE_RETENTION_SCHEDULES
+      - _APP_MAINTENANCE_DELAY
 
   appwrite-worker-usage:
     entrypoint: worker-usage
@@ -676,6 +678,38 @@ services:
       - _APP_LOGGING_CONFIG
       - _APP_USAGE_AGGREGATION_INTERVAL
 
+  appwrite-worker-usage-dump:
+    entrypoint: worker-usage-dump
+    <<: *x-logging
+    container_name: appwrite-worker-usage-dump
+    image: appwrite-dev
+    networks:
+      - appwrite
+    volumes:
+      - ./app:/usr/src/code/app
+      - ./src:/usr/src/code/src
+    depends_on:
+      - redis
+      - mariadb
+    environment:
+      - _APP_ENV
+      - _APP_WORKER_PER_CORE
+      - _APP_OPENSSL_KEY_V1
+      - _APP_DB_HOST
+      - _APP_DB_PORT
+      - _APP_DB_SCHEMA
+      - _APP_DB_USER
+      - _APP_DB_PASS
+      - _APP_REDIS_HOST
+      - _APP_REDIS_PORT
+      - _APP_REDIS_USER
+      - _APP_REDIS_PASS
+      - _APP_USAGE_STATS
+      - _APP_LOGGING_PROVIDER
+      - _APP_LOGGING_CONFIG
+      - _APP_USAGE_AGGREGATION_INTERVAL
+
+
   appwrite-schedule:
     entrypoint: schedule
     <<: *x-logging

docs/references/health/get-certificate.md

@@ -0,0 +1 @@
+Get the SSL certificate for a domain

docs/references/health/get-failed-queue-jobs.md

@@ -0,0 +1 @@
+Returns the amount of failed jobs in a given queue.

src/Appwrite/Event/Delete.php

@@ -10,6 +10,7 @@ class Delete extends Event
 {
     protected string $type = '';
     protected ?Document $document = null;
+    protected ?string $resourceType = null;
     protected ?string $resource = null;
     protected ?string $datetime = null;
     protected ?string $hourlyUsageRetentionDatetime = null;
@@ -107,6 +108,19 @@ class Delete extends Event
         return $this;
     }
 
+    /**
+     * Sets the resource type for the delete event.
+     *
+     * @param string $resourceType
+     * @return self
+     */
+    public function setResourceType(string $resourceType): self
+    {
+        $this->resourceType = $resourceType;
+
+        return $this;
+    }
+
     /**
      * Returns the set document for the delete event.
      *
@@ -133,6 +147,7 @@ class Delete extends Event
             'type' => $this->type,
             'document' => $this->document,
             'resource' => $this->resource,
+            'resourceType' => $this->resourceType,
             'datetime' => $this->datetime,
             'hourlyUsageRetentionDatetime' => $this->hourlyUsageRetentionDatetime
         ]);

src/Appwrite/Event/Event.php

@@ -27,6 +27,9 @@ class Event
     public const USAGE_QUEUE_NAME = 'v1-usage';
     public const USAGE_CLASS_NAME = 'UsageV1';
 
+    public const USAGE_DUMP_QUEUE_NAME = 'v1-usage-dump';
+    public const USAGE_DUMP_CLASS_NAME = 'UsageDumpV1';
+
     public const WEBHOOK_QUEUE_NAME = 'v1-webhooks';
     public const WEBHOOK_CLASS_NAME = 'WebhooksV1';
 

src/Appwrite/Event/Usage.php

@@ -2,6 +2,7 @@
 
 namespace Appwrite\Event;
 
+use Utopia\CLI\Console;
 use Utopia\Queue\Client;
 use Utopia\Queue\Connection;
 use Utopia\Database\Document;

src/Appwrite/Event/UsageDump.php

@@ -0,0 +1,47 @@
+<?php
+
+namespace Appwrite\Event;
+
+use Utopia\Queue\Client;
+use Utopia\Queue\Connection;
+
+class UsageDump extends Event
+{
+    protected array $stats;
+
+    public function __construct(protected Connection $connection)
+    {
+        parent::__construct($connection);
+
+        $this
+            ->setQueue(Event::USAGE_DUMP_QUEUE_NAME)
+            ->setClass(Event::USAGE_DUMP_CLASS_NAME);
+    }
+
+    /**
+     * Add Stats.
+     *
+     * @param array $stats
+     * @return self
+     */
+    public function setStats(array $stats): self
+    {
+        $this->stats = $stats;
+
+        return $this;
+    }
+
+    /**
+     * Sends metrics to the usage worker.
+     *
+     * @return string|bool
+     */
+    public function trigger(): string|bool
+    {
+        $client = new Client($this->queue, $this->connection);
+
+        return $client->enqueue([
+            'stats' => $this->stats,
+        ]);
+    }
+}

src/Appwrite/Extend/Exception.php

@@ -84,8 +84,9 @@ class Exception extends \Exception
     public const USER_OAUTH2_BAD_REQUEST           = 'user_oauth2_bad_request';
     public const USER_OAUTH2_UNAUTHORIZED          = 'user_oauth2_unauthorized';
     public const USER_OAUTH2_PROVIDER_ERROR        = 'user_oauth2_provider_error';
-    public const USER_EMAIL_ALREADY_VERIFIED        = 'user_email_alread_verified';
-    public const USER_PHONE_ALREADY_VERIFIED        = 'user_phone_already_verified';
+    public const USER_EMAIL_ALREADY_VERIFIED       = 'user_email_alread_verified';
+    public const USER_PHONE_ALREADY_VERIFIED       = 'user_phone_already_verified';
+    public const USER_DELETION_PROHIBITED          = 'user_deletion_prohibited';
 
     /** Teams */
     public const TEAM_NOT_FOUND                    = 'team_not_found';
@@ -234,25 +235,22 @@ class Exception extends \Exception
     public const REALTIME_POLICY_VIOLATION          = 'realtime_policy_violation';
 
     /** Health */
-    public const QUEUE_SIZE_EXCEEDED                 = 'queue_size_exceeded';
+    public const HEALTH_QUEUE_SIZE_EXCEEDED          = 'health_queue_size_exceeded';
+    public const HEALTH_CERTIFICATE_EXPIRED          = 'health_certificate_expired';
+    public const HEALTH_INVALID_HOST                 = 'health_invalid_host';
 
     protected string $type = '';
     protected array $errors = [];
-    protected bool $publish = true;
+    protected bool $publish;
 
     public function __construct(string $type = Exception::GENERAL_UNKNOWN, string $message = null, int $code = null, \Throwable $previous = null)
     {
         $this->errors = Config::getParam('errors');
         $this->type = $type;
+        $this->code = $code ?? $this->errors[$type]['code'];
+        $this->message = $message ?? $this->errors[$type]['description'];
 
-        if (isset($this->errors[$type])) {
-            $this->code = $this->errors[$type]['code'];
-            $this->message = $this->errors[$type]['description'];
-            $this->publish = $this->errors[$type]['publish'] ?? true;
-        }
-
-        $this->message = $message ?? $this->message;
-        $this->code = $code ?? $this->code;
+        $this->publish = $this->errors[$type]['publish'] ?? ($this->code >= 500);
 
         parent::__construct($this->message, $this->code, $previous);
     }

src/Appwrite/Network/Validator/CNAME.php

@@ -6,6 +6,11 @@ use Utopia\Validator;
 
 class CNAME extends Validator
 {
+    /**
+     * @var mixed
+     */
+    protected mixed $logs;
+
     /**
      * @var string
      */
@@ -27,6 +32,14 @@ class CNAME extends Validator
         return 'Invalid CNAME record';
     }
 
+    /**
+     * @return mixed
+     */
+    public function getLogs(): mixed
+    {
+        return $this->logs;
+    }
+
     /**
      * Check if CNAME record target value matches selected target
      *
@@ -42,6 +55,7 @@ class CNAME extends Validator
 
         try {
             $records = \dns_get_record($domain, DNS_CNAME);
+            $this->logs = $records;
         } catch (\Throwable $th) {
             return false;
         }

src/Appwrite/Platform/Services/Tasks.php

@@ -20,7 +20,9 @@ use Appwrite\Platform\Tasks\Upgrade;
 use Appwrite\Platform\Tasks\DeleteOrphanedProjects;
 use Appwrite\Platform\Tasks\GetMigrationStats;
 use Appwrite\Platform\Tasks\PatchRecreateRepositoriesDocuments;
+use Appwrite\Platform\Tasks\QueueRetry;
 use Appwrite\Platform\Tasks\CreateInfMetric;
+use Appwrite\Platform\Tasks\QueueCount;
 
 class Tasks extends Service
 {
@@ -45,8 +47,9 @@ class Tasks extends Service
             ->addAction(DeleteOrphanedProjects::getName(), new DeleteOrphanedProjects())
             ->addAction(PatchRecreateRepositoriesDocuments::getName(), new PatchRecreateRepositoriesDocuments())
             ->addAction(GetMigrationStats::getName(), new GetMigrationStats())
+            ->addAction(QueueRetry::getName(), new QueueRetry())
+            ->addAction(QueueCount::getName(), new QueueCount())
             ->addAction(CreateInfMetric::getName(), new CreateInfMetric())
-
         ;
     }
 }

src/Appwrite/Platform/Services/Workers.php

@@ -14,7 +14,7 @@ use Appwrite\Platform\Workers\Builds;
 use Appwrite\Platform\Workers\Deletes;
 use Appwrite\Platform\Workers\Hamster;
 use Appwrite\Platform\Workers\Usage;
-use Appwrite\Platform\Workers\UsageHook;
+use Appwrite\Platform\Workers\UsageDump;
 use Appwrite\Platform\Workers\Migrations;
 
 class Workers extends Service
@@ -33,7 +33,7 @@ class Workers extends Service
             ->addAction(Builds::getName(), new Builds())
             ->addAction(Deletes::getName(), new Deletes())
             ->addAction(Hamster::getName(), new Hamster())
-            ->addAction(UsageHook::getName(), new UsageHook())
+            ->addAction(UsageDump::getName(), new UsageDump())
             ->addAction(Usage::getName(), new Usage())
             ->addAction(Migrations::getName(), new Migrations())
 

src/Appwrite/Platform/Tasks/CalcTierStats.php

@@ -270,7 +270,7 @@ class CalcTierStats extends Action
                 $limit = $periods[$range]['limit'];
                 $period = $periods[$range]['period'];
 
-                $requestDocs = $dbForProject->find('stats_v2', [
+                $requestDocs = $dbForProject->find('stats', [
                     Query::equal('metric', [$metric]),
                     Query::equal('period', [$period]),
                     Query::limit($limit),

src/Appwrite/Platform/Tasks/CreateInfMetric.php

@@ -167,8 +167,8 @@ class CreateInfMetric extends Action
 
         try {
             $id = \md5("_inf_{$metric}");
-            $dbForProject->deleteDocument('stats_v2', $id);
-            $dbForProject->createDocument('stats_v2', new Document([
+            $dbForProject->deleteDocument('stats', $id);
+            $dbForProject->createDocument('stats', new Document([
                 '$id' => $id,
                 'metric' => $metric,
                 'period' => 'inf',
@@ -190,7 +190,7 @@ class CreateInfMetric extends Action
     protected function getFromMetric(database $dbForProject, string $metric): int|float
     {
 
-        return  $dbForProject->sum('stats_v2', 'value', [
+        return  $dbForProject->sum('stats', 'value', [
             Query::equal('metric', [
                 $metric,
             ]),

src/Appwrite/Platform/Tasks/Maintenance.php

@@ -36,6 +36,7 @@ class Maintenance extends Action
 
         // # of days in seconds (1 day = 86400s)
         $interval = (int) App::getEnv('_APP_MAINTENANCE_INTERVAL', '86400');
+        $delay = (int) App::getEnv('_APP_MAINTENANCE_DELAY', '0');
         $usageStatsRetentionHourly = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_HOURLY', '8640000'); //100 days
         $cacheRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_CACHE', '2592000'); // 30 days
         $schedulesDeletionRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_SCHEDULES', '86400'); // 1 Day
@@ -59,7 +60,7 @@ class Maintenance extends Action
             $this->renewCertificates($dbForConsole, $queueForCertificates);
             $this->notifyDeleteCache($cacheRetention, $queueForDeletes);
             $this->notifyDeleteSchedules($schedulesDeletionRetention, $queueForDeletes);
-        }, $interval);
+        }, $interval, $delay);
     }
 
     protected function foreachProject(Database $dbForConsole, callable $callback): void

src/Appwrite/Platform/Tasks/QueueCount.php

@@ -0,0 +1,70 @@
+<?php
+
+namespace Appwrite\Platform\Tasks;
+
+use Appwrite\Event\Event;
+use Utopia\CLI\Console;
+use Utopia\Platform\Action;
+use Utopia\Queue\Client;
+use Utopia\Queue\Connection;
+use Utopia\Validator\WhiteList;
+
+class QueueCount extends Action
+{
+    public static function getName(): string
+    {
+        return 'queue-count';
+    }
+
+
+    public function __construct()
+    {
+        $this
+            ->desc('Return the number of from a specific queue identified by the name parameter with a specific type')
+            ->param('name', '', new WhiteList([
+                Event::DATABASE_QUEUE_NAME,
+                Event::DELETE_QUEUE_NAME,
+                Event::AUDITS_QUEUE_NAME,
+                Event::MAILS_QUEUE_NAME,
+                Event::FUNCTIONS_QUEUE_NAME,
+                Event::USAGE_QUEUE_NAME,
+                Event::WEBHOOK_QUEUE_NAME,
+                Event::CERTIFICATES_QUEUE_NAME,
+                Event::BUILDS_QUEUE_NAME,
+                Event::MESSAGING_QUEUE_NAME,
+                Event::MIGRATIONS_QUEUE_NAME,
+                Event::HAMSTER_QUEUE_NAME
+            ]), 'Queue name')
+            ->param('type', '', new WhiteList([
+                'success',
+                'failed',
+                'processing',
+            ]), 'Queue type')
+            ->inject('queue')
+            ->callback(fn ($name, $type, $queue) => $this->action($name, $type, $queue));
+    }
+
+    /**
+     * @param string $name The name of the queue to count the jobs from
+     * @param string $type The type of jobs to count
+     * @param Connection $queue
+     */
+    public function action(string $name, string $type, Connection $queue): void
+    {
+        if (!$name) {
+            Console::error('Missing required parameter $name');
+            return;
+        }
+
+        $queueClient = new Client($name, $queue);
+
+        $count = match ($type) {
+            'success' => $queueClient->countSuccessfulJobs(),
+            'failed' => $queueClient->countFailedJobs(),
+            'processing' => $queueClient->countProcessingJobs(),
+            default => 0
+        };
+
+        Console::log("Queue: '{$name}' has {$count} {$type} jobs.");
+    }
+}

src/Appwrite/Platform/Tasks/QueueRetry.php

@@ -0,0 +1,64 @@
+<?php
+
+namespace Appwrite\Platform\Tasks;
+
+use Appwrite\Event\Event;
+use Utopia\CLI\Console;
+use Utopia\Platform\Action;
+use Utopia\Queue\Client;
+use Utopia\Queue\Connection;
+use Utopia\Validator\WhiteList;
+
+class QueueRetry extends Action
+{
+    public static function getName(): string
+    {
+        return 'queue-retry';
+    }
+
+
+    public function __construct()
+    {
+        $this
+            ->desc('Retry failed jobs from a specific queue identified by the name parameter')
+            ->param('name', '', new WhiteList([
+                Event::DATABASE_QUEUE_NAME,
+                Event::DELETE_QUEUE_NAME,
+                Event::AUDITS_QUEUE_NAME,
+                Event::MAILS_QUEUE_NAME,
+                Event::FUNCTIONS_QUEUE_NAME,
+                Event::USAGE_QUEUE_NAME,
+                Event::WEBHOOK_CLASS_NAME,
+                Event::CERTIFICATES_QUEUE_NAME,
+                Event::BUILDS_QUEUE_NAME,
+                Event::MESSAGING_QUEUE_NAME,
+                Event::MIGRATIONS_QUEUE_NAME,
+                Event::HAMSTER_CLASS_NAME
+            ]), 'Queue name')
+            ->inject('queue')
+            ->callback(fn ($name, $queue) => $this->action($name, $queue));
+    }
+
+    /**
+     * @param string $name The name of the queue to retry jobs from
+     * @param Connection $queue
+     */
+    public function action(string $name, Connection $queue): void
+    {
+        if (!$name) {
+            Console::error('Missing required parameter $name');
+            return;
+        }
+
+        $queueClient = new Client($name, $queue);
+
+        if ($queueClient->countFailedJobs() === 0) {
+            Console::error('No failed jobs found.');
+            return;
+        }
+
+        Console::log('Retrying failed jobs...');
+
+        $queueClient->retry();
+    }
+}

src/Appwrite/Platform/Tasks/SSL.php

@@ -7,6 +7,7 @@ use Appwrite\Event\Certificate;
 use Utopia\App;
 use Utopia\CLI\Console;
 use Utopia\Database\Document;
+use Utopia\Validator\Boolean;
 use Utopia\Validator\Hostname;
 
 class SSL extends Action
@@ -21,19 +22,22 @@ class SSL extends Action
         $this
             ->desc('Validate server certificates')
             ->param('domain', App::getEnv('_APP_DOMAIN', ''), new Hostname(), 'Domain to generate certificate for. If empty, main domain will be used.', true)
+            ->param('skip-check', true, new Boolean(true), 'If DNS and renew check should be skipped. Defaults to true, and when true, all jobs will result in certificate generation attempt.', true)
             ->inject('queueForCertificates')
-            ->callback(fn (string $domain, Certificate $queueForCertificates) => $this->action($domain, $queueForCertificates));
+            ->callback(fn (string $domain, bool|string $skipCheck, Certificate $queueForCertificates) => $this->action($domain, $skipCheck, $queueForCertificates));
     }
 
-    public function action(string $domain, Certificate $queueForCertificates): void
+    public function action(string $domain, bool|string $skipCheck, Certificate $queueForCertificates): void
     {
+        $skipCheck = \strval($skipCheck) === 'true';
+
         Console::success('Scheduling a job to issue a TLS certificate for domain: ' . $domain);
 
         $queueForCertificates
             ->setDomain(new Document([
                 'domain' => $domain
             ]))
-            ->setSkipRenewCheck(true)
+            ->setSkipRenewCheck($skipCheck)
             ->trigger();
     }
 }

src/Appwrite/Platform/Workers/Certificates.php

@@ -23,6 +23,7 @@ use Utopia\Database\Helpers\ID;
 use Utopia\Database\Query;
 use Utopia\Domains\Domain;
 use Utopia\Locale\Locale;
+use Utopia\Logger\Log;
 use Utopia\Platform\Action;
 use Utopia\Queue\Message;
 
@@ -45,7 +46,8 @@ class Certificates extends Action
             ->inject('queueForMails')
             ->inject('queueForEvents')
             ->inject('queueForFunctions')
-            ->callback(fn(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions) => $this->action($message, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions));
+            ->inject('log')
+            ->callback(fn(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log) => $this->action($message, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log));
     }
 
     /**
@@ -58,7 +60,7 @@ class Certificates extends Action
      * @throws Throwable
      * @throws \Utopia\Database\Exception
      */
-    public function action(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions): void
+    public function action(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log): void
     {
         $payload = $message->getPayload() ?? [];
 
@@ -70,7 +72,7 @@ class Certificates extends Action
         $domain   = new Domain($document->getAttribute('domain', ''));
         $skipRenewCheck = $payload['skipRenewCheck'] ?? false;
 
-        $this->execute($domain, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $skipRenewCheck);
+        $this->execute($domain, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log, $skipRenewCheck);
     }
 
     /**
@@ -84,7 +86,7 @@ class Certificates extends Action
      * @throws Throwable
      * @throws \Utopia\Database\Exception
      */
-    private function execute(Domain $domain, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, bool $skipRenewCheck = false): void
+    private function execute(Domain $domain, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, bool $skipRenewCheck = false): void
     {
         /**
          * 1. Read arguments and validate domain
@@ -138,11 +140,11 @@ class Certificates extends Action
             if (!$skipRenewCheck) {
                 $mainDomain = $this->getMainDomain();
                 $isMainDomain = !isset($mainDomain) || $domain->get() === $mainDomain;
-                $this->validateDomain($domain, $isMainDomain);
+                $this->validateDomain($domain, $isMainDomain, $log);
             }
 
             // If certificate exists already, double-check expiry date. Skip if job is forced
-            if (!$skipRenewCheck && !$this->isRenewRequired($domain->get())) {
+            if (!$skipRenewCheck && !$this->isRenewRequired($domain->get(), $log)) {
                 throw new Exception('Renew isn\'t required.');
             }
 
@@ -180,6 +182,8 @@ class Certificates extends Action
 
             // Send email to security email
             $this->notifyError($domain->get(), $e->getMessage(), $attempts, $queueForMails);
+
+            throw $e;
         } finally {
             // All actions result in new updatedAt date
             $certificate->setAttribute('updated', DateTime::now());
@@ -247,7 +251,7 @@ class Certificates extends Action
      * @return void
      * @throws Exception
      */
-    private function validateDomain(Domain $domain, bool $isMainDomain): void
+    private function validateDomain(Domain $domain, bool $isMainDomain, Log $log): void
     {
         if (empty($domain->get())) {
             throw new Exception('Missing certificate domain.');
@@ -267,8 +271,15 @@ class Certificates extends Action
             }
 
             // Verify domain with DNS records
+            $validationStart = \microtime(true);
             $validator = new CNAME($target->get());
             if (!$validator->isValid($domain->get())) {
+                $log->addExtra('dnsTiming', \strval(\microtime(true) - $validationStart));
+                $log->addTag('dnsDomain', $domain->get());
+
+                $error = $validator->getLogs();
+                $log->addExtra('dnsResponse', \is_array($error) ? \json_encode($error) : \strval($error));
+
                 throw new Exception('Failed to verify domain DNS records.');
             }
         } else {
@@ -284,7 +295,7 @@ class Certificates extends Action
      * @return bool True, if certificate needs to be renewed
      * @throws Exception
      */
-    private function isRenewRequired(string $domain): bool
+    private function isRenewRequired(string $domain, Log $log): bool
     {
         $certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem';
         if (\file_exists($certPath)) {
@@ -294,12 +305,15 @@ class Certificates extends Action
             $validTo = $certData['validTo_time_t'] ?? 0;
 
             if (empty($validTo)) {
+                $log->addTag('certificateDomain', $domain);
                 throw new Exception('Unable to read certificate file (cert.pem).');
             }
 
             // LetsEncrypt allows renewal 30 days before expiry
             $expiryInAdvance = (60 * 60 * 24 * 30);
             if ($validTo - $expiryInAdvance > \time()) {
+                $log->addTag('certificateDomain', $domain);
+                $log->addExtra('certificateData', \is_array($certData) ? \json_encode($certData) : \strval($certData));
                 return false;
             }
         }

src/Appwrite/Platform/Workers/Deletes.php

@@ -68,18 +68,13 @@ class Deletes extends Action
         $datetime = $payload['datetime'] ?? null;
         $hourlyUsageRetentionDatetime = $payload['hourlyUsageRetentionDatetime'] ?? null;
         $resource = $payload['resource'] ?? null;
+        $resourceType = $payload['resourceType'] ?? null;
         $document = new Document($payload['document'] ?? []);
         $project  = new Document($payload['project'] ?? []);
 
         switch (strval($type)) {
             case DELETE_TYPE_DOCUMENT:
                 switch ($document->getCollection()) {
-                    case DELETE_TYPE_DATABASES:
-                        $this->deleteDatabase($getProjectDB, $document, $project);
-                        break;
-                    case DELETE_TYPE_COLLECTIONS:
-                        $this->deleteCollection($getProjectDB, $document, $project);
-                        break;
                     case DELETE_TYPE_PROJECTS:
                         $this->deleteProject($dbForConsole, $getProjectDB, $getFilesDevice, $getFunctionsDevice, $getBuildsDevice, $getCacheDevice, $document);
                         break;
@@ -108,10 +103,6 @@ class Deletes extends Action
                         $this->deleteRule($dbForConsole, $document);
                         break;
                     default:
-                        if (\str_starts_with($document->getCollection(), 'database_')) {
-                            $this->deleteCollection($getProjectDB, $document, $project);
-                            break;
-                        }
                         Console::error('No lazy delete operation available for document of type: ' . $document->getCollection());
                         break;
                 }
@@ -145,7 +136,7 @@ class Deletes extends Action
                 $this->deleteUsageStats($project, $getProjectDB, $hourlyUsageRetentionDatetime);
                 break;
             case DELETE_TYPE_CACHE_BY_RESOURCE:
-                $this->deleteCacheByResource($project, $getProjectDB, $resource);
+                $this->deleteCacheByResource($project, $getProjectDB, $resource, $resourceType);
                 break;
             case DELETE_TYPE_CACHE_BY_TIMESTAMP:
                 $this->deleteCacheByDate($project, $getProjectDB, $datetime);
@@ -203,32 +194,37 @@ class Deletes extends Action
      * @param string $resource
      * @return void
      * @throws Authorization
+     * @param string|null $resourceType
+     * @throws Exception
      */
-    private function deleteCacheByResource(Document $project, callable $getProjectDB, string $resource): void
+    private function deleteCacheByResource(Document $project, callable $getProjectDB, string $resource, string $resourceType = null): void
     {
         $projectId = $project->getId();
         $dbForProject = $getProjectDB($project);
-        $document = $dbForProject->findOne('cache', [Query::equal('resource', [$resource])]);
 
-        if ($document) {
-            $cache = new Cache(
-                new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId)
-            );
+        $cache = new Cache(
+            new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId)
+        );
 
-            $this->deleteById(
-                $document,
-                $dbForProject,
-                function ($document) use ($cache, $projectId) {
-                    $path = APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId . DIRECTORY_SEPARATOR . $document->getId();
-
-                    if ($cache->purge($document->getId())) {
-                        Console::success('Deleting cache file: ' . $path);
-                    } else {
-                        Console::error('Failed to delete cache file: ' . $path);
-                    }
-                }
-            );
+        $query[] = Query::equal('resource', [$resource]);
+        if (!empty($resourceType)) {
+            $query[] = Query::equal('resourceType', [$resourceType]);
         }
+
+        $this->deleteByGroup(
+            'cache',
+            $query,
+            $dbForProject,
+            function (Document $document) use ($cache, $projectId) {
+                $path = APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId . DIRECTORY_SEPARATOR . $document->getId();
+
+                if ($cache->purge($document->getId())) {
+                    Console::success('Deleting cache file: ' . $path);
+                } else {
+                    Console::error('Failed to delete cache file: ' . $path);
+                }
+            }
+        );
     }
 
     /**
@@ -268,72 +264,6 @@ class Deletes extends Action
         );
     }
 
-    /**
-     * @param callable $getProjectDB
-     * @param Document $document
-     * @param Document $project
-     * @return void
-     * @throws Exception
-     */
-    private function deleteDatabase(callable $getProjectDB, Document $document, Document $project): void
-    {
-        $databaseId = $document->getId();
-        $dbForProject = $getProjectDB($project);
-
-        $this->deleteByGroup('database_' . $document->getInternalId(), [], $dbForProject, function ($document) use ($getProjectDB, $project) {
-            $this->deleteCollection($getProjectDB, $document, $project);
-        });
-
-        $dbForProject->deleteCollection('database_' . $document->getInternalId());
-        $this->deleteAuditLogsByResource($getProjectDB, 'database/' . $databaseId, $project);
-    }
-
-    /**
-     * @param callable $getProjectDB
-     * @param Document $document teams document
-     * @param Document $project
-     * @return void
-     * @throws Exception
-     */
-    private function deleteCollection(callable $getProjectDB, Document $document, Document $project): void
-    {
-        $collectionId = $document->getId();
-        $collectionInternalId = $document->getInternalId();
-        $databaseId = $document->getAttribute('databaseId');
-        $databaseInternalId = $document->getAttribute('databaseInternalId');
-
-        $dbForProject = $getProjectDB($project);
-
-        $relationships = \array_filter(
-            $document->getAttribute('attributes'),
-            fn ($attribute) => $attribute['type'] === Database::VAR_RELATIONSHIP
-        );
-
-        foreach ($relationships as $relationship) {
-            if (!$relationship['twoWay']) {
-                continue;
-            }
-            $relatedCollection = $dbForProject->getDocument('database_' . $databaseInternalId, $relationship['relatedCollection']);
-            $dbForProject->deleteDocument('attributes', $databaseInternalId . '_' . $relatedCollection->getInternalId() . '_' . $relationship['twoWayKey']);
-            $dbForProject->deleteCachedDocument('database_' . $databaseInternalId, $relatedCollection->getId());
-            $dbForProject->deleteCachedCollection('database_' . $databaseInternalId . '_collection_' . $relatedCollection->getInternalId());
-        }
-
-        $dbForProject->deleteCollection('database_' . $databaseInternalId . '_collection_' . $document->getInternalId());
-
-        $this->deleteByGroup('attributes', [
-            Query::equal('databaseInternalId', [$databaseInternalId]),
-            Query::equal('collectionInternalId', [$collectionInternalId])
-        ], $dbForProject);
-
-        $this->deleteByGroup('indexes', [
-            Query::equal('databaseInternalId', [$databaseInternalId]),
-            Query::equal('collectionInternalId', [$collectionInternalId])
-        ], $dbForProject);
-
-        $this->deleteAuditLogsByResource($getProjectDB, 'database/' . $databaseId . '/collection/' . $collectionId, $project);
-    }
-
     /**
      * @param Database $dbForConsole
      * @param callable $getProjectDB
@@ -345,7 +275,7 @@ class Deletes extends Action
     {
         $dbForProject = $getProjectDB($project);
         // Delete Usage stats
-        $this->deleteByGroup('stats_v2', [
+        $this->deleteByGroup('stats', [
             Query::lessThan('time', $hourlyUsageRetentionDatetime),
             Query::equal('period', ['1h']),
         ], $dbForProject);
@@ -517,12 +447,7 @@ class Deletes extends Action
                 $teamId = $document->getAttribute('teamId');
                 $team = $dbForProject->getDocument('teams', $teamId);
                 if (!$team->isEmpty()) {
-                    $team = $dbForProject->updateDocument(
-                        'teams',
-                        $teamId,
-                        // Ensure that total >= 0
-                        $team->setAttribute('total', \max($team->getAttribute('total', 0) - 1, 0))
-                    );
+                    $dbForProject->decreaseDocumentAttribute('teams', $teamId, 'total', 1, 0);
                 }
             }
         });

src/Appwrite/Platform/Workers/Hamster.php

@@ -286,7 +286,7 @@ class Hamster extends Action
                         $limit = $periodValue['limit'];
                         $period = $periodValue['period'];
 
-                        $requestDocs = $dbForProject->find('stats_v2', [
+                        $requestDocs = $dbForProject->find('stats', [
                             Query::equal('period', [$period]),
                             Query::equal('metric', [$metric]),
                             Query::limit($limit),

src/Appwrite/Platform/Workers/Messaging.php

@@ -5,6 +5,7 @@ namespace Appwrite\Platform\Workers;
 use Exception;
 use Utopia\App;
 use Utopia\CLI\Console;
+use Utopia\Database\Document;
 use Utopia\DSN\DSN;
 use Utopia\Messaging\Messages\SMS;
 use Utopia\Messaging\Adapters\SMS\Mock;
@@ -15,6 +16,7 @@ use Utopia\Messaging\Adapters\SMS\Twilio;
 use Utopia\Messaging\Adapters\SMS\Vonage;
 use Utopia\Platform\Action;
 use Utopia\Queue\Message;
+use Appwrite\Event\Usage;
 
 class Messaging extends Action
 {
@@ -43,20 +45,36 @@ class Messaging extends Action
         $this
             ->desc('Messaging worker')
             ->inject('message')
-            ->callback(fn($message) => $this->action($message));
+            ->inject('queueForUsage')
+            ->callback(fn(Message $message, Usage $queueForUsage) => $this->action($message, $queueForUsage));
     }
 
     /**
      * @param Message $message
+     * @param Usage $queueForUsage
      * @return void
      * @throws Exception
      */
-    public function action(Message $message): void
+    public function action(Message $message, Usage $queueForUsage): void
     {
         $payload = $message->getPayload() ?? [];
 
         if (empty($payload)) {
-            Console::error('Payload arg not found');
+            throw new Exception('Missing payload');
+        }
+
+        if (empty($payload['project'])) {
+            throw new Exception('Project not set in payload');
+        }
+
+        $project = new Document($payload['project'] ?? []);
+
+        Console::log('Project: ' . $project->getId());
+
+        $denyList = App::getEnv('_APP_SMS_PROJECTS_DENY_LIST', '');
+        $denyList = explode(',', $denyList);
+        if (in_array($project->getId(), $denyList)) {
+            Console::error("Project is in the deny list. Skipping ...");
             return;
         }
 
@@ -100,6 +118,11 @@ class Messaging extends Action
 
         try {
             $sms->send($message);
+
+            $queueForUsage
+                ->setProject($project)
+                ->addMetric(METRIC_MESSAGES, 1)
+                ->trigger();
         } catch (\Exception $error) {
             throw new Exception('Error sending message: ' . $error->getMessage(), 500);
         }

src/Appwrite/Platform/Workers/Usage.php

@@ -3,24 +3,23 @@
 namespace Appwrite\Platform\Workers;
 
 use Exception;
+use Utopia\App;
 use Utopia\CLI\Console;
-use Utopia\Database\Database;
 use Utopia\Database\DateTime;
 use Utopia\Database\Document;
 use Utopia\Platform\Action;
+use Appwrite\Event\UsageDump;
 use Utopia\Queue\Message;
 
 class Usage extends Action
 {
-    protected static array $stats = [];
-    protected array $periods = [
-        '1h' => 'Y-m-d H:00',
-        '1d' => 'Y-m-d 00:00',
-        'inf' => '0000-00-00 00:00'
-    ];
+    private array $stats = [];
+    private int $lastTriggeredTime = 0;
+    private int $keys = 0;
+    private const INFINITY_PERIOD = '_inf_';
+    private const KEYS_THRESHOLD = 10000;
+
 
-    protected const INFINITY_PERIOD = '_inf_';
-    protected const DEBUG_PROJECT_ID = 85293;
     public static function getName(): string
     {
         return 'usage';
@@ -36,26 +35,31 @@ class Usage extends Action
         ->desc('Usage worker')
         ->inject('message')
         ->inject('getProjectDB')
-        ->callback(function (Message $message, callable $getProjectDB) {
-             $this->action($message, $getProjectDB);
+        ->inject('queueForUsageDump')
+        ->callback(function (Message $message, callable $getProjectDB, UsageDump $queueForUsageDump) {
+             $this->action($message, $getProjectDB, $queueForUsageDump);
         });
+
+        $this->lastTriggeredTime = time();
     }
 
     /**
      * @param Message $message
      * @param callable $getProjectDB
+     * @param UsageDump $queueForUsageDump
      * @return void
      * @throws \Utopia\Database\Exception
      * @throws Exception
      */
-    public function action(Message $message, callable $getProjectDB): void
+    public function action(Message $message, callable $getProjectDB, UsageDump $queueForUsageDump): void
     {
         $payload = $message->getPayload() ?? [];
         if (empty($payload)) {
             throw new Exception('Missing payload');
         }
+        //Todo Figure out way to preserve keys when the container is being recreated @shimonewman
 
-        $payload = $message->getPayload() ?? [];
+        $aggregationInterval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '20');
         $project = new Document($payload['project'] ?? []);
         $projectId = $project->getInternalId();
         foreach ($payload['reduce'] ?? [] as $document) {
@@ -71,13 +75,31 @@ class Usage extends Action
             );
         }
 
-        self::$stats[$projectId]['project'] = $project;
+        $this->stats[$projectId]['project'] = $project;
         foreach ($payload['metrics'] ?? [] as $metric) {
-            if (!isset(self::$stats[$projectId]['keys'][$metric['key']])) {
-                self::$stats[$projectId]['keys'][$metric['key']] = $metric['value'];
+                 $this->keys++;
+            if (!isset($this->stats[$projectId]['keys'][$metric['key']])) {
+                $this->stats[$projectId]['keys'][$metric['key']] = $metric['value'];
                 continue;
             }
-            self::$stats[$projectId]['keys'][$metric['key']] += $metric['value'];
+
+            $this->stats[$projectId]['keys'][$metric['key']] += $metric['value'];
+        }
+
+        // if keys crossed threshold or X time passed since the last send and there are some keys in the array ($this->stats)
+        if (
+            $this->keys >= self::KEYS_THRESHOLD ||
+            (time() - $this->lastTriggeredTime > $aggregationInterval  && $this->keys > 0)
+        ) {
+            console::warning('[' . DateTime::now() . '] Aggregated ' . $this->keys . ' keys');
+
+            $queueForUsageDump
+                ->setStats($this->stats)
+                ->trigger();
+
+            $this->stats = [];
+            $this->keys = 0;
+            $this->lastTriggeredTime = time();
         }
     }
 
@@ -107,8 +129,8 @@ class Usage extends Action
                     }
                     break;
                 case $document->getCollection() === 'databases': // databases
-                    $collections = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS)));
-                    $documents = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_DOCUMENTS)));
+                    $collections = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS)));
+                    $documents = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_DOCUMENTS)));
                     if (!empty($collections['value'])) {
                         $metrics[] = [
                         'key' => METRIC_COLLECTIONS,
@@ -126,7 +148,7 @@ class Usage extends Action
                 case str_starts_with($document->getCollection(), 'database_') && !str_contains($document->getCollection(), 'collection'): //collections
                     $parts = explode('_', $document->getCollection());
                     $databaseInternalId = $parts[1] ?? 0;
-                    $documents = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $document->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS)));
+                    $documents = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $document->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS)));
 
                     if (!empty($documents['value'])) {
                         $metrics[] = [
@@ -141,8 +163,8 @@ class Usage extends Action
                     break;
 
                 case $document->getCollection() === 'buckets':
-                    $files = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES)));
-                    $storage = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE)));
+                    $files = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES)));
+                    $storage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE)));
 
                     if (!empty($files['value'])) {
                         $metrics[] = [
@@ -160,13 +182,13 @@ class Usage extends Action
                     break;
 
                 case $document->getCollection() === 'functions':
-                    $deployments = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS)));
-                    $deploymentsStorage = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE)));
-                    $builds = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS)));
-                    $buildsStorage = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE)));
-                    $buildsCompute = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE)));
-                    $executions = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS)));
-                    $executionsCompute = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE)));
+                    $deployments = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS)));
+                    $deploymentsStorage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE)));
+                    $builds = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS)));
+                    $buildsStorage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE)));
+                    $buildsCompute = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE)));
+                    $executions = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS)));
+                    $executionsCompute = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE)));
 
                     if (!empty($deployments['value'])) {
                         $metrics[] = [

src/Appwrite/Platform/Workers/UsageDump.php

@@ -0,0 +1,113 @@
+<?php
+
+namespace Appwrite\Platform\Workers;
+
+use Appwrite\Extend\Exception;
+use Utopia\App;
+use Utopia\Database\Document;
+use Utopia\Database\Exception\Duplicate;
+use Utopia\Platform\Action;
+use Utopia\CLI\Console;
+use Utopia\Database\DateTime;
+use Utopia\Queue\Message;
+
+class UsageDump extends Action
+{
+    protected array $stats = [];
+    protected array $periods = [
+        '1h' => 'Y-m-d H:00',
+        '1d' => 'Y-m-d 00:00',
+        'inf' => '0000-00-00 00:00'
+    ];
+
+    public static function getName(): string
+    {
+        return 'usage-dump';
+    }
+
+    /**
+     * @throws \Exception
+     */
+    public function __construct()
+    {
+
+        $this
+            ->inject('message')
+            ->inject('getProjectDB')
+            ->callback(function (Message $message, callable $getProjectDB) {
+                $this->action($message, $getProjectDB);
+            })
+        ;
+    }
+
+    /**
+     * @param Message $message
+     * @param callable $getProjectDB
+     * @return void
+     * @throws Exception
+     * @throws \Utopia\Database\Exception
+     */
+    public function action(Message $message, callable $getProjectDB): void
+    {
+
+        $payload = $message->getPayload() ?? [];
+        if (empty($payload)) {
+            throw new Exception('Missing payload');
+        }
+
+        //Todo rename both usage workers @shimonewman
+        foreach ($payload['stats'] ?? [] as $stats) {
+            $project = new Document($stats['project'] ?? []);
+            $numberOfKeys = !empty($stats['keys']) ? count($stats['keys']) : 0;
+
+            if ($numberOfKeys === 0) {
+                continue;
+            }
+
+            console::log('[' . DateTime::now() . '] ProjectId [' . $project->getInternalId() . '] Database [' . $project['database'] . '] ' . $numberOfKeys . ' keys');
+
+            try {
+                $dbForProject = $getProjectDB($project);
+                foreach ($stats['keys'] ?? [] as $key => $value) {
+                    if ($value == 0) {
+                        continue;
+                    }
+
+                    foreach ($this->periods as $period => $format) {
+                        $time = 'inf' === $period ? null : date($format, time());
+                        $id = \md5("{$time}_{$period}_{$key}");
+
+                        try {
+                            $dbForProject->createDocument('stats', new Document([
+                                '$id' => $id,
+                                'period' => $period,
+                                'time' => $time,
+                                'metric' => $key,
+                                'value' => $value,
+                                'region' => App::getEnv('_APP_REGION', 'default'),
+                            ]));
+                        } catch (Duplicate $th) {
+                            if ($value < 0) {
+                                $dbForProject->decreaseDocumentAttribute(
+                                    'stats',
+                                    $id,
+                                    'value',
+                                    abs($value)
+                                );
+                            } else {
+                                $dbForProject->increaseDocumentAttribute(
+                                    'stats',
+                                    $id,
+                                    'value',
+                                    $value
+                                );
+                            }
+                        }
+                    }
+                }
+            } catch (\Exception $e) {
+                console::error('[' . DateTime::now() . '] project [' . $project->getInternalId() . '] database [' . $project['database'] . '] ' . ' ' . $e->getMessage());
+            }
+        }
+    }
+}

src/Appwrite/Platform/Workers/UsageHook.php

@@ -1,103 +0,0 @@
-<?php
-
-namespace Appwrite\Platform\Workers;
-
-use Utopia\App;
-use Utopia\Database\Document;
-use Utopia\Database\Exception\Duplicate;
-use Utopia\Platform\Action;
-use Utopia\CLI\Console;
-use Swoole\Timer;
-use Utopia\Database\DateTime;
-
-class UsageHook extends Usage
-{
-    public static function getName(): string
-    {
-        return 'usageHook';
-    }
-
-    public function __construct()
-    {
-
-        $this
-            ->setType(Action::TYPE_WORKER_START)
-            ->inject('register')
-            ->inject('getProjectDB')
-            ->callback(function ($register, callable $getProjectDB) {
-                $this->action($register, $getProjectDB);
-            })
-        ;
-    }
-
-    /**
-     * @param $register
-     * @param $getProjectDB
-     * @return void
-     */
-    public function action($register, $getProjectDB): void
-    {
-
-        $interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '60000');
-        Timer::tick($interval, function () use ($register, $getProjectDB) {
-
-            $offset = count(self::$stats);
-            $projects = array_slice(self::$stats, 0, $offset, true);
-            array_splice(self::$stats, 0, $offset);
-            foreach ($projects as $data) {
-                $numberOfKeys = !empty($data['keys']) ? count($data['keys']) : 0;
-                $projectInternalId = $data['project']->getInternalId();
-                $database = $data['project']['database'] ?? '';
-
-                console::warning('Ticker started ' . DateTime::now());
-
-                if ($numberOfKeys === 0) {
-                    continue;
-                }
-
-                try {
-                    $dbForProject = $getProjectDB($data['project']);
-                    foreach ($data['keys'] ?? [] as $key => $value) {
-                        if ($value == 0) {
-                            continue;
-                        }
-
-                        foreach ($this->periods as $period => $format) {
-                            $time = 'inf' === $period ? null : date($format, time());
-                            $id = \md5("{$time}_{$period}_{$key}");
-
-                            try {
-                                $dbForProject->createDocument('stats_v2', new Document([
-                                    '$id' => $id,
-                                    'period' => $period,
-                                    'time' => $time,
-                                    'metric' => $key,
-                                    'value' => $value,
-                                    'region' => App::getEnv('_APP_REGION', 'default'),
-                                ]));
-                            } catch (Duplicate $th) {
-                                if ($value < 0) {
-                                    $dbForProject->decreaseDocumentAttribute(
-                                        'stats_v2',
-                                        $id,
-                                        'value',
-                                        abs($value)
-                                    );
-                                } else {
-                                    $dbForProject->increaseDocumentAttribute(
-                                        'stats_v2',
-                                        $id,
-                                        'value',
-                                        $value
-                                    );
-                                }
-                            }
-                        }
-                    }
-                } catch (\Exception $e) {
-                    console::error(DateTime::now() . ' ' . $projectInternalId . ' ' . $e->getMessage());
-                }
-            }
-        });
-    }
-}

src/Appwrite/Specification/Format/OpenAPI3.php

@@ -295,7 +295,7 @@ class OpenAPI3 extends Format
                 switch ((!empty($validator)) ? \get_class($validator) : '') {
                     case 'Utopia\Validator\Text':
                         $node['schema']['type'] = $validator->getType();
-                        $node['schema']['x-example'] = '[' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . ']';
+                        $node['schema']['x-example'] = '<' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . '>';
                         break;
                     case 'Utopia\Validator\Boolean':
                         $node['schema']['type'] = $validator->getType();
@@ -303,14 +303,14 @@ class OpenAPI3 extends Format
                         break;
                     case 'Utopia\Database\Validator\UID':
                         $node['schema']['type'] = $validator->getType();
-                        $node['schema']['x-example'] = '[' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . ']';
+                        $node['schema']['x-example'] = '<' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . '>';
                         break;
                     case 'Appwrite\Utopia\Database\Validator\CustomId':
                         if ($route->getLabel('sdk.methodType', '') === 'upload') {
                             $node['schema']['x-upload-id'] = true;
                         }
                         $node['schema']['type'] = $validator->getType();
-                        $node['schema']['x-example'] = '[' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . ']';
+                        $node['schema']['x-example'] = '<' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . '>';
                         break;
                     case 'Utopia\Database\Validator\DatetimeValidator':
                         $node['schema']['type'] = $validator->getType();

src/Appwrite/Specification/Format/Swagger2.php

@@ -297,7 +297,7 @@ class Swagger2 extends Format
                 switch ((!empty($validator)) ? \get_class($validator) : '') {
                     case 'Utopia\Validator\Text':
                         $node['type'] = $validator->getType();
-                        $node['x-example'] = '[' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . ']';
+                        $node['x-example'] = '<' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . '>';
                         break;
                     case 'Utopia\Validator\Boolean':
                         $node['type'] = $validator->getType();
@@ -308,11 +308,11 @@ class Swagger2 extends Format
                             $node['x-upload-id'] = true;
                         }
                         $node['type'] = $validator->getType();
-                        $node['x-example'] = '[' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . ']';
+                        $node['x-example'] = '<' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . '>';
                         break;
                     case 'Utopia\Database\Validator\UID':
                         $node['type'] = $validator->getType();
-                        $node['x-example'] = '[' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . ']';
+                        $node['x-example'] = '<' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . '>';
                         break;
                     case 'Utopia\Database\Validator\DatetimeValidator':
                         $node['type'] = $validator->getType();

src/Appwrite/Utopia/Response.php

@@ -70,6 +70,7 @@ use Appwrite\Utopia\Response\Model\Token;
 use Appwrite\Utopia\Response\Model\Webhook;
 use Appwrite\Utopia\Response\Model\Preferences;
 use Appwrite\Utopia\Response\Model\HealthAntivirus;
+use Appwrite\Utopia\Response\Model\HealthCertificate;
 use Appwrite\Utopia\Response\Model\HealthQueue;
 use Appwrite\Utopia\Response\Model\HealthStatus;
 use Appwrite\Utopia\Response\Model\HealthTime;
@@ -253,6 +254,7 @@ class Response extends SwooleResponse
     public const MODEL_HEALTH_QUEUE = 'healthQueue';
     public const MODEL_HEALTH_TIME = 'healthTime';
     public const MODEL_HEALTH_ANTIVIRUS = 'healthAntivirus';
+    public const MODEL_HEALTH_CERTIFICATE = 'healthCertificate';
     public const MODEL_HEALTH_STATUS_LIST = 'healthStatusList';
 
     // Console
@@ -390,6 +392,7 @@ class Response extends SwooleResponse
             ->setModel(new HealthAntivirus())
             ->setModel(new HealthQueue())
             ->setModel(new HealthStatus())
+            ->setModel(new HealthCertificate())
             ->setModel(new HealthTime())
             ->setModel(new HealthVersion())
             ->setModel(new Metric())

src/Appwrite/Utopia/Response/Model/HealthCertificate.php

@@ -0,0 +1,71 @@
+<?php
+
+namespace Appwrite\Utopia\Response\Model;
+
+use Appwrite\Utopia\Response;
+use Appwrite\Utopia\Response\Model;
+
+class HealthCertificate extends Model
+{
+    public function __construct()
+    {
+        $this
+            ->addRule('name', [
+                'type' => self::TYPE_STRING,
+                'description' => 'Certificate name',
+                'default' => '',
+                'example' => '/CN=www.google.com',
+            ])
+            ->addRule('subjectSN', [
+                'type' => self::TYPE_STRING,
+                'description' => 'Subject SN',
+                'default' => 'www.google.com',
+                'example' => '',
+            ])
+            ->addRule('issuerOrganisation', [
+                'type' => self::TYPE_STRING,
+                'description' => 'Issuer organisation',
+                'default' => 'Google Trust Services LLC',
+                'example' => '',
+            ])
+            ->addRule('validFrom', [
+                'type' => self::TYPE_STRING,
+                'description' => 'Valid from',
+                'default' => '',
+                'example' => '1704200998',
+            ])
+            ->addRule('validTo', [
+                'type' => self::TYPE_STRING,
+                'description' => 'Valid to',
+                'default' => '',
+                'example' => '1711458597',
+            ])
+            ->addRule('signatureTypeSN', [
+                'type' => self::TYPE_STRING,
+                'description' => 'Signature type SN',
+                'default' => '',
+                'example' => 'RSA-SHA256',
+            ])
+        ;
+    }
+
+    /**
+     * Get Name
+     *
+     * @return string
+     */
+    public function getName(): string
+    {
+        return 'Health Certificate';
+    }
+
+    /**
+     * Get Type
+     *
+     * @return string
+     */
+    public function getType(): string
+    {
+        return Response::MODEL_HEALTH_CERTIFICATE;
+    }
+}

tests/e2e/Services/Account/AccountConsoleClientTest.php

@@ -2,17 +2,90 @@
 
 namespace Tests\E2E\Services\Account;
 
-use Appwrite\Extend\Exception;
 use Tests\E2E\Scopes\Scope;
 use Tests\E2E\Scopes\ProjectConsole;
 use Tests\E2E\Scopes\SideClient;
 use Utopia\Database\Helpers\ID;
 use Tests\E2E\Client;
-use Utopia\Database\Validator\Datetime as DatetimeValidator;
 
 class AccountConsoleClientTest extends Scope
 {
     use AccountBase;
     use ProjectConsole;
     use SideClient;
+
+    public function testDeleteAccount(): void
+    {
+         $email = uniqid() . 'user@localhost.test';
+         $password = 'password';
+         $name = 'User Name';
+
+         $response = $this->client->call(Client::METHOD_POST, '/account', array_merge([
+             'origin' => 'http://localhost',
+             'content-type' => 'application/json',
+             'x-appwrite-project' => $this->getProject()['$id'],
+         ]), [
+             'userId' => ID::unique(),
+             'email' => $email,
+             'password' => $password,
+             'name' => $name,
+         ]);
+
+         $this->assertEquals($response['headers']['status-code'], 201);
+
+         $response = $this->client->call(Client::METHOD_POST, '/account/sessions/email', array_merge([
+             'origin' => 'http://localhost',
+             'content-type' => 'application/json',
+             'x-appwrite-project' => $this->getProject()['$id'],
+         ]), [
+             'email' => $email,
+             'password' => $password,
+         ]);
+
+         $this->assertEquals($response['headers']['status-code'], 201);
+
+         $session = $response['cookies']['a_session_' . $this->getProject()['$id']];
+
+         // create team
+         $team = $this->client->call(Client::METHOD_POST, '/teams', [
+            'origin' => 'http://localhost',
+             'content-type' => 'application/json',
+             'x-appwrite-project' => $this->getProject()['$id'],
+             'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session,
+         ], [
+            'teamId' => 'unique()',
+            'name' => 'myteam'
+         ]);
+         $this->assertEquals($team['headers']['status-code'], 201);
+
+         $teamId = $team['body']['$id'];
+
+         $response = $this->client->call(Client::METHOD_DELETE, '/account', array_merge([
+             'origin' => 'http://localhost',
+             'content-type' => 'application/json',
+             'x-appwrite-project' => $this->getProject()['$id'],
+             'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session,
+         ]));
+
+         $this->assertEquals($response['headers']['status-code'], 400);
+
+         // DELETE TEAM
+         $response = $this->client->call(Client::METHOD_DELETE, '/teams/' . $teamId, array_merge([
+             'origin' => 'http://localhost',
+             'content-type' => 'application/json',
+             'x-appwrite-project' => $this->getProject()['$id'],
+             'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session,
+         ]));
+         $this->assertEquals($response['headers']['status-code'], 204);
+         sleep(2);
+
+         $response = $this->client->call(Client::METHOD_DELETE, '/account', array_merge([
+             'origin' => 'http://localhost',
+             'content-type' => 'application/json',
+             'x-appwrite-project' => $this->getProject()['$id'],
+             'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session,
+         ]));
+
+         $this->assertEquals($response['headers']['status-code'], 204);
+    }
 }

tests/e2e/Services/Databases/DatabasesBase.php

@@ -978,7 +978,7 @@ trait DatabasesBase
         ]);
 
         $this->assertEquals(400, $badEnum['headers']['status-code']);
-        $this->assertEquals('Invalid `elements` param: Value must a valid array and Value must be a valid string and at least 1 chars and no longer than 255 chars', $badEnum['body']['message']);
+        $this->assertEquals('Invalid `elements` param: Value must a valid array no longer than 100 items and Value must be a valid string and at least 1 chars and no longer than 255 chars', $badEnum['body']['message']);
 
         return $data;
     }

tests/e2e/Services/Health/HealthCustomServerTest.php

@@ -424,4 +424,74 @@ class HealthCustomServerTest extends Scope
 
         return [];
     }
+
+    public function testCertificateValidity(): array
+    {
+        /**
+         * Test for SUCCESS
+         */
+        $response = $this->client->call(Client::METHOD_GET, '/health/certificate?domain=www.google.com', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), []);
+
+        $this->assertEquals(200, $response['headers']['status-code']);
+        $this->assertEquals('/CN=www.google.com', $response['body']['name']);
+        $this->assertEquals('www.google.com', $response['body']['subjectSN']);
+        $this->assertEquals('Google Trust Services LLC', $response['body']['issuerOrganisation']);
+        $this->assertIsInt($response['body']['validFrom']);
+        $this->assertIsInt($response['body']['validTo']);
+
+        $response = $this->client->call(Client::METHOD_GET, '/health/certificate?domain=appwrite.io', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), []);
+
+        $this->assertEquals(200, $response['headers']['status-code']);
+        $this->assertEquals('/CN=appwrite.io', $response['body']['name']);
+        $this->assertEquals('appwrite.io', $response['body']['subjectSN']);
+        $this->assertEquals("Let's Encrypt", $response['body']['issuerOrganisation']);
+        $this->assertIsInt($response['body']['validFrom']);
+        $this->assertIsInt($response['body']['validTo']);
+
+        $response = $this->client->call(Client::METHOD_GET, '/health/certificate?domain=https://google.com', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), []);
+
+        $this->assertEquals(200, $response['headers']['status-code']);
+
+        /**
+         * Test for FAILURE
+         */
+        $response = $this->client->call(Client::METHOD_GET, '/health/certificate?domain=localhost', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), []);
+
+        $this->assertEquals(400, $response['headers']['status-code']);
+
+        $response = $this->client->call(Client::METHOD_GET, '/health/certificate?domain=doesnotexist.com', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), []);
+
+        $this->assertEquals(404, $response['headers']['status-code']);
+
+        $response = $this->client->call(Client::METHOD_GET, '/health/certificate?domain=www.google.com/usr/src/local', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), []);
+
+        $this->assertEquals(400, $response['headers']['status-code']);
+
+        $response = $this->client->call(Client::METHOD_GET, '/health/certificate?domain=', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), []);
+
+        $this->assertEquals(400, $response['headers']['status-code']);
+
+        return [];
+    }
 }