Merge remote-tracking branch 'origin/feat-db-pools-master' into feat-db-pools-db-pools-master-sync

2026-05-26 13:51:13 +00:00 · 2023-07-10 03:06:48 +00:00
parent 9fcf4400d8 c6be81592f
commit f59d3563f4
90 changed files with 4065 additions and 2030 deletions
@@ -158,6 +158,18 @@ class Github extends OAuth2
        return $user['name'] ?? '';
    }

+    /**
+     * @param string $accessToken
+     *
+     * @return string
+     */
+    public function getUserSlug(string $accessToken): string
+    {
+        $user = $this->getUser($accessToken);
+
+        return $user['login'] ?? '';
+    }
+
    /**
     * @param string $accessToken
     *
@@ -171,13 +183,27 @@ class Github extends OAuth2
            $emails = $this->request('GET', 'https://api.github.com/user/emails', ['Authorization: token ' . \urlencode($accessToken)]);

            $emails = \json_decode($emails, true);
+
+            $verifiedEmail = null;
+            $primaryEmail = null;
+
            foreach ($emails as $email) {
                if (isset($email['verified']) && $email['verified'] === true) {
-                    $this->user['email'] = $email['email'];
-                    $this->user['verified'] = $email['verified'];
-                    break;
+                    $verifiedEmail = $email;
+
+                    if (isset($email['primary']) && $email['primary'] === true) {
+                        $primaryEmail = $email;
+                    }
                }
            }
+
+            if (!empty($primaryEmail)) {
+                $this->user['email'] = $primaryEmail['email'];
+                $this->user['verified'] = $primaryEmail['verified'];
+            } elseif (!empty($verifiedEmail)) {
+                $this->user['email'] = $verifiedEmail['email'];
+                $this->user['verified'] = $verifiedEmail['verified'];
+            }
        }

        return $this->user;
@@ -7,6 +7,11 @@ use Utopia\Validator;

 class Event extends Validator
 {
+    /**
+     * @var string
+     */
+    protected string $message = 'Event is not valid.';
+
    /**
     * Get Description.
     *
@@ -16,7 +21,7 @@ class Event extends Validator
     */
    public function getDescription(): string
    {
-        return 'Event is not valid.';
+        return $this->message;
    }

    /**
@@ -40,6 +45,12 @@ class Event extends Validator
         * Identify all sections of the pattern.
         */
        $type = $parts[0] ?? false;
+
+        if ($type == 'functions') {
+            $this->message = 'Triggering a function on a function event is not allowed.';
+            return false;
+        }
+
        $resource = $parts[1] ?? false;
        $hasSubResource = $count > 3 && ($events[$type]['$resource'] ?? false) && ($events[$type][$parts[2]]['$resource'] ?? false);
        $hasSubSubResource = $count > 5 && $hasSubResource && ($events[$type][$parts[2]][$parts[4]]['$resource'] ?? false);
@@ -157,6 +157,7 @@ class Exception extends \Exception
    public const PROJECT_UNKNOWN                   = 'project_unknown';
    public const PROJECT_PROVIDER_DISABLED         = 'project_provider_disabled';
    public const PROJECT_PROVIDER_UNSUPPORTED      = 'project_provider_unsupported';
+    public const PROJECT_ALREADY_EXISTS            = 'project_already_exists';
    public const PROJECT_INVALID_SUCCESS_URL       = 'project_invalid_success_url';
    public const PROJECT_INVALID_FAILURE_URL       = 'project_invalid_failure_url';
    public const PROJECT_RESERVED_PROJECT          = 'project_reserved_project';
@@ -178,6 +179,7 @@ class Exception extends \Exception
    /** Domain */
    public const DOMAIN_NOT_FOUND                  = 'domain_not_found';
    public const DOMAIN_ALREADY_EXISTS             = 'domain_already_exists';
+    public const DOMAIN_FORBIDDEN                  = 'domain_forbidden';
    public const DOMAIN_VERIFICATION_FAILED        = 'domain_verification_failed';
    public const DOMAIN_TARGET_INVALID             = 'domain_target_invalid';

@@ -14,10 +14,14 @@ use Appwrite\Platform\Tasks\Specs;
 use Appwrite\Platform\Tasks\SSL;
 use Appwrite\Platform\Tasks\Hamster;
 use Appwrite\Platform\Tasks\PatchDeleteScheduleUpdatedAtAttribute;
+use Appwrite\Platform\Tasks\ClearCardCache;
 use Appwrite\Platform\Tasks\Usage;
 use Appwrite\Platform\Tasks\Vars;
 use Appwrite\Platform\Tasks\Version;
 use Appwrite\Platform\Tasks\VolumeSync;
+use Appwrite\Platform\Tasks\CalcUsersStats;
+use Appwrite\Platform\Tasks\CalcTierStats;
+use Appwrite\Platform\Tasks\PatchDeleteProjectCollections;

 class Tasks extends Service
 {
@@ -33,11 +37,16 @@ class Tasks extends Service
            ->addAction(Install::getName(), new Install())
            ->addAction(Maintenance::getName(), new Maintenance())
            ->addAction(PatchCreateMissingSchedules::getName(), new PatchCreateMissingSchedules())
+            ->addAction(ClearCardCache::getName(), new ClearCardCache())
            ->addAction(PatchDeleteScheduleUpdatedAtAttribute::getName(), new PatchDeleteScheduleUpdatedAtAttribute())
            ->addAction(Schedule::getName(), new Schedule())
            ->addAction(Migrate::getName(), new Migrate())
            ->addAction(SDKs::getName(), new SDKs())
            ->addAction(VolumeSync::getName(), new VolumeSync())
-            ->addAction(Specs::getName(), new Specs());
+            ->addAction(Specs::getName(), new Specs())
+            ->addAction(CalcUsersStats::getName(), new CalcUsersStats())
+            ->addAction(CalcTierStats::getName(), new CalcTierStats())
+            ->addAction(PatchDeleteProjectCollections::getName(), new PatchDeleteProjectCollections())
+        ;
    }
 }
@@ -0,0 +1,351 @@
+<?php
+
+namespace Appwrite\Platform\Tasks;
+
+use Exception;
+use League\Csv\CannotInsertRecord;
+use Utopia\App;
+use Utopia\Database\Validator\Authorization;
+use Utopia\Platform\Action;
+use Utopia\Cache\Cache;
+use Utopia\CLI\Console;
+use Utopia\Database\Database;
+use Utopia\Database\Query;
+use League\Csv\Writer;
+use PHPMailer\PHPMailer\PHPMailer;
+use Utopia\Pools\Group;
+use Utopia\Registry\Registry;
+
+class CalcTierStats extends Action
+{
+    /*
+     * Csv cols headers
+     */
+    private array $columns = [
+        'Project ID',
+        'Organization ID',
+        'Organization Members',
+        'Teams',
+        'Users',
+        'Requests',
+        'Bandwidth',
+        'Domains',
+        'Api keys',
+        'Webhooks',
+        'Platforms',
+        'Buckets',
+        'Files',
+        'Storage (bytes)',
+        'Max File Size (bytes)',
+        'Databases',
+        'Functions',
+        'Deployments',
+        'Executions',
+    ];
+
+    protected string $directory = '/usr/local';
+    protected string $path;
+    protected string $date;
+
+    private array $usageStats = [
+        'project.$all.network.requests'  => 'Requests',
+        'project.$all.network.bandwidth' => 'Bandwidth',
+
+    ];
+
+    public static function getName(): string
+    {
+        return 'calc-tier-stats';
+    }
+
+    public function __construct()
+    {
+
+        $this
+            ->desc('Get stats for projects')
+            ->inject('pools')
+            ->inject('cache')
+            ->inject('dbForConsole')
+            ->inject('register')
+            ->callback(function (Group $pools, Cache $cache, Database $dbForConsole, Registry $register) {
+                $this->action($pools, $cache, $dbForConsole, $register);
+            });
+    }
+
+    /**
+     * @throws \Utopia\Exception
+     * @throws CannotInsertRecord
+     */
+    public function action(Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void
+    {
+        //docker compose exec -t appwrite calc-tier-stats
+
+        Console::title('Cloud free tier  stats calculation V1');
+        Console::success(APP_NAME . ' cloud free tier  stats calculation has started');
+
+        /* Initialise new Utopia app */
+        $app = new App('UTC');
+        $console = $app->getResource('console');
+
+        /** CSV stuff */
+        $this->date = date('Y-m-d');
+        $this->path = "{$this->directory}/tier_stats_{$this->date}.csv";
+        $csv = Writer::createFromPath($this->path, 'w');
+        $csv->insertOne($this->columns);
+
+        /** Database connections */
+        $totalProjects = $dbForConsole->count('projects');
+        Console::success("Found a total of: {$totalProjects} projects");
+
+        $projects = [$console];
+        $count = 0;
+        $limit = 30;
+        $sum = 30;
+        $offset = 0;
+        while (!empty($projects)) {
+            foreach ($projects as $project) {
+
+                /**
+                 * Skip user projects with id 'console'
+                 */
+                if ($project->getId() === 'console') {
+                    continue;
+                }
+
+                Console::info("Getting stats for {$project->getId()}");
+
+                try {
+                    $db = $project->getAttribute('database');
+                    $adapter = $pools
+                        ->get($db)
+                        ->pop()
+                        ->getResource();
+
+                    $dbForProject = new Database($adapter, $cache);
+                    $dbForProject->setDefaultDatabase('appwrite');
+                    $dbForProject->setNamespace('_' . $project->getInternalId());
+
+                    /** Get Project ID */
+                    $stats['Project ID'] = $project->getId();
+
+                    $stats['Organization ID']   = $project->getAttribute('teamId', null);
+
+                    /** Get Total Members */
+                    $teamInternalId = $project->getAttribute('teamInternalId', null);
+                    if ($teamInternalId) {
+                        $stats['Organization Members'] = $dbForConsole->count('memberships', [
+                            Query::equal('teamInternalId', [$teamInternalId])
+                        ]);
+                    } else {
+                        $stats['Organization Members'] = 0;
+                    }
+
+                    /** Get Total internal Teams */
+                    try {
+                        $stats['Teams'] = $dbForProject->count('teams', []);
+                    } catch (\Throwable) {
+                        $stats['Teams'] = 0;
+                    }
+
+                    /** Get Total users */
+                    try {
+                        $stats['Users'] = $dbForProject->count('users', []);
+                    } catch (\Throwable) {
+                        $stats['Users'] = 0;
+                    }
+
+                    /** Get Usage stats */
+                    $range = '90d';
+                    $periods = [
+                        '90d' => [
+                            'period' => '1d',
+                            'limit' => 90,
+                        ],
+                    ];
+
+                    $tmp = [];
+                    $metrics = $this->usageStats;
+                    Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$tmp) {
+                        foreach ($metrics as $metric => $name) {
+                            $limit = $periods[$range]['limit'];
+                            $period = $periods[$range]['period'];
+
+                            $requestDocs = $dbForProject->find('stats', [
+                                Query::equal('period', [$period]),
+                                Query::equal('metric', [$metric]),
+                                Query::limit($limit),
+                                Query::orderDesc('time'),
+                            ]);
+
+                            $tmp[$metric] = [];
+                            foreach ($requestDocs as $requestDoc) {
+                                if (empty($requestDoc)) {
+                                    continue;
+                                }
+
+                                $tmp[$metric][] = [
+                                    'value' => $requestDoc->getAttribute('value'),
+                                    'date' => $requestDoc->getAttribute('time'),
+                                ];
+                            }
+
+                            $tmp[$metric] = array_reverse($tmp[$metric]);
+                            $tmp[$metric] = array_sum(array_column($tmp[$metric], 'value'));
+                        }
+                    });
+
+                    foreach ($tmp as $key => $value) {
+                        $stats[$metrics[$key]]  = $value;
+                    }
+
+                    try {
+                        /** Get Domains */
+                        $stats['Domains'] = $dbForConsole->count('domains', [
+                            Query::equal('projectInternalId', [$project->getInternalId()]),
+                        ]);
+                    } catch (\Throwable) {
+                        $stats['Domains'] = 0;
+                    }
+
+                    try {
+                    /** Get Api keys */
+                        $stats['Api keys'] = $dbForConsole->count('keys', [
+                        Query::equal('projectInternalId', [$project->getInternalId()]),
+                        ]);
+                    } catch (\Throwable) {
+                        $stats['Api keys'] = 0;
+                    }
+
+                    try {
+                    /** Get Webhooks */
+                        $stats['Webhooks'] = $dbForConsole->count('webhooks', [
+                        Query::equal('projectInternalId', [$project->getInternalId()]),
+                        ]);
+                    } catch (\Throwable) {
+                        $stats['Webhooks'] = 0;
+                    }
+
+                    try {
+                    /** Get Platforms */
+                        $stats['Platforms'] = $dbForConsole->count('platforms', [
+                        Query::equal('projectInternalId', [$project->getInternalId()]),
+                        ]);
+                    } catch (\Throwable) {
+                        $stats['Platforms'] = 0;
+                    }
+
+                    /** Get Files & Buckets */
+                    $filesCount = 0;
+                    $filesSum = 0;
+                    $maxFileSize = 0;
+                    $counter = 0;
+                    try {
+                        $buckets = $dbForProject->find('buckets', []);
+                        foreach ($buckets as $bucket) {
+                            $file = $dbForProject->findOne('bucket_' . $bucket->getInternalId(), [Query::orderDesc('sizeOriginal'),]);
+                            if (empty($file)) {
+                                continue;
+                            }
+                            $filesSum   += $dbForProject->sum('bucket_' . $bucket->getInternalId(), 'sizeOriginal', [], 0);
+                            $filesCount += $dbForProject->count('bucket_' . $bucket->getInternalId(), []);
+                            if ($file->getAttribute('sizeOriginal') > $maxFileSize) {
+                                $maxFileSize = $file->getAttribute('sizeOriginal');
+                            }
+                            $counter++;
+                        }
+                    } catch (\Throwable) {
+                        ;
+                    }
+                    $stats['Buckets'] = $counter;
+                    $stats['Files'] = $filesCount;
+                    $stats['Storage (bytes)'] = $filesSum;
+                    $stats['Max File Size (bytes)'] = $maxFileSize;
+
+
+                    try {
+                    /** Get Total Functions */
+                        $stats['Databases'] = $dbForProject->count('databases', []);
+                    } catch (\Throwable) {
+                        $stats['Databases'] = 0;
+                    }
+
+                    /** Get Total Functions */
+                    try {
+                        $stats['Functions'] = $dbForProject->count('functions', []);
+                    } catch (\Throwable) {
+                        $stats['Functions'] = 0;
+                    }
+
+                    /** Get Total Deployments */
+                    try {
+                        $stats['Deployments'] = $dbForProject->count('deployments', []);
+                    } catch (\Throwable) {
+                        $stats['Deployments'] = 0;
+                    }
+
+                    /** Get Total Executions */
+                    try {
+                        $stats['Executions'] = $dbForProject->count('executions', []);
+                    } catch (\Throwable) {
+                        $stats['Executions'] = 0;
+                    }
+
+                    $csv->insertOne(array_values($stats));
+                } catch (\Throwable $th) {
+                    Console::error('Failed on project ("' . $project->getId() . '") version with error on File: ' . $th->getFile() . '  line no: ' . $th->getline() . ' with message: ' . $th->getMessage());
+                } finally {
+                    $pools
+                        ->get($db)
+                        ->reclaim();
+                }
+            }
+
+            $sum = \count($projects);
+
+            $projects = $dbForConsole->find('projects', [
+                Query::limit($limit),
+                Query::offset($offset),
+            ]);
+
+            $offset = $offset + $limit;
+            $count = $count + $sum;
+        }
+
+        Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects...');
+
+        $pools
+            ->get('console')
+            ->reclaim();
+
+        /** @var PHPMailer $mail */
+        $mail = $register->get('smtp');
+
+        $mail->clearAddresses();
+        $mail->clearAllRecipients();
+        $mail->clearReplyTos();
+        $mail->clearAttachments();
+        $mail->clearBCCs();
+        $mail->clearCCs();
+
+        try {
+            /** Addresses */
+            $mail->setFrom(App::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM), 'Appwrite Cloud Hamster');
+            $recipients = explode(',', App::getEnv('_APP_USERS_STATS_RECIPIENTS', ''));
+
+            foreach ($recipients as $recipient) {
+                $mail->addAddress($recipient);
+            }
+
+            /** Attachments */
+            $mail->addAttachment($this->path);
+
+            /** Content */
+            $mail->Subject = "Cloud Report for {$this->date}";
+            $mail->Body = "Please find the daily cloud report atttached";
+            $mail->send();
+            Console::success('Email has been sent!');
+        } catch (Exception $e) {
+            Console::error("Message could not be sent. Mailer Error: {$mail->ErrorInfo}");
+        }
+    }
+}
@@ -0,0 +1,176 @@
+<?php
+
+namespace Appwrite\Platform\Tasks;
+
+use Exception;
+use Utopia\App;
+use Utopia\Platform\Action;
+use Utopia\Cache\Cache;
+use Utopia\CLI\Console;
+use Utopia\Database\Database;
+use Utopia\Database\Query;
+use League\Csv\Writer;
+use PHPMailer\PHPMailer\PHPMailer;
+use Utopia\Pools\Group;
+use Utopia\Registry\Registry;
+
+class CalcUsersStats extends Action
+{
+    private array $columns = [
+        'Project ID',
+        'Project Name',
+        'Team ID',
+        'Team name',
+        'Users'
+    ];
+
+    protected string $directory = '/usr/local';
+    protected string $path;
+    protected string $date;
+
+    public static function getName(): string
+    {
+        return 'calc-users-stats';
+    }
+
+    public function __construct()
+    {
+
+        $this
+            ->desc('Get stats for projects')
+            ->inject('pools')
+            ->inject('cache')
+            ->inject('dbForConsole')
+            ->inject('register')
+            ->callback(function (Group $pools, Cache $cache, Database $dbForConsole, Registry $register) {
+                $this->action($pools, $cache, $dbForConsole, $register);
+            });
+    }
+
+    public function action(Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void
+    {
+        //docker compose exec -t appwrite calc-users-stats
+
+        Console::title('Cloud Users calculation V1');
+        Console::success(APP_NAME . ' cloud Users calculation has started');
+
+        /* Initialise new Utopia app */
+        $app = new App('UTC');
+        $console = $app->getResource('console');
+
+        /** CSV stuff */
+        $this->date = date('Y-m-d');
+        $this->path = "{$this->directory}/users_stats_{$this->date}.csv";
+        $csv = Writer::createFromPath($this->path, 'w');
+        $csv->insertOne($this->columns);
+
+        /** Database connections */
+        $totalProjects = $dbForConsole->count('projects');
+        Console::success("Found a total of: {$totalProjects} projects");
+
+        $projects = [$console];
+        $count = 0;
+        $limit = 30;
+        $sum = 30;
+        $offset = 0;
+        while (!empty($projects)) {
+            foreach ($projects as $project) {
+
+                /**
+                 * Skip user projects with id 'console'
+                 */
+                if ($project->getId() === 'console') {
+                    continue;
+                }
+
+                Console::info("Getting stats for {$project->getId()}");
+
+                try {
+                    $db = $project->getAttribute('database');
+                    $adapter = $pools
+                        ->get($db)
+                        ->pop()
+                        ->getResource();
+
+                    $dbForProject = new Database($adapter, $cache);
+                    $dbForProject->setDefaultDatabase('appwrite');
+                    $dbForProject->setNamespace('_' . $project->getInternalId());
+
+                    /** Get Project ID */
+                    $stats['Project ID'] = $project->getId();
+
+                    /** Get Project Name */
+                    $stats['Project Name'] = $project->getAttribute('name');
+
+
+                    /** Get Team Name and Id */
+                    $teamId = $project->getAttribute('teamId', null);
+                    $teamName = null;
+                    if ($teamId) {
+                        $team = $dbForConsole->getDocument('teams', $teamId);
+                        $teamName = $team->getAttribute('name');
+                    }
+
+                    $stats['Team ID']   = $teamId;
+                    $stats['Team name'] = $teamName;
+
+                    /** Get Total Users */
+                    $stats['users'] = $dbForProject->count('users', []);
+
+                    $csv->insertOne(array_values($stats));
+                } catch (\Throwable $th) {
+                    Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage());
+                } finally {
+                    $pools
+                        ->get($db)
+                        ->reclaim();
+                }
+            }
+
+            $sum = \count($projects);
+
+            $projects = $dbForConsole->find('projects', [
+                Query::limit($limit),
+                Query::offset($offset),
+            ]);
+
+            $offset = $offset + $limit;
+            $count = $count + $sum;
+        }
+        Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects...');
+        $pools
+            ->get('console')
+            ->reclaim();
+
+        /** @var PHPMailer $mail */
+        $mail = $register->get('smtp');
+
+        $mail->clearAddresses();
+        $mail->clearAllRecipients();
+        $mail->clearReplyTos();
+        $mail->clearAttachments();
+        $mail->clearBCCs();
+        $mail->clearCCs();
+
+        try {
+            /** Addresses */
+            $mail->setFrom(App::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM), 'Appwrite Cloud Hamster');
+            $recipients = explode(',', App::getEnv('_APP_USERS_STATS_RECIPIENTS', ''));
+
+            foreach ($recipients as $recipient) {
+                $mail->addAddress($recipient);
+            }
+
+            /** Attachments */
+            $mail->addAttachment($this->path);
+
+            /** Content */
+            $mail->Subject = "Cloud Report for {$this->date}";
+            $mail->Body = "Please find the daily cloud report atttached";
+            $mail->send();
+            Console::success('Email has been sent!');
+        } catch (Exception $e) {
+            Console::error("Message could not be sent. Mailer Error: {$mail->ErrorInfo}");
+        }
+    }
+}
@@ -0,0 +1,62 @@
+<?php
+
+namespace Appwrite\Platform\Tasks;
+
+use Utopia\Cache\Adapter\Filesystem;
+use Utopia\Cache\Cache;
+use Utopia\Platform\Action;
+use Utopia\CLI\Console;
+use Utopia\Database\Query;
+use Utopia\Database\Database;
+use Utopia\Database\Validator\Authorization;
+use Utopia\Database\Validator\UID;
+
+class ClearCardCache extends Action
+{
+    public static function getName(): string
+    {
+        return 'clear-card-cache';
+    }
+
+    public function __construct()
+    {
+        $this
+            ->desc('Deletes card cache for specific user')
+            ->param('userId', '', new UID(), 'User UID.', false)
+            ->inject('dbForConsole')
+            ->callback(fn (string $userId, Database $dbForConsole) => $this->action($userId, $dbForConsole));
+    }
+
+    public function action(string $userId, Database $dbForConsole): void
+    {
+        Authorization::disable();
+        Authorization::setDefaultStatus(false);
+
+        Console::title('ClearCardCache V1');
+        Console::success(APP_NAME . ' ClearCardCache v1 has started');
+        $resources = ['card/' . $userId, 'card-back/' . $userId, 'card-og/' . $userId];
+
+        $caches = Authorization::skip(fn () => $dbForConsole->find('cache', [
+            Query::equal('resource', $resources),
+            Query::limit(100)
+        ]));
+
+        $count = \count($caches);
+        Console::info("Going to delete {$count} cache records in 10 seconds...");
+        \sleep(10);
+
+        foreach ($caches as $cache) {
+            $key = $cache->getId();
+
+            $cacheFolder = new Cache(
+                new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-console')
+            );
+
+            $cacheFolder->purge($key);
+
+            Authorization::skip(fn () => $dbForConsole->deleteDocument('cache', $cache->getId()));
+        }
+
+        Console::success(APP_NAME . ' ClearCardCache v1 has finished');
+    }
+}
@@ -2,6 +2,7 @@

 namespace Appwrite\Platform\Tasks;

+use Appwrite\Network\Validator\Origin;
 use Exception;
 use Utopia\App;
 use Utopia\Platform\Action;
@@ -12,6 +13,7 @@ use Utopia\Database\Query;
 use Utopia\Database\Validator\Authorization;
 use Utopia\Analytics\Adapter\Mixpanel;
 use Utopia\Analytics\Event;
+use Utopia\Config\Config;
 use Utopia\Database\Document;
 use Utopia\Pools\Group;

@@ -98,6 +100,12 @@ class Hamster extends Action
                /** Get Total Functions */
                $statsPerProject['custom_functions'] = $dbForProject->count('functions', [], APP_LIMIT_COUNT);

+                foreach (\array_keys(Config::getParam('runtimes')) as $runtime) {
+                    $statsPerProject['custom_functions_' . $runtime] = $dbForProject->count('functions', [
+                        Query::equal('runtime', [$runtime]),
+                    ], APP_LIMIT_COUNT);
+                }
+
                /** Get Total Deployments */
                $statsPerProject['custom_deployments'] = $dbForProject->count('deployments', [], APP_LIMIT_COUNT);

@@ -136,7 +144,10 @@ class Hamster extends Action
                }

                /** Get Domains */
-                $statsPerProject['custom_domains'] = $dbForProject->count('domains', [], APP_LIMIT_COUNT);
+                $statsPerProject['custom_domains'] = $dbForConsole->count('domains', [
+                    Query::equal('projectInternalId', [$project->getInternalId()]),
+                    Query::limit(APP_LIMIT_COUNT)
+                ]);

                /** Get Platforms */
                $platforms = $dbForConsole->find('platforms', [
@@ -152,7 +163,7 @@ class Hamster extends Action
                    return $platform['type'] === 'android';
                }));

-                $statsPerProject['custom_platforms_iOS'] = sizeof(array_filter($platforms, function ($platform) {
+                $statsPerProject['custom_platforms_apple'] = sizeof(array_filter($platforms, function ($platform) {
                    return str_contains($platform['type'], 'apple');
                }));

@@ -160,6 +171,19 @@ class Hamster extends Action
                    return str_contains($platform['type'], 'flutter');
                }));

+                $flutterPlatforms = [Origin::CLIENT_TYPE_FLUTTER_ANDROID, Origin::CLIENT_TYPE_FLUTTER_IOS, Origin::CLIENT_TYPE_FLUTTER_MACOS, Origin::CLIENT_TYPE_FLUTTER_WINDOWS, Origin::CLIENT_TYPE_FLUTTER_LINUX];
+
+                foreach ($flutterPlatforms as $flutterPlatform) {
+                    $statsPerProject['custom_platforms_' . $flutterPlatform] = sizeof(array_filter($platforms, function ($platform) use ($flutterPlatform) {
+                        return $platform['type'] === $flutterPlatform;
+                    }));
+                }
+
+                $statsPerProject['custom_platforms_api_keys'] = $dbForConsole->count('keys', [
+                    Query::equal('projectInternalId', [$project->getInternalId()]),
+                    Query::limit(APP_LIMIT_COUNT)
+                ]);
+
                /** Get Usage $statsPerProject */
                $periods = [
                    'infinity' => [
@@ -0,0 +1,129 @@
+<?php
+
+namespace Appwrite\Platform\Tasks;
+
+use Utopia\App;
+use Utopia\Platform\Action;
+use Utopia\Cache\Cache;
+use Utopia\CLI\Console;
+use Utopia\Database\Database;
+use Utopia\Database\Query;
+use Utopia\Pools\Group;
+use Utopia\Validator\Numeric;
+
+class PatchDeleteProjectCollections extends Action
+{
+    private array $names = [
+        'webhooks',
+        'platforms',
+        'schedules',
+        'projects',
+        'domains',
+        'certificates',
+        'keys',
+        'realtime',
+    ];
+
+    public static function getName(): string
+    {
+        return 'patch-delete-project-collections';
+    }
+
+    public function __construct()
+    {
+
+        $this
+            ->desc('Delete unnecessary project collections')
+            ->param('offset', 0, new Numeric(), 'Resume deletion from param pos', true)
+            ->inject('pools')
+            ->inject('cache')
+            ->inject('dbForConsole')
+            ->callback(function (int $offset, Group $pools, Cache $cache, Database $dbForConsole) {
+                $this->action($offset, $pools, $cache, $dbForConsole);
+            });
+    }
+
+    public function action(int $offset, Group $pools, Cache $cache, Database $dbForConsole): void
+    {
+        //docker compose exec -t appwrite patch-delete-project-collections
+
+        Console::title('Delete project collections V1');
+        Console::success(APP_NAME . ' delete project collections has started');
+
+        /* Initialise new Utopia app */
+        $app = new App('UTC');
+        $console = $app->getResource('console');
+
+        /** Database connections */
+        $totalProjects = $dbForConsole->count('projects');
+        Console::success("Found a total of: {$totalProjects} projects");
+
+        $projects = [$console];
+        $count = 0;
+        $limit = 50;
+        $sum = 50;
+        $offset = $offset;
+        while (!empty($projects)) {
+            foreach ($projects as $project) {
+
+                /**
+                 * Skip user projects with id 'console'
+                 */
+                if ($project->getId() === 'console') {
+                    continue;
+                }
+
+                Console::info("Deleting collections for {$project->getId()}");
+
+                try {
+                    $db = $project->getAttribute('database');
+                    $adapter = $pools
+                        ->get($db)
+                        ->pop()
+                        ->getResource();
+
+                    $dbForProject = new Database($adapter, $cache);
+                    $dbForProject->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
+                    $dbForProject->setNamespace('_' . $project->getInternalId());
+
+                    foreach ($this->names as $name) {
+                        if (empty($name)) {
+                            continue;
+                        }
+                        if ($dbForProject->exists(App::getEnv('_APP_DB_SCHEMA', 'appwrite'), $name)) {
+                            if ($dbForProject->deleteCollection($name)) {
+                                Console::log('Deleted ' . $name);
+                            } else {
+                                Console::error('Failed to delete ' . $name);
+                            }
+                        }
+                    }
+                } catch (\Throwable $th) {
+                    Console::error('Failed  on project ("' . $project->getId() . '") version with error: ' . $th->getMessage());
+                } finally {
+                    $pools
+                        ->get($db)
+                        ->reclaim();
+                }
+            }
+
+            $sum = \count($projects);
+
+            $projects = $dbForConsole->find('projects', [
+                Query::limit($limit),
+                Query::offset($offset),
+            ]);
+
+            if (!empty($projects)) {
+                Console::log('Querying..... offset=' . $offset . ' , limit=' . $limit  . ', count=' . $count);
+            }
+
+            $offset = $offset + $limit;
+            $count = $count + $sum;
+        }
+        Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects...');
+        $pools
+            ->get('console')
+            ->reclaim();
+    }
+}
@@ -174,6 +174,7 @@ abstract class Worker
     * @throws Exception
     */
    protected static $databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools
+
    protected function getProjectDB(Document $project): Database
    {
        global $register;
@@ -218,6 +219,14 @@ abstract class Worker

        $pools = $register->get('pools'); /** @var Group $pools */

+        $databaseName = 'console';
+
+        if (isset(self::$databases[$databaseName])) {
+            $database = self::$databases[$databaseName];
+            $database->setNamespace('console');
+            return $database;
+        }
+
        $dbAdapter = $pools
            ->get('console')
            ->pop()
@@ -226,6 +235,8 @@ abstract class Worker

        $database = new Database($dbAdapter, $this->getCache());

+        self::$databases[$databaseName] = $database;
+
        $database->setNamespace('console');

        return $database;
@@ -22,7 +22,9 @@ class Base extends Queries
     */
    public function __construct(string $collection, array $allowedAttributes)
    {
-        $collection = Config::getParam('collections', [])[$collection];
+        $config = Config::getParam('collections', []);
+        $collections = array_merge($config['console'], $config['projects'], $config['buckets'], $config['databases']);
+        $collection = $collections[$collection];
        // array for constant lookup time
        $allowedAttributesLookup = [];
        foreach ($allowedAttributes as $attribute) {