From 1da476e5acf1e52342d2371914abb923ac73f389 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 9 Jan 2024 11:13:58 +0200 Subject: [PATCH] remove cloud related scripts --- Dockerfile | 14 +- bin/calc-tier-stats | 3 - bin/calc-users-stats | 3 - bin/clear-card-cache | 3 - bin/delete-orphaned-projects | 3 - bin/get-migration-stats | 3 - bin/hamster | 3 - bin/patch-delete-project-collections | 3 - ...patch-delete-schedule-updated-at-attribute | 3 - bin/patch-recreate-repositories-documents | 3 - bin/volume-sync | 3 - bin/worker-hamster | 3 - docker-compose.yml | 57 --- src/Appwrite/Event/Event.php | 3 - src/Appwrite/Event/Hamster.php | 157 ------- src/Appwrite/Platform/Services/Tasks.php | 11 - src/Appwrite/Platform/Tasks/CalcTierStats.php | 359 -------------- .../Platform/Tasks/DeleteOrphanedProjects.php | 161 ------- .../Platform/Tasks/GetMigrationStats.php | 187 -------- src/Appwrite/Platform/Tasks/Hamster.php | 158 ------- .../PatchRecreateRepositoriesDocuments.php | 169 ------- src/Appwrite/Platform/Tasks/VolumeSync.php | 59 --- src/Appwrite/Platform/Workers/Hamster.php | 437 ------------------ 23 files changed, 1 insertion(+), 1804 deletions(-) delete mode 100644 bin/calc-tier-stats delete mode 100644 bin/calc-users-stats delete mode 100644 bin/clear-card-cache delete mode 100644 bin/delete-orphaned-projects delete mode 100644 bin/get-migration-stats delete mode 100644 bin/hamster delete mode 100644 bin/patch-delete-project-collections delete mode 100644 bin/patch-delete-schedule-updated-at-attribute delete mode 100644 bin/patch-recreate-repositories-documents delete mode 100644 bin/volume-sync delete mode 100644 bin/worker-hamster delete mode 100644 src/Appwrite/Event/Hamster.php delete mode 100644 src/Appwrite/Platform/Tasks/CalcTierStats.php delete mode 100644 src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php delete mode 100644 src/Appwrite/Platform/Tasks/GetMigrationStats.php delete mode 100644 src/Appwrite/Platform/Tasks/Hamster.php delete mode 100644 src/Appwrite/Platform/Tasks/PatchRecreateRepositoriesDocuments.php delete mode 100644 src/Appwrite/Platform/Tasks/VolumeSync.php delete mode 100644 src/Appwrite/Platform/Workers/Hamster.php diff --git a/Dockerfile b/Dockerfile index 2f85f2cc43..4fba47cdbf 100755 --- a/Dockerfile +++ b/Dockerfile @@ -94,20 +94,8 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/worker-mails && \ chmod +x /usr/local/bin/worker-messaging && \ chmod +x /usr/local/bin/worker-webhooks && \ - chmod +x /usr/local/bin/worker-migrations && \ - chmod +x /usr/local/bin/worker-hamster + chmod +x /usr/local/bin/worker-migrations -# Cloud Executabless -RUN chmod +x /usr/local/bin/hamster && \ - chmod +x /usr/local/bin/volume-sync && \ - chmod +x /usr/local/bin/patch-delete-schedule-updated-at-attribute && \ - chmod +x /usr/local/bin/patch-recreate-repositories-documents && \ - chmod +x /usr/local/bin/patch-delete-project-collections && \ - chmod +x /usr/local/bin/delete-orphaned-projects && \ - chmod +x /usr/local/bin/clear-card-cache && \ - chmod +x /usr/local/bin/calc-users-stats && \ - chmod +x /usr/local/bin/calc-tier-stats && \ - chmod +x /usr/local/bin/get-migration-stats # Letsencrypt Permissions RUN mkdir -p /etc/letsencrypt/live/ && chmod -Rf 755 /etc/letsencrypt/live/ diff --git a/bin/calc-tier-stats b/bin/calc-tier-stats deleted file mode 100644 index c7fb71e6fd..0000000000 --- a/bin/calc-tier-stats +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/cli.php calc-tier-stats $@ \ No newline at end of file diff --git a/bin/calc-users-stats b/bin/calc-users-stats deleted file mode 100644 index 60472ed7a2..0000000000 --- a/bin/calc-users-stats +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/cli.php calc-users-stats $@ \ No newline at end of file diff --git a/bin/clear-card-cache b/bin/clear-card-cache deleted file mode 100644 index b39bc13651..0000000000 --- a/bin/clear-card-cache +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/cli.php clear-card-cache $@ \ No newline at end of file diff --git a/bin/delete-orphaned-projects b/bin/delete-orphaned-projects deleted file mode 100644 index 16b9e3184d..0000000000 --- a/bin/delete-orphaned-projects +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/cli.php delete-orphaned-projects $@ \ No newline at end of file diff --git a/bin/get-migration-stats b/bin/get-migration-stats deleted file mode 100644 index efc1934e15..0000000000 --- a/bin/get-migration-stats +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/cli.php get-migration-stats $@ \ No newline at end of file diff --git a/bin/hamster b/bin/hamster deleted file mode 100644 index dcc7ed308d..0000000000 --- a/bin/hamster +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/cli.php hamster $@ \ No newline at end of file diff --git a/bin/patch-delete-project-collections b/bin/patch-delete-project-collections deleted file mode 100644 index 8bf0736cf3..0000000000 --- a/bin/patch-delete-project-collections +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/cli.php patch-delete-project-collections $@ \ No newline at end of file diff --git a/bin/patch-delete-schedule-updated-at-attribute b/bin/patch-delete-schedule-updated-at-attribute deleted file mode 100644 index 3e28289cbe..0000000000 --- a/bin/patch-delete-schedule-updated-at-attribute +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/cli.php patch-delete-schedule-updated-at-attribute $@ \ No newline at end of file diff --git a/bin/patch-recreate-repositories-documents b/bin/patch-recreate-repositories-documents deleted file mode 100644 index 8c6c4157f4..0000000000 --- a/bin/patch-recreate-repositories-documents +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/cli.php patch-recreate-repositories-documents $@ \ No newline at end of file diff --git a/bin/volume-sync b/bin/volume-sync deleted file mode 100644 index 5190750a24..0000000000 --- a/bin/volume-sync +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/cli.php volume-sync $@ \ No newline at end of file diff --git a/bin/worker-hamster b/bin/worker-hamster deleted file mode 100644 index b388dd13c9..0000000000 --- a/bin/worker-hamster +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/worker.php hamster $@ \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 97cf7e5136..42091e5e46 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -717,63 +717,6 @@ services: environment: - _APP_ASSISTANT_OPENAI_API_KEY - appwrite-worker-hamster: - entrypoint: worker-hamster - <<: *x-logging - container_name: appwrite-worker-hamster - image: appwrite-dev - networks: - - appwrite - volumes: - - ./app:/usr/src/code/app - - ./src:/usr/src/code/src - depends_on: - - redis - - mariadb - environment: - - _APP_ENV - - _APP_WORKER_PER_CORE - - _APP_OPENSSL_KEY_V1 - - _APP_DB_HOST - - _APP_DB_PORT - - _APP_DB_SCHEMA - - _APP_DB_USER - - _APP_DB_PASS - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - - _APP_MIXPANEL_TOKEN - - appwrite-hamster-scheduler: - entrypoint: hamster - <<: *x-logging - container_name: appwrite-hamster-scheduler - image: appwrite-dev - networks: - - appwrite - volumes: - - ./app:/usr/src/code/app - - ./src:/usr/src/code/src - depends_on: - - redis - - mariadb - environment: - - _APP_ENV - - _APP_WORKER_PER_CORE - - _APP_OPENSSL_KEY_V1 - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - - _APP_DB_HOST - - _APP_DB_PORT - - _APP_DB_SCHEMA - - _APP_DB_USER - - _APP_DB_PASS - - _APP_HAMSTER_TIME - - _APP_HAMSTER_INTERVAL - openruntimes-executor: container_name: openruntimes-executor hostname: appwrite-executor diff --git a/src/Appwrite/Event/Event.php b/src/Appwrite/Event/Event.php index fc12c5b5b3..46b430d122 100644 --- a/src/Appwrite/Event/Event.php +++ b/src/Appwrite/Event/Event.php @@ -42,9 +42,6 @@ class Event public const MIGRATIONS_QUEUE_NAME = 'v1-migrations'; public const MIGRATIONS_CLASS_NAME = 'MigrationsV1'; - public const HAMSTER_QUEUE_NAME = 'v1-hamster'; - public const HAMSTER_CLASS_NAME = 'HamsterV1'; - protected string $queue = ''; protected string $class = ''; protected string $event = ''; diff --git a/src/Appwrite/Event/Hamster.php b/src/Appwrite/Event/Hamster.php deleted file mode 100644 index 5d79fce568..0000000000 --- a/src/Appwrite/Event/Hamster.php +++ /dev/null @@ -1,157 +0,0 @@ -setQueue(Event::HAMSTER_QUEUE_NAME) - ->setClass(Event::HAMSTER_CLASS_NAME); - } - - /** - * Sets the type for the hamster event. - * - * @param string $type - * @return self - */ - public function setType(string $type): self - { - $this->type = $type; - - return $this; - } - - /** - * Returns the set type for the hamster event. - * - * @return string - */ - public function getType(): string - { - return $this->type; - } - - /** - * Sets the project for the hamster event. - * - * @param Document $project - */ - public function setProject(Document $project): self - { - $this->project = $project; - - return $this; - } - - /** - * Returns the set project for the hamster event. - * - * @return Document - */ - public function getProject(): Document - { - return $this->project; - } - - /** - * Sets the organization for the hamster event. - * - * @param Document $organization - */ - public function setOrganization(Document $organization): self - { - $this->organization = $organization; - - return $this; - } - - /** - * Returns the set organization for the hamster event. - * - * @return string - */ - public function getOrganization(): Document - { - return $this->organization; - } - - /** - * Sets the user for the hamster event. - * - * @param Document $user - */ - public function setUser(Document $user): self - { - $this->user = $user; - - return $this; - } - - /** - * Returns the set user for the hamster event. - * - * @return Document - */ - public function getUser(): Document - { - return $this->user; - } - - /** - * Executes the function event and sends it to the functions worker. - * - * @return string|bool - * @throws \InvalidArgumentException - */ - public function trigger(): string|bool - { - if ($this->paused) { - return false; - } - - $client = new Client($this->queue, $this->connection); - - $events = $this->getEvent() ? Event::generateEvents($this->getEvent(), $this->getParams()) : null; - - return $client->enqueue([ - 'type' => $this->type, - 'project' => $this->project, - 'organization' => $this->organization, - 'user' => $this->user, - 'events' => $events, - ]); - } - - /** - * Generate a function event from a base event - * - * @param Event $event - * - * @return self - * - */ - public function from(Event $event): self - { - $this->event = $event->getEvent(); - $this->params = $event->getParams(); - return $this; - } -} diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index dc6ddc1a5b..0da42e3545 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -15,12 +15,7 @@ use Appwrite\Platform\Tasks\Hamster; use Appwrite\Platform\Tasks\Usage; use Appwrite\Platform\Tasks\Vars; use Appwrite\Platform\Tasks\Version; -use Appwrite\Platform\Tasks\VolumeSync; -use Appwrite\Platform\Tasks\CalcTierStats; use Appwrite\Platform\Tasks\Upgrade; -use Appwrite\Platform\Tasks\DeleteOrphanedProjects; -use Appwrite\Platform\Tasks\GetMigrationStats; -use Appwrite\Platform\Tasks\PatchRecreateRepositoriesDocuments; class Tasks extends Service { @@ -40,13 +35,7 @@ class Tasks extends Service ->addAction(Schedule::getName(), new Schedule()) ->addAction(Migrate::getName(), new Migrate()) ->addAction(SDKs::getName(), new SDKs()) - ->addAction(VolumeSync::getName(), new VolumeSync()) ->addAction(Specs::getName(), new Specs()) - ->addAction(CalcTierStats::getName(), new CalcTierStats()) - ->addAction(DeleteOrphanedProjects::getName(), new DeleteOrphanedProjects()) - ->addAction(PatchRecreateRepositoriesDocuments::getName(), new PatchRecreateRepositoriesDocuments()) - ->addAction(GetMigrationStats::getName(), new GetMigrationStats()) - ; } } diff --git a/src/Appwrite/Platform/Tasks/CalcTierStats.php b/src/Appwrite/Platform/Tasks/CalcTierStats.php deleted file mode 100644 index e6559a05d7..0000000000 --- a/src/Appwrite/Platform/Tasks/CalcTierStats.php +++ /dev/null @@ -1,359 +0,0 @@ - 'Requests', - 'project.$all.network.bandwidth' => 'Bandwidth', - - ]; - - public static function getName(): string - { - return 'calc-tier-stats'; - } - - public function __construct() - { - - $this - ->desc('Get stats for projects') - ->inject('pools') - ->inject('cache') - ->inject('dbForConsole') - ->inject('register') - ->callback(function (Group $pools, Cache $cache, Database $dbForConsole, Registry $register) { - $this->action($pools, $cache, $dbForConsole, $register); - }); - } - - /** - * @throws \Utopia\Exception - * @throws CannotInsertRecord - */ - public function action(Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void - { - //docker compose exec -t appwrite calc-tier-stats - - Console::title('Cloud free tier stats calculation V1'); - Console::success(APP_NAME . ' cloud free tier stats calculation has started'); - - /* Initialise new Utopia app */ - $app = new App('UTC'); - $console = $app->getResource('console'); - - /** CSV stuff */ - $this->date = date('Y-m-d'); - $this->path = "{$this->directory}/tier_stats_{$this->date}.csv"; - $csv = Writer::createFromPath($this->path, 'w'); - $csv->insertOne($this->columns); - - /** Database connections */ - $totalProjects = $dbForConsole->count('projects'); - Console::success("Found a total of: {$totalProjects} projects"); - - $projects = [$console]; - $count = 0; - $limit = 100; - $sum = 100; - $offset = 0; - while (!empty($projects)) { - foreach ($projects as $project) { - - /** - * Skip user projects with id 'console' - */ - if ($project->getId() === 'console') { - continue; - } - - Console::info("Getting stats for {$project->getId()}"); - - try { - $db = $project->getAttribute('database'); - $adapter = $pools - ->get($db) - ->pop() - ->getResource(); - - $dbForProject = new Database($adapter, $cache); - $dbForProject->setDefaultDatabase('appwrite'); - $dbForProject->setNamespace('_' . $project->getInternalId()); - - /** Get Project ID */ - $stats['Project ID'] = $project->getId(); - - $stats['Organization ID'] = $project->getAttribute('teamId', null); - - /** Get Total Members */ - $teamInternalId = $project->getAttribute('teamInternalId', null); - if ($teamInternalId) { - $stats['Organization Members'] = $dbForConsole->count('memberships', [ - Query::equal('teamInternalId', [$teamInternalId]) - ]); - } else { - $stats['Organization Members'] = 0; - } - - /** Get Total internal Teams */ - try { - $stats['Teams'] = $dbForProject->count('teams', []); - } catch (\Throwable) { - $stats['Teams'] = 0; - } - - /** Get Total users */ - try { - $stats['Users'] = $dbForProject->count('users', []); - } catch (\Throwable) { - $stats['Users'] = 0; - } - - /** Get Usage stats */ - $range = '30d'; - $periods = [ - '30d' => [ - 'period' => '1d', - 'limit' => 30, - ] - ]; - - $tmp = []; - $metrics = $this->usageStats; - Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$tmp) { - foreach ($metrics as $metric => $name) { - $limit = $periods[$range]['limit']; - $period = $periods[$range]['period']; - - $requestDocs = $dbForProject->find('stats', [ - Query::equal('period', [$period]), - Query::equal('metric', [$metric]), - Query::limit($limit), - Query::orderDesc('time'), - ]); - - $tmp[$metric] = []; - foreach ($requestDocs as $requestDoc) { - if (empty($requestDoc)) { - continue; - } - - $tmp[$metric][] = [ - 'value' => $requestDoc->getAttribute('value'), - 'date' => $requestDoc->getAttribute('time'), - ]; - } - - $tmp[$metric] = array_reverse($tmp[$metric]); - $tmp[$metric] = array_sum(array_column($tmp[$metric], 'value')); - } - }); - - foreach ($tmp as $key => $value) { - $stats[$metrics[$key]] = $value; - } - - try { - /** Get Domains */ - $stats['Domains'] = $dbForConsole->count('rules', [ - Query::equal('projectInternalId', [$project->getInternalId()]), - ]); - } catch (\Throwable) { - $stats['Domains'] = 0; - } - - try { - /** Get Api keys */ - $stats['Api keys'] = $dbForConsole->count('keys', [ - Query::equal('projectInternalId', [$project->getInternalId()]), - ]); - } catch (\Throwable) { - $stats['Api keys'] = 0; - } - - try { - /** Get Webhooks */ - $stats['Webhooks'] = $dbForConsole->count('webhooks', [ - Query::equal('projectInternalId', [$project->getInternalId()]), - ]); - } catch (\Throwable) { - $stats['Webhooks'] = 0; - } - - try { - /** Get Platforms */ - $stats['Platforms'] = $dbForConsole->count('platforms', [ - Query::equal('projectInternalId', [$project->getInternalId()]), - ]); - } catch (\Throwable) { - $stats['Platforms'] = 0; - } - - /** Get Files & Buckets */ - $filesCount = 0; - $filesSum = 0; - $maxFileSize = 0; - $counter = 0; - try { - $buckets = $dbForProject->find('buckets', []); - foreach ($buckets as $bucket) { - $file = $dbForProject->findOne('bucket_' . $bucket->getInternalId(), [Query::orderDesc('sizeOriginal'),]); - if (empty($file)) { - continue; - } - $filesSum += $dbForProject->sum('bucket_' . $bucket->getInternalId(), 'sizeOriginal', []); - $filesCount += $dbForProject->count('bucket_' . $bucket->getInternalId(), []); - if ($file->getAttribute('sizeOriginal') > $maxFileSize) { - $maxFileSize = $file->getAttribute('sizeOriginal'); - } - $counter++; - } - } catch (\Throwable) { - ; - } - $stats['Buckets'] = $counter; - $stats['Files'] = $filesCount; - $stats['Storage (bytes)'] = $filesSum; - $stats['Max File Size (bytes)'] = $maxFileSize; - - - try { - /** Get Total Functions */ - $stats['Databases'] = $dbForProject->count('databases', []); - } catch (\Throwable) { - $stats['Databases'] = 0; - } - - /** Get Total Functions */ - try { - $stats['Functions'] = $dbForProject->count('functions', []); - } catch (\Throwable) { - $stats['Functions'] = 0; - } - - /** Get Total Deployments */ - try { - $stats['Deployments'] = $dbForProject->count('deployments', []); - } catch (\Throwable) { - $stats['Deployments'] = 0; - } - - /** Get Total Executions */ - try { - $stats['Executions'] = $dbForProject->count('executions', []); - } catch (\Throwable) { - $stats['Executions'] = 0; - } - - /** Get Total Migrations */ - try { - $stats['Migrations'] = $dbForProject->count('migrations', []); - } catch (\Throwable) { - $stats['Migrations'] = 0; - } - - $csv->insertOne(array_values($stats)); - } catch (\Throwable $th) { - Console::error('Failed on project ("' . $project->getId() . '") version with error on File: ' . $th->getFile() . ' line no: ' . $th->getline() . ' with message: ' . $th->getMessage()); - } finally { - $pools - ->get($db) - ->reclaim(); - } - } - - $sum = \count($projects); - - $projects = $dbForConsole->find('projects', [ - Query::limit($limit), - Query::offset($offset), - ]); - - $offset = $offset + $limit; - $count = $count + $sum; - } - - Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects...'); - - $pools - ->get('console') - ->reclaim(); - - /** @var PHPMailer $mail */ - $mail = $register->get('smtp'); - - $mail->clearAddresses(); - $mail->clearAllRecipients(); - $mail->clearReplyTos(); - $mail->clearAttachments(); - $mail->clearBCCs(); - $mail->clearCCs(); - - try { - /** Addresses */ - $mail->setFrom(App::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM), 'Appwrite Cloud Hamster'); - $recipients = explode(',', App::getEnv('_APP_USERS_STATS_RECIPIENTS', '')); - - foreach ($recipients as $recipient) { - $mail->addAddress($recipient); - } - - /** Attachments */ - $mail->addAttachment($this->path); - - /** Content */ - $mail->Subject = "Cloud Report for {$this->date}"; - $mail->Body = "Please find the daily cloud report atttached"; - $mail->send(); - Console::success('Email has been sent!'); - } catch (Exception $e) { - Console::error("Message could not be sent. Mailer Error: {$mail->ErrorInfo}"); - } - } -} diff --git a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php deleted file mode 100644 index 757b29c1b6..0000000000 --- a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php +++ /dev/null @@ -1,161 +0,0 @@ -desc('Delete orphaned projects') - ->param('commit', false, new Boolean(true), 'Commit project deletion', true) - ->inject('pools') - ->inject('cache') - ->inject('dbForConsole') - ->inject('register') - ->callback(function (bool $commit, Group $pools, Cache $cache, Database $dbForConsole, Registry $register) { - $this->action($commit, $pools, $cache, $dbForConsole, $register); - }); - } - - - public function action(bool $commit, Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void - { - - Console::title('Delete orphaned projects V1'); - Console::success(APP_NAME . ' Delete orphaned projects started'); - - /** @var array $collections */ - $collectionsConfig = Config::getParam('collections', [])['projects'] ?? []; - - $collectionsConfig = array_merge([ - 'audit' => [ - '$id' => ID::custom('audit'), - '$collection' => Database::METADATA - ], - 'abuse' => [ - '$id' => ID::custom('abuse'), - '$collection' => Database::METADATA - ] - ], $collectionsConfig); - - /* Initialise new Utopia app */ - $app = new App('UTC'); - $console = $app->getResource('console'); - $projects = [$console]; - - /** Database connections */ - $totalProjects = $dbForConsole->count('projects'); - Console::success("Found a total of: {$totalProjects} projects"); - - $orphans = 1; - $cnt = 0; - $count = 0; - $limit = 30; - $sum = 30; - $offset = 0; - while (!empty($projects)) { - foreach ($projects as $project) { - - /** - * Skip user projects with id 'console' - */ - if ($project->getId() === 'console') { - continue; - } - - try { - $db = $project->getAttribute('database'); - $adapter = $pools - ->get($db) - ->pop() - ->getResource(); - - $dbForProject = new Database($adapter, $cache); - $dbForProject->setDefaultDatabase('appwrite'); - $dbForProject->setNamespace('_' . $project->getInternalId()); - - $collectionsCreated = 0; - $cnt++; - if ($dbForProject->exists($dbForProject->getDefaultDatabase(), Database::METADATA)) { - $collectionsCreated = $dbForProject->count(Database::METADATA); - } - - $msg = '(' . $cnt . ') found (' . $collectionsCreated . ') collections on project (' . $project->getInternalId() . ') , database (' . $project['database'] . ')'; - - if ($collectionsCreated >= count($collectionsConfig)) { - Console::log($msg . ' ignoring....'); - continue; - } - - Console::log($msg); - - if ($collectionsCreated > 0) { - $collections = $dbForProject->find(Database::METADATA, []); - foreach ($collections as $collection) { - if ($commit) { - $dbForProject->deleteCollection($collection->getId()); - $dbForConsole->deleteCachedCollection($collection->getId()); - } - Console::info('--Deleting collection (' . $collection->getId() . ') project no (' . $project->getInternalId() . ')'); - } - } - - if ($commit) { - $dbForConsole->deleteDocument('projects', $project->getId()); - $dbForConsole->deleteCachedDocument('projects', $project->getId()); - - if ($dbForProject->exists($dbForProject->getDefaultDatabase(), Database::METADATA)) { - try { - $dbForProject->deleteCollection(Database::METADATA); - $dbForProject->deleteCachedCollection(Database::METADATA); - } catch (\Throwable $th) { - Console::warning('Metadata collection does not exist'); - } - } - } - - Console::info('--Deleting project no (' . $project->getInternalId() . ')'); - - $orphans++; - } catch (\Throwable $th) { - Console::error('Error: ' . $th->getMessage() . ' ' . $th->getTraceAsString()); - } finally { - $pools - ->get($db) - ->reclaim(); - } - } - - $sum = \count($projects); - - $projects = $dbForConsole->find('projects', [ - Query::limit($limit), - Query::offset($offset), - ]); - - $offset = $offset + $limit; - $count = $count + $sum; - } - - Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects found ' . $orphans - 1 . ' orphans'); - } -} diff --git a/src/Appwrite/Platform/Tasks/GetMigrationStats.php b/src/Appwrite/Platform/Tasks/GetMigrationStats.php deleted file mode 100644 index b76e0428d7..0000000000 --- a/src/Appwrite/Platform/Tasks/GetMigrationStats.php +++ /dev/null @@ -1,187 +0,0 @@ -desc('Get stats for projects') - ->inject('pools') - ->inject('cache') - ->inject('dbForConsole') - ->inject('register') - ->callback(function (Group $pools, Cache $cache, Database $dbForConsole, Registry $register) { - $this->action($pools, $cache, $dbForConsole, $register); - }); - } - - /** - * @throws \Utopia\Exception - * @throws CannotInsertRecord - */ - public function action(Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void - { - //docker compose exec -t appwrite get-migration-stats - - Console::title('Migration stats calculation V1'); - Console::success(APP_NAME . ' Migration stats calculation has started'); - - /* Initialise new Utopia app */ - $app = new App('UTC'); - $console = $app->getResource('console'); - - /** CSV stuff */ - $this->date = date('Y-m-d'); - $this->path = "{$this->directory}/migration_stats_{$this->date}.csv"; - $csv = Writer::createFromPath($this->path, 'w'); - $csv->insertOne($this->columns); - - /** Database connections */ - $totalProjects = $dbForConsole->count('projects'); - Console::success("Found a total of: {$totalProjects} projects"); - - $projects = [$console]; - $count = 0; - $limit = 100; - $sum = 100; - $offset = 0; - while (!empty($projects)) { - foreach ($projects as $project) { - - /** - * Skip user projects with id 'console' - */ - if ($project->getId() === 'console') { - continue; - } - - Console::info("Getting stats for {$project->getId()}"); - - try { - $db = $project->getAttribute('database'); - $adapter = $pools - ->get($db) - ->pop() - ->getResource(); - - $dbForProject = new Database($adapter, $cache); - $dbForProject->setDefaultDatabase('appwrite'); - $dbForProject->setNamespace('_' . $project->getInternalId()); - - /** Get Project ID */ - $stats['Project ID'] = $project->getId(); - - /** Get Migration details */ - $migrations = $dbForProject->find('migrations', [ - Query::limit(500) - ]); - - $migrations = array_map(function ($migration) use ($project) { - return [ - $project->getId(), - $migration->getAttribute('$id'), - $migration->getAttribute('$createdAt'), - $migration->getAttribute('status'), - $migration->getAttribute('stage'), - $migration->getAttribute('source'), - ]; - }, $migrations); - - if (!empty($migrations)) { - $csv->insertAll($migrations); - } - } catch (\Throwable $th) { - Console::error('Failed on project ("' . $project->getId() . '") with error on File: ' . $th->getFile() . ' line no: ' . $th->getline() . ' with message: ' . $th->getMessage()); - } finally { - $pools - ->get($db) - ->reclaim(); - } - } - - $sum = \count($projects); - - $projects = $dbForConsole->find('projects', [ - Query::limit($limit), - Query::offset($offset), - ]); - - $offset = $offset + $limit; - $count = $count + $sum; - } - - Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects...'); - - $pools - ->get('console') - ->reclaim(); - - /** @var PHPMailer $mail */ - $mail = $register->get('smtp'); - - $mail->clearAddresses(); - $mail->clearAllRecipients(); - $mail->clearReplyTos(); - $mail->clearAttachments(); - $mail->clearBCCs(); - $mail->clearCCs(); - - try { - /** Addresses */ - $mail->setFrom(App::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM), 'Appwrite Cloud Hamster'); - $recipients = explode(',', App::getEnv('_APP_USERS_STATS_RECIPIENTS', '')); - - foreach ($recipients as $recipient) { - $mail->addAddress($recipient); - } - - /** Attachments */ - $mail->addAttachment($this->path); - - /** Content */ - $mail->Subject = "Migration Report for {$this->date}"; - $mail->Body = "Please find the migration report atttached"; - $mail->send(); - Console::success('Email has been sent!'); - } catch (Exception $e) { - Console::error("Message could not be sent. Mailer Error: {$mail->ErrorInfo}"); - } - } -} diff --git a/src/Appwrite/Platform/Tasks/Hamster.php b/src/Appwrite/Platform/Tasks/Hamster.php deleted file mode 100644 index 1dca095e93..0000000000 --- a/src/Appwrite/Platform/Tasks/Hamster.php +++ /dev/null @@ -1,158 +0,0 @@ -desc('Get stats for projects') - ->inject('queueForHamster') - ->inject('dbForConsole') - ->callback(function (EventHamster $queueForHamster, Database $dbForConsole) { - $this->action($queueForHamster, $dbForConsole); - }); - } - - public function action(EventHamster $queueForHamster, Database $dbForConsole): void - { - Console::title('Cloud Hamster V1'); - Console::success(APP_NAME . ' cloud hamster process has started'); - - $sleep = (int) App::getEnv('_APP_HAMSTER_INTERVAL', '30'); // 30 seconds (by default) - - $jobInitTime = App::getEnv('_APP_HAMSTER_TIME', '22:00'); // (hour:minutes) - - $now = new \DateTime(); - $now->setTimezone(new \DateTimeZone(date_default_timezone_get())); - - $next = new \DateTime($now->format("Y-m-d $jobInitTime")); - $next->setTimezone(new \DateTimeZone(date_default_timezone_get())); - - $delay = $next->getTimestamp() - $now->getTimestamp(); - /** - * If time passed for the target day. - */ - if ($delay <= 0) { - $next->add(\DateInterval::createFromDateString('1 days')); - $delay = $next->getTimestamp() - $now->getTimestamp(); - } - - Console::log('[' . $now->format("Y-m-d H:i:s.v") . '] Delaying for ' . $delay . ' setting loop to [' . $next->format("Y-m-d H:i:s.v") . ']'); - - Console::loop(function () use ($queueForHamster, $dbForConsole, $sleep) { - $now = date('d-m-Y H:i:s', time()); - Console::info("[{$now}] Queuing Cloud Usage Stats every {$sleep} seconds"); - $loopStart = microtime(true); - - Console::info('Queuing stats for all projects'); - $this->getStatsPerProject($queueForHamster, $dbForConsole, $loopStart); - Console::success('Completed queuing stats for all projects'); - - Console::info('Queuing stats for all organizations'); - $this->getStatsPerOrganization($queueForHamster, $dbForConsole, $loopStart); - Console::success('Completed queuing stats for all organizations'); - - Console::info('Queuing stats for all users'); - $this->getStatsPerUser($queueForHamster, $dbForConsole, $loopStart); - Console::success('Completed queuing stats for all users'); - - $loopTook = microtime(true) - $loopStart; - $now = date('d-m-Y H:i:s', time()); - Console::info("[{$now}] Cloud Stats took {$loopTook} seconds"); - }, $sleep, $delay); - } - - protected function calculateByGroup(string $collection, Database $database, callable $callback) - { - $count = 0; - $chunk = 0; - $limit = 50; - $results = []; - $sum = $limit; - - $executionStart = \microtime(true); - - while ($sum === $limit) { - $chunk++; - - $results = $database->find($collection, \array_merge([ - Query::limit($limit), - Query::offset($count) - ])); - - $sum = count($results); - - Console::log('Processing chunk #' . $chunk . '. Found ' . $sum . ' documents'); - - foreach ($results as $document) { - call_user_func($callback, $database, $document); - $count++; - } - } - - $executionEnd = \microtime(true); - - Console::log("Processed {$count} document by group in " . ($executionEnd - $executionStart) . " seconds"); - } - - protected function getStatsPerOrganization(EventHamster $hamster, Database $dbForConsole, float $loopStart) - { - $this->calculateByGroup('teams', $dbForConsole, function (Database $dbForConsole, Document $organization) use ($hamster, $loopStart) { - try { - $organization->setAttribute('$time', $loopStart); - $hamster - ->setType(EventHamster::TYPE_ORGANISATION) - ->setOrganization($organization) - ->trigger(); - } catch (Exception $e) { - Console::error($e->getMessage()); - } - }); - } - - private function getStatsPerProject(EventHamster $hamster, Database $dbForConsole, float $loopStart) - { - $this->calculateByGroup('projects', $dbForConsole, function (Database $dbForConsole, Document $project) use ($hamster, $loopStart) { - try { - $project->setAttribute('$time', $loopStart); - $hamster - ->setType(EventHamster::TYPE_PROJECT) - ->setProject($project) - ->trigger(); - } catch (Exception $e) { - Console::error($e->getMessage()); - } - }); - } - - protected function getStatsPerUser(EventHamster $hamster, Database $dbForConsole, float $loopStart) - { - $this->calculateByGroup('users', $dbForConsole, function (Database $dbForConsole, Document $user) use ($hamster, $loopStart) { - try { - $user->setAttribute('$time', $loopStart); - $hamster - ->setType(EventHamster::TYPE_USER) - ->setUser($user) - ->trigger(); - } catch (Exception $e) { - Console::error($e->getMessage()); - } - }); - } -} diff --git a/src/Appwrite/Platform/Tasks/PatchRecreateRepositoriesDocuments.php b/src/Appwrite/Platform/Tasks/PatchRecreateRepositoriesDocuments.php deleted file mode 100644 index 93e6c527bb..0000000000 --- a/src/Appwrite/Platform/Tasks/PatchRecreateRepositoriesDocuments.php +++ /dev/null @@ -1,169 +0,0 @@ -desc('Recreate missing repositories in consoleDB from projectDBs. They can be missing if you used Appwrite 1.4.10 or 1.4.11, and deleted a function.') - ->param('after', '', new Text(36), 'After cursor', true) - ->param('projectId', '', new Text(36), 'Select project to validate', true) - ->inject('dbForConsole') - ->inject('getProjectDB') - ->callback(fn ($after, $projectId, $dbForConsole, $getProjectDB) => $this->action($after, $projectId, $dbForConsole, $getProjectDB)); - } - - public function action($after, $projectId, Database $dbForConsole, callable $getProjectDB): void - { - Console::info("Starting the patch"); - - $startTime = microtime(true); - - if (!empty($projectId)) { - try { - $project = $dbForConsole->getDocument('projects', $projectId); - $dbForProject = call_user_func($getProjectDB, $project); - $this->recreateRepositories($dbForConsole, $dbForProject, $project); - } catch (\Throwable $th) { - Console::error("Unexpected error occured with Project ID {$projectId}"); - Console::error('[Error] Type: ' . get_class($th)); - Console::error('[Error] Message: ' . $th->getMessage()); - Console::error('[Error] File: ' . $th->getFile()); - Console::error('[Error] Line: ' . $th->getLine()); - } - } else { - $queries = []; - if (!empty($after)) { - Console::info("Iterating remaining projects after project with ID {$after}"); - $project = $dbForConsole->getDocument('projects', $after); - $queries = [Query::cursorAfter($project)]; - } else { - Console::info("Iterating all projects"); - } - $this->foreachDocument($dbForConsole, 'projects', $queries, function (Document $project) use ($getProjectDB, $dbForConsole) { - $projectId = $project->getId(); - - try { - $dbForProject = call_user_func($getProjectDB, $project); - $this->recreateRepositories($dbForConsole, $dbForProject, $project); - } catch (\Throwable $th) { - Console::error("Unexpected error occured with Project ID {$projectId}"); - Console::error('[Error] Type: ' . get_class($th)); - Console::error('[Error] Message: ' . $th->getMessage()); - Console::error('[Error] File: ' . $th->getFile()); - Console::error('[Error] Line: ' . $th->getLine()); - } - }); - } - - $endTime = microtime(true); - $timeTaken = $endTime - $startTime; - - $hours = (int)($timeTaken / 3600); - $timeTaken -= $hours * 3600; - $minutes = (int)($timeTaken / 60); - $timeTaken -= $minutes * 60; - $seconds = (int)$timeTaken; - $milliseconds = ($timeTaken - $seconds) * 1000; - Console::info("Recreate patch completed in $hours h, $minutes m, $seconds s, $milliseconds mis ( total $timeTaken milliseconds)"); - } - - protected function foreachDocument(Database $database, string $collection, array $queries = [], callable $callback = null): void - { - $limit = 1000; - $results = []; - $sum = $limit; - $latestDocument = null; - - while ($sum === $limit) { - $newQueries = $queries; - - if ($latestDocument != null) { - array_unshift($newQueries, Query::cursorAfter($latestDocument)); - } - $newQueries[] = Query::limit($limit); - $results = $database->find($collection, $newQueries); - - if (empty($results)) { - return; - } - - $sum = count($results); - - foreach ($results as $document) { - if (is_callable($callback)) { - $callback($document); - } - } - $latestDocument = $results[array_key_last($results)]; - } - } - - public function recreateRepositories(Database $dbForConsole, Database $dbForProject, Document $project): void - { - $projectId = $project->getId(); - Console::log("Running patch for project {$projectId}"); - - $this->foreachDocument($dbForProject, 'functions', [], function (Document $function) use ($dbForProject, $dbForConsole, $project) { - $isConnected = !empty($function->getAttribute('providerRepositoryId', '')); - - if ($isConnected) { - $repository = $dbForConsole->getDocument('repositories', $function->getAttribute('repositoryId', '')); - - if ($repository->isEmpty()) { - $projectId = $project->getId(); - $functionId = $function->getId(); - Console::success("Recreating repositories document for project ID {$projectId}, function ID {$functionId}"); - - $repository = $dbForConsole->createDocument('repositories', new Document([ - '$id' => ID::unique(), - '$permissions' => [ - Permission::read(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()), - ], - 'installationId' => $function->getAttribute('installationId', ''), - 'installationInternalId' => $function->getAttribute('installationInternalId', ''), - 'projectId' => $project->getId(), - 'projectInternalId' => $project->getInternalId(), - 'providerRepositoryId' => $function->getAttribute('providerRepositoryId', ''), - 'resourceId' => $function->getId(), - 'resourceInternalId' => $function->getInternalId(), - 'resourceType' => 'function', - 'providerPullRequestIds' => [] - ])); - - $function = $dbForProject->updateDocument('functions', $function->getId(), $function - ->setAttribute('repositoryId', $repository->getId()) - ->setAttribute('repositoryInternalId', $repository->getInternalId())); - - $this->foreachDocument($dbForProject, 'deployments', [ - Query::equal('resourceInternalId', [$function->getInternalId()]), - Query::equal('resourceType', ['functions']) - ], function (Document $deployment) use ($dbForProject, $repository) { - $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment - ->setAttribute('repositoryId', $repository->getId()) - ->setAttribute('repositoryInternalId', $repository->getInternalId())); - }); - } - } - }); - } -} diff --git a/src/Appwrite/Platform/Tasks/VolumeSync.php b/src/Appwrite/Platform/Tasks/VolumeSync.php deleted file mode 100644 index 6197b20fbd..0000000000 --- a/src/Appwrite/Platform/Tasks/VolumeSync.php +++ /dev/null @@ -1,59 +0,0 @@ -desc('Runs rsync to sync certificates between the storage mount and traefik.') - ->param('source', null, new Text(255), 'Source path to sync from.', false) - ->param('destination', null, new Text(255), 'Destination path to sync to.', false) - ->param('interval', null, new Integer(true), 'Interval to run rsync', false) - ->callback(fn ($source, $destination, $interval) => $this->action($source, $destination, $interval)); - } - - public function action(string $source, string $destination, int $interval) - { - - Console::title('RSync V1'); - Console::success(APP_NAME . ' rsync process v1 has started'); - - if (!file_exists($source)) { - Console::error('Source directory does not exist. Exiting ... '); - Console::exit(0); - } - - Console::loop(function () use ($interval, $source, $destination) { - $time = DateTime::now(); - - Console::info("[{$time}] Executing rsync every {$interval} seconds"); - Console::info("Syncing between $source and $destination"); - - if (!file_exists($source)) { - Console::error('Source directory does not exist. Skipping ... '); - return; - } - - $stdin = ""; - $stdout = ""; - $stderr = ""; - - Console::execute("rsync -av $source $destination", $stdin, $stdout, $stderr); - Console::success($stdout); - Console::error($stderr); - }, $interval); - } -} diff --git a/src/Appwrite/Platform/Workers/Hamster.php b/src/Appwrite/Platform/Workers/Hamster.php deleted file mode 100644 index e911bb6c7a..0000000000 --- a/src/Appwrite/Platform/Workers/Hamster.php +++ /dev/null @@ -1,437 +0,0 @@ - 'files.$all.count.total', - 'usage_buckets' => 'buckets.$all.count.total', - 'usage_databases' => 'databases.$all.count.total', - 'usage_documents' => 'documents.$all.count.total', - 'usage_collections' => 'collections.$all.count.total', - 'usage_storage' => 'project.$all.storage.size', - 'usage_requests' => 'project.$all.network.requests', - 'usage_bandwidth' => 'project.$all.network.bandwidth', - 'usage_users' => 'users.$all.count.total', - 'usage_sessions' => 'sessions.email.requests.create', - 'usage_executions' => 'executions.$all.compute.total', - ]; - - protected Mixpanel $mixpanel; - - public static function getName(): string - { - return 'hamster'; - } - - /** - * @throws \Exception - */ - public function __construct() - { - $this - ->desc('Hamster worker') - ->inject('message') - ->inject('pools') - ->inject('cache') - ->inject('dbForConsole') - ->callback(fn (Message $message, Group $pools, Cache $cache, Database $dbForConsole) => $this->action($message, $pools, $cache, $dbForConsole)); - } - - /** - * @param Message $message - * @param Group $pools - * @param Cache $cache - * @param Database $dbForConsole - * - * @return void - * @throws \Utopia\Database\Exception - */ - public function action(Message $message, Group $pools, Cache $cache, Database $dbForConsole): void - { - $token = App::getEnv('_APP_MIXPANEL_TOKEN', ''); - if (empty($token)) { - throw new \Exception('Missing MixPanel Token'); - } - $this->mixpanel = new Mixpanel($token); - - $payload = $message->getPayload() ?? []; - - if (empty($payload)) { - throw new \Exception('Missing payload'); - } - - $type = $payload['type'] ?? ''; - - switch ($type) { - case EventHamster::TYPE_PROJECT: - $this->getStatsForProject(new Document($payload['project']), $pools, $cache, $dbForConsole); - break; - case EventHamster::TYPE_ORGANISATION: - $this->getStatsForOrganization(new Document($payload['organization']), $dbForConsole); - break; - case EventHamster::TYPE_USER: - $this->getStatsPerUser(new Document($payload['user']), $dbForConsole); - break; - } - } - - /** - * @param Document $project - * @param Group $pools - * @param Cache $cache - * @param Database $dbForConsole - * @throws \Utopia\Database\Exception - */ - private function getStatsForProject(Document $project, Group $pools, Cache $cache, Database $dbForConsole): void - { - /** - * Skip user projects with id 'console' - */ - if ($project->getId() === 'console') { - Console::info("Skipping project console"); - return; - } - - Console::log("Getting stats for Project {$project->getId()}"); - - try { - $db = $project->getAttribute('database'); - $adapter = $pools - ->get($db) - ->pop() - ->getResource(); - - $dbForProject = new Database($adapter, $cache); - $dbForProject->setDefaultDatabase('appwrite'); - $dbForProject->setNamespace('_' . $project->getInternalId()); - - $statsPerProject = []; - - $statsPerProject['time'] = $project->getAttribute('$time'); - - /** Get Project ID */ - $statsPerProject['project_id'] = $project->getId(); - - /** Get project created time */ - $statsPerProject['project_created'] = $project->getAttribute('$createdAt'); - - /** Get Project Name */ - $statsPerProject['project_name'] = $project->getAttribute('name'); - - /** Total Project Variables */ - $statsPerProject['custom_variables'] = $dbForProject->count('variables', [], APP_LIMIT_COUNT); - - /** Total Migrations */ - $statsPerProject['custom_migrations'] = $dbForProject->count('migrations', [], APP_LIMIT_COUNT); - - /** Get Custom SMTP */ - $smtp = $project->getAttribute('smtp', null); - if ($smtp) { - $statsPerProject['custom_smtp_status'] = $smtp['enabled'] === true ? 'enabled' : 'disabled'; - - /** Get Custom Templates Count */ - $templates = array_keys($project->getAttribute('templates', [])); - $statsPerProject['custom_email_templates'] = array_filter($templates, function ($template) { - return str_contains($template, 'email'); - }); - $statsPerProject['custom_sms_templates'] = array_filter($templates, function ($template) { - return str_contains($template, 'sms'); - }); - } - - /** Get total relationship attributes */ - $statsPerProject['custom_relationship_attributes'] = $dbForProject->count('attributes', [ - Query::equal('type', ['relationship']) - ], APP_LIMIT_COUNT); - - /** Get Total Functions */ - $statsPerProject['custom_functions'] = $dbForProject->count('functions', [], APP_LIMIT_COUNT); - - foreach (\array_keys(Config::getParam('runtimes')) as $runtime) { - $statsPerProject['custom_functions_' . $runtime] = $dbForProject->count('functions', [ - Query::equal('runtime', [$runtime]), - ], APP_LIMIT_COUNT); - } - - /** Get Total Deployments */ - $statsPerProject['custom_deployments'] = $dbForProject->count('deployments', [], APP_LIMIT_COUNT); - $statsPerProject['custom_deployments_manual'] = $dbForProject->count('deployments', [ - Query::equal('type', ['manual']) - ], APP_LIMIT_COUNT); - $statsPerProject['custom_deployments_git'] = $dbForProject->count('deployments', [ - Query::equal('type', ['vcs']) - ], APP_LIMIT_COUNT); - - /** Get VCS repos connected */ - $statsPerProject['custom_vcs_repositories'] = $dbForConsole->count('repositories', [ - Query::equal('projectInternalId', [$project->getInternalId()]) - ], APP_LIMIT_COUNT); - - /** Get Total Teams */ - $statsPerProject['custom_teams'] = $dbForProject->count('teams', [], APP_LIMIT_COUNT); - - /** Get Total Members */ - $teamInternalId = $project->getAttribute('teamInternalId', null); - if ($teamInternalId) { - $statsPerProject['custom_organization_members'] = $dbForConsole->count('memberships', [ - Query::equal('teamInternalId', [$teamInternalId]) - ], APP_LIMIT_COUNT); - } else { - $statsPerProject['custom_organization_members'] = 0; - } - - /** Get Email and Name of the project owner */ - if ($teamInternalId) { - $membership = $dbForConsole->findOne('memberships', [ - Query::equal('teamInternalId', [$teamInternalId]), - ]); - - if (!$membership || $membership->isEmpty()) { - throw new \Exception('Membership not found. Skipping project : ' . $project->getId()); - } - - $userId = $membership->getAttribute('userId', null); - if ($userId) { - $user = $dbForConsole->getDocument('users', $userId); - $statsPerProject['email'] = $user->getAttribute('email', null); - $statsPerProject['name'] = $user->getAttribute('name', null); - } - } - - /** Get Domains */ - $statsPerProject['custom_domains'] = $dbForConsole->count('rules', [ - Query::equal('projectInternalId', [$project->getInternalId()]), - Query::limit(APP_LIMIT_COUNT) - ]); - - /** Get Platforms */ - $platforms = $dbForConsole->find('platforms', [ - Query::equal('projectInternalId', [$project->getInternalId()]), - Query::limit(APP_LIMIT_COUNT) - ]); - - $statsPerProject['custom_platforms_web'] = sizeof(array_filter($platforms, function ($platform) { - return $platform['type'] === 'web'; - })); - - $statsPerProject['custom_platforms_android'] = sizeof(array_filter($platforms, function ($platform) { - return $platform['type'] === 'android'; - })); - - $statsPerProject['custom_platforms_apple'] = sizeof(array_filter($platforms, function ($platform) { - return str_contains($platform['type'], 'apple'); - })); - - $statsPerProject['custom_platforms_flutter'] = sizeof(array_filter($platforms, function ($platform) { - return str_contains($platform['type'], 'flutter'); - })); - - $flutterPlatforms = [Origin::CLIENT_TYPE_FLUTTER_ANDROID, Origin::CLIENT_TYPE_FLUTTER_IOS, Origin::CLIENT_TYPE_FLUTTER_MACOS, Origin::CLIENT_TYPE_FLUTTER_WINDOWS, Origin::CLIENT_TYPE_FLUTTER_LINUX]; - - foreach ($flutterPlatforms as $flutterPlatform) { - $statsPerProject['custom_platforms_' . $flutterPlatform] = sizeof(array_filter($platforms, function ($platform) use ($flutterPlatform) { - return $platform['type'] === $flutterPlatform; - })); - } - - $statsPerProject['custom_platforms_api_keys'] = $dbForConsole->count('keys', [ - Query::equal('projectInternalId', [$project->getInternalId()]), - Query::limit(APP_LIMIT_COUNT) - ]); - - /** Get Usage $statsPerProject */ - $periods = [ - 'infinity' => [ - 'period' => '1d', - 'limit' => 90, - ], - '24h' => [ - 'period' => '1h', - 'limit' => 24, - ], - ]; - - Authorization::skip(function () use ($dbForProject, $periods, &$statsPerProject) { - foreach ($this->metrics as $key => $metric) { - foreach ($periods as $periodKey => $periodValue) { - $limit = $periodValue['limit']; - $period = $periodValue['period']; - - $requestDocs = $dbForProject->find('stats', [ - Query::equal('period', [$period]), - Query::equal('metric', [$metric]), - Query::limit($limit), - Query::orderDesc('time'), - ]); - - $statsPerProject[$key . '_' . $periodKey] = []; - foreach ($requestDocs as $requestDoc) { - $statsPerProject[$key . '_' . $periodKey][] = [ - 'value' => $requestDoc->getAttribute('value'), - 'date' => $requestDoc->getAttribute('time'), - ]; - } - - $statsPerProject[$key . '_' . $periodKey] = array_reverse($statsPerProject[$key . '_' . $periodKey]); - // Calculate aggregate of each metric - $statsPerProject[$key . '_' . $periodKey] = array_sum(array_column($statsPerProject[$key . '_' . $periodKey], 'value')); - } - } - }); - - if (isset($statsPerProject['email'])) { - /** Send data to mixpanel */ - $res = $this->mixpanel->createProfile($statsPerProject['email'], '', [ - 'name' => $statsPerProject['name'], - 'email' => $statsPerProject['email'] - ]); - - if (!$res) { - Console::error('Failed to create user profile for project: ' . $project->getId()); - } - } - - $event = new AnalyticsEvent(); - $event - ->setName('Project Daily Usage') - ->setProps($statsPerProject); - $res = $this->mixpanel->createEvent($event); - - if (!$res) { - Console::error('Failed to create event for project: ' . $project->getId()); - } - } catch (\Exception $e) { - Console::error('Failed to send stats for project: ' . $project->getId()); - Console::error($e->getMessage()); - } finally { - $pools - ->get($db) - ->reclaim(); - } - } - - /** - * @param Document $organization - * @param Database $dbForConsole - * @throws \Utopia\Database\Exception - */ - private function getStatsForOrganization(Document $organization, Database $dbForConsole): void - { - Console::log("Getting stats for Organization {$organization->getId()}"); - - try { - $statsPerOrganization = []; - - $statsPerOrganization['time'] = $organization->getAttribute('$time'); - - /** Organization name */ - $statsPerOrganization['name'] = $organization->getAttribute('name'); - - - /** Get Email and of the organization owner */ - $membership = $dbForConsole->findOne('memberships', [ - Query::equal('teamInternalId', [$organization->getInternalId()]), - ]); - - if (!$membership || $membership->isEmpty()) { - throw new \Exception('Membership not found. Skipping organization : ' . $organization->getId()); - } - - $userId = $membership->getAttribute('userId', null); - if ($userId) { - $user = $dbForConsole->getDocument('users', $userId); - $statsPerOrganization['email'] = $user->getAttribute('email', null); - } - - /** Organization Creation Date */ - $statsPerOrganization['created'] = $organization->getAttribute('$createdAt'); - - /** Number of team members */ - $statsPerOrganization['members'] = $organization->getAttribute('total'); - - /** Number of projects in this organization */ - $statsPerOrganization['projects'] = $dbForConsole->count('projects', [ - Query::equal('teamId', [$organization->getId()]), - Query::limit(APP_LIMIT_COUNT) - ]); - - if (!isset($statsPerOrganization['email'])) { - throw new \Exception('Email not found. Skipping organization : ' . $organization->getId()); - } - - $event = new AnalyticsEvent(); - $event - ->setName('Organization Daily Usage') - ->setProps($statsPerOrganization); - $res = $this->mixpanel->createEvent($event); - if (!$res) { - throw new \Exception('Failed to create event for organization : ' . $organization->getId()); - } - } catch (\Exception $e) { - Console::error($e->getMessage()); - } - } - - protected function getStatsPerUser(Document $user, Database $dbForConsole) - { - Console::log("Getting stats for User {$user->getId()}"); - - try { - $statsPerUser = []; - - $statsPerUser['time'] = $user->getAttribute('$time'); - - /** Organization name */ - $statsPerUser['name'] = $user->getAttribute('name'); - - /** Organization ID (needs to be stored as an email since mixpanel uses the email attribute as a distinctID) */ - $statsPerUser['email'] = $user->getAttribute('email'); - - /** Organization Creation Date */ - $statsPerUser['created'] = $user->getAttribute('$createdAt'); - - /** Number of teams this user is a part of */ - $statsPerUser['memberships'] = $dbForConsole->count('memberships', [ - Query::equal('userInternalId', [$user->getInternalId()]), - Query::limit(APP_LIMIT_COUNT) - ]); - - if (!isset($statsPerUser['email'])) { - throw new \Exception('User has no email: ' . $user->getId()); - } - - /** Send data to mixpanel */ - $event = new AnalyticsEvent(); - $event - ->setName('User Daily Usage') - ->setProps($statsPerUser); - - $res = $this->mixpanel->createEvent($event); - - if (!$res) { - throw new \Exception('Failed to create user profile for user: ' . $user->getId()); - } - } catch (\Exception $e) { - Console::error($e->getMessage()); - } - } -}