From 7ea8d761e475603a97d9b7340c53586ae3a6e91c Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 21 Dec 2022 02:08:37 +0530 Subject: [PATCH 1/5] feat: aggregate stats --- Dockerfile | 1 + bin/stat | 3 + src/Appwrite/Platform/Services/Tasks.php | 2 + src/Appwrite/Platform/Tasks/Stat.php | 212 +++++++++++++++++++++++ 4 files changed, 218 insertions(+) create mode 100644 bin/stat create mode 100644 src/Appwrite/Platform/Tasks/Stat.php diff --git a/Dockerfile b/Dockerfile index 54aa327769..9a8e35f730 100755 --- a/Dockerfile +++ b/Dockerfile @@ -314,6 +314,7 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/sdks && \ chmod +x /usr/local/bin/specs && \ chmod +x /usr/local/bin/ssl && \ + chmod +x /usr/local/bin/stat && \ chmod +x /usr/local/bin/test && \ chmod +x /usr/local/bin/vars && \ chmod +x /usr/local/bin/worker-audits && \ diff --git a/bin/stat b/bin/stat new file mode 100644 index 0000000000..d7030a0958 --- /dev/null +++ b/bin/stat @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php stat $@ \ No newline at end of file diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index 2e15cd015c..2cb0878589 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -12,6 +12,7 @@ use Appwrite\Platform\Tasks\PatchCreateMissingSchedules; use Appwrite\Platform\Tasks\SDKs; use Appwrite\Platform\Tasks\Specs; use Appwrite\Platform\Tasks\SSL; +use Appwrite\Platform\Tasks\Stat; use Appwrite\Platform\Tasks\Usage; use Appwrite\Platform\Tasks\Vars; use Appwrite\Platform\Tasks\Version; @@ -27,6 +28,7 @@ class Tasks extends Service ->addAction(Usage::getName(), new Usage()) ->addAction(Vars::getName(), new Vars()) ->addAction(SSL::getName(), new SSL()) + ->addAction(Stat::getName(), new Stat()) ->addAction(Doctor::getName(), new Doctor()) ->addAction(Install::getName(), new Install()) ->addAction(Maintenance::getName(), new Maintenance()) diff --git a/src/Appwrite/Platform/Tasks/Stat.php b/src/Appwrite/Platform/Tasks/Stat.php new file mode 100644 index 0000000000..8ac5687bae --- /dev/null +++ b/src/Appwrite/Platform/Tasks/Stat.php @@ -0,0 +1,212 @@ +desc('Get stats for project') + ->callback(fn () => $this->action()); + } + + function getConnection(string $dsn): PDO + { + if (empty($dsn)) { + throw new Exception("Missing value for DSN connection"); + } + $dsn = new DSN($dsn); + $dsnHost = $dsn->getHost(); + $dsnPort = $dsn->getPort(); + $dsnUser = $dsn->getUser(); + $dsnPass = $dsn->getPassword(); + $dsnScheme = $dsn->getScheme(); + $dsnDatabase = $dsn->getPath(); + + $connection = new PDO("mysql:host={$dsnHost};port={$dsnPort};dbname={$dsnDatabase};charset=utf8mb4", $dsnUser, $dsnPass, array( + PDO::ATTR_TIMEOUT => 3, // Seconds + PDO::ATTR_PERSISTENT => true, + PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, + PDO::ATTR_ERRMODE => PDO::ERRMODE_WARNING, + PDO::ATTR_EMULATE_PREPARES => true, + PDO::ATTR_STRINGIFY_FETCHES => true + )); + + return $connection; + } + + + function getStats(Database $dbForProject): array + { + $range = '90d'; + $periods = [ + '90d' => [ + 'period' => '1d', + 'limit' => 90, + ], + ]; + + $metrics = [ + 'files.$all.count.total', + 'buckets.$all.count.total', + 'databases.$all.count.total', + 'documents.$all.count.total', + 'collections.$all.count.total', + 'project.$all.storage.size', + 'project.$all.network.requests', + 'project.$all.network.bandwidth', + 'users.$all.count.total', + 'sessions.$all.requests.create', + 'executions.$all.compute.total', + ]; + + $stats = []; + + Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) { + foreach ($metrics as $metric) { + $limit = $periods[$range]['limit']; + $period = $periods[$range]['period']; + + $requestDocs = $dbForProject->find('stats', [ + Query::equal('period', [$period]), + Query::equal('metric', [$metric]), + Query::limit($limit), + Query::orderDesc('time'), + ]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + + $stats[$metric] = array_reverse($stats[$metric]); + // Calculate aggregate of each metric + $stats[$metric . '.sum'] = array_sum(array_column($stats[$metric], 'value')); + } + }); + + // return only the ahhggregate values + return array_filter($stats, fn ($key) => strpos($key, '.sum') !== false, ARRAY_FILTER_USE_KEY); + } + + + public function action(): void + { + Console::success('Getting stats...'); + + $databases = [ + 'console' => [ + 'type' => 'database', + 'dsns' => '', + 'multiple' => false, + 'schemes' => ['mariadb', 'mysql'], + ], + 'projects' => [ + 'type' => 'database', + 'dsns' => '', + 'multiple' => true, + 'schemes' => ['mariadb', 'mysql'], + ], + ]; + + $dsns = explode(',', $databases['projects']['dsns']); + $projectdsns = []; + foreach ($dsns as &$dsn) { + $dsn = explode('=', $dsn); + $name = 'database' . '_' . $dsn[0]; + $dsn = $dsn[1] ?? ''; + $projectdsns[$name] = $dsn; + } + + $cache = new Cache(new None()); + $consoledsn = explode('=', $databases['console']['dsns']); + $consoledsn = $consoledsn[1] ?? ''; + $adapter = new MySQL($this->getConnection($consoledsn)); + $dbForConsole = new Database($adapter, $cache); + $dbForConsole->setDefaultDatabase('appwrite'); + $dbForConsole->setNamespace('console'); + + $totalProjects = $dbForConsole->count('projects') + 1; + Console::success("Iterating through : {$totalProjects} projects"); + + $app = new App('UTC'); + $console = $app->getResource('console'); + + $projects = [$console]; + $count = 0; + $limit = 30; + $sum = 30; + $offset = 0; + + $stats = []; + + while (!empty($projects)) { + foreach ($projects as $project) { + /** + * Skip user projects with id 'console' + */ + if ($project->getId() === 'console') { + continue; + } + Console::info("Getting stats for {$project->getId()}"); + + try { + // TODO: Iterate through all project DBs + $db = $project->getAttribute('database'); + $dsn = $projectdsns[$db] ?? ''; + $cache = new Cache(new None()); + $adapter = new MySQL($this->getConnection($dsn)); + $dbForProject = new Database($adapter, $cache); + $dbForProject->setDefaultDatabase('appwrite'); + $dbForProject->setNamespace('_' . $project->getInternalId()); + $statsPerProject = $this->getStats($dbForProject); + + foreach ($statsPerProject as $key => $value) { + $stats[$key] = isset($stats[$key]) ? $stats[$key] + $value : $value; + } + + } catch (\Throwable $th) { + throw $th; + Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage()); + } + + } + + $sum = \count($projects); + + $projects = $dbForConsole->find('projects', [ + Query::limit($limit), + Query::offset($offset), + ]); + + $offset = $offset + $limit; + $count = $count + $sum; + + Console::log('Iterated through ' . $count . '/' . $totalProjects . ' projects...'); + } + + var_dump($stats); + } +} From c97e7ce328dc0f6fc3691d7d252ea9661e937494 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 26 Jan 2023 02:26:33 +0530 Subject: [PATCH 2/5] feat: update scripts --- .env | 1 + Dockerfile | 2 +- bin/hamster | 3 + bin/stat | 3 - composer.json | 3 +- composer.lock | 222 ++++++++++++++------- docker-compose.yml | 1 + src/Appwrite/Platform/Services/Tasks.php | 4 +- src/Appwrite/Platform/Tasks/Hamster.php | 242 +++++++++++++++++++++++ src/Appwrite/Platform/Tasks/Stat.php | 212 -------------------- 10 files changed, 406 insertions(+), 287 deletions(-) create mode 100644 bin/hamster delete mode 100644 bin/stat create mode 100644 src/Appwrite/Platform/Tasks/Hamster.php delete mode 100644 src/Appwrite/Platform/Tasks/Stat.php diff --git a/.env b/.env index 51e6cc3705..6dcb013dff 100644 --- a/.env +++ b/.env @@ -44,6 +44,7 @@ _APP_SMTP_PORT=1025 _APP_SMTP_SECURE= _APP_SMTP_USERNAME= _APP_SMTP_PASSWORD= +_APP_HAMSTER_RECIPIENTS= _APP_SMS_PROVIDER=sms://username:password@mock _APP_SMS_FROM=+123456789 _APP_STORAGE_LIMIT=30000000 diff --git a/Dockerfile b/Dockerfile index 9a8e35f730..daeac6eb03 100755 --- a/Dockerfile +++ b/Dockerfile @@ -314,7 +314,7 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/sdks && \ chmod +x /usr/local/bin/specs && \ chmod +x /usr/local/bin/ssl && \ - chmod +x /usr/local/bin/stat && \ + chmod +x /usr/local/bin/hamster && \ chmod +x /usr/local/bin/test && \ chmod +x /usr/local/bin/vars && \ chmod +x /usr/local/bin/worker-audits && \ diff --git a/bin/hamster b/bin/hamster new file mode 100644 index 0000000000..dcc7ed308d --- /dev/null +++ b/bin/hamster @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php hamster $@ \ No newline at end of file diff --git a/bin/stat b/bin/stat deleted file mode 100644 index d7030a0958..0000000000 --- a/bin/stat +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/cli.php stat $@ \ No newline at end of file diff --git a/composer.json b/composer.json index 1d6d1be25b..f505291643 100644 --- a/composer.json +++ b/composer.json @@ -73,7 +73,8 @@ "phpmailer/phpmailer": "6.6.0", "chillerlan/php-qrcode": "4.3.3", "adhocore/jwt": "1.1.2", - "slickdeals/statsd": "3.1.0" + "slickdeals/statsd": "3.1.0", + "league/csv": "^9.0.0" }, "repositories": [ { diff --git a/composer.lock b/composer.lock index c8b627849e..1324412b95 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "4893e1c13630239fe6a20d1c652eb484", + "content-hash": "55eaba9ed1fd51ed74a18ee488c3fdd1", "packages": [ { "name": "adhocore/jwt", @@ -801,20 +801,21 @@ "issues": "https://github.com/influxdata/influxdb-php/issues", "source": "https://github.com/influxdata/influxdb-php/tree/1.15.2" }, + "abandoned": true, "time": "2020-12-26T17:45:17+00:00" }, { "name": "laravel/pint", - "version": "v1.2.0", + "version": "v1.2.1", "source": { "type": "git", "url": "https://github.com/laravel/pint.git", - "reference": "1d276e4c803397a26cc337df908f55c2a4e90d86" + "reference": "e60e2112ee779ce60f253695b273d1646a17d6f1" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/pint/zipball/1d276e4c803397a26cc337df908f55c2a4e90d86", - "reference": "1d276e4c803397a26cc337df908f55c2a4e90d86", + "url": "https://api.github.com/repos/laravel/pint/zipball/e60e2112ee779ce60f253695b273d1646a17d6f1", + "reference": "e60e2112ee779ce60f253695b273d1646a17d6f1", "shasum": "" }, "require": { @@ -826,10 +827,10 @@ }, "require-dev": { "friendsofphp/php-cs-fixer": "^3.11.0", - "illuminate/view": "^9.27", - "laravel-zero/framework": "^9.1.3", - "mockery/mockery": "^1.5.0", - "nunomaduro/larastan": "^2.2", + "illuminate/view": "^9.32.0", + "laravel-zero/framework": "^9.2.0", + "mockery/mockery": "^1.5.1", + "nunomaduro/larastan": "^2.2.0", "nunomaduro/termwind": "^1.14.0", "pestphp/pest": "^1.22.1" }, @@ -867,7 +868,91 @@ "issues": "https://github.com/laravel/pint/issues", "source": "https://github.com/laravel/pint" }, - "time": "2022-09-13T15:07:15+00:00" + "time": "2022-11-29T16:25:20+00:00" + }, + { + "name": "league/csv", + "version": "9.8.0", + "source": { + "type": "git", + "url": "https://github.com/thephpleague/csv.git", + "reference": "9d2e0265c5d90f5dd601bc65ff717e05cec19b47" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/thephpleague/csv/zipball/9d2e0265c5d90f5dd601bc65ff717e05cec19b47", + "reference": "9d2e0265c5d90f5dd601bc65ff717e05cec19b47", + "shasum": "" + }, + "require": { + "ext-json": "*", + "ext-mbstring": "*", + "php": "^7.4 || ^8.0" + }, + "require-dev": { + "ext-curl": "*", + "ext-dom": "*", + "friendsofphp/php-cs-fixer": "^v3.4.0", + "phpstan/phpstan": "^1.3.0", + "phpstan/phpstan-phpunit": "^1.0.0", + "phpstan/phpstan-strict-rules": "^1.1.0", + "phpunit/phpunit": "^9.5.11" + }, + "suggest": { + "ext-dom": "Required to use the XMLConverter and or the HTMLConverter classes", + "ext-iconv": "Needed to ease transcoding CSV using iconv stream filters" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "9.x-dev" + } + }, + "autoload": { + "files": [ + "src/functions_include.php" + ], + "psr-4": { + "League\\Csv\\": "src" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Ignace Nyamagana Butera", + "email": "nyamsprod@gmail.com", + "homepage": "https://github.com/nyamsprod/", + "role": "Developer" + } + ], + "description": "CSV data manipulation made easy in PHP", + "homepage": "https://csv.thephpleague.com", + "keywords": [ + "convert", + "csv", + "export", + "filter", + "import", + "read", + "transform", + "write" + ], + "support": { + "docs": "https://csv.thephpleague.com", + "issues": "https://github.com/thephpleague/csv/issues", + "rss": "https://github.com/thephpleague/csv/releases.atom", + "source": "https://github.com/thephpleague/csv" + }, + "funding": [ + { + "url": "https://github.com/sponsors/nyamsprod", + "type": "github" + } + ], + "time": "2022-01-04T00:13:07+00:00" }, { "name": "matomo/device-detector", @@ -1461,16 +1546,16 @@ }, { "name": "symfony/deprecation-contracts", - "version": "v3.1.1", + "version": "v3.2.0", "source": { "type": "git", "url": "https://github.com/symfony/deprecation-contracts.git", - "reference": "07f1b9cc2ffee6aaafcf4b710fbc38ff736bd918" + "reference": "1ee04c65529dea5d8744774d474e7cbd2f1206d3" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/deprecation-contracts/zipball/07f1b9cc2ffee6aaafcf4b710fbc38ff736bd918", - "reference": "07f1b9cc2ffee6aaafcf4b710fbc38ff736bd918", + "url": "https://api.github.com/repos/symfony/deprecation-contracts/zipball/1ee04c65529dea5d8744774d474e7cbd2f1206d3", + "reference": "1ee04c65529dea5d8744774d474e7cbd2f1206d3", "shasum": "" }, "require": { @@ -1479,7 +1564,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-main": "3.1-dev" + "dev-main": "3.3-dev" }, "thanks": { "name": "symfony/contracts", @@ -1508,7 +1593,7 @@ "description": "A generic function and convention to trigger deprecation notices", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/deprecation-contracts/tree/v3.1.1" + "source": "https://github.com/symfony/deprecation-contracts/tree/v3.2.0" }, "funding": [ { @@ -1524,7 +1609,7 @@ "type": "tidelift" } ], - "time": "2022-02-25T11:15:52+00:00" + "time": "2022-11-25T10:21:52+00:00" }, { "name": "utopia-php/abuse", @@ -2574,16 +2659,16 @@ }, { "name": "utopia-php/storage", - "version": "0.13.0", + "version": "0.13.2", "source": { "type": "git", "url": "https://github.com/utopia-php/storage.git", - "reference": "f34c010e4f8394a6b4aff70b6de55041d9a145d3" + "reference": "ad1c00f24ca56e73888acc2af3deee4919b1194b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/storage/zipball/f34c010e4f8394a6b4aff70b6de55041d9a145d3", - "reference": "f34c010e4f8394a6b4aff70b6de55041d9a145d3", + "url": "https://api.github.com/repos/utopia-php/storage/zipball/ad1c00f24ca56e73888acc2af3deee4919b1194b", + "reference": "ad1c00f24ca56e73888acc2af3deee4919b1194b", "shasum": "" }, "require": { @@ -2623,9 +2708,9 @@ ], "support": { "issues": "https://github.com/utopia-php/storage/issues", - "source": "https://github.com/utopia-php/storage/tree/0.13.0" + "source": "https://github.com/utopia-php/storage/tree/0.13.2" }, - "time": "2022-11-17T15:10:18+00:00" + "time": "2022-12-20T11:11:35+00:00" }, { "name": "utopia-php/swoole", @@ -2905,30 +2990,30 @@ }, { "name": "doctrine/instantiator", - "version": "1.4.1", + "version": "1.5.0", "source": { "type": "git", "url": "https://github.com/doctrine/instantiator.git", - "reference": "10dcfce151b967d20fde1b34ae6640712c3891bc" + "reference": "0a0fa9780f5d4e507415a065172d26a98d02047b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/doctrine/instantiator/zipball/10dcfce151b967d20fde1b34ae6640712c3891bc", - "reference": "10dcfce151b967d20fde1b34ae6640712c3891bc", + "url": "https://api.github.com/repos/doctrine/instantiator/zipball/0a0fa9780f5d4e507415a065172d26a98d02047b", + "reference": "0a0fa9780f5d4e507415a065172d26a98d02047b", "shasum": "" }, "require": { "php": "^7.1 || ^8.0" }, "require-dev": { - "doctrine/coding-standard": "^9", + "doctrine/coding-standard": "^9 || ^11", "ext-pdo": "*", "ext-phar": "*", "phpbench/phpbench": "^0.16 || ^1", "phpstan/phpstan": "^1.4", "phpstan/phpstan-phpunit": "^1", "phpunit/phpunit": "^7.5 || ^8.5 || ^9.5", - "vimeo/psalm": "^4.22" + "vimeo/psalm": "^4.30 || ^5.4" }, "type": "library", "autoload": { @@ -2955,7 +3040,7 @@ ], "support": { "issues": "https://github.com/doctrine/instantiator/issues", - "source": "https://github.com/doctrine/instantiator/tree/1.4.1" + "source": "https://github.com/doctrine/instantiator/tree/1.5.0" }, "funding": [ { @@ -2971,20 +3056,20 @@ "type": "tidelift" } ], - "time": "2022-03-03T08:28:38+00:00" + "time": "2022-12-30T00:15:36+00:00" }, { "name": "matthiasmullie/minify", - "version": "1.3.69", + "version": "1.3.70", "source": { "type": "git", "url": "https://github.com/matthiasmullie/minify.git", - "reference": "a61c949cccd086808063611ef9698eabe42ef22f" + "reference": "2807d9f9bece6877577ad44acb5c801bb3ae536b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/matthiasmullie/minify/zipball/a61c949cccd086808063611ef9698eabe42ef22f", - "reference": "a61c949cccd086808063611ef9698eabe42ef22f", + "url": "https://api.github.com/repos/matthiasmullie/minify/zipball/2807d9f9bece6877577ad44acb5c801bb3ae536b", + "reference": "2807d9f9bece6877577ad44acb5c801bb3ae536b", "shasum": "" }, "require": { @@ -2993,9 +3078,10 @@ "php": ">=5.3.0" }, "require-dev": { - "friendsofphp/php-cs-fixer": "~2.0", - "matthiasmullie/scrapbook": "dev-master", - "phpunit/phpunit": ">=4.8" + "friendsofphp/php-cs-fixer": ">=2.0", + "matthiasmullie/scrapbook": ">=1.3", + "phpunit/phpunit": ">=4.8", + "squizlabs/php_codesniffer": ">=3.0" }, "suggest": { "psr/cache-implementation": "Cache implementation to use with Minify::cache" @@ -3018,12 +3104,12 @@ { "name": "Matthias Mullie", "email": "minify@mullie.eu", - "homepage": "http://www.mullie.eu", + "homepage": "https://www.mullie.eu", "role": "Developer" } ], "description": "CSS & JavaScript minifier, in PHP. Removes whitespace, strips comments, combines files (incl. @import statements and small assets in CSS files), and optimizes/shortens a few common programming patterns.", - "homepage": "http://www.minifier.org", + "homepage": "https://github.com/matthiasmullie/minify", "keywords": [ "JS", "css", @@ -3033,7 +3119,7 @@ ], "support": { "issues": "https://github.com/matthiasmullie/minify/issues", - "source": "https://github.com/matthiasmullie/minify/tree/1.3.69" + "source": "https://github.com/matthiasmullie/minify/tree/1.3.70" }, "funding": [ { @@ -3041,7 +3127,7 @@ "type": "github" } ], - "time": "2022-08-01T09:00:18+00:00" + "time": "2022-12-09T12:56:44+00:00" }, { "name": "matthiasmullie/path-converter", @@ -3157,16 +3243,16 @@ }, { "name": "nikic/php-parser", - "version": "v4.15.2", + "version": "v4.15.3", "source": { "type": "git", "url": "https://github.com/nikic/PHP-Parser.git", - "reference": "f59bbe44bf7d96f24f3e2b4ddc21cd52c1d2adbc" + "reference": "570e980a201d8ed0236b0a62ddf2c9cbb2034039" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/f59bbe44bf7d96f24f3e2b4ddc21cd52c1d2adbc", - "reference": "f59bbe44bf7d96f24f3e2b4ddc21cd52c1d2adbc", + "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/570e980a201d8ed0236b0a62ddf2c9cbb2034039", + "reference": "570e980a201d8ed0236b0a62ddf2c9cbb2034039", "shasum": "" }, "require": { @@ -3207,9 +3293,9 @@ ], "support": { "issues": "https://github.com/nikic/PHP-Parser/issues", - "source": "https://github.com/nikic/PHP-Parser/tree/v4.15.2" + "source": "https://github.com/nikic/PHP-Parser/tree/v4.15.3" }, - "time": "2022-11-12T15:38:23+00:00" + "time": "2023-01-16T22:05:37+00:00" }, { "name": "phar-io/manifest", @@ -3489,21 +3575,21 @@ }, { "name": "phpspec/prophecy", - "version": "v1.15.0", + "version": "v1.16.0", "source": { "type": "git", "url": "https://github.com/phpspec/prophecy.git", - "reference": "bbcd7380b0ebf3961ee21409db7b38bc31d69a13" + "reference": "be8cac52a0827776ff9ccda8c381ac5b71aeb359" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpspec/prophecy/zipball/bbcd7380b0ebf3961ee21409db7b38bc31d69a13", - "reference": "bbcd7380b0ebf3961ee21409db7b38bc31d69a13", + "url": "https://api.github.com/repos/phpspec/prophecy/zipball/be8cac52a0827776ff9ccda8c381ac5b71aeb359", + "reference": "be8cac52a0827776ff9ccda8c381ac5b71aeb359", "shasum": "" }, "require": { "doctrine/instantiator": "^1.2", - "php": "^7.2 || ~8.0, <8.2", + "php": "^7.2 || 8.0.* || 8.1.* || 8.2.*", "phpdocumentor/reflection-docblock": "^5.2", "sebastian/comparator": "^3.0 || ^4.0", "sebastian/recursion-context": "^3.0 || ^4.0" @@ -3550,22 +3636,22 @@ ], "support": { "issues": "https://github.com/phpspec/prophecy/issues", - "source": "https://github.com/phpspec/prophecy/tree/v1.15.0" + "source": "https://github.com/phpspec/prophecy/tree/v1.16.0" }, - "time": "2021-12-08T12:19:24+00:00" + "time": "2022-11-29T15:06:56+00:00" }, { "name": "phpunit/php-code-coverage", - "version": "9.2.19", + "version": "9.2.23", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-code-coverage.git", - "reference": "c77b56b63e3d2031bd8997fcec43c1925ae46559" + "reference": "9f1f0f9a2fbb680b26d1cf9b61b6eac43a6e4e9c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/c77b56b63e3d2031bd8997fcec43c1925ae46559", - "reference": "c77b56b63e3d2031bd8997fcec43c1925ae46559", + "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/9f1f0f9a2fbb680b26d1cf9b61b6eac43a6e4e9c", + "reference": "9f1f0f9a2fbb680b26d1cf9b61b6eac43a6e4e9c", "shasum": "" }, "require": { @@ -3621,7 +3707,7 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/php-code-coverage/issues", - "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.19" + "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.23" }, "funding": [ { @@ -3629,7 +3715,7 @@ "type": "github" } ], - "time": "2022-11-18T07:47:47+00:00" + "time": "2022-12-28T12:41:10+00:00" }, { "name": "phpunit/php-file-iterator", @@ -5303,16 +5389,16 @@ }, { "name": "twig/twig", - "version": "v3.4.3", + "version": "v3.5.0", "source": { "type": "git", "url": "https://github.com/twigphp/Twig.git", - "reference": "c38fd6b0b7f370c198db91ffd02e23b517426b58" + "reference": "3ffcf4b7d890770466da3b2666f82ac054e7ec72" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/twigphp/Twig/zipball/c38fd6b0b7f370c198db91ffd02e23b517426b58", - "reference": "c38fd6b0b7f370c198db91ffd02e23b517426b58", + "url": "https://api.github.com/repos/twigphp/Twig/zipball/3ffcf4b7d890770466da3b2666f82ac054e7ec72", + "reference": "3ffcf4b7d890770466da3b2666f82ac054e7ec72", "shasum": "" }, "require": { @@ -5327,7 +5413,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-master": "3.4-dev" + "dev-master": "3.5-dev" } }, "autoload": { @@ -5363,7 +5449,7 @@ ], "support": { "issues": "https://github.com/twigphp/Twig/issues", - "source": "https://github.com/twigphp/Twig/tree/v3.4.3" + "source": "https://github.com/twigphp/Twig/tree/v3.5.0" }, "funding": [ { @@ -5375,7 +5461,7 @@ "type": "tidelift" } ], - "time": "2022-09-28T08:42:51+00:00" + "time": "2022-12-27T12:28:18+00:00" } ], "aliases": [], diff --git a/docker-compose.yml b/docker-compose.yml index feac2c89d0..f831feca67 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -134,6 +134,7 @@ services: - _APP_SMTP_SECURE - _APP_SMTP_USERNAME - _APP_SMTP_PASSWORD + - _APP_HAMSTER_RECIPIENTS - _APP_USAGE_STATS - _APP_INFLUXDB_HOST - _APP_INFLUXDB_PORT diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index 2cb0878589..00cf3f89c5 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -12,7 +12,7 @@ use Appwrite\Platform\Tasks\PatchCreateMissingSchedules; use Appwrite\Platform\Tasks\SDKs; use Appwrite\Platform\Tasks\Specs; use Appwrite\Platform\Tasks\SSL; -use Appwrite\Platform\Tasks\Stat; +use Appwrite\Platform\Tasks\Hamster; use Appwrite\Platform\Tasks\Usage; use Appwrite\Platform\Tasks\Vars; use Appwrite\Platform\Tasks\Version; @@ -28,7 +28,7 @@ class Tasks extends Service ->addAction(Usage::getName(), new Usage()) ->addAction(Vars::getName(), new Vars()) ->addAction(SSL::getName(), new SSL()) - ->addAction(Stat::getName(), new Stat()) + ->addAction(Hamster::getName(), new Hamster()) ->addAction(Doctor::getName(), new Doctor()) ->addAction(Install::getName(), new Install()) ->addAction(Maintenance::getName(), new Maintenance()) diff --git a/src/Appwrite/Platform/Tasks/Hamster.php b/src/Appwrite/Platform/Tasks/Hamster.php new file mode 100644 index 0000000000..650afa7f94 --- /dev/null +++ b/src/Appwrite/Platform/Tasks/Hamster.php @@ -0,0 +1,242 @@ + 'files.$all.count.total', + 'Buckets' => 'buckets.$all.count.total', + 'Databases' => 'databases.$all.count.total', + 'Documents' => 'documents.$all.count.total', + 'Collections' => 'collections.$all.count.total', + 'Storage' => 'project.$all.storage.size', + 'Requests' => 'project.$all.network.requests', + 'Bandwidth' => 'project.$all.network.bandwidth', + 'Users' => 'users.$all.count.total', + 'Sessions' => 'sessions.$all.requests.create', + 'Executions' => 'executions.$all.compute.total', + ]; + + protected string $directory = '/usr/local/dev'; + protected string $path; + + protected string $date; + + public static function getName(): string + { + return 'hamster'; + } + + public function __construct() + { + $this + ->desc('Get stats for project') + ->inject('register') + ->inject('pools') + ->inject('cache') + ->inject('dbForConsole') + ->callback(function (Registry $register, Group $pools, Cache $cache, Database $dbForConsole) { + $this->action($register, $pools, $cache, $dbForConsole); + }); + } + + private function getStats(Database $dbForConsole, Database $dbForProject, Document $project): array + { + $stats = []; + + /** Get Project ID */ + $stats['Project ID'] = $project->getId(); + + /** Get Project Name */ + $stats['Project Name'] = $project->getAttribute('name'); + + /** Get Total Functions */ + $stats['Functions'] = $dbForProject->count('functions', [], APP_LIMIT_COUNT); + + /** Get Total Deployments */ + $stats['Deployments'] = $dbForProject->count('deployments', [], APP_LIMIT_COUNT); + + /** Get Total Members */ + $teamInternalId = $project->getAttribute('teamInternalId', null); + if ($teamInternalId) { + $stats['Members'] = $dbForConsole->count('memberships', [ + Query::equal('teamInternalId', [$teamInternalId]) + ], APP_LIMIT_COUNT); + } else { + $stats['Members'] = 0; + } + + /** Get Domains */ + $stats['Domains'] = $dbForProject->count('domains', [], APP_LIMIT_COUNT); + + /** Get Usage stats */ + $range = '90d'; + $periods = [ + '90d' => [ + 'period' => '1d', + 'limit' => 90, + ], + ]; + + $metrics = array_values($this->usageStats); + Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) { + foreach ($metrics as $metric) { + $limit = $periods[$range]['limit']; + $period = $periods[$range]['period']; + + $requestDocs = $dbForProject->find('stats', [ + Query::equal('period', [$period]), + Query::equal('metric', [$metric]), + Query::limit($limit), + Query::orderDesc('time'), + ]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + + $stats[$metric] = array_reverse($stats[$metric]); + // Calculate aggregate of each metric + $stats[$metric] = array_sum(array_column($stats[$metric], 'value')); + } + }); + + return $stats; + } + + public function action(Registry $register, Group $pools, Cache $cache, Database $dbForConsole): void + { + Console::success('Getting stats...'); + + /* Initialise new Utopia app */ + $app = new App('UTC'); + $console = $app->getResource('console'); + + /** CSV stuff */ + $this->date = date('Y-m-d'); + $this->path = "{$this->directory}/stats_{$this->date}.csv"; + $csv = Writer::createFromPath($this->path, 'w'); + $csv->insertOne($this->columns); + + /** Database connections */ + $totalProjects = $dbForConsole->count('projects') + 1; + Console::success("Found a total of: {$totalProjects} projects"); + + $projects = [$console]; + $count = 0; + $limit = 30; + $sum = 30; + $offset = 0; + while (!empty($projects)) { + foreach ($projects as $project) { + /** + * Skip user projects with id 'console' + */ + if ($project->getId() === 'console') { + continue; + } + + Console::info("Getting stats for {$project->getId()}"); + + try { + $db = $project->getAttribute('database'); + $adapter = $pools + ->get($db) + ->pop() + ->getResource(); + + $dbForProject = new Database($adapter, $cache); + $dbForProject->setDefaultDatabase('appwrite'); + $dbForProject->setNamespace('_' . $project->getInternalId()); + + $statsPerProject = $this->getStats($dbForConsole, $dbForProject, $project); + $csv->insertOne(array_values($statsPerProject)); + + } catch (\Throwable $th) { + throw $th; + Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage()); + } + } + + $sum = \count($projects); + + $projects = $dbForConsole->find('projects', [ + Query::limit($limit), + Query::offset($offset), + ]); + + $offset = $offset + $limit; + $count = $count + $sum; + + Console::log('Iterated through ' . $count . '/' . $totalProjects . ' projects...'); + } + + $this->sendEmail($register); + } + + private function sendEmail(Registry $register) + { + /** @var \PHPMailer\PHPMailer\PHPMailer $mail */ + $mail = $register->get('smtp'); + + try { + /** Addresses */ + $mail->setFrom(App::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM), 'Appwrite Cloud Hamster'); + $recipients = explode(',', App::getEnv('_APP_HAMSTER_RECIPIENTS', '')); + foreach ($recipients as $recipient) { + $mail->addAddress($recipient); + } + + /** Attachments */ + $mail->addAttachment($this->path); + + /** Content */ + $mail->Subject = "Cloud Report for {$this->date}"; + $mail->Body = "Please find the daily cloud report atttached"; + + $mail->send(); + Console::success('Email has been sent!'); + } catch (Exception $e) { + Console::error("Message could not be sent. Mailer Error: {$mail->ErrorInfo}"); + } + } +} diff --git a/src/Appwrite/Platform/Tasks/Stat.php b/src/Appwrite/Platform/Tasks/Stat.php deleted file mode 100644 index 8ac5687bae..0000000000 --- a/src/Appwrite/Platform/Tasks/Stat.php +++ /dev/null @@ -1,212 +0,0 @@ -desc('Get stats for project') - ->callback(fn () => $this->action()); - } - - function getConnection(string $dsn): PDO - { - if (empty($dsn)) { - throw new Exception("Missing value for DSN connection"); - } - $dsn = new DSN($dsn); - $dsnHost = $dsn->getHost(); - $dsnPort = $dsn->getPort(); - $dsnUser = $dsn->getUser(); - $dsnPass = $dsn->getPassword(); - $dsnScheme = $dsn->getScheme(); - $dsnDatabase = $dsn->getPath(); - - $connection = new PDO("mysql:host={$dsnHost};port={$dsnPort};dbname={$dsnDatabase};charset=utf8mb4", $dsnUser, $dsnPass, array( - PDO::ATTR_TIMEOUT => 3, // Seconds - PDO::ATTR_PERSISTENT => true, - PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, - PDO::ATTR_ERRMODE => PDO::ERRMODE_WARNING, - PDO::ATTR_EMULATE_PREPARES => true, - PDO::ATTR_STRINGIFY_FETCHES => true - )); - - return $connection; - } - - - function getStats(Database $dbForProject): array - { - $range = '90d'; - $periods = [ - '90d' => [ - 'period' => '1d', - 'limit' => 90, - ], - ]; - - $metrics = [ - 'files.$all.count.total', - 'buckets.$all.count.total', - 'databases.$all.count.total', - 'documents.$all.count.total', - 'collections.$all.count.total', - 'project.$all.storage.size', - 'project.$all.network.requests', - 'project.$all.network.bandwidth', - 'users.$all.count.total', - 'sessions.$all.requests.create', - 'executions.$all.compute.total', - ]; - - $stats = []; - - Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) { - foreach ($metrics as $metric) { - $limit = $periods[$range]['limit']; - $period = $periods[$range]['period']; - - $requestDocs = $dbForProject->find('stats', [ - Query::equal('period', [$period]), - Query::equal('metric', [$metric]), - Query::limit($limit), - Query::orderDesc('time'), - ]); - - $stats[$metric] = []; - foreach ($requestDocs as $requestDoc) { - $stats[$metric][] = [ - 'value' => $requestDoc->getAttribute('value'), - 'date' => $requestDoc->getAttribute('time'), - ]; - } - - $stats[$metric] = array_reverse($stats[$metric]); - // Calculate aggregate of each metric - $stats[$metric . '.sum'] = array_sum(array_column($stats[$metric], 'value')); - } - }); - - // return only the ahhggregate values - return array_filter($stats, fn ($key) => strpos($key, '.sum') !== false, ARRAY_FILTER_USE_KEY); - } - - - public function action(): void - { - Console::success('Getting stats...'); - - $databases = [ - 'console' => [ - 'type' => 'database', - 'dsns' => '', - 'multiple' => false, - 'schemes' => ['mariadb', 'mysql'], - ], - 'projects' => [ - 'type' => 'database', - 'dsns' => '', - 'multiple' => true, - 'schemes' => ['mariadb', 'mysql'], - ], - ]; - - $dsns = explode(',', $databases['projects']['dsns']); - $projectdsns = []; - foreach ($dsns as &$dsn) { - $dsn = explode('=', $dsn); - $name = 'database' . '_' . $dsn[0]; - $dsn = $dsn[1] ?? ''; - $projectdsns[$name] = $dsn; - } - - $cache = new Cache(new None()); - $consoledsn = explode('=', $databases['console']['dsns']); - $consoledsn = $consoledsn[1] ?? ''; - $adapter = new MySQL($this->getConnection($consoledsn)); - $dbForConsole = new Database($adapter, $cache); - $dbForConsole->setDefaultDatabase('appwrite'); - $dbForConsole->setNamespace('console'); - - $totalProjects = $dbForConsole->count('projects') + 1; - Console::success("Iterating through : {$totalProjects} projects"); - - $app = new App('UTC'); - $console = $app->getResource('console'); - - $projects = [$console]; - $count = 0; - $limit = 30; - $sum = 30; - $offset = 0; - - $stats = []; - - while (!empty($projects)) { - foreach ($projects as $project) { - /** - * Skip user projects with id 'console' - */ - if ($project->getId() === 'console') { - continue; - } - Console::info("Getting stats for {$project->getId()}"); - - try { - // TODO: Iterate through all project DBs - $db = $project->getAttribute('database'); - $dsn = $projectdsns[$db] ?? ''; - $cache = new Cache(new None()); - $adapter = new MySQL($this->getConnection($dsn)); - $dbForProject = new Database($adapter, $cache); - $dbForProject->setDefaultDatabase('appwrite'); - $dbForProject->setNamespace('_' . $project->getInternalId()); - $statsPerProject = $this->getStats($dbForProject); - - foreach ($statsPerProject as $key => $value) { - $stats[$key] = isset($stats[$key]) ? $stats[$key] + $value : $value; - } - - } catch (\Throwable $th) { - throw $th; - Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage()); - } - - } - - $sum = \count($projects); - - $projects = $dbForConsole->find('projects', [ - Query::limit($limit), - Query::offset($offset), - ]); - - $offset = $offset + $limit; - $count = $count + $sum; - - Console::log('Iterated through ' . $count . '/' . $totalProjects . ' projects...'); - } - - var_dump($stats); - } -} From a3cf72bc15d9d05cd271d907b1b37342200a37f8 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 26 Jan 2023 15:36:28 +0530 Subject: [PATCH 3/5] Update src/Appwrite/Platform/Tasks/Hamster.php MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matej Bačo --- src/Appwrite/Platform/Tasks/Hamster.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Tasks/Hamster.php b/src/Appwrite/Platform/Tasks/Hamster.php index 650afa7f94..cd85a0d5cc 100644 --- a/src/Appwrite/Platform/Tasks/Hamster.php +++ b/src/Appwrite/Platform/Tasks/Hamster.php @@ -65,7 +65,7 @@ class Hamster extends Action public function __construct() { $this - ->desc('Get stats for project') + ->desc('Get stats for projects') ->inject('register') ->inject('pools') ->inject('cache') From f6bef128e4391bbdc48c2da12c6946f33a50d9f3 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 26 Jan 2023 15:36:34 +0530 Subject: [PATCH 4/5] Update src/Appwrite/Platform/Tasks/Hamster.php MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matej Bačo --- src/Appwrite/Platform/Tasks/Hamster.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Tasks/Hamster.php b/src/Appwrite/Platform/Tasks/Hamster.php index cd85a0d5cc..17bbf99566 100644 --- a/src/Appwrite/Platform/Tasks/Hamster.php +++ b/src/Appwrite/Platform/Tasks/Hamster.php @@ -145,7 +145,7 @@ class Hamster extends Action public function action(Registry $register, Group $pools, Cache $cache, Database $dbForConsole): void { - Console::success('Getting stats...'); + Console::info'Getting stats...'); /* Initialise new Utopia app */ $app = new App('UTC'); From ac2e7efb3ce3cb5c7abd1cf09be2748b39e7cfbc Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 26 Jan 2023 16:29:24 +0530 Subject: [PATCH 5/5] feat: linter fixes --- src/Appwrite/Platform/Tasks/Hamster.php | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/Hamster.php b/src/Appwrite/Platform/Tasks/Hamster.php index 650afa7f94..bc03c8d824 100644 --- a/src/Appwrite/Platform/Tasks/Hamster.php +++ b/src/Appwrite/Platform/Tasks/Hamster.php @@ -140,7 +140,7 @@ class Hamster extends Action } }); - return $stats; + return $stats; } public function action(Registry $register, Group $pools, Cache $cache, Database $dbForConsole): void @@ -174,7 +174,7 @@ class Hamster extends Action if ($project->getId() === 'console') { continue; } - + Console::info("Getting stats for {$project->getId()}"); try { @@ -190,7 +190,6 @@ class Hamster extends Action $statsPerProject = $this->getStats($dbForConsole, $dbForProject, $project); $csv->insertOne(array_values($statsPerProject)); - } catch (\Throwable $th) { throw $th; Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage());